--- a
+++ b/Roberta+LLM/mistral7b-full-evaluation.ipynb
@@ -0,0 +1,3339 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "view-in-github",
+        "colab_type": "text"
+      },
+      "source": [
+        "<a href=\"https://colab.research.google.com/github/jlopetegui98/NER-ClinicalTrials-Eligibility-Criteria/blob/main/Roberta%2BLLM/mistral7b-full-evaluation.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 1,
+      "metadata": {
+        "id": "GREqJcq4OLO6",
+        "outputId": "511e84c9-0109-411a-b44e-0801766d0fe9",
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        }
+      },
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
+          ]
+        }
+      ],
+      "source": [
+        "# uncomment if working in colab\n",
+        "from google.colab import drive\n",
+        "drive.mount('/content/drive')"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 2,
+      "metadata": {
+        "id": "ICZaolBLOLO8",
+        "outputId": "c8c38b1c-e0e3-4258-cc8b-d7d684d9d089",
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        }
+      },
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "Requirement already satisfied: seqeval in /usr/local/lib/python3.10/dist-packages (1.2.2)\n",
+            "Requirement already satisfied: numpy>=1.14.0 in /usr/local/lib/python3.10/dist-packages (from seqeval) (1.25.2)\n",
+            "Requirement already satisfied: scikit-learn>=0.21.3 in /usr/local/lib/python3.10/dist-packages (from seqeval) (1.2.2)\n",
+            "Requirement already satisfied: scipy>=1.3.2 in /usr/local/lib/python3.10/dist-packages (from scikit-learn>=0.21.3->seqeval) (1.11.4)\n",
+            "Requirement already satisfied: joblib>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from scikit-learn>=0.21.3->seqeval) (1.4.2)\n",
+            "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn>=0.21.3->seqeval) (3.5.0)\n",
+            "Requirement already satisfied: torch in /usr/local/lib/python3.10/dist-packages (2.3.0)\n",
+            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from torch) (3.14.0)\n",
+            "Requirement already satisfied: typing-extensions>=4.8.0 in /usr/local/lib/python3.10/dist-packages (from torch) (4.11.0)\n",
+            "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch) (1.12)\n",
+            "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch) (3.3)\n",
+            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch) (3.1.4)\n",
+            "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from torch) (2023.6.0)\n",
+            "Requirement already satisfied: nvidia-cuda-nvrtc-cu12==12.1.105 in /usr/local/lib/python3.10/dist-packages (from torch) (12.1.105)\n",
+            "Requirement already satisfied: nvidia-cuda-runtime-cu12==12.1.105 in /usr/local/lib/python3.10/dist-packages (from torch) (12.1.105)\n",
+            "Requirement already satisfied: nvidia-cuda-cupti-cu12==12.1.105 in /usr/local/lib/python3.10/dist-packages (from torch) (12.1.105)\n",
+            "Requirement already satisfied: nvidia-cudnn-cu12==8.9.2.26 in /usr/local/lib/python3.10/dist-packages (from torch) (8.9.2.26)\n",
+            "Requirement already satisfied: nvidia-cublas-cu12==12.1.3.1 in /usr/local/lib/python3.10/dist-packages (from torch) (12.1.3.1)\n",
+            "Requirement already satisfied: nvidia-cufft-cu12==11.0.2.54 in /usr/local/lib/python3.10/dist-packages (from torch) (11.0.2.54)\n",
+            "Requirement already satisfied: nvidia-curand-cu12==10.3.2.106 in /usr/local/lib/python3.10/dist-packages (from torch) (10.3.2.106)\n",
+            "Requirement already satisfied: nvidia-cusolver-cu12==11.4.5.107 in /usr/local/lib/python3.10/dist-packages (from torch) (11.4.5.107)\n",
+            "Requirement already satisfied: nvidia-cusparse-cu12==12.1.0.106 in /usr/local/lib/python3.10/dist-packages (from torch) (12.1.0.106)\n",
+            "Requirement already satisfied: nvidia-nccl-cu12==2.20.5 in /usr/local/lib/python3.10/dist-packages (from torch) (2.20.5)\n",
+            "Requirement already satisfied: nvidia-nvtx-cu12==12.1.105 in /usr/local/lib/python3.10/dist-packages (from torch) (12.1.105)\n",
+            "Requirement already satisfied: triton==2.3.0 in /usr/local/lib/python3.10/dist-packages (from torch) (2.3.0)\n",
+            "Requirement already satisfied: nvidia-nvjitlink-cu12 in /usr/local/lib/python3.10/dist-packages (from nvidia-cusolver-cu12==11.4.5.107->torch) (12.4.127)\n",
+            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch) (2.1.5)\n",
+            "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->torch) (1.3.0)\n",
+            "  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
+            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
+            "  Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
+            "  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
+            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
+            "  Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
+            "  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
+            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
+            "  Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n"
+          ]
+        }
+      ],
+      "source": [
+        "# uncomment if using colab\n",
+        "!pip install -q -U datasets\n",
+        "!pip install seqeval\n",
+        "!pip install -U torch\n",
+        "!pip install -q -U evaluate\n",
+        "!pip install -q -U git+https://github.com/huggingface/transformers.git\n",
+        "# !pip install -q -U transformers\n",
+        "!pip install -q -U bitsandbytes\n",
+        "!pip install -q -U git+https://github.com/huggingface/peft.git\n",
+        "!pip install -q -U git+https://github.com/huggingface/accelerate.git\n",
+        "# !pip install -q -U accelerate"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 2,
+      "metadata": {
+        "id": "Km86o_uHOLO8"
+      },
+      "outputs": [],
+      "source": [
+        "import torch\n",
+        "from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig, TextGenerationPipeline\n",
+        "import os\n",
+        "from utils import *"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "import pandas as pd\n",
+        "from datasets import Dataset, DatasetDict, load_dataset"
+      ],
+      "metadata": {
+        "id": "L0zkxHTmOxkJ"
+      },
+      "execution_count": 3,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "import re"
+      ],
+      "metadata": {
+        "id": "uoWkyegpPCOZ"
+      },
+      "execution_count": 4,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 5,
+      "metadata": {
+        "id": "L0zE4thvOLO9",
+        "outputId": "75716581-4307-4d98-fddf-887bc825e808",
+        "colab": {
+          "base_uri": "https://localhost:8080/",
+          "height": 145,
+          "referenced_widgets": [
+            "009541513b8e451886782e457d7cd479",
+            "519375173fe64735a4faccf127abcf90",
+            "484017270bb646119bc605fccb88fa4e",
+            "23dc47bf61314784b7ffb42d995bbce9",
+            "7fdc12ed98be4101a7ff8d15272d5033",
+            "683aa055aae842adabca4ee3fa9a16bf",
+            "6ab04465b07747a592cb28f77c6b5e20",
+            "2379403cd2df4ffd9f7d4eecbf571d7e",
+            "382eff5973b942678fde4a9c6e9c8150",
+            "aec5500c07b54365ab252b5746e08127",
+            "86ed1786dfcc49148bc65e88627a34f3",
+            "36b52f9261ae4c589efc8faab5806509",
+            "8c0ca5e87a1243ba94d821d87dfd7cc1",
+            "1d62a5d167c04e839152933688357dbe",
+            "20614482956a4d359d7e6470fe3b5236",
+            "b46a3ac13be24c04a0c47d424e54d256",
+            "94885d557c64483bbcb842c570e380a5",
+            "0ca2c9f7ad19437886ef4a9aa74cf952",
+            "50557c598d9246599d7e894137d2df26",
+            "ab9a60b256554da0a101dc9d541cbb8c",
+            "523cf83bffdc4d8bbe06c9d886e5d884",
+            "0a508a2d17d04b76aa6bad327e25da96",
+            "7fd41498666942149a04c076f61cb6ad",
+            "32cfc02171c1420599bbbeca98ebfc3b",
+            "c72717e7d2714c7aa695cf5b6ba2481c",
+            "45c84a726d5e470babfb03cd5b139de7",
+            "ec182a1a030c46ac87b2fa998ea9d868",
+            "22972fbed4cc4b408be8abe9f9bb2847",
+            "09fc71a94b2f4766a02b5879ba232e95",
+            "da47463752ff4288a73b1802d37941aa",
+            "9e89840302da47e3a383888883bbadea",
+            "5ecf0ca50841453d85a082217c033fae"
+          ]
+        }
+      },
+      "outputs": [
+        {
+          "output_type": "display_data",
+          "data": {
+            "text/plain": [
+              "VBox(children=(HTML(value='<center> <img\\nsrc=https://huggingface.co/front/assets/huggingface_logo-noborder.sv…"
+            ],
+            "application/vnd.jupyter.widget-view+json": {
+              "version_major": 2,
+              "version_minor": 0,
+              "model_id": "009541513b8e451886782e457d7cd479"
+            }
+          },
+          "metadata": {}
+        }
+      ],
+      "source": [
+        "from huggingface_hub import notebook_login\n",
+        "\n",
+        "notebook_login()"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 6,
+      "metadata": {
+        "id": "6DQED9SWOLO9"
+      },
+      "outputs": [],
+      "source": [
+        "# dict for the entities (entity to int value)\n",
+        "simple_ent = {\"Condition\", \"Value\", \"Drug\", \"Procedure\", \"Measurement\", \"Temporal\", \"Observation\", \"Person\", \"Device\"}\n",
+        "sel_ent = {\n",
+        "    \"O\": 0,\n",
+        "    \"B-Condition\": 1,\n",
+        "    \"I-Condition\": 2,\n",
+        "    \"B-Value\": 3,\n",
+        "    \"I-Value\": 4,\n",
+        "    \"B-Drug\": 5,\n",
+        "    \"I-Drug\": 6,\n",
+        "    \"B-Procedure\": 7,\n",
+        "    \"I-Procedure\": 8,\n",
+        "    \"B-Measurement\": 9,\n",
+        "    \"I-Measurement\": 10,\n",
+        "    \"B-Temporal\": 11,\n",
+        "    \"I-Temporal\": 12,\n",
+        "    \"B-Observation\": 13,\n",
+        "    \"I-Observation\": 14,\n",
+        "    \"B-Person\": 15,\n",
+        "    \"I-Person\": 16,\n",
+        "    \"B-Device\": 17,\n",
+        "    \"I-Device\": 18\n",
+        "}\n",
+        "\n",
+        "entities_list = list(sel_ent.keys())\n",
+        "sel_ent_inv = {v: k for k, v in sel_ent.items()}"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 7,
+      "metadata": {
+        "id": "fLKSRg5XOLO9"
+      },
+      "outputs": [],
+      "source": [
+        "root = '..'\n",
+        "root = './drive/MyDrive/TER-LISN-2024'\n",
+        "data_path = f'{root}/data'\n",
+        "models_path = f'{root}/models'"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 8,
+      "metadata": {
+        "id": "ZZnHifTsOLO9"
+      },
+      "outputs": [],
+      "source": [
+        "model_name = \"mistralai/Mistral-7B-v0.1\""
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 9,
+      "metadata": {
+        "id": "ytfHR4b2OLO-",
+        "outputId": "7f66d4d3-1867-4d9f-c298-42b2368c50df",
+        "colab": {
+          "base_uri": "https://localhost:8080/",
+          "height": 173,
+          "referenced_widgets": [
+            "1ca11a0dedb44e6897fd914763d6731d",
+            "b418a6bbce1346bd9c0076f2d31bad26",
+            "b7975e8109d74252ae427c2507b70c2c",
+            "f0af7ea71ed14275bb36b55c87d6c785",
+            "ddb109aa9fed447b9e8929f2ce49b8d2",
+            "6e57a33e6f4641bd85d68b6773657962",
+            "901e5be78e494518924d3237c3398b09",
+            "4fdf171fbb704213a0df598adb0c918c",
+            "40c20831011e4706aedbe2ed73219dfd",
+            "83849f3935464b2ab0f77925aab9d7f3",
+            "e7c31cc6d3a64a4fb911f3d2023ecfb1"
+          ]
+        }
+      },
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stderr",
+          "text": [
+            "/usr/local/lib/python3.10/dist-packages/huggingface_hub/utils/_token.py:89: UserWarning: \n",
+            "The secret `HF_TOKEN` does not exist in your Colab secrets.\n",
+            "To authenticate with the Hugging Face Hub, create a token in your settings tab (https://huggingface.co/settings/tokens), set it as secret in your Google Colab and restart your session.\n",
+            "You will be able to reuse this secret in all of your notebooks.\n",
+            "Please note that authentication is recommended but still optional to access public models or datasets.\n",
+            "  warnings.warn(\n"
+          ]
+        },
+        {
+          "output_type": "display_data",
+          "data": {
+            "text/plain": [
+              "Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]"
+            ],
+            "application/vnd.jupyter.widget-view+json": {
+              "version_major": 2,
+              "version_minor": 0,
+              "model_id": "1ca11a0dedb44e6897fd914763d6731d"
+            }
+          },
+          "metadata": {}
+        }
+      ],
+      "source": [
+        "# Load base model(Mistral 7B)\n",
+        "bnb_config = BitsAndBytesConfig(\n",
+        "    load_in_4bit= True,\n",
+        "    bnb_4bit_quant_type= \"nf4\",\n",
+        "    bnb_4bit_compute_dtype= torch.bfloat16,\n",
+        "    bnb_4bit_use_double_quant= False,\n",
+        ")\n",
+        "model = AutoModelForCausalLM.from_pretrained(\n",
+        "   model_name,\n",
+        "    quantization_config=bnb_config,\n",
+        "    device_map={\"\": 0}\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 10,
+      "metadata": {
+        "id": "ageLiTUKOLO-",
+        "outputId": "8f9850bc-2dd1-4230-9b0d-62503154247e",
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        }
+      },
+      "outputs": [
+        {
+          "output_type": "execute_result",
+          "data": {
+            "text/plain": [
+              "(True, False)"
+            ]
+          },
+          "metadata": {},
+          "execution_count": 10
+        }
+      ],
+      "source": [
+        "# import tokenizer for mistral-7B\n",
+        "tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)\n",
+        "tokenizer.padding_side = 'left'\n",
+        "tokenizer.pad_token = tokenizer.eos_token\n",
+        "tokenizer.add_eos_token = False\n",
+        "tokenizer.add_bos_token, tokenizer.add_eos_token"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 11,
+      "metadata": {
+        "id": "qMfZMKhlOLO-"
+      },
+      "outputs": [],
+      "source": [
+        "pipe  = TextGenerationPipeline(model = model, tokenizer = tokenizer)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 12,
+      "metadata": {
+        "id": "E1T2wFahOLO-"
+      },
+      "outputs": [],
+      "source": [
+        "dataset = load_dataset('JavierLopetegui/chia_v1')"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 13,
+      "metadata": {
+        "id": "U37CsADiOLO-"
+      },
+      "outputs": [],
+      "source": [
+        "# for each sentence save the text\n",
+        "def generate_sentences_from_tokens(sentences):\n",
+        "    texts_sentences = []\n",
+        "    sentences_tokens = sentences['tokens']\n",
+        "    for sentence in sentences_tokens:\n",
+        "        sent_text = \" \".join(sentence)\n",
+        "        texts_sentences.append(sent_text)\n",
+        "    sentences['text'] = texts_sentences\n",
+        "    return sentences"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 14,
+      "metadata": {
+        "id": "423oQ2BQOLO-"
+      },
+      "outputs": [],
+      "source": [
+        "def build_prompts(sentences, prompt_type=2):\n",
+        "    sentences_prompts = []\n",
+        "    for sent in sentences['text']:\n",
+        "        prompt = build_prompt(sent, prompt_type)\n",
+        "        sentences_prompts.append(prompt)\n",
+        "    sentences['prompt'] = sentences_prompts\n",
+        "    return sentences"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 15,
+      "metadata": {
+        "id": "kC1PSUY9OLO_"
+      },
+      "outputs": [],
+      "source": [
+        "dataset = dataset.map(lambda x: generate_sentences_from_tokens(x), batched = True)\n",
+        "dataset_prompt2 = dataset.map(lambda x: build_prompts(x, prompt_type=2), batched = True)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 16,
+      "metadata": {
+        "id": "IEyo7EiUOLO_"
+      },
+      "outputs": [],
+      "source": [
+        "test_dataset_p2 = dataset_prompt2['test']"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 17,
+      "metadata": {
+        "id": "47SUjSaWOLO_"
+      },
+      "outputs": [],
+      "source": [
+        "# keep just the prompt column\n",
+        "test_dataset_p2 = test_dataset_p2.remove_columns(['tokens', 'text', 'ner_tags', 'file'])"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "SAJgHjqnOLO_"
+      },
+      "outputs": [],
+      "source": [
+        "# data_loader_p2 = DataLoader(test_dataset_p2, batch_size=4, shuffle=False)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ps3MQNmIOLO_"
+      },
+      "outputs": [],
+      "source": [
+        "# generated_sentences_p2 = pipe(batch['prompt'], max_new_tokens = 500, return_full_text = False, handle_long_generation = \"hole\"))"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "from tqdm import tqdm"
+      ],
+      "metadata": {
+        "id": "VrXXsuAsSGVR"
+      },
+      "execution_count": 18,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "print(test_dataset_p2['prompt'][0])"
+      ],
+      "metadata": {
+        "id": "pULuqZPEB5Y4",
+        "outputId": "75d27b13-4396-4232-b8af-ca5aad7c3b5e",
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        }
+      },
+      "execution_count": 59,
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "I am working on a named entity recognition problem, in the context of clinical\n",
+            "    trials eligibility criteria. I will show you the list of entities:\n",
+            "    - Condition\n",
+            "    - Value\n",
+            "    - Drug\n",
+            "    - Procedure\n",
+            "    - Measurement\n",
+            "    - Temporal\n",
+            "    - Observation\n",
+            "    - Person\n",
+            "    - Device\n",
+            "\n",
+            "    Your task consists in annotate the named entities in a given sentence in the format I will explain you.\n",
+            "    I will explain you with some examples:\n",
+            "\n",
+            "    Example 1:\n",
+            "    Input: Patients who have received prior chemotherapy for unresectable disease.\n",
+            "    Output: Patients who have received prior <Procedure>chemotherapy</Procedure> for <Condition>unresectable disease</Condition>.\n",
+            "\n",
+            "    Example 2:\n",
+            "    Input: Patients with any other severe concurrent disease, which in the judgment of the investigator, would make the patient inappropriate for entry into this study.\n",
+            "    Ouput: Patients with any other severe <Condition>concurrent disease</Condition>, which in the judgment of the investigator, would make the patient inappropriate for <Observation>entry into this study</Observation>.\n",
+            "\n",
+            "    As you can see, in each example, the extracted entities are enclosed using the sintax: <ENT>text of the entity</ENT>.\n",
+            "\n",
+            "    Please now annotate as explained before the following sentence:\n",
+            "\n",
+            "    Input: self - reported healthy adults between the ages of 18 - 60 who are fluent in English .\n"
+          ]
+        }
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 88,
+      "metadata": {
+        "id": "QwPSO9NvOLO_",
+        "outputId": "b9b7841d-781a-47bf-92f0-a32c080343d1",
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        }
+      },
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stderr",
+          "text": [
+            "100%|██████████| 1307/1307 [15:16:55<00:00, 42.09s/it]\n"
+          ]
+        }
+      ],
+      "source": [
+        "generated_sentences_p2 = []\n",
+        "for sentence in tqdm(test_dataset_p2['prompt']):\n",
+        "    sentence += '\\noutput: '\n",
+        "    output = pipe(sentence, max_new_tokens = 500, return_full_text = False, handle_long_generation = \"hole\")[0]['generated_text']\n",
+        "    output = output.split('\\n')[0]\n",
+        "    generated_sentences_p2.append(output)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "len(generated_sentences_p2)"
+      ],
+      "metadata": {
+        "id": "uTSOYC9UhKxw",
+        "outputId": "3799dd2c-4f62-4856-a407-0825469f89d4",
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        }
+      },
+      "execution_count": 89,
+      "outputs": [
+        {
+          "output_type": "execute_result",
+          "data": {
+            "text/plain": [
+              "1307"
+            ]
+          },
+          "metadata": {},
+          "execution_count": 89
+        }
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 90,
+      "metadata": {
+        "id": "x0L_xzJXOLO_"
+      },
+      "outputs": [],
+      "source": [
+        "tokenizer = AutoTokenizer.from_pretrained('xlm-roberta-base')"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 91,
+      "metadata": {
+        "id": "6i9B0a6_OLO_"
+      },
+      "outputs": [],
+      "source": [
+        "# tokenize and align the labels in the dataset\n",
+        "def tokenize_and_align_labels(sentence, flag = 'I'):\n",
+        "    \"\"\"\n",
+        "    Tokenize the sentence and align the labels\n",
+        "    inputs:\n",
+        "        sentence: dict, the sentence from the dataset\n",
+        "        flag: str, the flag to indicate how to deal with the labels for subwords\n",
+        "            - 'I': use the label of the first subword for all subwords but as intermediate (I-ENT)\n",
+        "            - 'B': use the label of the first subword for all subwords as beginning (B-ENT)\n",
+        "            - None: use -100 for subwords\n",
+        "    outputs:\n",
+        "        tokenized_sentence: dict, the tokenized sentence now with a field for the labels\n",
+        "    \"\"\"\n",
+        "    tokenized_sentence = tokenizer(sentence['tokens'], is_split_into_words=True, truncation=True)\n",
+        "\n",
+        "    labels = []\n",
+        "    for i, labels_s in enumerate(sentence['ner_tags']):\n",
+        "        word_ids = tokenized_sentence.word_ids(batch_index=i)\n",
+        "        previous_word_idx = None\n",
+        "        label_ids = []\n",
+        "        for word_idx in word_ids:\n",
+        "            # if the word_idx is None, assign -100\n",
+        "            if word_idx is None:\n",
+        "                label_ids.append(-100)\n",
+        "            # if it is a new word, assign the corresponding label\n",
+        "            elif word_idx != previous_word_idx:\n",
+        "                label_ids.append(labels_s[word_idx])\n",
+        "            # if it is the same word, check the flag to assign\n",
+        "            else:\n",
+        "                if flag == 'I':\n",
+        "                    if entities_list[labels_s[word_idx]].startswith('I'):\n",
+        "                      label_ids.append(labels_s[word_idx])\n",
+        "                    else:\n",
+        "                      label_ids.append(labels_s[word_idx] + 1)\n",
+        "                elif flag == 'B':\n",
+        "                    label_ids.append(labels_s[word_idx])\n",
+        "                elif flag == None:\n",
+        "                    label_ids.append(-100)\n",
+        "            previous_word_idx = word_idx\n",
+        "        labels.append(label_ids)\n",
+        "    tokenized_sentence['labels'] = labels\n",
+        "    return tokenized_sentence"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ZpemgZm_OLPA"
+      },
+      "source": [
+        "**Standarizing true annotations**"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 92,
+      "metadata": {
+        "id": "-bbV-kgyOLPB"
+      },
+      "outputs": [],
+      "source": [
+        "def parse_ann2bio(sentence, pattern, pattern1, pattern2):\n",
+        "    # if sentence[-1] == \"\\n\":\n",
+        "    #     sentence = sentence[:-2] # remove the \\n and a final point wrongly added\n",
+        "    # else:\n",
+        "    #     sentence = sentence[:-1] # remove the final point wrongly added\n",
+        "\n",
+        "    # find the entities\n",
+        "    occurrences = re.finditer(pattern, sentence)\n",
+        "    indexes = [(match.start(), match.end()) for match in occurrences]\n",
+        "\n",
+        "    annotation = []\n",
+        "    i = 0\n",
+        "\n",
+        "\n",
+        "    # create the bio list\n",
+        "    for beg, end in indexes:\n",
+        "        if beg > i:\n",
+        "            annotation.extend([(word, \"O\") for word in sentence[i:beg].split()])\n",
+        "        entity = sentence[beg:end]\n",
+        "        entity_name = re.search(pattern1, entity).group(1)\n",
+        "        entity = entity.replace(f'<{entity_name}>', \"\").replace(f'</{entity_name}>', \"\")\n",
+        "        split_entity = entity.split()\n",
+        "        annotation.append((split_entity[0], \"B-\" + entity_name))\n",
+        "        annotation.extend([(word, \"I-\" + entity_name) for word in split_entity[1:]])\n",
+        "        i = end\n",
+        "    annotation.extend([(word, \"O\") for word in sentence[i:].split()])\n",
+        "\n",
+        "    # check punctuation sign in tokens and put them as individual tokens\n",
+        "    ps = r'(\\.|\\,|\\:|\\;|\\!|\\?|\\-|\\(|\\)|\\[|\\]|\\{|\\}|\\\")'\n",
+        "    new_annotation = []\n",
+        "    for i,(word, tag) in enumerate(annotation):\n",
+        "        if re.search(ps, word):\n",
+        "            # find the ocurrences of the punctuation signs\n",
+        "            occurrences = re.finditer(ps, word)\n",
+        "            indexes = [(match.start(), match.end()) for match in occurrences]\n",
+        "            # create the new tokens\n",
+        "            last = 0\n",
+        "            for j, (beg, end) in enumerate(indexes):\n",
+        "                if beg > last:\n",
+        "                    new_annotation.append((word[last:beg], tag))\n",
+        "                if tag != \"O\":\n",
+        "                    label = f'I-{tag.split(\"-\")[1]}'\n",
+        "                else:\n",
+        "                    label = \"O\"\n",
+        "                if end < len(word) or (i < len(annotation) - 1 and annotation[i+1][1] == label):\n",
+        "                    new_annotation.append((word[beg:end], label))\n",
+        "                else:\n",
+        "                    new_annotation.append((word[beg:end], 'O'))\n",
+        "                last = end\n",
+        "            if last < len(word):\n",
+        "                new_annotation.append((word[last:], label))\n",
+        "\n",
+        "        else:\n",
+        "            new_annotation.append((word, tag))\n",
+        "\n",
+        "\n",
+        "    return new_annotation"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "new_true_annotations = []\n",
+        "for sent in dataset['test']:\n",
+        "    annotation = []\n",
+        "    for word, tag in zip(sent['tokens'], sent['ner_tags']):\n",
+        "        annotation.append((word, entities_list[tag]))\n",
+        "    new_annotation = []\n",
+        "    ps = r'(\\.|\\,|\\:|\\;|\\!|\\?|\\-|\\(|\\)|\\[|\\]|\\{|\\}|\\\")'\n",
+        "    for i,(word, tag) in enumerate(annotation):\n",
+        "        if re.search(ps, word):\n",
+        "            # find the ocurrences of the punctuation signs\n",
+        "            occurrences = re.finditer(ps, word)\n",
+        "            indexes = [(match.start(), match.end()) for match in occurrences]\n",
+        "            # create the new tokens\n",
+        "            last = 0\n",
+        "            for j, (beg, end) in enumerate(indexes):\n",
+        "                if beg > last:\n",
+        "                    new_annotation.append((word[last:beg], tag))\n",
+        "                if tag != \"O\":\n",
+        "                    label = f'I-{tag.split(\"-\")[1]}'\n",
+        "                else:\n",
+        "                    label = \"O\"\n",
+        "                if end < len(word) or (i < len(annotation) - 1 and annotation[i+1][1] == label):\n",
+        "                    new_annotation.append((word[beg:end], label))\n",
+        "                else:\n",
+        "                    new_annotation.append((word[beg:end], 'O'))\n",
+        "                last = end\n",
+        "            if last < len(word):\n",
+        "                new_annotation.append((word[last:], label))\n",
+        "        else:\n",
+        "            new_annotation.append((word, tag))\n",
+        "    new_true_annotations.append(new_annotation)\n",
+        "len(new_true_annotations)"
+      ],
+      "metadata": {
+        "id": "Kgi4KQythezT",
+        "outputId": "51229f7b-ae1e-4b5c-d1dc-b477a365ab45",
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        }
+      },
+      "execution_count": 93,
+      "outputs": [
+        {
+          "output_type": "execute_result",
+          "data": {
+            "text/plain": [
+              "1307"
+            ]
+          },
+          "metadata": {},
+          "execution_count": 93
+        }
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 94,
+      "metadata": {
+        "id": "ng4AD9cxOLPB",
+        "outputId": "3c9b6d49-4b24-4377-8017-4dbc7aba9a3f",
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        }
+      },
+      "outputs": [
+        {
+          "output_type": "execute_result",
+          "data": {
+            "text/plain": [
+              "1307"
+            ]
+          },
+          "metadata": {},
+          "execution_count": 94
+        }
+      ],
+      "source": [
+        "true_annotations = []\n",
+        "for sent in new_true_annotations:\n",
+        "    dicc_sent = {\"tokens\":[], \"ner_tags\":[]}\n",
+        "    for word, tag in sent:\n",
+        "        dicc_sent[\"tokens\"].append(word)\n",
+        "        dicc_sent[\"ner_tags\"].append(sel_ent[tag])\n",
+        "    true_annotations.append(dicc_sent)\n",
+        "len(true_annotations)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 95,
+      "metadata": {
+        "id": "T3gzF1mvOLPB"
+      },
+      "outputs": [],
+      "source": [
+        "true_df = pd.DataFrame(true_annotations)\n",
+        "true_ann_dataset = Dataset.from_pandas(true_df)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 96,
+      "metadata": {
+        "id": "gtFN5ix8OLPB",
+        "outputId": "e2ccd19f-b58b-4a4a-c7e1-c3f118cda808",
+        "colab": {
+          "base_uri": "https://localhost:8080/",
+          "height": 49,
+          "referenced_widgets": [
+            "4a35141fd2544fd08d1c5748a16c28f6",
+            "d6e6a89659834daa88bcef28c1d5d220",
+            "b4f30646efe04849afd1b45f50dd3931",
+            "9a90159ea82b4c3f921839ea7b6f9c86",
+            "8698beb6d78342caaa49ea61c9ebcc58",
+            "5562bbdb7f064ba4867374297f2c7ef4",
+            "f5f58531d6474340a7e5d4ccdc171969",
+            "57e9591c1f1a4be58f8a826059e27d05",
+            "e52d41b0f7084b83abfc2dce289e9757",
+            "49662c4803d846d6bb15e8ba44e4b818",
+            "a09956aac8fb4447b241f1dc646c6429"
+          ]
+        }
+      },
+      "outputs": [
+        {
+          "output_type": "display_data",
+          "data": {
+            "text/plain": [
+              "Map:   0%|          | 0/1307 [00:00<?, ? examples/s]"
+            ],
+            "application/vnd.jupyter.widget-view+json": {
+              "version_major": 2,
+              "version_minor": 0,
+              "model_id": "4a35141fd2544fd08d1c5748a16c28f6"
+            }
+          },
+          "metadata": {}
+        }
+      ],
+      "source": [
+        "true_ann_dataset = true_ann_dataset.map(tokenize_and_align_labels, batched=True)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 145,
+      "metadata": {
+        "id": "qhcwK4tTOLPB"
+      },
+      "outputs": [],
+      "source": [
+        "def get_labels(p):\n",
+        "    predictions, labels = p\n",
+        "    # Remove ignored index (special tokens)\n",
+        "    # predictions = [\n",
+        "    #     [entities_list[p] for (p, l) in zip(prediction, label) if l != -100]\n",
+        "    #     for prediction, label in zip(predictions, labels)\n",
+        "    # ]\n",
+        "    predictions_ = []\n",
+        "    for (pred,lab) in zip(predictions, labels):\n",
+        "        predictions_.append([])\n",
+        "        for (p,l) in zip(pred, lab):\n",
+        "          if l != -100:\n",
+        "            if p == -100:\n",
+        "              predictions_[-1].append(entities_list[0])\n",
+        "            else:\n",
+        "              predictions_[-1].append(entities_list[p])\n",
+        "\n",
+        "    labels = [\n",
+        "        [entities_list[l] for (p, l) in zip(prediction, label) if l != -100]\n",
+        "        for prediction, label in zip(predictions, labels)\n",
+        "    ]\n",
+        "\n",
+        "    return predictions_, labels"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 98,
+      "metadata": {
+        "id": "kZqPzsPEOLPB"
+      },
+      "outputs": [],
+      "source": [
+        "from eval_file import *"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ICsodrKvOLPB"
+      },
+      "source": [
+        "**Evaluating prompt 2**"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 99,
+      "metadata": {
+        "id": "ox3BUHhrOLPC"
+      },
+      "outputs": [],
+      "source": [
+        "pattern1 = r'<(Person|Condition|Value|Drug|Procedure|Measurement|Temporal|Observation|Device)>'\n",
+        "pattern2 = r'</(Person|Condition|Value|Drug|Procedure|Measurement|Temporal|Observation|Device)>'\n",
+        "pattern = f'{pattern1}.*?{pattern2}'"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 100,
+      "metadata": {
+        "id": "NF6iDWPeOLPC",
+        "outputId": "cad977da-3c36-43ec-9bfa-482971505810",
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        }
+      },
+      "outputs": [
+        {
+          "output_type": "execute_result",
+          "data": {
+            "text/plain": [
+              "1307"
+            ]
+          },
+          "metadata": {},
+          "execution_count": 100
+        }
+      ],
+      "source": [
+        "new_p2_annotations = []\n",
+        "for sent in generated_sentences_p2:\n",
+        "    annotation = parse_ann2bio(sent, pattern, pattern1, pattern2)\n",
+        "    new_p2_annotations.append(annotation)\n",
+        "len(new_p2_annotations)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 101,
+      "metadata": {
+        "id": "kRxCaGoFOLPC",
+        "outputId": "6e580bb9-a3ca-40d6-8d23-3414e2da292f",
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        }
+      },
+      "outputs": [
+        {
+          "output_type": "execute_result",
+          "data": {
+            "text/plain": [
+              "1307"
+            ]
+          },
+          "metadata": {},
+          "execution_count": 101
+        }
+      ],
+      "source": [
+        "p2_annotations = []\n",
+        "for sent in new_p2_annotations:\n",
+        "    dicc_sent = {\"tokens\":[], \"ner_tags\":[]}\n",
+        "    for word, tag in sent:\n",
+        "        dicc_sent[\"tokens\"].append(word)\n",
+        "        dicc_sent[\"ner_tags\"].append(sel_ent[tag])\n",
+        "    p2_annotations.append(dicc_sent)\n",
+        "len(p2_annotations)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 102,
+      "metadata": {
+        "id": "JJapA5HhOLPC"
+      },
+      "outputs": [],
+      "source": [
+        "p2_df = pd.DataFrame(p2_annotations)\n",
+        "p2_dataset = Dataset.from_pandas(p2_df)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 103,
+      "metadata": {
+        "id": "Wh2C3LSmOLPC",
+        "outputId": "ebd4d143-619c-44f1-e84e-8a06cc2d0ef0",
+        "colab": {
+          "base_uri": "https://localhost:8080/",
+          "height": 49,
+          "referenced_widgets": [
+            "162bfe1f36444ce1861ceaf2c8f6a3f3",
+            "d33896d2c9c546e1b9d5cc12bb866d51",
+            "af607d56eb2c4de88e84315ebf4e5bed",
+            "0f469022363b4b0d96762dc3aafebffe",
+            "dc63518146884067aaebf37d682e83b5",
+            "676f0e00bfba466ea183f41ba064ff95",
+            "91796acd865b49e39500cbc7fdfa2e1e",
+            "48df188dce924c32b510d1ceb4d2450c",
+            "78810abaada343a998519b4cb128bf3a",
+            "5f6c93a23a55413f86aa4778e7d2a6c3",
+            "9b21d0ab5e874bf3aa723bb2114eef80"
+          ]
+        }
+      },
+      "outputs": [
+        {
+          "output_type": "display_data",
+          "data": {
+            "text/plain": [
+              "Map:   0%|          | 0/1307 [00:00<?, ? examples/s]"
+            ],
+            "application/vnd.jupyter.widget-view+json": {
+              "version_major": 2,
+              "version_minor": 0,
+              "model_id": "162bfe1f36444ce1861ceaf2c8f6a3f3"
+            }
+          },
+          "metadata": {}
+        }
+      ],
+      "source": [
+        "p2_dataset = p2_dataset.map(tokenize_and_align_labels, batched=True)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 104,
+      "metadata": {
+        "id": "wu_XrafjOLPC",
+        "outputId": "b9bd9c26-892e-45e8-eb58-b24c8e760d3f",
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        }
+      },
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "1.0\n"
+          ]
+        }
+      ],
+      "source": [
+        "# keep just sentences with the same length\n",
+        "sentences_to_evaluate_p2 = []\n",
+        "sentences_to_evaluate_true = []\n",
+        "\n",
+        "for i in range(len(p2_dataset)):\n",
+        "    # keep the min len among the true and annotated sentence\n",
+        "    l = min(len(p2_dataset['labels'][i]), len(true_ann_dataset['labels'][i]))\n",
+        "    sentences_to_evaluate_p2.append(p2_dataset['labels'][i][:l])\n",
+        "    sentences_to_evaluate_true.append(true_ann_dataset['labels'][i][:l])\n",
+        "    # if len(p2_dataset['labels'][i]) != len(true_ann_dataset['labels'][i]):\n",
+        "    #     print(p2_dataset['tokens'][i])\n",
+        "    #     print(true_ann_dataset['tokens'][i])\n",
+        "    #     print(get_labels(([p2_dataset['labels'][i]], [true_ann_dataset['labels'][i]])))\n",
+        "    #     # print(true_ann_dataset['labels'][i])\n",
+        "    #     # sentences_to_evaluate_p2.append(p2_dataset['labels'][i])\n",
+        "    #     # sentences_to_evaluate_true.append(true_ann_dataset['labels'][i])\n",
+        "\n",
+        "print(len(sentences_to_evaluate_p2)/len(p2_dataset))"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 105,
+      "metadata": {
+        "id": "BvXAM4BrOLPC"
+      },
+      "outputs": [],
+      "source": [
+        "evaluator = BioEval()"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "# sentences_to_evaluate_p2[:2]\n",
+        "for sentence in sentences_to_evaluate_p2:\n",
+        "  if len(sentence) == 0:\n",
+        "    print('len000')\n"
+      ],
+      "metadata": {
+        "id": "FgrghkwqfrGo"
+      },
+      "execution_count": 113,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 146,
+      "metadata": {
+        "id": "i3WlQ9nNOLPC"
+      },
+      "outputs": [],
+      "source": [
+        "pred_labels, true_labels = get_labels((sentences_to_evaluate_p2, sentences_to_evaluate_true))"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 140,
+      "metadata": {
+        "id": "50h006TSOLPC"
+      },
+      "outputs": [],
+      "source": [
+        "evaluator.evaluate_annotations(true_labels, pred_labels, do_lower=True)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 141,
+      "metadata": {
+        "id": "iYOfAO8qOLPK",
+        "outputId": "d630bfdc-faa4-4ba3-f817-a2885bc6e768",
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        }
+      },
+      "outputs": [
+        {
+          "output_type": "execute_result",
+          "data": {
+            "text/plain": [
+              "{'overall': {'acc': 0.6039,\n",
+              "  'strict': {'precision': 0.539911466514351,\n",
+              "   'recall': 0.5364642451759364,\n",
+              "   'f_score': 0.5381823357768131},\n",
+              "  'relax': {'precision': 0.6547194059688705,\n",
+              "   'recall': 0.6505391600454029,\n",
+              "   'f_score': 0.652622589139563}},\n",
+              " 'category': {'strict': {'condition': {'precision': 0.6026216555934638,\n",
+              "    'recall': 0.6669316375198728,\n",
+              "    'f_score': 0.6331478162437506},\n",
+              "   'measurement': {'precision': 0.14698162729658792,\n",
+              "    'recall': 0.19310344827586207,\n",
+              "    'f_score': 0.16691505216095381},\n",
+              "   'drug': {'precision': 0.562358276643991,\n",
+              "    'recall': 0.5598194130925508,\n",
+              "    'f_score': 0.5610859728506786},\n",
+              "   'procedure': {'precision': 0.3992248062015504,\n",
+              "    'recall': 0.329073482428115,\n",
+              "    'f_score': 0.36077057793345},\n",
+              "   'temporal': {'precision': 0.030303030303030304,\n",
+              "    'recall': 0.003367003367003367,\n",
+              "    'f_score': 0.006060606060606061},\n",
+              "   'person': {'precision': 0.053763440860215055,\n",
+              "    'recall': 0.03759398496240601,\n",
+              "    'f_score': 0.04424778761061946},\n",
+              "   'observation': {'precision': 0.016216216216216217,\n",
+              "    'recall': 0.018072289156626505,\n",
+              "    'f_score': 0.017094017094017092},\n",
+              "   'value': {'precision': 0.05555555555555555,\n",
+              "    'recall': 0.002849002849002849,\n",
+              "    'f_score': 0.005420054200542005},\n",
+              "   'device': {'precision': 0.32,\n",
+              "    'recall': 0.34782608695652173,\n",
+              "    'f_score': 0.3333333333333333}},\n",
+              "  'relax': {'condition': {'precision': 0.7141318010414797,\n",
+              "    'recall': 0.790341812400636,\n",
+              "    'f_score': 0.750306574851429},\n",
+              "   'measurement': {'precision': 0.25196850393700787,\n",
+              "    'recall': 0.3310344827586207,\n",
+              "    'f_score': 0.28614008941877794},\n",
+              "   'drug': {'precision': 0.6893424036281179,\n",
+              "    'recall': 0.6862302483069977,\n",
+              "    'f_score': 0.6877828054298643},\n",
+              "   'procedure': {'precision': 0.5271317829457365,\n",
+              "    'recall': 0.43450479233226835,\n",
+              "    'f_score': 0.4763572679509632},\n",
+              "   'temporal': {'precision': 0.5757575757575758,\n",
+              "    'recall': 0.06397306397306397,\n",
+              "    'f_score': 0.11515151515151514},\n",
+              "   'person': {'precision': 0.12903225806451613,\n",
+              "    'recall': 0.09022556390977443,\n",
+              "    'f_score': 0.10619469026548671},\n",
+              "   'observation': {'precision': 0.0918918918918919,\n",
+              "    'recall': 0.10240963855421686,\n",
+              "    'f_score': 0.09686609686609687},\n",
+              "   'value': {'precision': 0.7222222222222222,\n",
+              "    'recall': 0.037037037037037035,\n",
+              "    'f_score': 0.07046070460704607},\n",
+              "   'device': {'precision': 0.44,\n",
+              "    'recall': 0.4782608695652174,\n",
+              "    'f_score': 0.4583333333333333}}}}"
+            ]
+          },
+          "metadata": {},
+          "execution_count": 141
+        }
+      ],
+      "source": [
+        "evaluator.performance"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 143,
+      "metadata": {
+        "id": "vikGKa6fOLPK"
+      },
+      "outputs": [],
+      "source": [
+        "evaluator.save_evaluation('eval_paper_full_generative.json')"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "evaluator.get_counts()"
+      ],
+      "metadata": {
+        "id": "_nF89z1JiAqF",
+        "outputId": "94c401df-5c0a-46d7-b6f4-3a2b12aa39cd",
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        }
+      },
+      "execution_count": 142,
+      "outputs": [
+        {
+          "output_type": "execute_result",
+          "data": {
+            "text/plain": [
+              "{'expect': {'overall': 7048,\n",
+              "  'condition': 5032,\n",
+              "  'person': 133,\n",
+              "  'value': 351,\n",
+              "  'observation': 166,\n",
+              "  'drug': 443,\n",
+              "  'measurement': 290,\n",
+              "  'procedure': 313,\n",
+              "  'temporal': 297,\n",
+              "  'device': 23},\n",
+              " 'prediction': {'strict': {'overall': {'total': 7003,\n",
+              "    'true': 3781,\n",
+              "    'false': 3222},\n",
+              "   'condition': {'total': 5569, 'true': 3356, 'false': 2213},\n",
+              "   'measurement': {'total': 381, 'true': 56, 'false': 325},\n",
+              "   'drug': {'total': 441, 'true': 248, 'false': 193},\n",
+              "   'procedure': {'total': 258, 'true': 103, 'false': 155},\n",
+              "   'temporal': {'total': 33, 'true': 1, 'false': 32},\n",
+              "   'person': {'total': 93, 'true': 5, 'false': 88},\n",
+              "   'observation': {'total': 185, 'true': 3, 'false': 182},\n",
+              "   'value': {'total': 18, 'true': 1, 'false': 17},\n",
+              "   'device': {'total': 25, 'true': 8, 'false': 17}},\n",
+              "  'relax': {'overall': {'total': 7003, 'true': 4585, 'false': 2418},\n",
+              "   'condition': {'total': 5569, 'true': 3977, 'false': 1592},\n",
+              "   'measurement': {'total': 381, 'true': 96, 'false': 285},\n",
+              "   'drug': {'total': 441, 'true': 304, 'false': 137},\n",
+              "   'procedure': {'total': 258, 'true': 136, 'false': 122},\n",
+              "   'temporal': {'total': 33, 'true': 19, 'false': 14},\n",
+              "   'person': {'total': 93, 'true': 12, 'false': 81},\n",
+              "   'observation': {'total': 185, 'true': 17, 'false': 168},\n",
+              "   'value': {'total': 18, 'true': 13, 'false': 5},\n",
+              "   'device': {'total': 25, 'true': 11, 'false': 14}}}}"
+            ]
+          },
+          "metadata": {},
+          "execution_count": 142
+        }
+      ]
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "evaluator.save_evaluation('drive/MyDrive/TER-LISN-2024/eval_paper_full_generative.json')"
+      ],
+      "metadata": {
+        "id": "zeN6oRIzL9I7"
+      },
+      "execution_count": 144,
+      "outputs": []
+    }
+  ],
+  "metadata": {
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    },
+    "language_info": {
+      "codemirror_mode": {
+        "name": "ipython",
+        "version": 3
+      },
+      "file_extension": ".py",
+      "mimetype": "text/x-python",
+      "name": "python",
+      "nbconvert_exporter": "python",
+      "pygments_lexer": "ipython3",
+      "version": "3.10.13"
+    },
+    "colab": {
+      "provenance": [],
+      "gpuType": "T4",
+      "include_colab_link": true
+    },
+    "accelerator": "GPU",
+    "widgets": {
+      "application/vnd.jupyter.widget-state+json": {
+        "009541513b8e451886782e457d7cd479": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_name": "VBoxModel",
+          "model_module_version": "1.5.0",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "VBoxModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "VBoxView",
+            "box_style": "",
+            "children": [
+              "IPY_MODEL_523cf83bffdc4d8bbe06c9d886e5d884",
+              "IPY_MODEL_0a508a2d17d04b76aa6bad327e25da96",
+              "IPY_MODEL_7fd41498666942149a04c076f61cb6ad",
+              "IPY_MODEL_32cfc02171c1420599bbbeca98ebfc3b"
+            ],
+            "layout": "IPY_MODEL_6ab04465b07747a592cb28f77c6b5e20"
+          }
+        },
+        "519375173fe64735a4faccf127abcf90": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_name": "HTMLModel",
+          "model_module_version": "1.5.0",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "HTMLModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "HTMLView",
+            "description": "",
+            "description_tooltip": null,
+            "layout": "IPY_MODEL_2379403cd2df4ffd9f7d4eecbf571d7e",
+            "placeholder": "​",
+            "style": "IPY_MODEL_382eff5973b942678fde4a9c6e9c8150",
+            "value": "<center> <img\nsrc=https://huggingface.co/front/assets/huggingface_logo-noborder.svg\nalt='Hugging Face'> <br> Copy a token from <a\nhref=\"https://huggingface.co/settings/tokens\" target=\"_blank\">your Hugging Face\ntokens page</a> and paste it below. <br> Immediately click login after copying\nyour token or it might be stored in plain text in this notebook file. </center>"
+          }
+        },
+        "484017270bb646119bc605fccb88fa4e": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_name": "PasswordModel",
+          "model_module_version": "1.5.0",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "PasswordModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "PasswordView",
+            "continuous_update": true,
+            "description": "Token:",
+            "description_tooltip": null,
+            "disabled": false,
+            "layout": "IPY_MODEL_aec5500c07b54365ab252b5746e08127",
+            "placeholder": "​",
+            "style": "IPY_MODEL_86ed1786dfcc49148bc65e88627a34f3",
+            "value": ""
+          }
+        },
+        "23dc47bf61314784b7ffb42d995bbce9": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_name": "CheckboxModel",
+          "model_module_version": "1.5.0",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "CheckboxModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "CheckboxView",
+            "description": "Add token as git credential?",
+            "description_tooltip": null,
+            "disabled": false,
+            "indent": true,
+            "layout": "IPY_MODEL_36b52f9261ae4c589efc8faab5806509",
+            "style": "IPY_MODEL_8c0ca5e87a1243ba94d821d87dfd7cc1",
+            "value": true
+          }
+        },
+        "7fdc12ed98be4101a7ff8d15272d5033": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_name": "ButtonModel",
+          "model_module_version": "1.5.0",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "ButtonModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "ButtonView",
+            "button_style": "",
+            "description": "Login",
+            "disabled": false,
+            "icon": "",
+            "layout": "IPY_MODEL_1d62a5d167c04e839152933688357dbe",
+            "style": "IPY_MODEL_20614482956a4d359d7e6470fe3b5236",
+            "tooltip": ""
+          }
+        },
+        "683aa055aae842adabca4ee3fa9a16bf": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_name": "HTMLModel",
+          "model_module_version": "1.5.0",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "HTMLModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "HTMLView",
+            "description": "",
+            "description_tooltip": null,
+            "layout": "IPY_MODEL_b46a3ac13be24c04a0c47d424e54d256",
+            "placeholder": "​",
+            "style": "IPY_MODEL_94885d557c64483bbcb842c570e380a5",
+            "value": "\n<b>Pro Tip:</b> If you don't already have one, you can create a dedicated\n'notebooks' token with 'write' access, that you can then easily reuse for all\nnotebooks. </center>"
+          }
+        },
+        "6ab04465b07747a592cb28f77c6b5e20": {
+          "model_module": "@jupyter-widgets/base",
+          "model_name": "LayoutModel",
+          "model_module_version": "1.2.0",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": "center",
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": "flex",
+            "flex": null,
+            "flex_flow": "column",
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": "50%"
+          }
+        },
+        "2379403cd2df4ffd9f7d4eecbf571d7e": {
+          "model_module": "@jupyter-widgets/base",
+          "model_name": "LayoutModel",
+          "model_module_version": "1.2.0",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "382eff5973b942678fde4a9c6e9c8150": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_name": "DescriptionStyleModel",
+          "model_module_version": "1.5.0",
+          "state": {
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "DescriptionStyleModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "StyleView",
+            "description_width": ""
+          }
+        },
+        "aec5500c07b54365ab252b5746e08127": {
+          "model_module": "@jupyter-widgets/base",
+          "model_name": "LayoutModel",
+          "model_module_version": "1.2.0",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "86ed1786dfcc49148bc65e88627a34f3": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_name": "DescriptionStyleModel",
+          "model_module_version": "1.5.0",
+          "state": {
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "DescriptionStyleModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "StyleView",
+            "description_width": ""
+          }
+        },
+        "36b52f9261ae4c589efc8faab5806509": {
+          "model_module": "@jupyter-widgets/base",
+          "model_name": "LayoutModel",
+          "model_module_version": "1.2.0",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "8c0ca5e87a1243ba94d821d87dfd7cc1": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_name": "DescriptionStyleModel",
+          "model_module_version": "1.5.0",
+          "state": {
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "DescriptionStyleModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "StyleView",
+            "description_width": ""
+          }
+        },
+        "1d62a5d167c04e839152933688357dbe": {
+          "model_module": "@jupyter-widgets/base",
+          "model_name": "LayoutModel",
+          "model_module_version": "1.2.0",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "20614482956a4d359d7e6470fe3b5236": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_name": "ButtonStyleModel",
+          "model_module_version": "1.5.0",
+          "state": {
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "ButtonStyleModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "StyleView",
+            "button_color": null,
+            "font_weight": ""
+          }
+        },
+        "b46a3ac13be24c04a0c47d424e54d256": {
+          "model_module": "@jupyter-widgets/base",
+          "model_name": "LayoutModel",
+          "model_module_version": "1.2.0",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "94885d557c64483bbcb842c570e380a5": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_name": "DescriptionStyleModel",
+          "model_module_version": "1.5.0",
+          "state": {
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "DescriptionStyleModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "StyleView",
+            "description_width": ""
+          }
+        },
+        "0ca2c9f7ad19437886ef4a9aa74cf952": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_name": "LabelModel",
+          "model_module_version": "1.5.0",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "LabelModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "LabelView",
+            "description": "",
+            "description_tooltip": null,
+            "layout": "IPY_MODEL_50557c598d9246599d7e894137d2df26",
+            "placeholder": "​",
+            "style": "IPY_MODEL_ab9a60b256554da0a101dc9d541cbb8c",
+            "value": "Connecting..."
+          }
+        },
+        "50557c598d9246599d7e894137d2df26": {
+          "model_module": "@jupyter-widgets/base",
+          "model_name": "LayoutModel",
+          "model_module_version": "1.2.0",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "ab9a60b256554da0a101dc9d541cbb8c": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_name": "DescriptionStyleModel",
+          "model_module_version": "1.5.0",
+          "state": {
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "DescriptionStyleModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "StyleView",
+            "description_width": ""
+          }
+        },
+        "523cf83bffdc4d8bbe06c9d886e5d884": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_name": "LabelModel",
+          "model_module_version": "1.5.0",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "LabelModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "LabelView",
+            "description": "",
+            "description_tooltip": null,
+            "layout": "IPY_MODEL_c72717e7d2714c7aa695cf5b6ba2481c",
+            "placeholder": "​",
+            "style": "IPY_MODEL_45c84a726d5e470babfb03cd5b139de7",
+            "value": "Token is valid (permission: write)."
+          }
+        },
+        "0a508a2d17d04b76aa6bad327e25da96": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_name": "LabelModel",
+          "model_module_version": "1.5.0",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "LabelModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "LabelView",
+            "description": "",
+            "description_tooltip": null,
+            "layout": "IPY_MODEL_ec182a1a030c46ac87b2fa998ea9d868",
+            "placeholder": "​",
+            "style": "IPY_MODEL_22972fbed4cc4b408be8abe9f9bb2847",
+            "value": "Your token has been saved in your configured git credential helpers (store)."
+          }
+        },
+        "7fd41498666942149a04c076f61cb6ad": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_name": "LabelModel",
+          "model_module_version": "1.5.0",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "LabelModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "LabelView",
+            "description": "",
+            "description_tooltip": null,
+            "layout": "IPY_MODEL_09fc71a94b2f4766a02b5879ba232e95",
+            "placeholder": "​",
+            "style": "IPY_MODEL_da47463752ff4288a73b1802d37941aa",
+            "value": "Your token has been saved to /root/.cache/huggingface/token"
+          }
+        },
+        "32cfc02171c1420599bbbeca98ebfc3b": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_name": "LabelModel",
+          "model_module_version": "1.5.0",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "LabelModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "LabelView",
+            "description": "",
+            "description_tooltip": null,
+            "layout": "IPY_MODEL_9e89840302da47e3a383888883bbadea",
+            "placeholder": "​",
+            "style": "IPY_MODEL_5ecf0ca50841453d85a082217c033fae",
+            "value": "Login successful"
+          }
+        },
+        "c72717e7d2714c7aa695cf5b6ba2481c": {
+          "model_module": "@jupyter-widgets/base",
+          "model_name": "LayoutModel",
+          "model_module_version": "1.2.0",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "45c84a726d5e470babfb03cd5b139de7": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_name": "DescriptionStyleModel",
+          "model_module_version": "1.5.0",
+          "state": {
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "DescriptionStyleModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "StyleView",
+            "description_width": ""
+          }
+        },
+        "ec182a1a030c46ac87b2fa998ea9d868": {
+          "model_module": "@jupyter-widgets/base",
+          "model_name": "LayoutModel",
+          "model_module_version": "1.2.0",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "22972fbed4cc4b408be8abe9f9bb2847": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_name": "DescriptionStyleModel",
+          "model_module_version": "1.5.0",
+          "state": {
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "DescriptionStyleModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "StyleView",
+            "description_width": ""
+          }
+        },
+        "09fc71a94b2f4766a02b5879ba232e95": {
+          "model_module": "@jupyter-widgets/base",
+          "model_name": "LayoutModel",
+          "model_module_version": "1.2.0",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "da47463752ff4288a73b1802d37941aa": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_name": "DescriptionStyleModel",
+          "model_module_version": "1.5.0",
+          "state": {
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "DescriptionStyleModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "StyleView",
+            "description_width": ""
+          }
+        },
+        "9e89840302da47e3a383888883bbadea": {
+          "model_module": "@jupyter-widgets/base",
+          "model_name": "LayoutModel",
+          "model_module_version": "1.2.0",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "5ecf0ca50841453d85a082217c033fae": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_name": "DescriptionStyleModel",
+          "model_module_version": "1.5.0",
+          "state": {
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "DescriptionStyleModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "StyleView",
+            "description_width": ""
+          }
+        },
+        "1ca11a0dedb44e6897fd914763d6731d": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_name": "HBoxModel",
+          "model_module_version": "1.5.0",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "HBoxModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "HBoxView",
+            "box_style": "",
+            "children": [
+              "IPY_MODEL_b418a6bbce1346bd9c0076f2d31bad26",
+              "IPY_MODEL_b7975e8109d74252ae427c2507b70c2c",
+              "IPY_MODEL_f0af7ea71ed14275bb36b55c87d6c785"
+            ],
+            "layout": "IPY_MODEL_ddb109aa9fed447b9e8929f2ce49b8d2"
+          }
+        },
+        "b418a6bbce1346bd9c0076f2d31bad26": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_name": "HTMLModel",
+          "model_module_version": "1.5.0",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "HTMLModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "HTMLView",
+            "description": "",
+            "description_tooltip": null,
+            "layout": "IPY_MODEL_6e57a33e6f4641bd85d68b6773657962",
+            "placeholder": "​",
+            "style": "IPY_MODEL_901e5be78e494518924d3237c3398b09",
+            "value": "Loading checkpoint shards: 100%"
+          }
+        },
+        "b7975e8109d74252ae427c2507b70c2c": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_name": "FloatProgressModel",
+          "model_module_version": "1.5.0",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "FloatProgressModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "ProgressView",
+            "bar_style": "success",
+            "description": "",
+            "description_tooltip": null,
+            "layout": "IPY_MODEL_4fdf171fbb704213a0df598adb0c918c",
+            "max": 2,
+            "min": 0,
+            "orientation": "horizontal",
+            "style": "IPY_MODEL_40c20831011e4706aedbe2ed73219dfd",
+            "value": 2
+          }
+        },
+        "f0af7ea71ed14275bb36b55c87d6c785": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_name": "HTMLModel",
+          "model_module_version": "1.5.0",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "HTMLModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "HTMLView",
+            "description": "",
+            "description_tooltip": null,
+            "layout": "IPY_MODEL_83849f3935464b2ab0f77925aab9d7f3",
+            "placeholder": "​",
+            "style": "IPY_MODEL_e7c31cc6d3a64a4fb911f3d2023ecfb1",
+            "value": " 2/2 [01:15&lt;00:00, 35.11s/it]"
+          }
+        },
+        "ddb109aa9fed447b9e8929f2ce49b8d2": {
+          "model_module": "@jupyter-widgets/base",
+          "model_name": "LayoutModel",
+          "model_module_version": "1.2.0",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "6e57a33e6f4641bd85d68b6773657962": {
+          "model_module": "@jupyter-widgets/base",
+          "model_name": "LayoutModel",
+          "model_module_version": "1.2.0",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "901e5be78e494518924d3237c3398b09": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_name": "DescriptionStyleModel",
+          "model_module_version": "1.5.0",
+          "state": {
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "DescriptionStyleModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "StyleView",
+            "description_width": ""
+          }
+        },
+        "4fdf171fbb704213a0df598adb0c918c": {
+          "model_module": "@jupyter-widgets/base",
+          "model_name": "LayoutModel",
+          "model_module_version": "1.2.0",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "40c20831011e4706aedbe2ed73219dfd": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_name": "ProgressStyleModel",
+          "model_module_version": "1.5.0",
+          "state": {
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "ProgressStyleModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "StyleView",
+            "bar_color": null,
+            "description_width": ""
+          }
+        },
+        "83849f3935464b2ab0f77925aab9d7f3": {
+          "model_module": "@jupyter-widgets/base",
+          "model_name": "LayoutModel",
+          "model_module_version": "1.2.0",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "e7c31cc6d3a64a4fb911f3d2023ecfb1": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_name": "DescriptionStyleModel",
+          "model_module_version": "1.5.0",
+          "state": {
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "DescriptionStyleModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "StyleView",
+            "description_width": ""
+          }
+        },
+        "4a35141fd2544fd08d1c5748a16c28f6": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_name": "HBoxModel",
+          "model_module_version": "1.5.0",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "HBoxModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "HBoxView",
+            "box_style": "",
+            "children": [
+              "IPY_MODEL_d6e6a89659834daa88bcef28c1d5d220",
+              "IPY_MODEL_b4f30646efe04849afd1b45f50dd3931",
+              "IPY_MODEL_9a90159ea82b4c3f921839ea7b6f9c86"
+            ],
+            "layout": "IPY_MODEL_8698beb6d78342caaa49ea61c9ebcc58"
+          }
+        },
+        "d6e6a89659834daa88bcef28c1d5d220": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_name": "HTMLModel",
+          "model_module_version": "1.5.0",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "HTMLModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "HTMLView",
+            "description": "",
+            "description_tooltip": null,
+            "layout": "IPY_MODEL_5562bbdb7f064ba4867374297f2c7ef4",
+            "placeholder": "​",
+            "style": "IPY_MODEL_f5f58531d6474340a7e5d4ccdc171969",
+            "value": "Map: 100%"
+          }
+        },
+        "b4f30646efe04849afd1b45f50dd3931": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_name": "FloatProgressModel",
+          "model_module_version": "1.5.0",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "FloatProgressModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "ProgressView",
+            "bar_style": "success",
+            "description": "",
+            "description_tooltip": null,
+            "layout": "IPY_MODEL_57e9591c1f1a4be58f8a826059e27d05",
+            "max": 1307,
+            "min": 0,
+            "orientation": "horizontal",
+            "style": "IPY_MODEL_e52d41b0f7084b83abfc2dce289e9757",
+            "value": 1307
+          }
+        },
+        "9a90159ea82b4c3f921839ea7b6f9c86": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_name": "HTMLModel",
+          "model_module_version": "1.5.0",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "HTMLModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "HTMLView",
+            "description": "",
+            "description_tooltip": null,
+            "layout": "IPY_MODEL_49662c4803d846d6bb15e8ba44e4b818",
+            "placeholder": "​",
+            "style": "IPY_MODEL_a09956aac8fb4447b241f1dc646c6429",
+            "value": " 1307/1307 [00:00&lt;00:00, 1412.20 examples/s]"
+          }
+        },
+        "8698beb6d78342caaa49ea61c9ebcc58": {
+          "model_module": "@jupyter-widgets/base",
+          "model_name": "LayoutModel",
+          "model_module_version": "1.2.0",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "5562bbdb7f064ba4867374297f2c7ef4": {
+          "model_module": "@jupyter-widgets/base",
+          "model_name": "LayoutModel",
+          "model_module_version": "1.2.0",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "f5f58531d6474340a7e5d4ccdc171969": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_name": "DescriptionStyleModel",
+          "model_module_version": "1.5.0",
+          "state": {
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "DescriptionStyleModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "StyleView",
+            "description_width": ""
+          }
+        },
+        "57e9591c1f1a4be58f8a826059e27d05": {
+          "model_module": "@jupyter-widgets/base",
+          "model_name": "LayoutModel",
+          "model_module_version": "1.2.0",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "e52d41b0f7084b83abfc2dce289e9757": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_name": "ProgressStyleModel",
+          "model_module_version": "1.5.0",
+          "state": {
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "ProgressStyleModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "StyleView",
+            "bar_color": null,
+            "description_width": ""
+          }
+        },
+        "49662c4803d846d6bb15e8ba44e4b818": {
+          "model_module": "@jupyter-widgets/base",
+          "model_name": "LayoutModel",
+          "model_module_version": "1.2.0",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "a09956aac8fb4447b241f1dc646c6429": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_name": "DescriptionStyleModel",
+          "model_module_version": "1.5.0",
+          "state": {
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "DescriptionStyleModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "StyleView",
+            "description_width": ""
+          }
+        },
+        "162bfe1f36444ce1861ceaf2c8f6a3f3": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_name": "HBoxModel",
+          "model_module_version": "1.5.0",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "HBoxModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "HBoxView",
+            "box_style": "",
+            "children": [
+              "IPY_MODEL_d33896d2c9c546e1b9d5cc12bb866d51",
+              "IPY_MODEL_af607d56eb2c4de88e84315ebf4e5bed",
+              "IPY_MODEL_0f469022363b4b0d96762dc3aafebffe"
+            ],
+            "layout": "IPY_MODEL_dc63518146884067aaebf37d682e83b5"
+          }
+        },
+        "d33896d2c9c546e1b9d5cc12bb866d51": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_name": "HTMLModel",
+          "model_module_version": "1.5.0",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "HTMLModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "HTMLView",
+            "description": "",
+            "description_tooltip": null,
+            "layout": "IPY_MODEL_676f0e00bfba466ea183f41ba064ff95",
+            "placeholder": "​",
+            "style": "IPY_MODEL_91796acd865b49e39500cbc7fdfa2e1e",
+            "value": "Map: 100%"
+          }
+        },
+        "af607d56eb2c4de88e84315ebf4e5bed": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_name": "FloatProgressModel",
+          "model_module_version": "1.5.0",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "FloatProgressModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "ProgressView",
+            "bar_style": "success",
+            "description": "",
+            "description_tooltip": null,
+            "layout": "IPY_MODEL_48df188dce924c32b510d1ceb4d2450c",
+            "max": 1307,
+            "min": 0,
+            "orientation": "horizontal",
+            "style": "IPY_MODEL_78810abaada343a998519b4cb128bf3a",
+            "value": 1307
+          }
+        },
+        "0f469022363b4b0d96762dc3aafebffe": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_name": "HTMLModel",
+          "model_module_version": "1.5.0",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "HTMLModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "HTMLView",
+            "description": "",
+            "description_tooltip": null,
+            "layout": "IPY_MODEL_5f6c93a23a55413f86aa4778e7d2a6c3",
+            "placeholder": "​",
+            "style": "IPY_MODEL_9b21d0ab5e874bf3aa723bb2114eef80",
+            "value": " 1307/1307 [00:00&lt;00:00, 3328.06 examples/s]"
+          }
+        },
+        "dc63518146884067aaebf37d682e83b5": {
+          "model_module": "@jupyter-widgets/base",
+          "model_name": "LayoutModel",
+          "model_module_version": "1.2.0",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "676f0e00bfba466ea183f41ba064ff95": {
+          "model_module": "@jupyter-widgets/base",
+          "model_name": "LayoutModel",
+          "model_module_version": "1.2.0",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "91796acd865b49e39500cbc7fdfa2e1e": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_name": "DescriptionStyleModel",
+          "model_module_version": "1.5.0",
+          "state": {
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "DescriptionStyleModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "StyleView",
+            "description_width": ""
+          }
+        },
+        "48df188dce924c32b510d1ceb4d2450c": {
+          "model_module": "@jupyter-widgets/base",
+          "model_name": "LayoutModel",
+          "model_module_version": "1.2.0",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "78810abaada343a998519b4cb128bf3a": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_name": "ProgressStyleModel",
+          "model_module_version": "1.5.0",
+          "state": {
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "ProgressStyleModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "StyleView",
+            "bar_color": null,
+            "description_width": ""
+          }
+        },
+        "5f6c93a23a55413f86aa4778e7d2a6c3": {
+          "model_module": "@jupyter-widgets/base",
+          "model_name": "LayoutModel",
+          "model_module_version": "1.2.0",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "9b21d0ab5e874bf3aa723bb2114eef80": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_name": "DescriptionStyleModel",
+          "model_module_version": "1.5.0",
+          "state": {
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "DescriptionStyleModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "StyleView",
+            "description_width": ""
+          }
+        }
+      }
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}
\ No newline at end of file