[357738]: / Roberta+LLM / mistral7b-full-evaluation.ipynb

Download this file

3339 lines (3339 with data), 122.5 kB

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/jlopetegui98/NER-ClinicalTrials-Eligibility-Criteria/blob/main/Roberta%2BLLM/mistral7b-full-evaluation.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "GREqJcq4OLO6",
        "outputId": "511e84c9-0109-411a-b44e-0801766d0fe9",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
          ]
        }
      ],
      "source": [
        "# uncomment if working in colab\n",
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "id": "ICZaolBLOLO8",
        "outputId": "c8c38b1c-e0e3-4258-cc8b-d7d684d9d089",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: seqeval in /usr/local/lib/python3.10/dist-packages (1.2.2)\n",
            "Requirement already satisfied: numpy>=1.14.0 in /usr/local/lib/python3.10/dist-packages (from seqeval) (1.25.2)\n",
            "Requirement already satisfied: scikit-learn>=0.21.3 in /usr/local/lib/python3.10/dist-packages (from seqeval) (1.2.2)\n",
            "Requirement already satisfied: scipy>=1.3.2 in /usr/local/lib/python3.10/dist-packages (from scikit-learn>=0.21.3->seqeval) (1.11.4)\n",
            "Requirement already satisfied: joblib>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from scikit-learn>=0.21.3->seqeval) (1.4.2)\n",
            "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn>=0.21.3->seqeval) (3.5.0)\n",
            "Requirement already satisfied: torch in /usr/local/lib/python3.10/dist-packages (2.3.0)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from torch) (3.14.0)\n",
            "Requirement already satisfied: typing-extensions>=4.8.0 in /usr/local/lib/python3.10/dist-packages (from torch) (4.11.0)\n",
            "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch) (1.12)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch) (3.3)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch) (3.1.4)\n",
            "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from torch) (2023.6.0)\n",
            "Requirement already satisfied: nvidia-cuda-nvrtc-cu12==12.1.105 in /usr/local/lib/python3.10/dist-packages (from torch) (12.1.105)\n",
            "Requirement already satisfied: nvidia-cuda-runtime-cu12==12.1.105 in /usr/local/lib/python3.10/dist-packages (from torch) (12.1.105)\n",
            "Requirement already satisfied: nvidia-cuda-cupti-cu12==12.1.105 in /usr/local/lib/python3.10/dist-packages (from torch) (12.1.105)\n",
            "Requirement already satisfied: nvidia-cudnn-cu12==8.9.2.26 in /usr/local/lib/python3.10/dist-packages (from torch) (8.9.2.26)\n",
            "Requirement already satisfied: nvidia-cublas-cu12==12.1.3.1 in /usr/local/lib/python3.10/dist-packages (from torch) (12.1.3.1)\n",
            "Requirement already satisfied: nvidia-cufft-cu12==11.0.2.54 in /usr/local/lib/python3.10/dist-packages (from torch) (11.0.2.54)\n",
            "Requirement already satisfied: nvidia-curand-cu12==10.3.2.106 in /usr/local/lib/python3.10/dist-packages (from torch) (10.3.2.106)\n",
            "Requirement already satisfied: nvidia-cusolver-cu12==11.4.5.107 in /usr/local/lib/python3.10/dist-packages (from torch) (11.4.5.107)\n",
            "Requirement already satisfied: nvidia-cusparse-cu12==12.1.0.106 in /usr/local/lib/python3.10/dist-packages (from torch) (12.1.0.106)\n",
            "Requirement already satisfied: nvidia-nccl-cu12==2.20.5 in /usr/local/lib/python3.10/dist-packages (from torch) (2.20.5)\n",
            "Requirement already satisfied: nvidia-nvtx-cu12==12.1.105 in /usr/local/lib/python3.10/dist-packages (from torch) (12.1.105)\n",
            "Requirement already satisfied: triton==2.3.0 in /usr/local/lib/python3.10/dist-packages (from torch) (2.3.0)\n",
            "Requirement already satisfied: nvidia-nvjitlink-cu12 in /usr/local/lib/python3.10/dist-packages (from nvidia-cusolver-cu12==11.4.5.107->torch) (12.4.127)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch) (2.1.5)\n",
            "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->torch) (1.3.0)\n",
            "  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "  Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "  Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "  Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n"
          ]
        }
      ],
      "source": [
        "# uncomment if using colab\n",
        "!pip install -q -U datasets\n",
        "!pip install seqeval\n",
        "!pip install -U torch\n",
        "!pip install -q -U evaluate\n",
        "!pip install -q -U git+https://github.com/huggingface/transformers.git\n",
        "# !pip install -q -U transformers\n",
        "!pip install -q -U bitsandbytes\n",
        "!pip install -q -U git+https://github.com/huggingface/peft.git\n",
        "!pip install -q -U git+https://github.com/huggingface/accelerate.git\n",
        "# !pip install -q -U accelerate"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "id": "Km86o_uHOLO8"
      },
      "outputs": [],
      "source": [
        "import torch\n",
        "from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig, TextGenerationPipeline\n",
        "import os\n",
        "from utils import *"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "from datasets import Dataset, DatasetDict, load_dataset"
      ],
      "metadata": {
        "id": "L0zkxHTmOxkJ"
      },
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import re"
      ],
      "metadata": {
        "id": "uoWkyegpPCOZ"
      },
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "id": "L0zE4thvOLO9",
        "outputId": "75716581-4307-4d98-fddf-887bc825e808",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 145,
          "referenced_widgets": [
            "009541513b8e451886782e457d7cd479",
            "519375173fe64735a4faccf127abcf90",
            "484017270bb646119bc605fccb88fa4e",
            "23dc47bf61314784b7ffb42d995bbce9",
            "7fdc12ed98be4101a7ff8d15272d5033",
            "683aa055aae842adabca4ee3fa9a16bf",
            "6ab04465b07747a592cb28f77c6b5e20",
            "2379403cd2df4ffd9f7d4eecbf571d7e",
            "382eff5973b942678fde4a9c6e9c8150",
            "aec5500c07b54365ab252b5746e08127",
            "86ed1786dfcc49148bc65e88627a34f3",
            "36b52f9261ae4c589efc8faab5806509",
            "8c0ca5e87a1243ba94d821d87dfd7cc1",
            "1d62a5d167c04e839152933688357dbe",
            "20614482956a4d359d7e6470fe3b5236",
            "b46a3ac13be24c04a0c47d424e54d256",
            "94885d557c64483bbcb842c570e380a5",
            "0ca2c9f7ad19437886ef4a9aa74cf952",
            "50557c598d9246599d7e894137d2df26",
            "ab9a60b256554da0a101dc9d541cbb8c",
            "523cf83bffdc4d8bbe06c9d886e5d884",
            "0a508a2d17d04b76aa6bad327e25da96",
            "7fd41498666942149a04c076f61cb6ad",
            "32cfc02171c1420599bbbeca98ebfc3b",
            "c72717e7d2714c7aa695cf5b6ba2481c",
            "45c84a726d5e470babfb03cd5b139de7",
            "ec182a1a030c46ac87b2fa998ea9d868",
            "22972fbed4cc4b408be8abe9f9bb2847",
            "09fc71a94b2f4766a02b5879ba232e95",
            "da47463752ff4288a73b1802d37941aa",
            "9e89840302da47e3a383888883bbadea",
            "5ecf0ca50841453d85a082217c033fae"
          ]
        }
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "VBox(children=(HTML(value='<center> <img\\nsrc=https://huggingface.co/front/assets/huggingface_logo-noborder.sv…"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "009541513b8e451886782e457d7cd479"
            }
          },
          "metadata": {}
        }
      ],
      "source": [
        "from huggingface_hub import notebook_login\n",
        "\n",
        "notebook_login()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "id": "6DQED9SWOLO9"
      },
      "outputs": [],
      "source": [
        "# dict for the entities (entity to int value)\n",
        "simple_ent = {\"Condition\", \"Value\", \"Drug\", \"Procedure\", \"Measurement\", \"Temporal\", \"Observation\", \"Person\", \"Device\"}\n",
        "sel_ent = {\n",
        "    \"O\": 0,\n",
        "    \"B-Condition\": 1,\n",
        "    \"I-Condition\": 2,\n",
        "    \"B-Value\": 3,\n",
        "    \"I-Value\": 4,\n",
        "    \"B-Drug\": 5,\n",
        "    \"I-Drug\": 6,\n",
        "    \"B-Procedure\": 7,\n",
        "    \"I-Procedure\": 8,\n",
        "    \"B-Measurement\": 9,\n",
        "    \"I-Measurement\": 10,\n",
        "    \"B-Temporal\": 11,\n",
        "    \"I-Temporal\": 12,\n",
        "    \"B-Observation\": 13,\n",
        "    \"I-Observation\": 14,\n",
        "    \"B-Person\": 15,\n",
        "    \"I-Person\": 16,\n",
        "    \"B-Device\": 17,\n",
        "    \"I-Device\": 18\n",
        "}\n",
        "\n",
        "entities_list = list(sel_ent.keys())\n",
        "sel_ent_inv = {v: k for k, v in sel_ent.items()}"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "id": "fLKSRg5XOLO9"
      },
      "outputs": [],
      "source": [
        "root = '..'\n",
        "root = './drive/MyDrive/TER-LISN-2024'\n",
        "data_path = f'{root}/data'\n",
        "models_path = f'{root}/models'"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "id": "ZZnHifTsOLO9"
      },
      "outputs": [],
      "source": [
        "model_name = \"mistralai/Mistral-7B-v0.1\""
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {
        "id": "ytfHR4b2OLO-",
        "outputId": "7f66d4d3-1867-4d9f-c298-42b2368c50df",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 173,
          "referenced_widgets": [
            "1ca11a0dedb44e6897fd914763d6731d",
            "b418a6bbce1346bd9c0076f2d31bad26",
            "b7975e8109d74252ae427c2507b70c2c",
            "f0af7ea71ed14275bb36b55c87d6c785",
            "ddb109aa9fed447b9e8929f2ce49b8d2",
            "6e57a33e6f4641bd85d68b6773657962",
            "901e5be78e494518924d3237c3398b09",
            "4fdf171fbb704213a0df598adb0c918c",
            "40c20831011e4706aedbe2ed73219dfd",
            "83849f3935464b2ab0f77925aab9d7f3",
            "e7c31cc6d3a64a4fb911f3d2023ecfb1"
          ]
        }
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/huggingface_hub/utils/_token.py:89: UserWarning: \n",
            "The secret `HF_TOKEN` does not exist in your Colab secrets.\n",
            "To authenticate with the Hugging Face Hub, create a token in your settings tab (https://huggingface.co/settings/tokens), set it as secret in your Google Colab and restart your session.\n",
            "You will be able to reuse this secret in all of your notebooks.\n",
            "Please note that authentication is recommended but still optional to access public models or datasets.\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "1ca11a0dedb44e6897fd914763d6731d"
            }
          },
          "metadata": {}
        }
      ],
      "source": [
        "# Load base model(Mistral 7B)\n",
        "bnb_config = BitsAndBytesConfig(\n",
        "    load_in_4bit= True,\n",
        "    bnb_4bit_quant_type= \"nf4\",\n",
        "    bnb_4bit_compute_dtype= torch.bfloat16,\n",
        "    bnb_4bit_use_double_quant= False,\n",
        ")\n",
        "model = AutoModelForCausalLM.from_pretrained(\n",
        "   model_name,\n",
        "    quantization_config=bnb_config,\n",
        "    device_map={\"\": 0}\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "metadata": {
        "id": "ageLiTUKOLO-",
        "outputId": "8f9850bc-2dd1-4230-9b0d-62503154247e",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(True, False)"
            ]
          },
          "metadata": {},
          "execution_count": 10
        }
      ],
      "source": [
        "# import tokenizer for mistral-7B\n",
        "tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)\n",
        "tokenizer.padding_side = 'left'\n",
        "tokenizer.pad_token = tokenizer.eos_token\n",
        "tokenizer.add_eos_token = False\n",
        "tokenizer.add_bos_token, tokenizer.add_eos_token"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {
        "id": "qMfZMKhlOLO-"
      },
      "outputs": [],
      "source": [
        "pipe  = TextGenerationPipeline(model = model, tokenizer = tokenizer)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {
        "id": "E1T2wFahOLO-"
      },
      "outputs": [],
      "source": [
        "dataset = load_dataset('JavierLopetegui/chia_v1')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 13,
      "metadata": {
        "id": "U37CsADiOLO-"
      },
      "outputs": [],
      "source": [
        "# for each sentence save the text\n",
        "def generate_sentences_from_tokens(sentences):\n",
        "    texts_sentences = []\n",
        "    sentences_tokens = sentences['tokens']\n",
        "    for sentence in sentences_tokens:\n",
        "        sent_text = \" \".join(sentence)\n",
        "        texts_sentences.append(sent_text)\n",
        "    sentences['text'] = texts_sentences\n",
        "    return sentences"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 14,
      "metadata": {
        "id": "423oQ2BQOLO-"
      },
      "outputs": [],
      "source": [
        "def build_prompts(sentences, prompt_type=2):\n",
        "    sentences_prompts = []\n",
        "    for sent in sentences['text']:\n",
        "        prompt = build_prompt(sent, prompt_type)\n",
        "        sentences_prompts.append(prompt)\n",
        "    sentences['prompt'] = sentences_prompts\n",
        "    return sentences"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 15,
      "metadata": {
        "id": "kC1PSUY9OLO_"
      },
      "outputs": [],
      "source": [
        "dataset = dataset.map(lambda x: generate_sentences_from_tokens(x), batched = True)\n",
        "dataset_prompt2 = dataset.map(lambda x: build_prompts(x, prompt_type=2), batched = True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 16,
      "metadata": {
        "id": "IEyo7EiUOLO_"
      },
      "outputs": [],
      "source": [
        "test_dataset_p2 = dataset_prompt2['test']"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 17,
      "metadata": {
        "id": "47SUjSaWOLO_"
      },
      "outputs": [],
      "source": [
        "# keep just the prompt column\n",
        "test_dataset_p2 = test_dataset_p2.remove_columns(['tokens', 'text', 'ner_tags', 'file'])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "SAJgHjqnOLO_"
      },
      "outputs": [],
      "source": [
        "# data_loader_p2 = DataLoader(test_dataset_p2, batch_size=4, shuffle=False)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ps3MQNmIOLO_"
      },
      "outputs": [],
      "source": [
        "# generated_sentences_p2 = pipe(batch['prompt'], max_new_tokens = 500, return_full_text = False, handle_long_generation = \"hole\"))"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from tqdm import tqdm"
      ],
      "metadata": {
        "id": "VrXXsuAsSGVR"
      },
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(test_dataset_p2['prompt'][0])"
      ],
      "metadata": {
        "id": "pULuqZPEB5Y4",
        "outputId": "75d27b13-4396-4232-b8af-ca5aad7c3b5e",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 59,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "I am working on a named entity recognition problem, in the context of clinical\n",
            "    trials eligibility criteria. I will show you the list of entities:\n",
            "    - Condition\n",
            "    - Value\n",
            "    - Drug\n",
            "    - Procedure\n",
            "    - Measurement\n",
            "    - Temporal\n",
            "    - Observation\n",
            "    - Person\n",
            "    - Device\n",
            "\n",
            "    Your task consists in annotate the named entities in a given sentence in the format I will explain you.\n",
            "    I will explain you with some examples:\n",
            "\n",
            "    Example 1:\n",
            "    Input: Patients who have received prior chemotherapy for unresectable disease.\n",
            "    Output: Patients who have received prior <Procedure>chemotherapy</Procedure> for <Condition>unresectable disease</Condition>.\n",
            "\n",
            "    Example 2:\n",
            "    Input: Patients with any other severe concurrent disease, which in the judgment of the investigator, would make the patient inappropriate for entry into this study.\n",
            "    Ouput: Patients with any other severe <Condition>concurrent disease</Condition>, which in the judgment of the investigator, would make the patient inappropriate for <Observation>entry into this study</Observation>.\n",
            "\n",
            "    As you can see, in each example, the extracted entities are enclosed using the sintax: <ENT>text of the entity</ENT>.\n",
            "\n",
            "    Please now annotate as explained before the following sentence:\n",
            "\n",
            "    Input: self - reported healthy adults between the ages of 18 - 60 who are fluent in English .\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 88,
      "metadata": {
        "id": "QwPSO9NvOLO_",
        "outputId": "b9b7841d-781a-47bf-92f0-a32c080343d1",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 1307/1307 [15:16:55<00:00, 42.09s/it]\n"
          ]
        }
      ],
      "source": [
        "generated_sentences_p2 = []\n",
        "for sentence in tqdm(test_dataset_p2['prompt']):\n",
        "    sentence += '\\noutput: '\n",
        "    output = pipe(sentence, max_new_tokens = 500, return_full_text = False, handle_long_generation = \"hole\")[0]['generated_text']\n",
        "    output = output.split('\\n')[0]\n",
        "    generated_sentences_p2.append(output)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "len(generated_sentences_p2)"
      ],
      "metadata": {
        "id": "uTSOYC9UhKxw",
        "outputId": "3799dd2c-4f62-4856-a407-0825469f89d4",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 89,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "1307"
            ]
          },
          "metadata": {},
          "execution_count": 89
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 90,
      "metadata": {
        "id": "x0L_xzJXOLO_"
      },
      "outputs": [],
      "source": [
        "tokenizer = AutoTokenizer.from_pretrained('xlm-roberta-base')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 91,
      "metadata": {
        "id": "6i9B0a6_OLO_"
      },
      "outputs": [],
      "source": [
        "# tokenize and align the labels in the dataset\n",
        "def tokenize_and_align_labels(sentence, flag = 'I'):\n",
        "    \"\"\"\n",
        "    Tokenize the sentence and align the labels\n",
        "    inputs:\n",
        "        sentence: dict, the sentence from the dataset\n",
        "        flag: str, the flag to indicate how to deal with the labels for subwords\n",
        "            - 'I': use the label of the first subword for all subwords but as intermediate (I-ENT)\n",
        "            - 'B': use the label of the first subword for all subwords as beginning (B-ENT)\n",
        "            - None: use -100 for subwords\n",
        "    outputs:\n",
        "        tokenized_sentence: dict, the tokenized sentence now with a field for the labels\n",
        "    \"\"\"\n",
        "    tokenized_sentence = tokenizer(sentence['tokens'], is_split_into_words=True, truncation=True)\n",
        "\n",
        "    labels = []\n",
        "    for i, labels_s in enumerate(sentence['ner_tags']):\n",
        "        word_ids = tokenized_sentence.word_ids(batch_index=i)\n",
        "        previous_word_idx = None\n",
        "        label_ids = []\n",
        "        for word_idx in word_ids:\n",
        "            # if the word_idx is None, assign -100\n",
        "            if word_idx is None:\n",
        "                label_ids.append(-100)\n",
        "            # if it is a new word, assign the corresponding label\n",
        "            elif word_idx != previous_word_idx:\n",
        "                label_ids.append(labels_s[word_idx])\n",
        "            # if it is the same word, check the flag to assign\n",
        "            else:\n",
        "                if flag == 'I':\n",
        "                    if entities_list[labels_s[word_idx]].startswith('I'):\n",
        "                      label_ids.append(labels_s[word_idx])\n",
        "                    else:\n",
        "                      label_ids.append(labels_s[word_idx] + 1)\n",
        "                elif flag == 'B':\n",
        "                    label_ids.append(labels_s[word_idx])\n",
        "                elif flag == None:\n",
        "                    label_ids.append(-100)\n",
        "            previous_word_idx = word_idx\n",
        "        labels.append(label_ids)\n",
        "    tokenized_sentence['labels'] = labels\n",
        "    return tokenized_sentence"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZpemgZm_OLPA"
      },
      "source": [
        "**Standarizing true annotations**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 92,
      "metadata": {
        "id": "-bbV-kgyOLPB"
      },
      "outputs": [],
      "source": [
        "def parse_ann2bio(sentence, pattern, pattern1, pattern2):\n",
        "    # if sentence[-1] == \"\\n\":\n",
        "    #     sentence = sentence[:-2] # remove the \\n and a final point wrongly added\n",
        "    # else:\n",
        "    #     sentence = sentence[:-1] # remove the final point wrongly added\n",
        "\n",
        "    # find the entities\n",
        "    occurrences = re.finditer(pattern, sentence)\n",
        "    indexes = [(match.start(), match.end()) for match in occurrences]\n",
        "\n",
        "    annotation = []\n",
        "    i = 0\n",
        "\n",
        "\n",
        "    # create the bio list\n",
        "    for beg, end in indexes:\n",
        "        if beg > i:\n",
        "            annotation.extend([(word, \"O\") for word in sentence[i:beg].split()])\n",
        "        entity = sentence[beg:end]\n",
        "        entity_name = re.search(pattern1, entity).group(1)\n",
        "        entity = entity.replace(f'<{entity_name}>', \"\").replace(f'</{entity_name}>', \"\")\n",
        "        split_entity = entity.split()\n",
        "        annotation.append((split_entity[0], \"B-\" + entity_name))\n",
        "        annotation.extend([(word, \"I-\" + entity_name) for word in split_entity[1:]])\n",
        "        i = end\n",
        "    annotation.extend([(word, \"O\") for word in sentence[i:].split()])\n",
        "\n",
        "    # check punctuation sign in tokens and put them as individual tokens\n",
        "    ps = r'(\\.|\\,|\\:|\\;|\\!|\\?|\\-|\\(|\\)|\\[|\\]|\\{|\\}|\\\")'\n",
        "    new_annotation = []\n",
        "    for i,(word, tag) in enumerate(annotation):\n",
        "        if re.search(ps, word):\n",
        "            # find the ocurrences of the punctuation signs\n",
        "            occurrences = re.finditer(ps, word)\n",
        "            indexes = [(match.start(), match.end()) for match in occurrences]\n",
        "            # create the new tokens\n",
        "            last = 0\n",
        "            for j, (beg, end) in enumerate(indexes):\n",
        "                if beg > last:\n",
        "                    new_annotation.append((word[last:beg], tag))\n",
        "                if tag != \"O\":\n",
        "                    label = f'I-{tag.split(\"-\")[1]}'\n",
        "                else:\n",
        "                    label = \"O\"\n",
        "                if end < len(word) or (i < len(annotation) - 1 and annotation[i+1][1] == label):\n",
        "                    new_annotation.append((word[beg:end], label))\n",
        "                else:\n",
        "                    new_annotation.append((word[beg:end], 'O'))\n",
        "                last = end\n",
        "            if last < len(word):\n",
        "                new_annotation.append((word[last:], label))\n",
        "\n",
        "        else:\n",
        "            new_annotation.append((word, tag))\n",
        "\n",
        "\n",
        "    return new_annotation"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "new_true_annotations = []\n",
        "for sent in dataset['test']:\n",
        "    annotation = []\n",
        "    for word, tag in zip(sent['tokens'], sent['ner_tags']):\n",
        "        annotation.append((word, entities_list[tag]))\n",
        "    new_annotation = []\n",
        "    ps = r'(\\.|\\,|\\:|\\;|\\!|\\?|\\-|\\(|\\)|\\[|\\]|\\{|\\}|\\\")'\n",
        "    for i,(word, tag) in enumerate(annotation):\n",
        "        if re.search(ps, word):\n",
        "            # find the ocurrences of the punctuation signs\n",
        "            occurrences = re.finditer(ps, word)\n",
        "            indexes = [(match.start(), match.end()) for match in occurrences]\n",
        "            # create the new tokens\n",
        "            last = 0\n",
        "            for j, (beg, end) in enumerate(indexes):\n",
        "                if beg > last:\n",
        "                    new_annotation.append((word[last:beg], tag))\n",
        "                if tag != \"O\":\n",
        "                    label = f'I-{tag.split(\"-\")[1]}'\n",
        "                else:\n",
        "                    label = \"O\"\n",
        "                if end < len(word) or (i < len(annotation) - 1 and annotation[i+1][1] == label):\n",
        "                    new_annotation.append((word[beg:end], label))\n",
        "                else:\n",
        "                    new_annotation.append((word[beg:end], 'O'))\n",
        "                last = end\n",
        "            if last < len(word):\n",
        "                new_annotation.append((word[last:], label))\n",
        "        else:\n",
        "            new_annotation.append((word, tag))\n",
        "    new_true_annotations.append(new_annotation)\n",
        "len(new_true_annotations)"
      ],
      "metadata": {
        "id": "Kgi4KQythezT",
        "outputId": "51229f7b-ae1e-4b5c-d1dc-b477a365ab45",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 93,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "1307"
            ]
          },
          "metadata": {},
          "execution_count": 93
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 94,
      "metadata": {
        "id": "ng4AD9cxOLPB",
        "outputId": "3c9b6d49-4b24-4377-8017-4dbc7aba9a3f",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "1307"
            ]
          },
          "metadata": {},
          "execution_count": 94
        }
      ],
      "source": [
        "true_annotations = []\n",
        "for sent in new_true_annotations:\n",
        "    dicc_sent = {\"tokens\":[], \"ner_tags\":[]}\n",
        "    for word, tag in sent:\n",
        "        dicc_sent[\"tokens\"].append(word)\n",
        "        dicc_sent[\"ner_tags\"].append(sel_ent[tag])\n",
        "    true_annotations.append(dicc_sent)\n",
        "len(true_annotations)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 95,
      "metadata": {
        "id": "T3gzF1mvOLPB"
      },
      "outputs": [],
      "source": [
        "true_df = pd.DataFrame(true_annotations)\n",
        "true_ann_dataset = Dataset.from_pandas(true_df)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 96,
      "metadata": {
        "id": "gtFN5ix8OLPB",
        "outputId": "e2ccd19f-b58b-4a4a-c7e1-c3f118cda808",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 49,
          "referenced_widgets": [
            "4a35141fd2544fd08d1c5748a16c28f6",
            "d6e6a89659834daa88bcef28c1d5d220",
            "b4f30646efe04849afd1b45f50dd3931",
            "9a90159ea82b4c3f921839ea7b6f9c86",
            "8698beb6d78342caaa49ea61c9ebcc58",
            "5562bbdb7f064ba4867374297f2c7ef4",
            "f5f58531d6474340a7e5d4ccdc171969",
            "57e9591c1f1a4be58f8a826059e27d05",
            "e52d41b0f7084b83abfc2dce289e9757",
            "49662c4803d846d6bb15e8ba44e4b818",
            "a09956aac8fb4447b241f1dc646c6429"
          ]
        }
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Map:   0%|          | 0/1307 [00:00<?, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "4a35141fd2544fd08d1c5748a16c28f6"
            }
          },
          "metadata": {}
        }
      ],
      "source": [
        "true_ann_dataset = true_ann_dataset.map(tokenize_and_align_labels, batched=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 145,
      "metadata": {
        "id": "qhcwK4tTOLPB"
      },
      "outputs": [],
      "source": [
        "def get_labels(p):\n",
        "    predictions, labels = p\n",
        "    # Remove ignored index (special tokens)\n",
        "    # predictions = [\n",
        "    #     [entities_list[p] for (p, l) in zip(prediction, label) if l != -100]\n",
        "    #     for prediction, label in zip(predictions, labels)\n",
        "    # ]\n",
        "    predictions_ = []\n",
        "    for (pred,lab) in zip(predictions, labels):\n",
        "        predictions_.append([])\n",
        "        for (p,l) in zip(pred, lab):\n",
        "          if l != -100:\n",
        "            if p == -100:\n",
        "              predictions_[-1].append(entities_list[0])\n",
        "            else:\n",
        "              predictions_[-1].append(entities_list[p])\n",
        "\n",
        "    labels = [\n",
        "        [entities_list[l] for (p, l) in zip(prediction, label) if l != -100]\n",
        "        for prediction, label in zip(predictions, labels)\n",
        "    ]\n",
        "\n",
        "    return predictions_, labels"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 98,
      "metadata": {
        "id": "kZqPzsPEOLPB"
      },
      "outputs": [],
      "source": [
        "from eval_file import *"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ICsodrKvOLPB"
      },
      "source": [
        "**Evaluating prompt 2**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 99,
      "metadata": {
        "id": "ox3BUHhrOLPC"
      },
      "outputs": [],
      "source": [
        "pattern1 = r'<(Person|Condition|Value|Drug|Procedure|Measurement|Temporal|Observation|Device)>'\n",
        "pattern2 = r'</(Person|Condition|Value|Drug|Procedure|Measurement|Temporal|Observation|Device)>'\n",
        "pattern = f'{pattern1}.*?{pattern2}'"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 100,
      "metadata": {
        "id": "NF6iDWPeOLPC",
        "outputId": "cad977da-3c36-43ec-9bfa-482971505810",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "1307"
            ]
          },
          "metadata": {},
          "execution_count": 100
        }
      ],
      "source": [
        "new_p2_annotations = []\n",
        "for sent in generated_sentences_p2:\n",
        "    annotation = parse_ann2bio(sent, pattern, pattern1, pattern2)\n",
        "    new_p2_annotations.append(annotation)\n",
        "len(new_p2_annotations)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 101,
      "metadata": {
        "id": "kRxCaGoFOLPC",
        "outputId": "6e580bb9-a3ca-40d6-8d23-3414e2da292f",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "1307"
            ]
          },
          "metadata": {},
          "execution_count": 101
        }
      ],
      "source": [
        "p2_annotations = []\n",
        "for sent in new_p2_annotations:\n",
        "    dicc_sent = {\"tokens\":[], \"ner_tags\":[]}\n",
        "    for word, tag in sent:\n",
        "        dicc_sent[\"tokens\"].append(word)\n",
        "        dicc_sent[\"ner_tags\"].append(sel_ent[tag])\n",
        "    p2_annotations.append(dicc_sent)\n",
        "len(p2_annotations)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 102,
      "metadata": {
        "id": "JJapA5HhOLPC"
      },
      "outputs": [],
      "source": [
        "p2_df = pd.DataFrame(p2_annotations)\n",
        "p2_dataset = Dataset.from_pandas(p2_df)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 103,
      "metadata": {
        "id": "Wh2C3LSmOLPC",
        "outputId": "ebd4d143-619c-44f1-e84e-8a06cc2d0ef0",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 49,
          "referenced_widgets": [
            "162bfe1f36444ce1861ceaf2c8f6a3f3",
            "d33896d2c9c546e1b9d5cc12bb866d51",
            "af607d56eb2c4de88e84315ebf4e5bed",
            "0f469022363b4b0d96762dc3aafebffe",
            "dc63518146884067aaebf37d682e83b5",
            "676f0e00bfba466ea183f41ba064ff95",
            "91796acd865b49e39500cbc7fdfa2e1e",
            "48df188dce924c32b510d1ceb4d2450c",
            "78810abaada343a998519b4cb128bf3a",
            "5f6c93a23a55413f86aa4778e7d2a6c3",
            "9b21d0ab5e874bf3aa723bb2114eef80"
          ]
        }
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Map:   0%|          | 0/1307 [00:00<?, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "162bfe1f36444ce1861ceaf2c8f6a3f3"
            }
          },
          "metadata": {}
        }
      ],
      "source": [
        "p2_dataset = p2_dataset.map(tokenize_and_align_labels, batched=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 104,
      "metadata": {
        "id": "wu_XrafjOLPC",
        "outputId": "b9bd9c26-892e-45e8-eb58-b24c8e760d3f",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "1.0\n"
          ]
        }
      ],
      "source": [
        "# keep just sentences with the same length\n",
        "sentences_to_evaluate_p2 = []\n",
        "sentences_to_evaluate_true = []\n",
        "\n",
        "for i in range(len(p2_dataset)):\n",
        "    # keep the min len among the true and annotated sentence\n",
        "    l = min(len(p2_dataset['labels'][i]), len(true_ann_dataset['labels'][i]))\n",
        "    sentences_to_evaluate_p2.append(p2_dataset['labels'][i][:l])\n",
        "    sentences_to_evaluate_true.append(true_ann_dataset['labels'][i][:l])\n",
        "    # if len(p2_dataset['labels'][i]) != len(true_ann_dataset['labels'][i]):\n",
        "    #     print(p2_dataset['tokens'][i])\n",
        "    #     print(true_ann_dataset['tokens'][i])\n",
        "    #     print(get_labels(([p2_dataset['labels'][i]], [true_ann_dataset['labels'][i]])))\n",
        "    #     # print(true_ann_dataset['labels'][i])\n",
        "    #     # sentences_to_evaluate_p2.append(p2_dataset['labels'][i])\n",
        "    #     # sentences_to_evaluate_true.append(true_ann_dataset['labels'][i])\n",
        "\n",
        "print(len(sentences_to_evaluate_p2)/len(p2_dataset))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 105,
      "metadata": {
        "id": "BvXAM4BrOLPC"
      },
      "outputs": [],
      "source": [
        "evaluator = BioEval()"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# sentences_to_evaluate_p2[:2]\n",
        "for sentence in sentences_to_evaluate_p2:\n",
        "  if len(sentence) == 0:\n",
        "    print('len000')\n"
      ],
      "metadata": {
        "id": "FgrghkwqfrGo"
      },
      "execution_count": 113,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": 146,
      "metadata": {
        "id": "i3WlQ9nNOLPC"
      },
      "outputs": [],
      "source": [
        "pred_labels, true_labels = get_labels((sentences_to_evaluate_p2, sentences_to_evaluate_true))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 140,
      "metadata": {
        "id": "50h006TSOLPC"
      },
      "outputs": [],
      "source": [
        "evaluator.evaluate_annotations(true_labels, pred_labels, do_lower=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 141,
      "metadata": {
        "id": "iYOfAO8qOLPK",
        "outputId": "d630bfdc-faa4-4ba3-f817-a2885bc6e768",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'overall': {'acc': 0.6039,\n",
              "  'strict': {'precision': 0.539911466514351,\n",
              "   'recall': 0.5364642451759364,\n",
              "   'f_score': 0.5381823357768131},\n",
              "  'relax': {'precision': 0.6547194059688705,\n",
              "   'recall': 0.6505391600454029,\n",
              "   'f_score': 0.652622589139563}},\n",
              " 'category': {'strict': {'condition': {'precision': 0.6026216555934638,\n",
              "    'recall': 0.6669316375198728,\n",
              "    'f_score': 0.6331478162437506},\n",
              "   'measurement': {'precision': 0.14698162729658792,\n",
              "    'recall': 0.19310344827586207,\n",
              "    'f_score': 0.16691505216095381},\n",
              "   'drug': {'precision': 0.562358276643991,\n",
              "    'recall': 0.5598194130925508,\n",
              "    'f_score': 0.5610859728506786},\n",
              "   'procedure': {'precision': 0.3992248062015504,\n",
              "    'recall': 0.329073482428115,\n",
              "    'f_score': 0.36077057793345},\n",
              "   'temporal': {'precision': 0.030303030303030304,\n",
              "    'recall': 0.003367003367003367,\n",
              "    'f_score': 0.006060606060606061},\n",
              "   'person': {'precision': 0.053763440860215055,\n",
              "    'recall': 0.03759398496240601,\n",
              "    'f_score': 0.04424778761061946},\n",
              "   'observation': {'precision': 0.016216216216216217,\n",
              "    'recall': 0.018072289156626505,\n",
              "    'f_score': 0.017094017094017092},\n",
              "   'value': {'precision': 0.05555555555555555,\n",
              "    'recall': 0.002849002849002849,\n",
              "    'f_score': 0.005420054200542005},\n",
              "   'device': {'precision': 0.32,\n",
              "    'recall': 0.34782608695652173,\n",
              "    'f_score': 0.3333333333333333}},\n",
              "  'relax': {'condition': {'precision': 0.7141318010414797,\n",
              "    'recall': 0.790341812400636,\n",
              "    'f_score': 0.750306574851429},\n",
              "   'measurement': {'precision': 0.25196850393700787,\n",
              "    'recall': 0.3310344827586207,\n",
              "    'f_score': 0.28614008941877794},\n",
              "   'drug': {'precision': 0.6893424036281179,\n",
              "    'recall': 0.6862302483069977,\n",
              "    'f_score': 0.6877828054298643},\n",
              "   'procedure': {'precision': 0.5271317829457365,\n",
              "    'recall': 0.43450479233226835,\n",
              "    'f_score': 0.4763572679509632},\n",
              "   'temporal': {'precision': 0.5757575757575758,\n",
              "    'recall': 0.06397306397306397,\n",
              "    'f_score': 0.11515151515151514},\n",
              "   'person': {'precision': 0.12903225806451613,\n",
              "    'recall': 0.09022556390977443,\n",
              "    'f_score': 0.10619469026548671},\n",
              "   'observation': {'precision': 0.0918918918918919,\n",
              "    'recall': 0.10240963855421686,\n",
              "    'f_score': 0.09686609686609687},\n",
              "   'value': {'precision': 0.7222222222222222,\n",
              "    'recall': 0.037037037037037035,\n",
              "    'f_score': 0.07046070460704607},\n",
              "   'device': {'precision': 0.44,\n",
              "    'recall': 0.4782608695652174,\n",
              "    'f_score': 0.4583333333333333}}}}"
            ]
          },
          "metadata": {},
          "execution_count": 141
        }
      ],
      "source": [
        "evaluator.performance"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 143,
      "metadata": {
        "id": "vikGKa6fOLPK"
      },
      "outputs": [],
      "source": [
        "evaluator.save_evaluation('eval_paper_full_generative.json')"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "evaluator.get_counts()"
      ],
      "metadata": {
        "id": "_nF89z1JiAqF",
        "outputId": "94c401df-5c0a-46d7-b6f4-3a2b12aa39cd",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 142,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'expect': {'overall': 7048,\n",
              "  'condition': 5032,\n",
              "  'person': 133,\n",
              "  'value': 351,\n",
              "  'observation': 166,\n",
              "  'drug': 443,\n",
              "  'measurement': 290,\n",
              "  'procedure': 313,\n",
              "  'temporal': 297,\n",
              "  'device': 23},\n",
              " 'prediction': {'strict': {'overall': {'total': 7003,\n",
              "    'true': 3781,\n",
              "    'false': 3222},\n",
              "   'condition': {'total': 5569, 'true': 3356, 'false': 2213},\n",
              "   'measurement': {'total': 381, 'true': 56, 'false': 325},\n",
              "   'drug': {'total': 441, 'true': 248, 'false': 193},\n",
              "   'procedure': {'total': 258, 'true': 103, 'false': 155},\n",
              "   'temporal': {'total': 33, 'true': 1, 'false': 32},\n",
              "   'person': {'total': 93, 'true': 5, 'false': 88},\n",
              "   'observation': {'total': 185, 'true': 3, 'false': 182},\n",
              "   'value': {'total': 18, 'true': 1, 'false': 17},\n",
              "   'device': {'total': 25, 'true': 8, 'false': 17}},\n",
              "  'relax': {'overall': {'total': 7003, 'true': 4585, 'false': 2418},\n",
              "   'condition': {'total': 5569, 'true': 3977, 'false': 1592},\n",
              "   'measurement': {'total': 381, 'true': 96, 'false': 285},\n",
              "   'drug': {'total': 441, 'true': 304, 'false': 137},\n",
              "   'procedure': {'total': 258, 'true': 136, 'false': 122},\n",
              "   'temporal': {'total': 33, 'true': 19, 'false': 14},\n",
              "   'person': {'total': 93, 'true': 12, 'false': 81},\n",
              "   'observation': {'total': 185, 'true': 17, 'false': 168},\n",
              "   'value': {'total': 18, 'true': 13, 'false': 5},\n",
              "   'device': {'total': 25, 'true': 11, 'false': 14}}}}"
            ]
          },
          "metadata": {},
          "execution_count": 142
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "evaluator.save_evaluation('drive/MyDrive/TER-LISN-2024/eval_paper_full_generative.json')"
      ],
      "metadata": {
        "id": "zeN6oRIzL9I7"
      },
      "execution_count": 144,
      "outputs": []
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.10.13"
    },
    "colab": {
      "provenance": [],
      "gpuType": "T4",
      "include_colab_link": true
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "009541513b8e451886782e457d7cd479": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "VBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "VBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "VBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_523cf83bffdc4d8bbe06c9d886e5d884",
              "IPY_MODEL_0a508a2d17d04b76aa6bad327e25da96",
              "IPY_MODEL_7fd41498666942149a04c076f61cb6ad",
              "IPY_MODEL_32cfc02171c1420599bbbeca98ebfc3b"
            ],
            "layout": "IPY_MODEL_6ab04465b07747a592cb28f77c6b5e20"
          }
        },
        "519375173fe64735a4faccf127abcf90": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_2379403cd2df4ffd9f7d4eecbf571d7e",
            "placeholder": "​",
            "style": "IPY_MODEL_382eff5973b942678fde4a9c6e9c8150",
            "value": "<center> <img\nsrc=https://huggingface.co/front/assets/huggingface_logo-noborder.svg\nalt='Hugging Face'> <br> Copy a token from <a\nhref=\"https://huggingface.co/settings/tokens\" target=\"_blank\">your Hugging Face\ntokens page</a> and paste it below. <br> Immediately click login after copying\nyour token or it might be stored in plain text in this notebook file. </center>"
          }
        },
        "484017270bb646119bc605fccb88fa4e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "PasswordModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "PasswordModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "PasswordView",
            "continuous_update": true,
            "description": "Token:",
            "description_tooltip": null,
            "disabled": false,
            "layout": "IPY_MODEL_aec5500c07b54365ab252b5746e08127",
            "placeholder": "​",
            "style": "IPY_MODEL_86ed1786dfcc49148bc65e88627a34f3",
            "value": ""
          }
        },
        "23dc47bf61314784b7ffb42d995bbce9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "CheckboxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "CheckboxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "CheckboxView",
            "description": "Add token as git credential?",
            "description_tooltip": null,
            "disabled": false,
            "indent": true,
            "layout": "IPY_MODEL_36b52f9261ae4c589efc8faab5806509",
            "style": "IPY_MODEL_8c0ca5e87a1243ba94d821d87dfd7cc1",
            "value": true
          }
        },
        "7fdc12ed98be4101a7ff8d15272d5033": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ButtonModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ButtonModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ButtonView",
            "button_style": "",
            "description": "Login",
            "disabled": false,
            "icon": "",
            "layout": "IPY_MODEL_1d62a5d167c04e839152933688357dbe",
            "style": "IPY_MODEL_20614482956a4d359d7e6470fe3b5236",
            "tooltip": ""
          }
        },
        "683aa055aae842adabca4ee3fa9a16bf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b46a3ac13be24c04a0c47d424e54d256",
            "placeholder": "​",
            "style": "IPY_MODEL_94885d557c64483bbcb842c570e380a5",
            "value": "\n<b>Pro Tip:</b> If you don't already have one, you can create a dedicated\n'notebooks' token with 'write' access, that you can then easily reuse for all\nnotebooks. </center>"
          }
        },
        "6ab04465b07747a592cb28f77c6b5e20": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": "center",
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": "flex",
            "flex": null,
            "flex_flow": "column",
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": "50%"
          }
        },
        "2379403cd2df4ffd9f7d4eecbf571d7e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "382eff5973b942678fde4a9c6e9c8150": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "aec5500c07b54365ab252b5746e08127": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "86ed1786dfcc49148bc65e88627a34f3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "36b52f9261ae4c589efc8faab5806509": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8c0ca5e87a1243ba94d821d87dfd7cc1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "1d62a5d167c04e839152933688357dbe": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "20614482956a4d359d7e6470fe3b5236": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ButtonStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ButtonStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "button_color": null,
            "font_weight": ""
          }
        },
        "b46a3ac13be24c04a0c47d424e54d256": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "94885d557c64483bbcb842c570e380a5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "0ca2c9f7ad19437886ef4a9aa74cf952": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "LabelModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "LabelModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "LabelView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_50557c598d9246599d7e894137d2df26",
            "placeholder": "​",
            "style": "IPY_MODEL_ab9a60b256554da0a101dc9d541cbb8c",
            "value": "Connecting..."
          }
        },
        "50557c598d9246599d7e894137d2df26": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ab9a60b256554da0a101dc9d541cbb8c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "523cf83bffdc4d8bbe06c9d886e5d884": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "LabelModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "LabelModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "LabelView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c72717e7d2714c7aa695cf5b6ba2481c",
            "placeholder": "​",
            "style": "IPY_MODEL_45c84a726d5e470babfb03cd5b139de7",
            "value": "Token is valid (permission: write)."
          }
        },
        "0a508a2d17d04b76aa6bad327e25da96": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "LabelModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "LabelModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "LabelView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ec182a1a030c46ac87b2fa998ea9d868",
            "placeholder": "​",
            "style": "IPY_MODEL_22972fbed4cc4b408be8abe9f9bb2847",
            "value": "Your token has been saved in your configured git credential helpers (store)."
          }
        },
        "7fd41498666942149a04c076f61cb6ad": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "LabelModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "LabelModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "LabelView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_09fc71a94b2f4766a02b5879ba232e95",
            "placeholder": "​",
            "style": "IPY_MODEL_da47463752ff4288a73b1802d37941aa",
            "value": "Your token has been saved to /root/.cache/huggingface/token"
          }
        },
        "32cfc02171c1420599bbbeca98ebfc3b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "LabelModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "LabelModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "LabelView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_9e89840302da47e3a383888883bbadea",
            "placeholder": "​",
            "style": "IPY_MODEL_5ecf0ca50841453d85a082217c033fae",
            "value": "Login successful"
          }
        },
        "c72717e7d2714c7aa695cf5b6ba2481c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "45c84a726d5e470babfb03cd5b139de7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "ec182a1a030c46ac87b2fa998ea9d868": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "22972fbed4cc4b408be8abe9f9bb2847": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "09fc71a94b2f4766a02b5879ba232e95": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "da47463752ff4288a73b1802d37941aa": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "9e89840302da47e3a383888883bbadea": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5ecf0ca50841453d85a082217c033fae": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "1ca11a0dedb44e6897fd914763d6731d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_b418a6bbce1346bd9c0076f2d31bad26",
              "IPY_MODEL_b7975e8109d74252ae427c2507b70c2c",
              "IPY_MODEL_f0af7ea71ed14275bb36b55c87d6c785"
            ],
            "layout": "IPY_MODEL_ddb109aa9fed447b9e8929f2ce49b8d2"
          }
        },
        "b418a6bbce1346bd9c0076f2d31bad26": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_6e57a33e6f4641bd85d68b6773657962",
            "placeholder": "​",
            "style": "IPY_MODEL_901e5be78e494518924d3237c3398b09",
            "value": "Loading checkpoint shards: 100%"
          }
        },
        "b7975e8109d74252ae427c2507b70c2c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_4fdf171fbb704213a0df598adb0c918c",
            "max": 2,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_40c20831011e4706aedbe2ed73219dfd",
            "value": 2
          }
        },
        "f0af7ea71ed14275bb36b55c87d6c785": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_83849f3935464b2ab0f77925aab9d7f3",
            "placeholder": "​",
            "style": "IPY_MODEL_e7c31cc6d3a64a4fb911f3d2023ecfb1",
            "value": " 2/2 [01:15&lt;00:00, 35.11s/it]"
          }
        },
        "ddb109aa9fed447b9e8929f2ce49b8d2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6e57a33e6f4641bd85d68b6773657962": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "901e5be78e494518924d3237c3398b09": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "4fdf171fbb704213a0df598adb0c918c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "40c20831011e4706aedbe2ed73219dfd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "83849f3935464b2ab0f77925aab9d7f3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e7c31cc6d3a64a4fb911f3d2023ecfb1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "4a35141fd2544fd08d1c5748a16c28f6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_d6e6a89659834daa88bcef28c1d5d220",
              "IPY_MODEL_b4f30646efe04849afd1b45f50dd3931",
              "IPY_MODEL_9a90159ea82b4c3f921839ea7b6f9c86"
            ],
            "layout": "IPY_MODEL_8698beb6d78342caaa49ea61c9ebcc58"
          }
        },
        "d6e6a89659834daa88bcef28c1d5d220": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5562bbdb7f064ba4867374297f2c7ef4",
            "placeholder": "​",
            "style": "IPY_MODEL_f5f58531d6474340a7e5d4ccdc171969",
            "value": "Map: 100%"
          }
        },
        "b4f30646efe04849afd1b45f50dd3931": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_57e9591c1f1a4be58f8a826059e27d05",
            "max": 1307,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_e52d41b0f7084b83abfc2dce289e9757",
            "value": 1307
          }
        },
        "9a90159ea82b4c3f921839ea7b6f9c86": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_49662c4803d846d6bb15e8ba44e4b818",
            "placeholder": "​",
            "style": "IPY_MODEL_a09956aac8fb4447b241f1dc646c6429",
            "value": " 1307/1307 [00:00&lt;00:00, 1412.20 examples/s]"
          }
        },
        "8698beb6d78342caaa49ea61c9ebcc58": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5562bbdb7f064ba4867374297f2c7ef4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f5f58531d6474340a7e5d4ccdc171969": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "57e9591c1f1a4be58f8a826059e27d05": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e52d41b0f7084b83abfc2dce289e9757": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "49662c4803d846d6bb15e8ba44e4b818": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a09956aac8fb4447b241f1dc646c6429": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "162bfe1f36444ce1861ceaf2c8f6a3f3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_d33896d2c9c546e1b9d5cc12bb866d51",
              "IPY_MODEL_af607d56eb2c4de88e84315ebf4e5bed",
              "IPY_MODEL_0f469022363b4b0d96762dc3aafebffe"
            ],
            "layout": "IPY_MODEL_dc63518146884067aaebf37d682e83b5"
          }
        },
        "d33896d2c9c546e1b9d5cc12bb866d51": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_676f0e00bfba466ea183f41ba064ff95",
            "placeholder": "​",
            "style": "IPY_MODEL_91796acd865b49e39500cbc7fdfa2e1e",
            "value": "Map: 100%"
          }
        },
        "af607d56eb2c4de88e84315ebf4e5bed": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_48df188dce924c32b510d1ceb4d2450c",
            "max": 1307,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_78810abaada343a998519b4cb128bf3a",
            "value": 1307
          }
        },
        "0f469022363b4b0d96762dc3aafebffe": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5f6c93a23a55413f86aa4778e7d2a6c3",
            "placeholder": "​",
            "style": "IPY_MODEL_9b21d0ab5e874bf3aa723bb2114eef80",
            "value": " 1307/1307 [00:00&lt;00:00, 3328.06 examples/s]"
          }
        },
        "dc63518146884067aaebf37d682e83b5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "676f0e00bfba466ea183f41ba064ff95": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "91796acd865b49e39500cbc7fdfa2e1e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "48df188dce924c32b510d1ceb4d2450c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "78810abaada343a998519b4cb128bf3a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "5f6c93a23a55413f86aa4778e7d2a6c3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9b21d0ab5e874bf3aa723bb2114eef80": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}