{

  "cells": [

    {

      "cell_type": "markdown",

      "metadata": {

        "id": "view-in-github",

        "colab_type": "text"

      },

      "source": [

        "<a href=\"https://colab.research.google.com/github/jlopetegui98/NER-ClinicalTrials-Elegibility-Criteria/blob/main/Roberta%2BLLM/compare_two_prompts.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"

      ]

    },

    {

      "cell_type": "code",

      "execution_count": 1,

      "metadata": {

        "colab": {

          "base_uri": "https://localhost:8080/"

        },

        "id": "QO5obdMWWJJw",

        "outputId": "c0b8b68d-1af6-4308-8ea1-873bd428ecd5"

      },

      "outputs": [

        {

          "output_type": "stream",

          "name": "stdout",

          "text": [

            "Mounted at /content/drive\n"

          ]

        }

      ],

      "source": [

        "# uncomment if working in colab\n",

        "from google.colab import drive\n",

        "drive.mount('/content/drive')"

      ]

    },

    {

      "cell_type": "code",

      "execution_count": 2,

      "metadata": {

        "colab": {

          "base_uri": "https://localhost:8080/"

        },

        "id": "uXHZTGK9WJJx",

        "outputId": "c8a0372f-8774-40df-d275-03e90c2c8848"

      },

      "outputs": [

        {

          "output_type": "stream",

          "name": "stdout",

          "text": [

            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m510.5/510.5 kB\u001b[0m \u001b[31m8.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",

            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m116.3/116.3 kB\u001b[0m \u001b[31m13.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",

            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m194.1/194.1 kB\u001b[0m \u001b[31m11.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",

            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m134.8/134.8 kB\u001b[0m \u001b[31m12.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",

            "\u001b[?25hCollecting seqeval\n",

            "  Downloading seqeval-1.2.2.tar.gz (43 kB)\n",

            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m43.6/43.6 kB\u001b[0m \u001b[31m1.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",

            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",

            "Requirement already satisfied: numpy>=1.14.0 in /usr/local/lib/python3.10/dist-packages (from seqeval) (1.25.2)\n",

            "Requirement already satisfied: scikit-learn>=0.21.3 in /usr/local/lib/python3.10/dist-packages (from seqeval) (1.2.2)\n",

            "Requirement already satisfied: scipy>=1.3.2 in /usr/local/lib/python3.10/dist-packages (from scikit-learn>=0.21.3->seqeval) (1.11.4)\n",

            "Requirement already satisfied: joblib>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from scikit-learn>=0.21.3->seqeval) (1.4.0)\n",

            "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn>=0.21.3->seqeval) (3.4.0)\n",

            "Building wheels for collected packages: seqeval\n",

            "  Building wheel for seqeval (setup.py) ... \u001b[?25l\u001b[?25hdone\n",

            "  Created wheel for seqeval: filename=seqeval-1.2.2-py3-none-any.whl size=16161 sha256=3d05e378a95e6360b53d3fd878ed43d9796294678465f3f139c5e30bef6ab718\n",

            "  Stored in directory: /root/.cache/pip/wheels/1a/67/4a/ad4082dd7dfc30f2abfe4d80a2ed5926a506eb8a972b4767fa\n",

            "Successfully built seqeval\n",

            "Installing collected packages: seqeval\n",

            "Successfully installed seqeval-1.2.2\n",

            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m84.1/84.1 kB\u001b[0m \u001b[31m2.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",

            "\u001b[?25h  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",

            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",

            "  Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",

            "  Building wheel for transformers (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",

            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m119.8/119.8 MB\u001b[0m \u001b[31m8.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",

            "\u001b[?25h  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",

            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",

            "  Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",

            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m297.4/297.4 kB\u001b[0m \u001b[31m6.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",

            "\u001b[?25h  Building wheel for peft (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",

            "  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",

            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",

            "  Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",

            "  Building wheel for accelerate (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n"

          ]

        }

      ],

      "source": [

        "# uncomment if using colab\n",

        "!pip install -q -U datasets\n",

        "!pip install seqeval\n",

        "!pip install -q -U evaluate\n",

        "!pip install -q -U git+https://github.com/huggingface/transformers.git\n",

        "!pip install -q -U bitsandbytes\n",

        "# !pip install -i https://pypi.org/simple/ bitsandbytes\n",

        "!pip install -q -U git+https://github.com/huggingface/peft.git\n",

        "!pip install -q -U git+https://github.com/huggingface/accelerate.git"

      ]

    },

    {

      "cell_type": "code",

      "execution_count": 1,

      "metadata": {

        "id": "iwRXECp_WJJx"

      },

      "outputs": [],

      "source": [

        "from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig, TextGenerationPipeline\n",

        "import torch\n",

        "import accelerate\n",

        "import os\n",

        "from utils import *"

      ]

    },

    {

      "cell_type": "code",

      "source": [

        "import pandas as pd\n",

        "from datasets import Dataset, DatasetDict"

      ],

      "metadata": {

        "id": "-De8g6lgpWtZ"

      },

      "execution_count": 2,

      "outputs": []

    },

    {

      "cell_type": "code",

      "source": [

        "from datasets import load_dataset, load_metric"

      ],

      "metadata": {

        "id": "e7Bpoz11bRe3"

      },

      "execution_count": 3,

      "outputs": []

    },

    {

      "cell_type": "code",

      "execution_count": 4,

      "metadata": {

        "colab": {

          "base_uri": "https://localhost:8080/",

          "height": 159,

          "referenced_widgets": [

            "0f4e594a6aa64dd697fb841d4207d4b7",

            "413c1cd08da84e32ba2de36d891bf86a",

            "ba7f75db7de145abb174658862ef50a3",

            "d55f2bd66fe044e4816348c1b4b032bf",

            "d1fd6600f76a4bf7968b0d13de093148",

            "b00aef23e75445c785a6b3b8756b9c94",

            "1fa716b65c7f47989184cc24bd56e5bd",

            "494b9dacddb948f7b08b509e7a79f3f1",

            "e3e2f93363b74e24bf22abcaf021b3dc",

            "b607f0954b3844e18ed2a887372d42c5",

            "95176ba773274023b4e3356bb3cb4cc9",

            "ac4c65187e864fd09e296fcbbe3ad6d8",

            "540f2e543376445d849e1a56ac38e4d4",

            "a9d9e19fa5ae41109e7c7cf9a4d8a13c",

            "4fd000047a0a402194cb3b99bb59d8f9",

            "9d08585ce6fd40538dc10d1c5d1aee70",

            "f4642fc990e94bf99c7d26017d8771ba",

            "9c49b620b2c04b83ae01bf4ab06e7270",

            "d47d0c2d528c40a38965fac763e450cf",

            "374d4f7c917d48c2b81f24cbfcfce062",

            "9f151f163b2c4966b9460dd31e184a84",

            "67a9e41b56a741baa4a953905d11c263",

            "e32ad767760d4bddb2f3a0a38327339b",

            "681dac30b90b4b02b38f9e5bb99429c0",

            "dd0df631524e42e48299283f1a9555e9",

            "609d5b49e085499994a11342e8dd425f",

            "3e7792c5d2484e24950f1f83f899ba4a",

            "8fae6bd7ebc54636a14b375fa46ec0dc",

            "7fa56a404f8847239109441a3b043a77",

            "d34e4794cb944c1cb8ae88845cad54fb",

            "4856843799e242ac8b72cf37ad1a159f",

            "c3c6d1e8aeb14e999ea3fadc10539861"

          ]

        },

        "id": "SMrGGcyPWJJy",

        "outputId": "1988c833-06f1-44c2-c832-525b2dabee55"

      },

      "outputs": [

        {

          "output_type": "display_data",

          "data": {

            "text/plain": [

              "VBox(children=(HTML(value='<center> <img\\nsrc=https://huggingface.co/front/assets/huggingface_logo-noborder.sv…"

            ],

            "application/vnd.jupyter.widget-view+json": {

              "version_major": 2,

              "version_minor": 0,

              "model_id": "0f4e594a6aa64dd697fb841d4207d4b7"

            }

          },

          "metadata": {}

        }

      ],

      "source": [

        "from huggingface_hub import notebook_login\n",

        "\n",

        "notebook_login()"

      ]

    },

    {

      "cell_type": "code",

      "execution_count": 5,

      "metadata": {

        "id": "eOQPAyi5WJJy"

      },

      "outputs": [],

      "source": [

        "# dict for the entities (entity to int value)\n",

        "simple_ent = {\"Condition\", \"Value\", \"Drug\", \"Procedure\", \"Measurement\", \"Temporal\", \"Observation\", \"Person\", \"Device\"}\n",

        "sel_ent = {\n",

        "    \"O\": 0,\n",

        "    \"B-Condition\": 1,\n",

        "    \"I-Condition\": 2,\n",

        "    \"B-Value\": 3,\n",

        "    \"I-Value\": 4,\n",

        "    \"B-Drug\": 5,\n",

        "    \"I-Drug\": 6,\n",

        "    \"B-Procedure\": 7,\n",

        "    \"I-Procedure\": 8,\n",

        "    \"B-Measurement\": 9,\n",

        "    \"I-Measurement\": 10,\n",

        "    \"B-Temporal\": 11,\n",

        "    \"I-Temporal\": 12,\n",

        "    \"B-Observation\": 13,\n",

        "    \"I-Observation\": 14,\n",

        "    \"B-Person\": 15,\n",

        "    \"I-Person\": 16,\n",

        "    \"B-Device\": 17,\n",

        "    \"I-Device\": 18\n",

        "}\n",

        "\n",

        "entities_list = list(sel_ent.keys())\n",

        "sel_ent_inv = {v: k for k, v in sel_ent.items()}"

      ]

    },

    {

      "cell_type": "code",

      "execution_count": 6,

      "metadata": {

        "id": "5mCzQOK7WJJy"

      },

      "outputs": [],

      "source": [

        "root = '..'\n",

        "root = './drive/MyDrive/TER-LISN-2024'\n",

        "data_path = f'{root}/data'\n",

        "models_path = f'{root}/models'"

      ]

    },

    {

      "cell_type": "code",

      "execution_count": 7,

      "metadata": {

        "id": "9WoScuG9WJJy"

      },

      "outputs": [],

      "source": [

        "model_name = \"mistralai/Mistral-7B-v0.1\""

      ]

    },

    {

      "cell_type": "code",

      "execution_count": 8,

      "metadata": {

        "colab": {

          "base_uri": "https://localhost:8080/",

          "height": 173,

          "referenced_widgets": [

            "e1b0ea4e59bd4bebb31544a4ca43fae6",

            "bb7c0220e2404676a2a37b5f8e45a560",

            "d9a38641722d4d1698c9eb12c9ba2aa2",

            "726b3712020e475e8e5734e51ad3255b",

            "b834f5ce5d4b4e8eac3688c2119e0e6e",

            "99fea94eaf344a1687764756225e9a88",

            "640023c221074c73a38d147a05cfedaf",

            "e2a1e70aa375450299d81baa7a007f1f",

            "4f06ce02f7b8452981731c87ac3814bc",

            "ed7294f2acf84015ac0c0c3b78c629cb",

            "1cf1101362c545738dcbb5e528f1af33"

          ]

        },

        "id": "-R1Ht6AqWJJy",

        "outputId": "9b883de3-f658-4402-c902-cfd946986427"

      },

      "outputs": [

        {

          "output_type": "stream",

          "name": "stderr",

          "text": [

            "/usr/local/lib/python3.10/dist-packages/huggingface_hub/utils/_token.py:88: UserWarning: \n",

            "The secret `HF_TOKEN` does not exist in your Colab secrets.\n",

            "To authenticate with the Hugging Face Hub, create a token in your settings tab (https://huggingface.co/settings/tokens), set it as secret in your Google Colab and restart your session.\n",

            "You will be able to reuse this secret in all of your notebooks.\n",

            "Please note that authentication is recommended but still optional to access public models or datasets.\n",

            "  warnings.warn(\n"

          ]

        },

        {

          "output_type": "display_data",

          "data": {

            "text/plain": [

              "Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]"

            ],

            "application/vnd.jupyter.widget-view+json": {

              "version_major": 2,

              "version_minor": 0,

              "model_id": "e1b0ea4e59bd4bebb31544a4ca43fae6"

            }

          },

          "metadata": {}

        }

      ],

      "source": [

        "# Load base model(Mistral 7B)\n",

        "bnb_config = BitsAndBytesConfig(\n",

        "    load_in_4bit= True,\n",

        "    bnb_4bit_quant_type= \"nf4\",\n",

        "    bnb_4bit_compute_dtype= torch.bfloat16,\n",

        "    bnb_4bit_use_double_quant= False,\n",

        ")\n",

        "model = AutoModelForCausalLM.from_pretrained(\n",

        "   model_name,\n",

        "    quantization_config=bnb_config,\n",

        "    device_map={\"\": 0}\n",

        ")"

      ]

    },

    {

      "cell_type": "code",

      "execution_count": 9,

      "metadata": {

        "colab": {

          "base_uri": "https://localhost:8080/"

        },

        "id": "zjt27HXtWJJz",

        "outputId": "0df3c4ce-03eb-4f64-9546-b790d5fdd3a8"

      },

      "outputs": [

        {

          "output_type": "execute_result",

          "data": {

            "text/plain": [

              "(True, True)"

            ]

          },

          "metadata": {},

          "execution_count": 9

        }

      ],

      "source": [

        "# import tokenizer for mistral-7B\n",

        "tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)\n",

        "tokenizer.padding_side = 'left'\n",

        "tokenizer.pad_token = tokenizer.eos_token\n",

        "tokenizer.add_eos_token = True\n",

        "tokenizer.add_bos_token, tokenizer.add_eos_token"

      ]

    },

    {

      "cell_type": "code",

      "execution_count": 10,

      "metadata": {

        "id": "_CvK-Os_WJJz"

      },

      "outputs": [],

      "source": [

        "pipe  = TextGenerationPipeline(model = model, tokenizer = tokenizer)"

      ]

    },

    {

      "cell_type": "code",

      "source": [],

      "metadata": {

        "id": "ooEQwK2tSDat"

      },

      "execution_count": null,

      "outputs": []

    },

    {

      "cell_type": "code",

      "execution_count": 11,

      "metadata": {

        "id": "DjRonD-qWJJz"

      },

      "outputs": [],

      "source": [

        "dataset = load_dataset('JavierLopetegui/chia_v1')"

      ]

    },

    {

      "cell_type": "code",

      "execution_count": 12,

      "metadata": {

        "id": "4OY0LfqXWJJz"

      },

      "outputs": [],

      "source": [

        "# for each sentence save the text\n",

        "def generate_sentences_from_tokens(sentences):\n",

        "    texts_sentences = []\n",

        "    sentences_tokens = sentences['tokens']\n",

        "    for sentence in sentences_tokens:\n",

        "        sent_text = \" \".join(sentence)\n",

        "        texts_sentences.append(sent_text)\n",

        "    sentences['text'] = texts_sentences\n",

        "    return sentences"

      ]

    },

    {

      "cell_type": "code",

      "execution_count": 13,

      "metadata": {

        "id": "3RB2Ao-7WJJz"

      },

      "outputs": [],

      "source": [

        "def build_prompts(sentences, prompt_type=2):\n",

        "    sentences_prompts = []\n",

        "    for sent in sentences['text']:\n",

        "        prompt = build_prompt(sent, prompt_type)\n",

        "        sentences_prompts.append(prompt)\n",

        "    sentences['prompt'] = sentences_prompts\n",

        "    return sentences"

      ]

    },

    {

      "cell_type": "code",

      "execution_count": 14,

      "metadata": {

        "id": "brYkkOKPWJJ0"

      },

      "outputs": [],

      "source": [

        "dataset = dataset.map(lambda x: generate_sentences_from_tokens(x), batched = True)\n",

        "dataset_prompt1 = dataset.map(lambda x: build_prompts(x, prompt_type=1), batched = True)\n",

        "dataset_prompt2 = dataset.map(lambda x: build_prompts(x, prompt_type=2), batched = True)"

      ]

    },

    {

      "cell_type": "code",

      "execution_count": 15,

      "metadata": {

        "id": "-DsYscp_WJJ0"

      },

      "outputs": [],

      "source": [

        "test_dataset_p1 = dataset_prompt1['test']\n",

        "test_dataset_p2 = dataset_prompt2['test']"

      ]

    },

    {

      "cell_type": "code",

      "source": [

        "test_dataset_p1['prompt'][0]"

      ],

      "metadata": {

        "colab": {

          "base_uri": "https://localhost:8080/",

          "height": 209

        },

        "id": "brQSqtjxbbJ8",

        "outputId": "2da15406-ca32-490f-8349-c5f0b78fb1b9"

      },

      "execution_count": 16,

      "outputs": [

        {

          "output_type": "execute_result",

          "data": {

            "text/plain": [

              "'I need to perform a named entity recognition task on a  text related with inclusion criteria in clinical trials.\\n    The entities you need to recognize are: Condition, Value, Drug, Procedure, Measurement, Temporal, Observation, Person, Mood, Device and Pregnancy_considerations.\\n    Particularly you have to produce the ouput in the BIO format. I will show you an example of the expected output.\\n    Input text: Patients with symptomatic CNS metastases or leptomeningeal involvement\\n    Output:\\n    Patients O\\n    with O\\n    symptomatic O\\n    CNS B-Condition\\n    metastases I-Condition\\n    or O\\n    leptomeningeal B-Condition\\n    involvement I-Condition\\n\\n    You can see that tokens without any entity are labeled as O, and the tokens that are part of an entity are labeled as B-<entity> or I-<entity> depending on if they are the beginning or the inside of the entity.\\n    Please, just answer the question for this specific example and stop writting after that.\\n    Input text: self - reported healthy adults between the ages of 18 - 60 who are fluent in English .'"

            ],

            "application/vnd.google.colaboratory.intrinsic+json": {

              "type": "string"

            }

          },

          "metadata": {},

          "execution_count": 16

        }

      ]

    },

    {

      "cell_type": "code",

      "source": [

        "test_dataset_p2['prompt'][0]"

      ],

      "metadata": {

        "colab": {

          "base_uri": "https://localhost:8080/",

          "height": 209

        },

        "id": "tJzeOg9vbh3j",

        "outputId": "aaeb49fb-3d83-4832-abda-034df0f7aede"

      },

      "execution_count": 17,

      "outputs": [

        {

          "output_type": "execute_result",

          "data": {

            "text/plain": [

              "'I am working on a named entity recognition problem, in the context of clinical\\n    trials eligibility criteria. I will show you the list of entities:\\n    - Condition\\n    - Value\\n    - Drug\\n    - Procedure\\n    - Measurement\\n    - Temporal\\n    - Observation\\n    - Person\\n    - Mood\\n    - Device\\n\\n    Your task consists in annotate the named entities in a given sentence in the format I will explain you.\\n    I will explain you with some examples:\\n\\n    Example 1:\\n    Input: Patients who have received prior chemotherapy for unresectable disease.\\n    Output: Patients who have received prior <Procedure>chemotherapy</Procedure> for <Condition>unresectable disease</Condition>.\\n\\n    Example 2:\\n    Input: Patients with any other severe concurrent disease, which in the judgment of the investigator, would make the patient inappropriate for entry into this study.\\n    Ouput: Patients with any other severe <Condition>concurrent disease</Condition>, which in the judgment of the investigator, would make the patient <Mood>inappropriate for <Observation>entry into this study</Observation>.\\n\\n    As you can see, in each example, the extracted entities are enclosed using the sintax: <ENT>text of the entity</ENT>.\\n\\n    Please now annotate as explained before the following sentence:\\n\\n    Input: self - reported healthy adults between the ages of 18 - 60 who are fluent in English .'"

            ],

            "application/vnd.google.colaboratory.intrinsic+json": {

              "type": "string"

            }

          },

          "metadata": {},

          "execution_count": 17

        }

      ]

    },

    {

      "cell_type": "code",

      "execution_count": null,

      "metadata": {

        "id": "ANrMTM9xWJJ0"

      },

      "outputs": [],

      "source": [

        "# # keep just the prompt column\n",

        "# test_dataset_p1 = test_dataset_p1.remove_columns(['tokens', 'text', 'ner_tags', 'file'])\n",

        "# test_dataset_p2 = test_dataset_p2.remove_columns(['tokens', 'text', 'ner_tags', 'file'])"

      ]

    },

    {

      "cell_type": "code",

      "source": [],

      "metadata": {

        "id": "fTKRysEQWaSM"

      },

      "execution_count": null,

      "outputs": []

    },

    {

      "cell_type": "code",

      "execution_count": null,

      "metadata": {

        "id": "W4SlSxy7WJJ0"

      },

      "outputs": [],

      "source": [

        "# data_loader_p1 = torch.utils.data.DataLoader(test_dataset_p1, batch_size=4, shuffle=False)\n",

        "# data_loader_p2 = torch.utils.data.DataLoader(test_dataset_p2, batch_size=4, shuffle=False)"

      ]

    },

    {

      "cell_type": "code",

      "execution_count": null,

      "metadata": {

        "id": "Kgd_0VL8WJJ0"

      },

      "outputs": [],

      "source": [

        "# generated_sentences_p1 = []\n",

        "# for batch in data_loader_p1:\n",

        "#     generated_sentences_p1.extend(pipe(batch['prompt'], max_new_tokens = 500, return_full_text = False, handle_long_generation = \"hole\"))"

      ]

    },

    {

      "cell_type": "code",

      "execution_count": null,

      "metadata": {

        "id": "nX5E-USMWJJ0"

      },

      "outputs": [],

      "source": [

        "# generated_sentences_p2 = []\n",

        "# for batch in data_loader_p2:\n",

        "#     generated_sentences_p2.extend(pipe(batch['prompt'], max_new_tokens = 500, return_full_text = False, handle_long_generation = \"hole\"))"

      ]

    },

    {

      "cell_type": "code",

      "source": [

        "from tqdm import tqdm"

      ],

      "metadata": {

        "id": "0JaNvN8Ec1Ko"

      },

      "execution_count": 21,

      "outputs": []

    },

    {

      "cell_type": "code",

      "execution_count": 31,

      "metadata": {

        "colab": {

          "base_uri": "https://localhost:8080/"

        },

        "id": "yGBo9jS8WJJ0",

        "outputId": "cc2c9e02-f75c-40ac-a262-7be58e3e09d0"

      },

      "outputs": [

        {

          "output_type": "stream",

          "name": "stderr",

          "text": [

            " 12%|█▏        | 6/50 [04:11<29:42, 40.51s/it]You seem to be using the pipelines sequentially on GPU. In order to maximize efficiency please use a dataset\n",

            "100%|██████████| 50/50 [33:08<00:00, 39.77s/it]\n"

          ]

        }

      ],

      "source": [

        "# generation one by one\n",

        "generated_sentences_p1 = []\n",

        "for sentence in tqdm(test_dataset_p1['prompt'][:50]):\n",

        "    output = pipe(sentence, max_new_tokens = 500, return_full_text = False, handle_long_generation = \"hole\")[0]['generated_text']\n",

        "    output = output.split('\\n\\n')[0]\n",

        "    generated_sentences_p1.append(output)"

      ]

    },

    {

      "cell_type": "code",

      "source": [

        "from tqdm import tqdm"

      ],

      "metadata": {

        "id": "rQ23uzq6A8NW"

      },

      "execution_count": 26,

      "outputs": []

    },

    {

      "cell_type": "code",

      "execution_count": 28,

      "metadata": {

        "colab": {

          "base_uri": "https://localhost:8080/"

        },

        "id": "fASHzsJCWJJ0",

        "outputId": "eb43ea5f-1eed-4794-945b-68fe63fd72a2"

      },

      "outputs": [

        {

          "output_type": "stream",

          "name": "stderr",

          "text": [

            " 14%|█▍        | 7/50 [04:47<29:07, 40.64s/it]You seem to be using the pipelines sequentially on GPU. In order to maximize efficiency please use a dataset\n",

            "100%|██████████| 50/50 [34:05<00:00, 40.91s/it]\n"

          ]

        }

      ],

      "source": [

        "generated_sentences_p2 = []\n",

        "for sentence in tqdm(test_dataset_p2['prompt'][:50]):\n",

        "    sentence += '\\noutput: '\n",

        "    output = pipe(sentence, max_new_tokens = 500, return_full_text = False, handle_long_generation = \"hole\")[0]['generated_text']\n",

        "    output = output.split('\\n')[0]\n",

        "    generated_sentences_p2.append(output)"

      ]

    },

    {

      "cell_type": "code",

      "source": [

        "generated_sentences_p2[0]"

      ],

      "metadata": {

        "colab": {

          "base_uri": "https://localhost:8080/",

          "height": 52

        },

        "id": "DoGkaFUrKux6",

        "outputId": "c42a73f4-0e9c-40a1-9137-cf4adc1f937c"

      },

      "execution_count": 29,

      "outputs": [

        {

          "output_type": "execute_result",

          "data": {

            "text/plain": [

              "' self - reported healthy adults between the ages of <Measurement>18 - 60</Measurement> who are fluent in <Language>English</Language>.'"

            ],

            "application/vnd.google.colaboratory.intrinsic+json": {

              "type": "string"

            }

          },

          "metadata": {},

          "execution_count": 29

        }

      ]

    },

    {

      "cell_type": "code",

      "execution_count": 30,

      "metadata": {

        "colab": {

          "base_uri": "https://localhost:8080/",

          "height": 145,

          "referenced_widgets": [

            "c8846cf2991a4e93a0e91bc1802f1079",

            "3b12538b8e254b4ebcc79d694b2b272c",

            "205f861cb2ed48209b795217f645e077",

            "25605c2b3e93464daedc1fcade1f47b4",

            "98919ba0701545fd997e3d71f4eff69e",

            "013efa5f201d460e9332bd047982c449",

            "7a0323004bd14c579c27df13bf3f861b",

            "ef49e0a4613f4207b11012c494b4ed15",

            "f88c80704c59405f9224c9a98105303d",

            "1a4eab919e9e475ea452687215355489",

            "cf5d7616d08944108ae6dcfdb487cd5a",

            "ae392574043b40c8bf461468e8f06f18",

            "8af0715bcf2647c390c073430ffbbe5d",

            "44e3af9f1124467b94cbb6f90f1669a3",

            "804dbc29e4a9404d935b8ab10dc481e9",

            "6ed75fda4c3f4fb2ab931ee7e162b3c4",

            "d71d92ee94994f93bbbdedb475dc5ba5",

            "1780c6cc045a49a6838bc1e43c89dca1",

            "51b7d216e4144dd0a5ea92eb12e410b7",

            "6e2bf10b566a453d8b342e824ec36fdf",

            "1a7b5758d56d4811902698d1ad696163",

            "11047f16bb534e70ad1d6bdde663cfa4",

            "3791512b6e92422799da4e950c6dde6d",

            "161970c8736a46899cd017c5e5906352",

            "cfafc68041754603b0916c8e28a90d0b",

            "a91cbbefe5ca4c07a46ad81424e2f0a7",

            "995f169324d647fbb1c09c7ea3ec22d6",

            "c617e79d6b4448f1af28c2d3c46d4ee5",

            "1d23076205f3470590110feb368afd52",

            "99a5dcb8b54c4e1da8747d16e1d54538",

            "a890ce1e992a4f04a524d3c76927d5f6",

            "4e12dbb6c203422193cf12d58338f1c3",

            "ee8308b8ca9a4dd8b9993824f2df176c",

            "3786b6a492114f58b8c92f904c80b9e2",

            "d6d29174233e4d1596dd391b9a22eac2",

            "fc135f2fe68d4b83ad57ecb3085febe1",

            "f539dc11a57e425ea714a681d5027960",

            "af7fa5b783c54fe4b62e53cd6641de2b",

            "f0e324c229f7477baf1bb26fa4bc85ed",

            "205312c7b62b4410a5aa1298deb1fb6f",

            "4847afa0d5044cb0a0d693495ff05c63",

            "0a7d5f7c3174491990f4f1018cbafe4f",

            "b01bb888c2a646c0af368da52b9b4f2f",

            "bf557e4621f04d56a736fda56470d77e"

          ]

        },

        "id": "3L_-8fahWJJ0",

        "outputId": "4366e679-185a-490a-8b61-2677a32c83c5"

      },

      "outputs": [

        {

          "output_type": "display_data",

          "data": {

            "text/plain": [

              "tokenizer_config.json:   0%|          | 0.00/25.0 [00:00<?, ?B/s]"

            ],

            "application/vnd.jupyter.widget-view+json": {

              "version_major": 2,

              "version_minor": 0,

              "model_id": "c8846cf2991a4e93a0e91bc1802f1079"

            }

          },

          "metadata": {}

        },

        {

          "output_type": "display_data",

          "data": {

            "text/plain": [

              "config.json:   0%|          | 0.00/615 [00:00<?, ?B/s]"

            ],

            "application/vnd.jupyter.widget-view+json": {

              "version_major": 2,

              "version_minor": 0,

              "model_id": "ae392574043b40c8bf461468e8f06f18"

            }

          },

          "metadata": {}

        },

        {

          "output_type": "display_data",

          "data": {

            "text/plain": [

              "sentencepiece.bpe.model:   0%|          | 0.00/5.07M [00:00<?, ?B/s]"

            ],

            "application/vnd.jupyter.widget-view+json": {

              "version_major": 2,

              "version_minor": 0,

              "model_id": "3791512b6e92422799da4e950c6dde6d"

            }

          },

          "metadata": {}

        },

        {

          "output_type": "display_data",

          "data": {

            "text/plain": [

              "tokenizer.json:   0%|          | 0.00/9.10M [00:00<?, ?B/s]"

            ],

            "application/vnd.jupyter.widget-view+json": {

              "version_major": 2,

              "version_minor": 0,

              "model_id": "3786b6a492114f58b8c92f904c80b9e2"

            }

          },

          "metadata": {}

        }

      ],

      "source": [

        "tokenizer = AutoTokenizer.from_pretrained('xlm-roberta-base')"

      ]

    },

    {

      "cell_type": "code",

      "execution_count": 31,

      "metadata": {

        "id": "9TAmhnSSWJJ1"

      },

      "outputs": [],

      "source": [

        "# tokenize and align the labels in the dataset\n",

        "def tokenize_and_align_labels(sentence, flag = 'I'):\n",

        "    \"\"\"\n",

        "    Tokenize the sentence and align the labels\n",

        "    inputs:\n",

        "        sentence: dict, the sentence from the dataset\n",

        "        flag: str, the flag to indicate how to deal with the labels for subwords\n",

        "            - 'I': use the label of the first subword for all subwords but as intermediate (I-ENT)\n",

        "            - 'B': use the label of the first subword for all subwords as beginning (B-ENT)\n",

        "            - None: use -100 for subwords\n",

        "    outputs:\n",

        "        tokenized_sentence: dict, the tokenized sentence now with a field for the labels\n",

        "    \"\"\"\n",

        "    tokenized_sentence = tokenizer(sentence['tokens'], is_split_into_words=True, truncation=True)\n",

        "\n",

        "    labels = []\n",

        "    for i, labels_s in enumerate(sentence['ner_tags']):\n",

        "        word_ids = tokenized_sentence.word_ids(batch_index=i)\n",

        "        previous_word_idx = None\n",

        "        label_ids = []\n",

        "        for word_idx in word_ids:\n",

        "            # if the word_idx is None, assign -100\n",

        "            if word_idx is None:\n",

        "                label_ids.append(-100)\n",

        "            # if it is a new word, assign the corresponding label\n",

        "            elif word_idx != previous_word_idx:\n",

        "                label_ids.append(labels_s[word_idx])\n",

        "            # if it is the same word, check the flag to assign\n",

        "            else:\n",

        "                if flag == 'I':\n",

        "                    if entities_list[labels_s[word_idx]].startswith('I'):\n",

        "                      label_ids.append(labels_s[word_idx])\n",

        "                    else:\n",

        "                      label_ids.append(labels_s[word_idx] + 1)\n",

        "                elif flag == 'B':\n",

        "                    label_ids.append(labels_s[word_idx])\n",

        "                elif flag == None:\n",

        "                    label_ids.append(-100)\n",

        "            previous_word_idx = word_idx\n",

        "        labels.append(label_ids)\n",

        "    tokenized_sentence['labels'] = labels\n",

        "    return tokenized_sentence"

      ]

    },

    {

      "cell_type": "markdown",

      "metadata": {

        "id": "0KSHfmoLWJJ1"

      },

      "source": [

        "**Standarizing true annotations**"

      ]

    },

    {

      "cell_type": "code",

      "source": [

        "import re"

      ],

      "metadata": {

        "id": "p0OeK1ero68J"

      },

      "execution_count": 32,

      "outputs": []

    },

    {

      "cell_type": "code",

      "execution_count": 33,

      "metadata": {

        "colab": {

          "base_uri": "https://localhost:8080/"

        },

        "id": "OVY071a-WJJ2",

        "outputId": "e40e2a96-71e6-45f6-82c6-0c343fdc3f0f"

      },

      "outputs": [

        {

          "output_type": "execute_result",

          "data": {

            "text/plain": [

              "1307"

            ]

          },

          "metadata": {},

          "execution_count": 33

        }

      ],

      "source": [

        "new_true_annotations = []\n",

        "for sent in dataset['test']:\n",

        "    annotation = []\n",

        "    for word, tag in zip(sent['tokens'], sent['ner_tags']):\n",

        "        annotation.append((word, entities_list[tag]))\n",

        "    new_annotation = []\n",

        "    ps = r'(\\.|\\,|\\:|\\;|\\!|\\?|\\-|\\(|\\)|\\[|\\]|\\{|\\}|\\\")'\n",

        "    for i,(word, tag) in enumerate(annotation):\n",

        "        if re.search(ps, word):\n",

        "            # find the ocurrences of the punctuation signs\n",

        "            occurrences = re.finditer(ps, word)\n",

        "            indexes = [(match.start(), match.end()) for match in occurrences]\n",

        "            # create the new tokens\n",

        "            last = 0\n",

        "            for j, (beg, end) in enumerate(indexes):\n",

        "                if beg > last:\n",

        "                    new_annotation.append((word[last:beg], tag))\n",

        "                if tag != \"O\":\n",

        "                    label = f'I-{tag.split(\"-\")[1]}'\n",

        "                else:\n",

        "                    label = \"O\"\n",

        "                if end < len(word) or (i < len(annotation) - 1 and annotation[i+1][1] == label):\n",

        "                    new_annotation.append((word[beg:end], label))\n",

        "                else:\n",

        "                    new_annotation.append((word[beg:end], 'O'))\n",

        "                last = end\n",

        "            if last < len(word):\n",

        "                new_annotation.append((word[last:], label))\n",

        "        else:\n",

        "            new_annotation.append((word, tag))\n",

        "    new_true_annotations.append(new_annotation)\n",

        "len(new_true_annotations)"

      ]

    },

    {

      "cell_type": "code",

      "execution_count": 34,

      "metadata": {

        "colab": {

          "base_uri": "https://localhost:8080/"

        },

        "id": "K0LaWT56WJJ2",

        "outputId": "5d996a00-56dc-421c-d81f-17ad3549330e"

      },

      "outputs": [

        {

          "output_type": "execute_result",

          "data": {

            "text/plain": [

              "1307"

            ]

          },

          "metadata": {},

          "execution_count": 34

        }

      ],

      "source": [

        "true_annotations = []\n",

        "for sent in new_true_annotations:\n",

        "    dicc_sent = {\"tokens\":[], \"ner_tags\":[]}\n",

        "    for word, tag in sent:\n",

        "        dicc_sent[\"tokens\"].append(word)\n",

        "        dicc_sent[\"ner_tags\"].append(sel_ent[tag])\n",

        "    true_annotations.append(dicc_sent)\n",

        "len(true_annotations)"

      ]

    },

    {

      "cell_type": "code",

      "execution_count": 35,

      "metadata": {

        "id": "FQgSF-gaWJJ2"

      },

      "outputs": [],

      "source": [

        "true_df = pd.DataFrame(true_annotations)\n",

        "true_ann_dataset = Dataset.from_pandas(true_df)"

      ]

    },

    {

      "cell_type": "code",

      "execution_count": 36,

      "metadata": {

        "colab": {

          "base_uri": "https://localhost:8080/",

          "height": 49,

          "referenced_widgets": [

            "cabdca2ad56e43eaa506e4588087c3f2",

            "116d6d8de1af401a971f3da77cb4b916",

            "5644e8831133414dbb54e899f1253e8b",

            "75cff91a18654513bc53045df04832f6",

            "883ee6d5131443eba2c4598c9fa0223e",

            "5454af7e0b964f388ecab1f83f74ecc9",

            "9f62543c61b04183b7784826e9178321",

            "dd31f0ed6eee4b6a97d4c9ee6addabe3",

            "9f072c7c1f7646ef876b49917e1eccff",

            "76f09404ea3d447782bb09cd27e9a723",

            "f2356694803b4c678de2ca0a11cb11e2"

          ]

        },

        "id": "5aO9u8ceWJJ2",

        "outputId": "6f999f1c-db84-4839-e645-aa81650f4168"

      },

      "outputs": [

        {

          "output_type": "display_data",

          "data": {

            "text/plain": [

              "Map:   0%|          | 0/1307 [00:00<?, ? examples/s]"

            ],

            "application/vnd.jupyter.widget-view+json": {

              "version_major": 2,

              "version_minor": 0,

              "model_id": "cabdca2ad56e43eaa506e4588087c3f2"

            }

          },

          "metadata": {}

        }

      ],

      "source": [

        "true_ann_dataset = true_ann_dataset.map(tokenize_and_align_labels, batched=True)"

      ]

    },

    {

      "cell_type": "code",

      "source": [

        "true_ann_dataset['ner_tags'][0]"

      ],

      "metadata": {

        "colab": {

          "base_uri": "https://localhost:8080/"

        },

        "id": "D2eYI1XdpirH",

        "outputId": "a65ef39b-f749-4ece-86e9-2ba03f39e4fe"

      },

      "execution_count": 37,

      "outputs": [

        {

          "output_type": "execute_result",

          "data": {

            "text/plain": [

              "[0, 0, 0, 1, 15, 3, 4, 4, 4, 4, 4, 4, 0, 0, 13, 14, 14, 0]"

            ]

          },

          "metadata": {},

          "execution_count": 37

        }

      ]

    },

    {

      "cell_type": "markdown",

      "metadata": {

        "id": "VMMWntNGWJJ2"

      },

      "source": [

        "**Evaluating prompt 1**"

      ]

    },

    {

      "cell_type": "code",

      "execution_count": 121,

      "metadata": {

        "id": "bTLXGyNwWJJ2"

      },

      "outputs": [],

      "source": [

        "new_p1_annotations = []\n",

        "for sent in generated_sentences_p1:\n",

        "    annotation = []\n",

        "    for line in sent.split('\\n')[2:]:\n",

        "        if line != '':\n",

        "            splited_line = line.split()\n",

        "            if len(splited_line) > 2:\n",

        "                splited_line = [' '.join(splited_line[:-1]), splited_line[-1]]\n",

        "            if len(splited_line) != 2:\n",

        "                continue\n",

        "            word, tag = splited_line\n",

        "            annotation.append((word, tag))\n",

        "    new_annotation = []\n",

        "    ps = r'(\\.|\\,|\\:|\\;|\\!|\\?|\\-|\\(|\\)|\\[|\\]|\\{|\\}|\\\")'\n",

        "    for i,(word, tag) in enumerate(annotation):\n",

        "        if tag not in sel_ent:\n",

        "            tag = \"O\"\n",

        "        if re.search(ps, word):\n",

        "            # find the ocurrences of the punctuation signs\n",

        "            occurrences = re.finditer(ps, word)\n",

        "            indexes = [(match.start(), match.end()) for match in occurrences]\n",

        "            # create the new tokens\n",

        "            last = 0\n",

        "            for j, (beg, end) in enumerate(indexes):\n",

        "                if beg > last:\n",

        "                    new_annotation.append((word[last:beg], tag))\n",

        "                if tag != \"O\":\n",

        "                    if not tag.startswith('I') and not tag.startswith('B'):\n",

        "                        label = \"O\"\n",

        "                    else:\n",

        "                        label = f'I-{tag.split(\"-\")[1]}'\n",

        "                else:\n",

        "                    label = \"O\"\n",

        "                if end < len(word) or (i < len(annotation) - 1 and annotation[i+1][1] == label):\n",

        "                    new_annotation.append((word[beg:end], label))\n",

        "                else:\n",

        "                    new_annotation.append((word[beg:end], 'O'))\n",

        "                last = end\n",

        "            if last < len(word):\n",

        "                new_annotation.append((word[last:], label))\n",

        "        else:\n",

        "            new_annotation.append((word, tag))\n",

        "    new_p1_annotations.append(new_annotation)\n",

        "len(new_p1_annotations)"

      ]

    },

    {

      "cell_type": "code",

      "execution_count": 73,

      "metadata": {

        "colab": {

          "base_uri": "https://localhost:8080/"

        },

        "id": "nICoEVoIWJJ3",

        "outputId": "7d65b295-6d8a-4cc1-b751-384433a1c7ea"

      },