{

  "cells": [

    {

      "cell_type": "markdown",

      "metadata": {

        "id": "view-in-github",

        "colab_type": "text"

      },

      "source": [

        "<a href=\"https://colab.research.google.com/github/jlopetegui98/NER-ClinicalTrials-Elegibility-Criteria/blob/main/LLM-Zero-shot_approach/generate_annotations.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"

      ]

    },

    {

      "cell_type": "markdown",

      "metadata": {

        "id": "h2dngSJ8J1h8"

      },

      "source": [

        "### Annotations for the CHIA dataset using mistral-7B"

      ]

    },

    {

      "cell_type": "code",

      "execution_count": 1,

      "metadata": {

        "colab": {

          "base_uri": "https://localhost:8080/"

        },

        "id": "y6X9ot2qJ1h-",

        "outputId": "7cd81d9a-d713-4ecb-c84a-8b452bb05301"

      },

      "outputs": [

        {

          "output_type": "stream",

          "name": "stdout",

          "text": [

            "  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",

            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",

            "  Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",

            "  Building wheel for transformers (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",

            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m105.0/105.0 MB\u001b[0m \u001b[31m8.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",

            "\u001b[?25h  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",

            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",

            "  Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",

            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m279.7/279.7 kB\u001b[0m \u001b[31m6.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",

            "\u001b[?25h  Building wheel for peft (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",

            "  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",

            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",

            "  Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",

            "  Building wheel for accelerate (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n"

          ]

        }

      ],

      "source": [

        "!pip install -q -U git+https://github.com/huggingface/transformers.git\n",

        "!pip install -q -U bitsandbytes\n",

        "!pip install -q -U git+https://github.com/huggingface/peft.git\n",

        "!pip install -q -U git+https://github.com/huggingface/accelerate.git"

      ]

    },

    {

      "cell_type": "code",

      "source": [

        "from google.colab import drive\n",

        "drive.mount('/content/drive')"

      ],

      "metadata": {

        "colab": {

          "base_uri": "https://localhost:8080/"

        },

        "id": "fEbMdJLLKCwX",

        "outputId": "23871af7-1e63-4d17-9b84-c659be597f11"

      },

      "execution_count": 2,

      "outputs": [

        {

          "output_type": "stream",

          "name": "stdout",

          "text": [

            "Mounted at /content/drive\n"

          ]

        }

      ]

    },

    {

      "cell_type": "code",

      "source": [

        "from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig, TextGenerationPipeline\n",

        "import torch\n",

        "import os\n",

        "from utils import *"

      ],

      "metadata": {

        "id": "yxdJkCNNKaem"

      },

      "execution_count": 7,

      "outputs": []

    },

    {

      "cell_type": "code",

      "source": [

        "model_name = \"mistralai/Mistral-7B-v0.1\""

      ],

      "metadata": {

        "id": "12ZiQ6jMKhTU"

      },

      "execution_count": 4,

      "outputs": []

    },

    {

      "cell_type": "code",

      "source": [

        "# Load base model(Mistral 7B)\n",

        "bnb_config = BitsAndBytesConfig(\n",

        "    load_in_4bit= True,\n",

        "    bnb_4bit_quant_type= \"nf4\",\n",

        "    bnb_4bit_compute_dtype= torch.bfloat16,\n",

        "    bnb_4bit_use_double_quant= False,\n",

        ")\n",

        "model = AutoModelForCausalLM.from_pretrained(\n",

        "   model_name,\n",

        "    quantization_config=bnb_config,\n",

        "    device_map={\"\": 0}\n",

        ")"

      ],

      "metadata": {

        "colab": {

          "base_uri": "https://localhost:8080/",

          "height": 368,

          "referenced_widgets": [

            "623351d84656420faa70ae4889536f78",

            "6e8db672b53f4ca3b65e861a8977e7f1",

            "c626b5aa49ba4a8ab4364d4a299baa1f",

            "d9a04c47e5aa4ccdafcff20d276bbbb1",

            "8d274858eef7465b8cb32b2556f65afb",

            "8665bd0d3d4d425786bbb64bdc57a36b",

            "ee3d9951387948089cd9360d9a9871ba",

            "b6269fd3f5324daa812c212b368ff0ad",

            "55b69b66ea004214b3d66af7843a7c99",

            "01e179ba9b6d4989bb908266137fab42",

            "51aacda23c324a9db4730410989c33e8",

            "f23ed67b7458460a86bc4c010542675a",

            "df6e7f48e48647cf9085a5143f7afd3c",

            "fca53ddc2b39498087fb8d76d4ccba83",

            "495a05fbb4984421897ee2beea733c63",

            "4290143e54344798bc7c79236338da91",

            "f3df546b544a4086a93e835f52f1374a",

            "044f3fdb95394ed4909e52d58761f60a",

            "44a254651d6d4d2f96627de75bc5e7dd",

            "a86ef33bb25644a5a313f1387bfab6e0",

            "39ea441013f94dd1b79abca45f7aa4e5",

            "f10d1774aa89441e9bbf7bd48633eeea",

            "2382371f37a74c86888483132a686039",

            "62c9043d2fb949998931c38ef6a84d66",

            "f549d20a326b4eeaa9e8f8694afc0f9d",

            "e6c39c8ec6d147e2b4376bf0a442ed1b",

            "8f2db049d8d24fc0b804a451b97f38b1",

            "cf2f5777aaa943218c01002705ad49a4",

            "a1d585f864ec4f96a1facb1c8fd1f228",

            "32fbb6cf09ae4485b5e800d739a11ef9",

            "4e98e5b117bd4a90866bc9bf8d397365",

            "6fa04422b4f2473ebed8cc51bfb8bbf9",

            "ebe4f4facd81490aaae3d9df1f84ef7a",

            "d5f20cf2d1354e54916a41d01958d663",

            "3f28950c5a1b433fad59457f3bfd0986",

            "060d6ac506cb40d4a01202a1f43773a0",

            "6718ab690cbd471a80b04df221148ccb",

            "d0a90bebc6ba424193998f6aa2871161",

            "630bbfdad2514be7bd6480610ed94aae",

            "61ced3c82e3d4299ac96838fee912cf7",

            "6b6cad15707a4fc4b9a6798d25ab0fec",

            "922a346ec1c443ce8c5082015e3b771d",

            "d54b32439cf5402b8f0a5a2a71d23ffb",

            "7867c50a7a46448a85e733a08a14074d",

            "ad18e4416984499686863f51bc363a75",

            "7c623e92720348ad8be453d2f9c8c10e",

            "67f9b070448648698e34ba6dce2140e4",

            "98c3b7b7483d40a7a92de523fa972dd0",

            "9ee6cee7294d45cb955f70fb1d7ec588",

            "c4040a2d270a4738b80e25d49db207d4",

            "c49a18d373fa4949b3be1c9ab15e99fd",

            "84d4c96405f945a58f3cbb6bd524ba08",

            "6f3ae0267d954f26bc961f6d645f2b9a",

            "972e6dc2d6e54a948e5e4690cc9b93ab",

            "d7db41f69a624e79bc57b15e9d2f9696",

            "91eebcebff2847dd923301c7722e2c99",

            "271682b7217b4205b43dfd89e801efe2",

            "b25a6da9b9cb48ebb01d8f1e8577cbd6",

            "1590c52cba7a41388b30be59b8225a3d",

            "aeb3fb1572034a84927e8df385126915",

            "a35ab30440bc40b7a6cb4262d6289c2c",

            "548e397f8e3a47a9bcaa7ba2c1f4ae48",

            "56fb139dd067428696acafeb2a9ee5e2",

            "7462a3aac96d442e9d6b93af57dc0be9",

            "4621e0960953441c9012a40de237e416",

            "4517f87fdc9e44aa9b8426d11613e76a",

            "d08ce15ed7ad4efb828b74f915784ec3",

            "7a99db0f716544448a19210f8690f9b0",

            "4ae5f91a5ddb47e6ad710dbba9e80187",

            "7d800a2659d44bec977835eb3db4326e",

            "81d1f7011258462699d12ee5b0a70a5d",

            "206fc938b0334936adb194547cfb7cc5",

            "72abe3b940ab473c82ba6b903530af65",

            "e56a79bf5c7f440c978478e88a26e0f2",

            "9c77ea2e33874b9e81d0d397a4662b43",

            "8772b2b1134a45bba71734c4400fa2c0",

            "8c914abb6ac5489e9c13046b66a7dd5d"

          ]

        },

        "id": "-lQ8vJ4sKiNh",

        "outputId": "0e0cbe37-d56f-4cef-da0d-c38fc06682f9"

      },

      "execution_count": 5,

      "outputs": [

        {

          "output_type": "stream",

          "name": "stderr",

          "text": [

            "/usr/local/lib/python3.10/dist-packages/huggingface_hub/utils/_token.py:88: UserWarning: \n",

            "The secret `HF_TOKEN` does not exist in your Colab secrets.\n",

            "To authenticate with the Hugging Face Hub, create a token in your settings tab (https://huggingface.co/settings/tokens), set it as secret in your Google Colab and restart your session.\n",

            "You will be able to reuse this secret in all of your notebooks.\n",

            "Please note that authentication is recommended but still optional to access public models or datasets.\n",

            "  warnings.warn(\n"

          ]

        },

        {

          "output_type": "display_data",

          "data": {

            "text/plain": [

              "config.json:   0%|          | 0.00/571 [00:00<?, ?B/s]"

            ],

            "application/vnd.jupyter.widget-view+json": {

              "version_major": 2,

              "version_minor": 0,

              "model_id": "623351d84656420faa70ae4889536f78"

            }

          },

          "metadata": {}

        },

        {

          "output_type": "display_data",

          "data": {

            "text/plain": [

              "model.safetensors.index.json:   0%|          | 0.00/25.1k [00:00<?, ?B/s]"

            ],

            "application/vnd.jupyter.widget-view+json": {

              "version_major": 2,

              "version_minor": 0,

              "model_id": "f23ed67b7458460a86bc4c010542675a"

            }

          },

          "metadata": {}

        },

        {

          "output_type": "display_data",

          "data": {

            "text/plain": [

              "Downloading shards:   0%|          | 0/2 [00:00<?, ?it/s]"

            ],

            "application/vnd.jupyter.widget-view+json": {

              "version_major": 2,

              "version_minor": 0,

              "model_id": "2382371f37a74c86888483132a686039"

            }

          },

          "metadata": {}

        },

        {

          "output_type": "display_data",

          "data": {

            "text/plain": [

              "model-00001-of-00002.safetensors:   0%|          | 0.00/9.94G [00:00<?, ?B/s]"

            ],

            "application/vnd.jupyter.widget-view+json": {

              "version_major": 2,

              "version_minor": 0,

              "model_id": "d5f20cf2d1354e54916a41d01958d663"

            }

          },

          "metadata": {}

        },

        {

          "output_type": "display_data",

          "data": {

            "text/plain": [

              "model-00002-of-00002.safetensors:   0%|          | 0.00/4.54G [00:00<?, ?B/s]"

            ],

            "application/vnd.jupyter.widget-view+json": {

              "version_major": 2,

              "version_minor": 0,

              "model_id": "ad18e4416984499686863f51bc363a75"

            }

          },

          "metadata": {}

        },

        {

          "output_type": "display_data",

          "data": {

            "text/plain": [

              "Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]"

            ],

            "application/vnd.jupyter.widget-view+json": {

              "version_major": 2,

              "version_minor": 0,

              "model_id": "91eebcebff2847dd923301c7722e2c99"

            }

          },

          "metadata": {}

        },

        {

          "output_type": "display_data",

          "data": {

            "text/plain": [

              "generation_config.json:   0%|          | 0.00/116 [00:00<?, ?B/s]"

            ],

            "application/vnd.jupyter.widget-view+json": {

              "version_major": 2,

              "version_minor": 0,

              "model_id": "d08ce15ed7ad4efb828b74f915784ec3"

            }

          },

          "metadata": {}

        }

      ]

    },

    {

      "cell_type": "code",

      "source": [

        "# import tokenizer for mistral-7B\n",

        "tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)\n",

        "tokenizer.padding_side = 'left'\n",

        "tokenizer.pad_token = tokenizer.eos_token\n",

        "tokenizer.add_eos_token = True\n",

        "tokenizer.add_bos_token, tokenizer.add_eos_token"

      ],

      "metadata": {

        "colab": {

          "base_uri": "https://localhost:8080/",

          "height": 163,

          "referenced_widgets": [

            "8ccdaa53fd4d44f6bebcc8b1daf613a7",

            "b9cdb24813514050adf15d3af2b9589b",

            "edb1863c025142908849cd8ce20505b1",

            "c971ffea2e4743b493ec3680261e0ff0",

            "551e6c40db8f430c880a57a3ab405f5b",

            "81630071ac784f72a7a0b27f8cf50ffa",

            "ff26ca039b0847a29502b2a73205ca21",

            "717f84e23b4147419344cdedb6be4fd1",

            "184db58f8d5241e7b2faf1e1bba700e0",

            "107fd3f5b956480197b8795ac5129734",

            "369133b2991542d68ef645538f353788",

            "24d9c53181ce49c2b91760bc386366b5",

            "ee1412a55f3d40e191083b2776650ac5",

            "a9cdbf1e2cd5499c9c9804eb55bc7da4",

            "a556a867bc7742f9b6810e6f63d216d7",

            "d1845140817b4118955f0c538e9a778e",

            "84891cf5bd124727bea560163935294a",

            "508144a592e54e2f9eb970a3418cbe2b",

            "775d8e4eb8fe4758b2b876d5538ad8eb",

            "a7c91bd72b65470db54b8461fba77a38",

            "e0a3b0edba7a4e54bb6ab892c7171113",

            "ae43479a1daa49d19d6d544accdc3124",

            "d7cdc98280e04394a80ba62d7b883cc2",

            "3fed97d8526d4a6195dc8c70c99eb3a3",

            "871a8f47d8bf4c0e957f575788a4a05a",

            "0c868554bdf64a43bf31571fe5a4e80a",

            "29618477f8e146c78ad6aaa4b61f6882",

            "0b789d08ba774ea89b9d79e3fec96fe4",

            "f74b8384258d4eba8ea0be83fcbd8a2b",

            "f68dd87f254a4136971047e6804964a2",

            "ead331636cf745d7b17a5d764939f5ed",

            "0388b415e4ba4afa8b7add0398afb5c1",

            "223b2343afde4167a53283fb6e9869c9",

            "e358fec2f71c418caac8d129444abcee",

            "f324f86a03924d538d4e394cb71219fb",

            "f6aad53167db4c4f90f560b28b35d7f0",

            "fefc941bdc1d4115a3d3a1f1dfc56fda",

            "2b0f02229d824c45a797d11a5b4c1703",

            "fee2d946671c4609b9e7c92725717e1c",

            "3c1fdeeae60d4ef5aa2997dce8614e67",

            "6568a690061d4f7d8127c63ce02bcbad",

            "e463325b65b643069760c4e9a15ec74d",

            "39a2137807ae4613a7b856765af6246c",

            "a91a6a026da94952b612ef0b21297f30"

          ]

        },

        "id": "l7lmarKbKnVg",

        "outputId": "cd2cd4af-0639-4813-f670-4f0cfddbcd4b"

      },

      "execution_count": 6,

      "outputs": [

        {

          "output_type": "display_data",

          "data": {

            "text/plain": [

              "tokenizer_config.json:   0%|          | 0.00/967 [00:00<?, ?B/s]"

            ],

            "application/vnd.jupyter.widget-view+json": {

              "version_major": 2,

              "version_minor": 0,

              "model_id": "8ccdaa53fd4d44f6bebcc8b1daf613a7"

            }

          },

          "metadata": {}

        },

        {

          "output_type": "display_data",

          "data": {

            "text/plain": [

              "tokenizer.model:   0%|          | 0.00/493k [00:00<?, ?B/s]"

            ],

            "application/vnd.jupyter.widget-view+json": {

              "version_major": 2,

              "version_minor": 0,

              "model_id": "24d9c53181ce49c2b91760bc386366b5"

            }

          },

          "metadata": {}

        },

        {

          "output_type": "display_data",

          "data": {

            "text/plain": [

              "tokenizer.json:   0%|          | 0.00/1.80M [00:00<?, ?B/s]"

            ],

            "application/vnd.jupyter.widget-view+json": {

              "version_major": 2,

              "version_minor": 0,

              "model_id": "d7cdc98280e04394a80ba62d7b883cc2"

            }

          },

          "metadata": {}

        },

        {

          "output_type": "display_data",

          "data": {

            "text/plain": [

              "special_tokens_map.json:   0%|          | 0.00/72.0 [00:00<?, ?B/s]"

            ],

            "application/vnd.jupyter.widget-view+json": {

              "version_major": 2,

              "version_minor": 0,

              "model_id": "e358fec2f71c418caac8d129444abcee"

            }

          },

          "metadata": {}

        },

        {

          "output_type": "execute_result",

          "data": {

            "text/plain": [

              "(True, True)"

            ]

          },

          "metadata": {},

          "execution_count": 6

        }

      ]

    },

    {

      "cell_type": "code",

      "source": [

        "pipe  = TextGenerationPipeline(model = model, tokenizer = tokenizer)"

      ],

      "metadata": {

        "id": "H1KZjewRKq4T"

      },

      "execution_count": 8,

      "outputs": []

    },

    {

      "cell_type": "code",

      "source": [

        "# load dataset for annotation (To make the model better for generalization we can generate a class that receive\n",

        "# the dataset in BIO format as input and make all parsing process and annotations)\n",

        "data_path = \"./drive/MyDrive/HandsOn-NLP/data/chia_criteria_texts/\""

      ],

      "metadata": {

        "id": "OQZxg3rLL4ZV"

      },

      "execution_count": 9,

      "outputs": []

    },

    {

      "cell_type": "code",

      "source": [

        "# get the list of files in the dataset\n",

        "files = os.listdir(data_path)\n",

        "files[:5]"

      ],

      "metadata": {

        "colab": {

          "base_uri": "https://localhost:8080/"

        },

        "id": "qJr3yMlCMwHL",

        "outputId": "fa0d19cd-7aec-4579-81c7-f2af42cce75d"

      },

      "execution_count": 12,

      "outputs": [

        {

          "output_type": "execute_result",

          "data": {

            "text/plain": [

              "['NCT03264911_inc.bio.txt',\n",

              " 'NCT03134378_inc.bio.txt',\n",

              " 'NCT02924090_exc.bio.txt',\n",

              " 'NCT02555163_inc.bio.txt',\n",

              " 'NCT02952378_exc.bio.txt']"

            ]

          },

          "metadata": {},

          "execution_count": 12

        }

      ]

    },

    {

      "cell_type": "code",

      "source": [

        "# example of input using first prompt\n",

        "with open(f\"{data_path}{files[1]}\", 'r+') as fd:\n",

        "  first_sentence = fd.readlines()[0]\n",

        "print(f\"Source sentence: {first_sentence}\")\n",

        "print(\"First prompt:\")\n",

        "print(build_prompt(first_sentence, 1))"

      ],

      "metadata": {

        "colab": {

          "base_uri": "https://localhost:8080/"

        },

        "id": "-99WLDuHM_Wn",

        "outputId": "48fb2fb8-b0e4-44b6-bb38-4fa7540efa30"

      },

      "execution_count": 20,

      "outputs": [

        {

          "output_type": "stream",

          "name": "stdout",

          "text": [

            "Source sentence: 18 years or older patients who are proven to be infected by Helicobacter pylori based on positive in Urea Breath Test or positive in histopathologic examination of biopsy in antrum and corpus of gaster through esophagoduodenoscopy .\n",

            "\n",

            "First prompt:\n",

            "I need to perform a named entity recognition task on a  text related with inclusion criteria in clinical trials.\n",

            "    The entities you need to recognize are: Condition, Value, Drug, Procedure, Measurement, Temporal, Observation, Person, Mood, Device and Pregnancy_considerations.\n",

            "    Particularly you have to produce the ouput in the BIO format. I will show you an example of the expected output.\n",

            "    Input text: Patients with symptomatic CNS metastases or leptomeningeal involvement\n",

            "    Output:\n",

            "    Patients O\n",

            "    with O\n",

            "    symptomatic O\n",

            "    CNS B-Condition\n",

            "    metastases I-Condition\n",

            "    or O\n",

            "    leptomeningeal B-Condition\n",

            "    involvement I-Condition\n",

            "\n",

            "    You can see that tokens without any entity are labeled as O, and the tokens that are part of an entity are labeled as B-<entity> or I-<entity> depending on if they are the beginning or the inside of the entity.\n",

            "    Please, just answer the question for this specific example and stop writting after that.\n",

            "    Input text: 18 years or older patients who are proven to be infected by Helicobacter pylori based on positive in Urea Breath Test or positive in histopathologic examination of biopsy in antrum and corpus of gaster through esophagoduodenoscopy .\n",

            "\n"

          ]

        }

      ]

    },

    {

      "cell_type": "code",

      "source": [

        "print(\"Second prompt:\")\n",

        "print(build_prompt(first_sentence, 2))"

      ],

      "metadata": {

        "colab": {

          "base_uri": "https://localhost:8080/"

        },

        "id": "n8X0eXX4PXwz",

        "outputId": "20374009-1ba6-4ed1-c057-0ca60257a776"

      },

      "execution_count": 21,

      "outputs": [

        {

          "output_type": "stream",

          "name": "stdout",

          "text": [

            "Second prompt:\n",

            "I am working on a named entity recognition problem, in the context of clinical\n",

            "    trials eligibility criteria. I will show you the list of entities:\n",

            "    - Condition\n",

            "    - Value\n",

            "    - Drug\n",

            "    - Procedure\n",

            "    - Measurement\n",

            "    - Temporal\n",

            "    - Observation\n",

            "    - Person\n",

            "    - Mood\n",

            "    - Device\n",

            "\n",

            "    Your task consists in annotate the named entities in a given sentence in the format I will explain you.\n",

            "    I will explain you with some examples:\n",

            "\n",

            "    Example 1:\n",

            "    Input: Patients who have received prior chemotherapy for unresectable disease.\n",

            "    Output: Patients who have received prior <Procedure>chemotherapy</Procedure> for <Condition>unresectable disease</Condition>.\n",

            "\n",

            "    Example 2:\n",

            "    Input: Patients with any other severe concurrent disease, which in the judgment of the investigator, would make the patient inappropriate for entry into this study.\n",

            "    Ouput: Patients with any other severe <Condition>concurrent disease</Condition>, which in the judgment of the investigator, would make the patient <Mood>inappropriate for <Observation>entry into this study</Observation>.\n",

            "\n",

            "    As you can see, in each example, the extracted entities are enclosed using the sintax: <ENT>text of the entity</ENT>.\n",

            "\n",

            "    Please now annotate as explained before the following sentence:\n",

            "\n",

            "    Input: 18 years or older patients who are proven to be infected by Helicobacter pylori based on positive in Urea Breath Test or positive in histopathologic examination of biopsy in antrum and corpus of gaster through esophagoduodenoscopy .\n",

            "\n"

          ]

        }

      ]

    },

    {

      "cell_type": "code",

      "source": [

        "from tqdm import tqdm"

      ],

      "metadata": {

        "id": "igfJpR5SVevy"

      },

      "execution_count": 27,

      "outputs": []

    },

    {

      "cell_type": "code",

      "source": [

        "# generate annotations for first prompt\n",

        "for file in tqdm(files[:20]):\n",

        "  sentences = []\n",

        "  with open(f\"{data_path}{file}\", 'r+', encoding='utf-8') as fd:\n",

        "    sentences = fd.readlines()\n",

        "  annotated_sentences = []\n",

        "  for sentence in sentences:\n",

        "    if len(sentence) == 0:\n",

        "      continue\n",

        "    input = build_prompt(sentence, 1)\n",

        "    output = pipe(input, max_new_tokens = 500, return_full_text = False, handle_long_generation = \"hole\")[0]['generated_text']\n",

        "    output = output.split('\\n\\n')[0]#to get just the first sentence in the ouput and avoid noise\n",

        "    annotated_sentences.append(output)\n",

        "  with open(f\"./drive/MyDrive/HandsOn-NLP/data/Annotations_Mistral_Prompt_1/{file}\", 'w+', encoding='utf-8') as fd:\n",

        "    fd.write('\\n\\n'.join(annotated_sentences))"

      ],

      "metadata": {

        "colab": {

          "base_uri": "https://localhost:8080/"

        },

        "id": "AVdymoJWQD0P",

        "outputId": "f7f0af37-04cb-4f51-874d-31e23dc1f79d"

      },

      "execution_count": 32,

      "outputs": [

        {

          "output_type": "stream",

          "name": "stderr",

          "text": [

            "\r  0%|          | 0/6 [00:00<?, ?it/s]/usr/local/lib/python3.10/dist-packages/transformers/pipelines/base.py:1157: UserWarning: You seem to be using the pipelines sequentially on GPU. In order to maximize efficiency please use a dataset\n",

            "  warnings.warn(\n",

            "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n",

            " 17%|█▋        | 1/6 [00:42<03:34, 42.80s/it]Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n",

            "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n",

            " 33%|███▎      | 2/6 [02:06<04:28, 67.01s/it]Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n",

            "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n",

            "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n",

            "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n",

            "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n",

            "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n",

            " 50%|█████     | 3/6 [06:16<07:31, 150.43s/it]Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n",

            "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n",

            "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n",

            " 67%|██████▋   | 4/6 [08:20<04:39, 139.84s/it]Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n",

            "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n",

            " 83%|████████▎ | 5/6 [09:42<01:59, 119.14s/it]Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n",

            "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n",

            "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n",

            "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n",

            "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n",

            "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n",

            "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n",

            "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n",

            "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n",

            "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n",

            "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n",

            "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n",

            "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n",

            "100%|██████████| 6/6 [18:36<00:00, 186.02s/it]\n"

          ]

        }

      ]

    },

    {

      "cell_type": "code",

      "source": [

        "# generate annotations for second prompt\n",

        "for file in tqdm(files[:20]):\n",

        "  print(file)\n",

        "  sentences = []\n",

        "  with open(f\"{data_path}{file}\", 'r+', encoding='utf-8') as fd:\n",

        "    sentences = fd.readlines()\n",

        "  annotated_sentences = []\n",

        "  for sentence in sentences:\n",

        "    if len(sentence) == 0:\n",

        "      continue\n",

        "    input = build_prompt(sentence, 2)\n",

        "    input += '\\noutput: '\n",

        "    output = pipe(input, max_new_tokens = 500, return_full_text = False, handle_long_generation = \"hole\")[0]['generated_text']\n",

        "    output = output.split('\\n')[0] + '.' #to get just the first sentence in the ouput and avoid noise\n",

        "    annotated_sentences.append(output)\n",

        "  with open(f\"./drive/MyDrive/HandsOn-NLP/data/Annotations_Mistral_Prompt_2/{file}\", 'w+', encoding='utf-8') as fd:\n",

        "    fd.write('\\n\\n'.join(annotated_sentences))"

      ],

      "metadata": {

        "colab": {

          "base_uri": "https://localhost:8080/"

        },

        "id": "3f539-JBSoJO",

        "outputId": "98db5051-f142-4ff2-ccdc-881b1952aa86"

      },

      "execution_count": 40,

      "outputs": [

        {

          "output_type": "stream",

          "name": "stderr",

          "text": [

            "\r  0%|          | 0/20 [00:00<?, ?it/s]Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n"

          ]

        },

        {

          "output_type": "stream",

          "name": "stdout",

          "text": [

            "NCT03264911_inc.bio.txt\n"

          ]

        },

        {

          "output_type": "stream",

          "name": "stderr",

          "text": [

            "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n",

            "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n",

            "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n",

            "  5%|▌         | 1/20 [02:48<53:28, 168.89s/it]Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n"

          ]

        },

        {

          "output_type": "stream",

          "name": "stdout",

          "text": [

            "NCT03134378_inc.bio.txt\n"

          ]

        },

        {

          "output_type": "stream",

          "name": "stderr",

          "text": [

            "\r 10%|█         | 2/20 [03:30<28:15, 94.20s/it] Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n"

          ]

        },

        {

          "output_type": "stream",

          "name": "stdout",

          "text": [

            "NCT02924090_exc.bio.txt\n"

          ]

        },

        {

          "output_type": "stream",

          "name": "stderr",

          "text": [

            "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n",

            "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n",

            "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n",

            " 15%|█▌        | 3/20 [06:17<36:00, 127.09s/it]Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n"

          ]

        },

        {

          "output_type": "stream",

          "name": "stdout",

          "text": [

            "NCT02555163_inc.bio.txt\n"

          ]

        },

        {

          "output_type": "stream",

          "name": "stderr",

          "text": [

            "\r 20%|██        | 4/20 [06:58<24:53, 93.31s/it] Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n"

          ]

        },

        {

          "output_type": "stream",

          "name": "stdout",

          "text": [

            "NCT02952378_exc.bio.txt\n"

          ]

        },

        {

          "output_type": "stream",

          "name": "stderr",

          "text": [

            "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n",

            "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n",

            " 25%|██▌       | 5/20 [09:02<26:07, 104.50s/it]Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n"

          ]

        },

        {

          "output_type": "stream",

          "name": "stdout",

          "text": [

            "NCT02918409_inc.bio.txt\n"

          ]

        },

        {

          "output_type": "stream",

          "name": "stderr",

          "text": [

            "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n",

            "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n",

            "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n",

            "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n",

            "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n",

            "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n",

            "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n",

            " 30%|███       | 6/20 [14:35<42:27, 182.00s/it]Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n"

          ]

        },

        {

          "output_type": "stream",

          "name": "stdout",

          "text": [

            "NCT03193684_exc.bio.txt\n"

          ]

        },

        {

          "output_type": "stream",

          "name": "stderr",

          "text": [

            "\r 35%|███▌      | 7/20 [15:16<29:28, 136.01s/it]Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n"

          ]

        },

        {

          "output_type": "stream",

          "name": "stdout",

          "text": [

            "NCT03472495_inc.bio.txt\n"

          ]

        },

        {

          "output_type": "stream",

          "name": "stderr",

          "text": [

            "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n",

            "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n",

            "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n",

            " 40%|████      | 8/20 [18:02<29:04, 145.38s/it]Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n"

          ]

        },

        {

          "output_type": "stream",

          "name": "stdout",

          "text": [

            "NCT01942109_inc.bio.txt\n"

          ]

        },

        {

          "output_type": "stream",

          "name": "stderr",

          "text": [

            "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n",

            "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n",

            " 45%|████▌     | 9/20 [20:06<25:26, 138.81s/it]Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n"

          ]

        },

        {

          "output_type": "stream",

          "name": "stdout",

          "text": [

            "NCT03187639_exc.bio.txt\n"

          ]

        },

        {

          "output_type": "stream",

          "name": "stderr",

          "text": [

            "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n",

            "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n",

            "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n",

            "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n",

            "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n",

            "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n",

            "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n",

            " 50%|█████     | 10/20 [25:39<33:07, 198.70s/it]Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n"

          ]

        },

        {

          "output_type": "stream",

          "name": "stdout",

          "text": [

            "NCT03193684_inc.bio.txt\n"

          ]

        },

        {

          "output_type": "stream",

          "name": "stderr",

          "text": [

            "\r 55%|█████▌    | 11/20 [26:21<22:36, 150.69s/it]Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n"

          ]

        },

        {

          "output_type": "stream",

          "name": "stdout",

          "text": [

            "NCT02339844_exc.bio.txt\n"

          ]

        },

        {

          "output_type": "stream",

          "name": "stderr",

          "text": [

            "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n",

            " 60%|██████    | 12/20 [27:31<16:50, 126.37s/it]Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n"

          ]

        },

        {

          "output_type": "stream",

          "name": "stdout",

          "text": [

            "NCT02102243_exc.bio.txt\n"

          ]

        },

        {

          "output_type": "stream",

          "name": "stderr",

          "text": [

            "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n",

            "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n",

            "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n",

            "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n",

            "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n",

            "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n",

            "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n",

            "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n",

            "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n",

            "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n",

            " 65%|██████▌   | 13/20 [35:06<26:20, 225.76s/it]Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n"

          ]

        },

        {

          "output_type": "stream",

          "name": "stdout",

          "text": [

            "NCT03156855_inc.bio.txt\n"

          ]

        },

        {

          "output_type": "stream",

          "name": "stderr",

          "text": [

            "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n",

            "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n",

            "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n",

            "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n",

            "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n",

            "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n",

            "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n",

            " 70%|███████   | 14/20 [40:36<25:43, 257.33s/it]Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n"

          ]

        },

        {

          "output_type": "stream",

          "name": "stdout",

          "text": [

            "NCT02553226_inc.bio.txt\n"

          ]

        },

        {

          "output_type": "stream",

          "name": "stderr",

          "text": [

            "\r 75%|███████▌  | 15/20 [41:18<16:01, 192.30s/it]Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n"

          ]

        },

        {

          "output_type": "stream",

          "name": "stdout",

          "text": [

            "NCT02606565_exc.bio.txt\n"

          ]

        },

        {

          "output_type": "stream",

          "name": "stderr",

          "text": [

            "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n",

            " 80%|████████  | 16/20 [42:40<10:36, 159.14s/it]Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n"

          ]

        },

        {

          "output_type": "stream",

          "name": "stdout",

          "text": [

            "NCT01857167_exc.bio.txt\n"

          ]

        },

        {

          "output_type": "stream",

          "name": "stderr",

          "text": [

            "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n",

            "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n",

            "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n",

            "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n",

            "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n",

            " 85%|████████▌ | 17/20 [46:42<09:11, 183.96s/it]Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n"

          ]

        },

        {

          "output_type": "stream",

          "name": "stdout",

          "text": [

            "NCT02939872_inc.bio.txt\n"

          ]

        },

        {

          "output_type": "stream",

          "name": "stderr",

          "text": [

            "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n",

            "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n",

            " 90%|█████████ | 18/20 [48:46<05:31, 165.99s/it]Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n"

          ]

        },

        {

          "output_type": "stream",

          "name": "stdout",

          "text": [

            "NCT02894372_inc.bio.txt\n"

          ]

        },

        {

          "output_type": "stream",

          "name": "stderr",

          "text": [

            "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n",

            " 95%|█████████▌| 19/20 [50:08<02:21, 141.01s/it]Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n"

          ]

        },

        {

          "output_type": "stream",

          "name": "stdout",

          "text": [

            "NCT00730301_inc.bio.txt\n"

          ]

        },

        {

          "output_type": "stream",

          "name": "stderr",

          "text": [

            "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n",

            "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n",

            "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n",

            "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n",

            "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n",

            "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n",

            "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n",

            "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n",

            "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n",

            "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n",

            "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n",

            "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n",

            "100%|██████████| 20/20 [59:05<00:00, 177.28s/it]\n"

          ]

        }

      ]

    }

  ],

  "metadata": {

    "kernelspec": {

      "display_name": "Python 3",

      "name": "python3"

    },

    "language_info": {

      "codemirror_mode": {

        "name": "ipython",

        "version": 3

      },

      "file_extension": ".py",

      "mimetype": "text/x-python",

      "name": "python",

      "nbconvert_exporter": "python",

      "pygments_lexer": "ipython3",

      "version": "3.10.13"

    },

    "colab": {

      "provenance": [],

      "gpuType": "T4",

      "include_colab_link": true

    },

    "accelerator": "GPU",

    "widgets": {

      "application/vnd.jupyter.widget-state+json": {

        "623351d84656420faa70ae4889536f78": {

          "model_module": "@jupyter-widgets/controls",

          "model_name": "HBoxModel",

          "model_module_version": "1.5.0",

          "state": {

            "_dom_classes": [],

            "_model_module": "@jupyter-widgets/controls",

            "_model_module_version": "1.5.0",

            "_model_name": "HBoxModel",

            "_view_count": null,

            "_view_module": "@jupyter-widgets/controls",

            "_view_module_version": "1.5.0",

            "_view_name": "HBoxView",

            "box_style": "",

            "children": [

              "IPY_MODEL_6e8db672b53f4ca3b65e861a8977e7f1",

              "IPY_MODEL_c626b5aa49ba4a8ab4364d4a299baa1f",

              "IPY_MODEL_d9a04c47e5aa4ccdafcff20d276bbbb1"