{

  "cells": [

    {

      "cell_type": "code",

      "execution_count": null,

      "id": "138778c4",

      "metadata": {

        "id": "138778c4"

      },

      "outputs": [],

      "source": [

        "from tqdm import tqdm\n",

        "import pandas as pd\n",

        "from sklearn import metrics\n",

        "from scipy.spatial.distance import cdist"

      ]

    },

    {

      "cell_type": "code",

      "execution_count": null,

      "id": "dTSILRD7hIHG",

      "metadata": {

        "colab": {

          "base_uri": "https://localhost:8080/"

        },

        "id": "dTSILRD7hIHG",

        "outputId": "9d7677f8-fc7d-403c-e8dd-6135bc581c01"

      },

      "outputs": [

        {

          "name": "stdout",

          "output_type": "stream",

          "text": [

            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",

            "Collecting transformers\n",

            "  Downloading transformers-4.24.0-py3-none-any.whl (5.5 MB)\n",

            "\u001b[K     |████████████████████████████████| 5.5 MB 4.7 MB/s \n",

            "\u001b[?25hCollecting huggingface-hub<1.0,>=0.10.0\n",

            "  Downloading huggingface_hub-0.10.1-py3-none-any.whl (163 kB)\n",

            "\u001b[K     |████████████████████████████████| 163 kB 88.9 MB/s \n",

            "\u001b[?25hRequirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers) (2.23.0)\n",

            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (1.21.6)\n",

            "Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from transformers) (4.13.0)\n",

            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.7/dist-packages (from transformers) (6.0)\n",

            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.7/dist-packages (from transformers) (21.3)\n",

            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (2022.6.2)\n",

            "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers) (3.8.0)\n",

            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers) (4.64.1)\n",

            "Collecting tokenizers!=0.11.3,<0.14,>=0.11.1\n",

            "  Downloading tokenizers-0.13.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (7.6 MB)\n",

            "\u001b[K     |████████████████████████████████| 7.6 MB 89.1 MB/s \n",

            "\u001b[?25hRequirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.7/dist-packages (from huggingface-hub<1.0,>=0.10.0->transformers) (4.1.1)\n",

            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging>=20.0->transformers) (3.0.9)\n",

            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->transformers) (3.10.0)\n",

            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2.10)\n",

            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (1.24.3)\n",

            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2022.9.24)\n",

            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (3.0.4)\n",

            "Installing collected packages: tokenizers, huggingface-hub, transformers\n",

            "Successfully installed huggingface-hub-0.10.1 tokenizers-0.13.1 transformers-4.24.0\n"

          ]

        }

      ],

      "source": [

        "!pip install transformers"

      ]

    },

    {

      "cell_type": "code",

      "execution_count": null,

      "id": "70512d4e",

      "metadata": {

        "id": "70512d4e",

        "outputId": "ebc36a95-bb47-425a-9c2c-7456141778e6"

      },

      "outputs": [

        {

          "name": "stdout",

          "output_type": "stream",

          "text": [

            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",

            "To disable this warning, you can either:\n",

            "\t- Avoid using `tokenizers` before the fork if possible\n",

            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",

            "Collecting parallelformers\n",

            "  Downloading parallelformers-1.2.7.tar.gz (48 kB)\n",

            "\u001b[K     |████████████████████████████████| 48 kB 557 kB/s eta 0:00:011\n",

            "\u001b[?25hRequirement already satisfied: transformers>=4.2 in /home2/sashank.sridhar/miniconda3/envs/TripletLoss/lib/python3.9/site-packages (from parallelformers) (4.24.0)\n",

            "Requirement already satisfied: torch in /home2/sashank.sridhar/miniconda3/envs/TripletLoss/lib/python3.9/site-packages (from parallelformers) (1.11.0)\n",

            "Collecting dacite\n",

            "  Downloading dacite-1.6.0-py3-none-any.whl (12 kB)\n",

            "Requirement already satisfied: filelock in /home2/sashank.sridhar/miniconda3/envs/TripletLoss/lib/python3.9/site-packages (from transformers>=4.2->parallelformers) (3.6.0)\n",

            "Requirement already satisfied: packaging>=20.0 in /home2/sashank.sridhar/miniconda3/envs/TripletLoss/lib/python3.9/site-packages (from transformers>=4.2->parallelformers) (21.3)\n",

            "Requirement already satisfied: huggingface-hub<1.0,>=0.10.0 in /home2/sashank.sridhar/miniconda3/envs/TripletLoss/lib/python3.9/site-packages (from transformers>=4.2->parallelformers) (0.10.1)\n",

            "Requirement already satisfied: tqdm>=4.27 in /home2/sashank.sridhar/miniconda3/envs/TripletLoss/lib/python3.9/site-packages (from transformers>=4.2->parallelformers) (4.64.1)\n",

            "Requirement already satisfied: regex!=2019.12.17 in /home2/sashank.sridhar/miniconda3/envs/TripletLoss/lib/python3.9/site-packages (from transformers>=4.2->parallelformers) (2022.7.25)\n",

            "Requirement already satisfied: requests in /home2/sashank.sridhar/miniconda3/envs/TripletLoss/lib/python3.9/site-packages (from transformers>=4.2->parallelformers) (2.27.1)\n",

            "Requirement already satisfied: pyyaml>=5.1 in /home2/sashank.sridhar/miniconda3/envs/TripletLoss/lib/python3.9/site-packages (from transformers>=4.2->parallelformers) (6.0)\n",

            "Requirement already satisfied: numpy>=1.17 in /home2/sashank.sridhar/miniconda3/envs/TripletLoss/lib/python3.9/site-packages (from transformers>=4.2->parallelformers) (1.21.5)\n",

            "Requirement already satisfied: tokenizers!=0.11.3,<0.14,>=0.11.1 in /home2/sashank.sridhar/miniconda3/envs/TripletLoss/lib/python3.9/site-packages (from transformers>=4.2->parallelformers) (0.13.1)\n",

            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /home2/sashank.sridhar/miniconda3/envs/TripletLoss/lib/python3.9/site-packages (from huggingface-hub<1.0,>=0.10.0->transformers>=4.2->parallelformers) (4.1.1)\n",

            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /home2/sashank.sridhar/miniconda3/envs/TripletLoss/lib/python3.9/site-packages (from packaging>=20.0->transformers>=4.2->parallelformers) (2.4.7)\n",

            "Requirement already satisfied: charset-normalizer~=2.0.0 in /home2/sashank.sridhar/miniconda3/envs/TripletLoss/lib/python3.9/site-packages (from requests->transformers>=4.2->parallelformers) (2.0.4)\n",

            "Requirement already satisfied: certifi>=2017.4.17 in /home2/sashank.sridhar/miniconda3/envs/TripletLoss/lib/python3.9/site-packages (from requests->transformers>=4.2->parallelformers) (2021.10.8)\n",

            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /home2/sashank.sridhar/miniconda3/envs/TripletLoss/lib/python3.9/site-packages (from requests->transformers>=4.2->parallelformers) (1.26.9)\n",

            "Requirement already satisfied: idna<4,>=2.5 in /home2/sashank.sridhar/miniconda3/envs/TripletLoss/lib/python3.9/site-packages (from requests->transformers>=4.2->parallelformers) (3.3)\n",

            "Building wheels for collected packages: parallelformers\n",

            "  Building wheel for parallelformers (setup.py) ... \u001b[?25ldone\n",

            "\u001b[?25h  Created wheel for parallelformers: filename=parallelformers-1.2.7-py3-none-any.whl size=117791 sha256=ce1c96f5d462c55210041d65abcc897059cc1349cfafe5afcc489f60b6fbb7c6\n",

            "  Stored in directory: /home2/sashank.sridhar/.cache/pip/wheels/4f/19/42/8d74380c84a1e93401ee163f3bd545853051a4d895fd95ca2e\n",

            "Successfully built parallelformers\n",

            "Installing collected packages: dacite, parallelformers\n",

            "Successfully installed dacite-1.6.0 parallelformers-1.2.7\n"

          ]

        }

      ],

      "source": [

        "!pip install parallelformers"

      ]

    },

    {

      "cell_type": "code",

      "execution_count": null,

      "id": "6b2ace9d",

      "metadata": {

        "id": "6b2ace9d"

      },

      "outputs": [],

      "source": [

        "!pip install faiss-gpu"

      ]

    },

    {

      "cell_type": "markdown",

      "id": "5a0d2481",

      "metadata": {

        "id": "5a0d2481"

      },

      "source": [

        "Download snomed term-concept file from UMLS website"

      ]

    },

    {

      "cell_type": "code",

      "execution_count": null,

      "id": "ea498c9d",

      "metadata": {

        "colab": {

          "base_uri": "https://localhost:8080/"

        },

        "id": "ea498c9d",

        "outputId": "49723e0b-aee4-46f9-b162-e319f10475d6"

      },

      "outputs": [

        {

          "name": "stdout",

          "output_type": "stream",

          "text": [

            "1569232\n"

          ]

        }

      ],

      "source": [

        "snomed_csv = pd.read_csv(\"data/sct2_Description_Snapshot-en_INT_20220831.txt\", delimiter=\"\\t\")\n",

        "print(len(snomed_csv))"

      ]

    },

    {

      "cell_type": "code",

      "execution_count": null,

      "id": "d811cd3c",

      "metadata": {

        "colab": {

          "base_uri": "https://localhost:8080/"

        },

        "id": "d811cd3c",

        "outputId": "2bce882d-ab28-4af6-b07f-ed73b1b5f4c8"

      },

      "outputs": [

        {

          "data": {

            "text/plain": [

              "Index(['id', 'effectiveTime', 'active', 'moduleId', 'conceptId',\n",

              "       'languageCode', 'typeId', 'term', 'caseSignificanceId'],\n",

              "      dtype='object')"

            ]

          },

          "execution_count": 3,

          "metadata": {},

          "output_type": "execute_result"

        }

      ],

      "source": [

        "snomed_csv.columns"

      ]

    },

    {

      "cell_type": "code",

      "execution_count": null,

      "id": "3327f0d9",

      "metadata": {

        "colab": {

          "base_uri": "https://localhost:8080/",

          "height": 530

        },

        "id": "3327f0d9",

        "outputId": "7bbd5033-5116-4b65-c66a-f2cb3365f859"

      },

      "outputs": [

        {

          "data": {

            "text/html": [

              "<div>\n",

              "<style scoped>\n",

              "    .dataframe tbody tr th:only-of-type {\n",

              "        vertical-align: middle;\n",

              "    }\n",

              "\n",

              "    .dataframe tbody tr th {\n",

              "        vertical-align: top;\n",

              "    }\n",

              "\n",

              "    .dataframe thead th {\n",

              "        text-align: right;\n",

              "    }\n",

              "</style>\n",

              "<table border=\"1\" class=\"dataframe\">\n",

              "  <thead>\n",

              "    <tr style=\"text-align: right;\">\n",

              "      <th></th>\n",

              "      <th>id</th>\n",

              "      <th>effectiveTime</th>\n",

              "      <th>active</th>\n",

              "      <th>moduleId</th>\n",

              "      <th>conceptId</th>\n",

              "      <th>languageCode</th>\n",

              "      <th>typeId</th>\n",

              "      <th>term</th>\n",

              "      <th>caseSignificanceId</th>\n",

              "    </tr>\n",

              "  </thead>\n",

              "  <tbody>\n",

              "    <tr>\n",

              "      <th>0</th>\n",

              "      <td>101013</td>\n",

              "      <td>20170731</td>\n",

              "      <td>1</td>\n",

              "      <td>900000000000207008</td>\n",

              "      <td>126813005</td>\n",

              "      <td>en</td>\n",

              "      <td>900000000000013009</td>\n",

              "      <td>Neoplasm of anterior aspect of epiglottis</td>\n",

              "      <td>900000000000448009</td>\n",

              "    </tr>\n",

              "    <tr>\n",

              "      <th>1</th>\n",

              "      <td>102018</td>\n",

              "      <td>20170731</td>\n",

              "      <td>1</td>\n",

              "      <td>900000000000207008</td>\n",

              "      <td>126814004</td>\n",

              "      <td>en</td>\n",

              "      <td>900000000000013009</td>\n",

              "      <td>Neoplasm of junctional region of epiglottis</td>\n",

              "      <td>900000000000448009</td>\n",

              "    </tr>\n",

              "    <tr>\n",

              "      <th>2</th>\n",

              "      <td>103011</td>\n",

              "      <td>20170731</td>\n",

              "      <td>1</td>\n",

              "      <td>900000000000207008</td>\n",

              "      <td>126815003</td>\n",

              "      <td>en</td>\n",

              "      <td>900000000000013009</td>\n",

              "      <td>Neoplasm of lateral wall of oropharynx</td>\n",

              "      <td>900000000000448009</td>\n",

              "    </tr>\n",

              "    <tr>\n",

              "      <th>3</th>\n",

              "      <td>104017</td>\n",

              "      <td>20170731</td>\n",

              "      <td>1</td>\n",

              "      <td>900000000000207008</td>\n",

              "      <td>126816002</td>\n",

              "      <td>en</td>\n",

              "      <td>900000000000013009</td>\n",

              "      <td>Neoplasm of posterior wall of oropharynx</td>\n",

              "      <td>900000000000448009</td>\n",

              "    </tr>\n",

              "    <tr>\n",

              "      <th>4</th>\n",

              "      <td>105016</td>\n",

              "      <td>20170731</td>\n",

              "      <td>1</td>\n",

              "      <td>900000000000207008</td>\n",

              "      <td>126817006</td>\n",

              "      <td>en</td>\n",

              "      <td>900000000000013009</td>\n",

              "      <td>Neoplasm of esophagus</td>\n",

              "      <td>900000000000448009</td>\n",

              "    </tr>\n",

              "  </tbody>\n",

              "</table>\n",

              "</div>"

            ],

            "text/plain": [

              "       id  effectiveTime  active            moduleId  conceptId languageCode  \\\n",

              "0  101013       20170731       1  900000000000207008  126813005           en   \n",

              "1  102018       20170731       1  900000000000207008  126814004           en   \n",

              "2  103011       20170731       1  900000000000207008  126815003           en   \n",

              "3  104017       20170731       1  900000000000207008  126816002           en   \n",

              "4  105016       20170731       1  900000000000207008  126817006           en   \n",

              "\n",

              "               typeId                                         term  \\\n",

              "0  900000000000013009    Neoplasm of anterior aspect of epiglottis   \n",

              "1  900000000000013009  Neoplasm of junctional region of epiglottis   \n",

              "2  900000000000013009       Neoplasm of lateral wall of oropharynx   \n",

              "3  900000000000013009     Neoplasm of posterior wall of oropharynx   \n",

              "4  900000000000013009                        Neoplasm of esophagus   \n",

              "\n",

              "   caseSignificanceId  \n",

              "0  900000000000448009  \n",

              "1  900000000000448009  \n",

              "2  900000000000448009  \n",

              "3  900000000000448009  \n",

              "4  900000000000448009  "

            ]

          },

          "execution_count": 4,

          "metadata": {},

          "output_type": "execute_result"

        }

      ],

      "source": [

        "snomed_csv.head()"

      ]

    },

    {

      "cell_type": "markdown",

      "id": "eee36071",

      "metadata": {

        "id": "eee36071"

      },

      "source": [

        "Process snomed terms"

      ]

    },

    {

      "cell_type": "code",

      "execution_count": null,

      "id": "fc74afa8",

      "metadata": {

        "id": "fc74afa8"

      },

      "outputs": [],

      "source": [

        "all_ids = snomed_csv['conceptId']\n",

        "all_names = []\n",

        "for i in snomed_csv['term']:\n",

        "    try:\n",

        "        all_names.append(i.lower())\n",

        "    except:\n",

        "        all_names.append('not applicable')\n",

        "#         print(i)"

      ]

    },

    {

      "cell_type": "code",

      "execution_count": null,

      "id": "ecbc8292",

      "metadata": {

        "colab": {

          "base_uri": "https://localhost:8080/"

        },

        "id": "ecbc8292",

        "outputId": "eb0c0228-aa13-4a37-e4d5-d1d5cee77806"

      },

      "outputs": [

        {

          "data": {

            "text/plain": [

              "id                            1491117014\n",

              "effectiveTime                   20030131\n",

              "active                                 1\n",

              "moduleId              900000000000207008\n",

              "conceptId                      385432009\n",

              "languageCode                          en\n",

              "typeId                900000000000013009\n",

              "term                                 NaN\n",

              "caseSignificanceId    900000000000020002\n",

              "Name: 906846, dtype: object"

            ]

          },

          "execution_count": 6,

          "metadata": {},

          "output_type": "execute_result"

        }

      ],

      "source": [

        "snomed_csv.iloc[906846]"

      ]

    },

    {

      "cell_type": "code",

      "execution_count": null,

      "id": "6d11f0d6",

      "metadata": {

        "id": "6d11f0d6"

      },

      "outputs": [],

      "source": [

        "snomed_name_id = [(all_names[i], all_ids[i]) for i in range(len(all_ids))]"

      ]

    },

    {

      "cell_type": "code",

      "execution_count": null,

      "id": "f61e031c",

      "metadata": {

        "colab": {

          "base_uri": "https://localhost:8080/"

        },

        "id": "f61e031c",

        "outputId": "3a768eb2-906b-4974-d468-c53b63cd057e"

      },

      "outputs": [

        {

          "data": {

            "text/plain": [

              "1569232"

            ]

          },

          "execution_count": 8,

          "metadata": {},

          "output_type": "execute_result"

        }

      ],

      "source": [

        "len(all_ids)"

      ]

    },

    {

      "cell_type": "code",

      "execution_count": null,

      "id": "8b2c1e53",

      "metadata": {

        "colab": {

          "base_uri": "https://localhost:8080/"

        },

        "id": "8b2c1e53",

        "outputId": "84851594-3060-4197-9886-78d9d1e443fa"

      },

      "outputs": [

        {

          "data": {

            "text/plain": [

              "['neoplasm of anterior aspect of epiglottis',\n",

              " 'neoplasm of junctional region of epiglottis',\n",

              " 'neoplasm of lateral wall of oropharynx',\n",

              " 'neoplasm of posterior wall of oropharynx',\n",

              " 'neoplasm of esophagus',\n",

              " 'neoplasm of cervical esophagus',\n",

              " 'neoplasm of thoracic esophagus',\n",

              " 'neoplasm of abdominal esophagus',\n",

              " 'neoplasm of middle third of esophagus',\n",

              " 'neoplasm of lower third of esophagus']"

            ]

          },

          "execution_count": 9,

          "metadata": {},

          "output_type": "execute_result"

        }

      ],

      "source": [

        "all_names[:10]"

      ]

    },

    {

      "cell_type": "code",

      "execution_count": null,

      "id": "4de928c7",

      "metadata": {

        "colab": {

          "base_uri": "https://localhost:8080/"

        },

        "id": "4de928c7",

        "outputId": "1f491d1a-1d2a-451c-86db-12b1bc92c536"

      },

      "outputs": [

        {

          "data": {

            "text/plain": [

              "0    126813005\n",

              "1    126814004\n",

              "2    126815003\n",

              "3    126816002\n",

              "4    126817006\n",

              "5    126818001\n",

              "6    126819009\n",

              "7    126820003\n",

              "8    126822006\n",

              "9    126823001\n",

              "Name: conceptId, dtype: int64"

            ]

          },

          "execution_count": 10,

          "metadata": {},

          "output_type": "execute_result"

        }

      ],

      "source": [

        "all_ids[:10]"

      ]

    },

    {

      "cell_type": "markdown",

      "id": "0b808263",

      "metadata": {

        "id": "0b808263"

      },

      "source": [

        "# load sapbert"

      ]

    },

    {

      "cell_type": "code",

      "execution_count": null,

      "id": "a7c7ac5b",

      "metadata": {

        "id": "a7c7ac5b"

      },

      "outputs": [],

      "source": [

        "import numpy as np\n",

        "import torch"

      ]

    },

    {

      "cell_type": "code",

      "execution_count": null,

      "id": "d2c96dea",

      "metadata": {

        "id": "d2c96dea"

      },

      "outputs": [],

      "source": [

        "GPU_COUNT = torch.cuda.device_count()"

      ]

    },

    {

      "cell_type": "code",

      "execution_count": null,

      "id": "5c3cfade",

      "metadata": {

        "id": "5c3cfade",

        "outputId": "511a715e-cad2-457c-9ed2-b2e4030cedac"

      },

      "outputs": [

        {

          "data": {

            "text/plain": [

              "4"

            ]

          },

          "execution_count": 13,

          "metadata": {},

          "output_type": "execute_result"

        }

      ],

      "source": [

        "GPU_COUNT"

      ]

    },

    {

      "cell_type": "code",

      "execution_count": null,

      "id": "7bd1e1f2",

      "metadata": {

        "id": "7bd1e1f2",

        "outputId": "1d07efbb-3eb4-48b3-fab2-b8ef364066bd"

      },

      "outputs": [

        {

          "data": {

            "text/plain": [

              "device(type='cuda')"

            ]

          },

          "execution_count": 14,

          "metadata": {},

          "output_type": "execute_result"

        }

      ],

      "source": [

        "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\") ## specify the GPU id's, GPU id's start from 0.\n",

        "device"

      ]

    },

    {

      "cell_type": "code",

      "execution_count": null,

      "id": "528023ac",

      "metadata": {

        "colab": {

          "base_uri": "https://localhost:8080/",

          "height": 177,

          "referenced_widgets": [

            "e6bdda56f1e14d45b1c99940a763e69f",

            "91742ec5cec441b5a94d3046c777b22c",

            "2634942c850f4597be75b3a91f2d75a7",

            "c6727941f53a42248764a106e13596c9",

            "c2f02ef98fc649fca32c157c0d048a89",

            "06f54b167c2e4289b78ba4822d2f98de",

            "9b0a676b668a4ff884a72196524a9229",

            "d76fe4566fb548188ecd8f0575fccb69",

            "394fa40a235147e49fad6edbae132702",

            "521596b5e09b4d8fa79410d3601dcc32",

            "e2258a4aa3ad44d182d19fa6cb7aa05e",

            "998be9daecca4ed6b835333b44a5ba9e",

            "7467ec745de94196b786bf7219744a42",

            "b2d2469f3f0e4639a51ee68b7b74d20a",

            "7eb1948251dd460aa0d42f2d98536ed1",

            "9cec228d653e40e8a3774021542ffb68",

            "81f4f0e1758e4d2498e62dd95c2ea6c6",

            "4db71f00bd814dc3a0e5004eb5a2ae63",

            "802c6b027b704fce99d59f7da5eb1955",

            "df0df21a4ca647e683889a221065b16f",

            "ac83aa99e714487a919ea7aebd0ca422",

            "b963ba7bcd0a481a99d32418a5411ea6",

            "b769f41cc70c4c65a25f0d4a94811bf6",

            "3389a5f07d1a4b4e88c118fe47f241aa",

            "79cac786f31e4ee083b5c637d28eb7fc",

            "ff99ea7820634adb98d061ef265f2ff4",

            "c91b598e790e4c6ba21aeab31c177bf0",

            "a9edca0387954a09ada230067e7355b2",

            "a8b2871e2a164c94a58762d5cd3a2d95",

            "d20681be016d438eb638f1c7108f0d6a",

            "5abf07ae3e3c4f0eb8c12334ed383bcf",

            "58efb1c2ea714ee0a373ecc67d72368f",

            "39801cbd09484e439afb2829be9a9d52",

            "17cd86d84b43442081de52f15f833a71",

            "dbfa65b199f44020be9db0547b7ab18e",

            "d4cb162e77d844d8a672746573950427",

            "f00d00978f0f4077a4d10eeec352734f",

            "94c0b3dad8754852b8f727454c03c814",

            "76ffde2b8d2a4e1ba6a237bda4f7d856",

            "ba7d87dddcaa4ab1b7b0d832b4b3c451",

            "243d9b8fbc0842cfb96b531e9762bb82",

            "7b3448ed1e9442fa92acd84ea146ac2d",

            "1f6c9e48bf54406da669ee35c3f61d11",

            "fb9ddde1db3f47c5b8d7febc0ab691de",

            "ba13e64517a740bc89b29e25dac9de4f",

            "3f87d08da51c42ff8c909b12ca2df863",

            "e7b62de8e4794cd599198ad69e062f2b",

            "0e853c8f845d44fd85b172c8200ec82a",

            "743e49cf0f5a464b99ab5e1b96c81543",

            "fb6b72144db74d8e85489007e0b97396",

            "18c2c03f320e4c59a6451e13d3ce9221",

            "862d52c5e86f49f68013d234d1cb80e1",

            "d2290c10bbff46548e613f5045eb7117",

            "615e44bd9b404f96ab1ca449cb14d30d",

            "a0329c633eb04e0594ed4f033d25bc37"

          ]

        },

        "id": "528023ac",

        "outputId": "deb976ff-2225-45d1-9bce-ced92357067b"

      },

      "outputs": [

        {

          "name": "stderr",

          "output_type": "stream",

          "text": [

            "2022-11-03 15:18:28.534264: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA\n",

            "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",

            "2022-11-03 15:18:28.796577: E tensorflow/stream_executor/cuda/cuda_blas.cc:2981] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n",

            "2022-11-03 15:18:30.642085: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /usr/local/cuda-10.2/lib64:/opt/cudnn-7.6.5.32-cuda-10.2/lib64\n",

            "2022-11-03 15:18:30.642256: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /usr/local/cuda-10.2/lib64:/opt/cudnn-7.6.5.32-cuda-10.2/lib64\n",

            "2022-11-03 15:18:30.642272: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly.\n"

          ]

        }

      ],

      "source": [

        "from transformers import AutoTokenizer, AutoModel\n",

        "tokenizer = AutoTokenizer.from_pretrained(\"cambridgeltl/SapBERT-from-PubMedBERT-fulltext\")\n",

        "model = AutoModel.from_pretrained(\"cambridgeltl/SapBERT-from-PubMedBERT-fulltext\")"

      ]

    },

    {

      "cell_type": "code",

      "execution_count": null,

      "id": "tlzJasirUq6Y",

      "metadata": {

        "id": "tlzJasirUq6Y",

        "outputId": "6fa6247e-3bb5-4e07-b976-09f3b62a26db"

      },

      "outputs": [

        {

          "name": "stderr",

          "output_type": "stream",

          "text": [

            "2022-11-03 15:18:42.016139: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA\n",

            "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",

            "2022-11-03 15:18:42.048513: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA\n",

            "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",

            "2022-11-03 15:18:42.083803: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA\n",

            "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",

            "2022-11-03 15:18:42.090996: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA\n",

            "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",

            "2022-11-03 15:18:42.302304: E tensorflow/stream_executor/cuda/cuda_blas.cc:2981] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n",

            "2022-11-03 15:18:42.302975: E tensorflow/stream_executor/cuda/cuda_blas.cc:2981] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n",

            "2022-11-03 15:18:42.342529: E tensorflow/stream_executor/cuda/cuda_blas.cc:2981] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n",

            "2022-11-03 15:18:42.344770: E tensorflow/stream_executor/cuda/cuda_blas.cc:2981] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n",

            "2022-11-03 15:18:43.882634: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /usr/local/cuda-10.2/lib64:/opt/cudnn-7.6.5.32-cuda-10.2/lib64\n",

            "2022-11-03 15:18:43.882867: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /usr/local/cuda-10.2/lib64:/opt/cudnn-7.6.5.32-cuda-10.2/lib64\n",

            "2022-11-03 15:18:43.882893: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly.\n",

            "2022-11-03 15:18:43.895654: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /usr/local/cuda-10.2/lib64:/opt/cudnn-7.6.5.32-cuda-10.2/lib64\n",

            "2022-11-03 15:18:43.895895: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /usr/local/cuda-10.2/lib64:/opt/cudnn-7.6.5.32-cuda-10.2/lib64\n",

            "2022-11-03 15:18:43.895920: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly.\n",

            "2022-11-03 15:18:43.897874: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /usr/local/cuda-10.2/lib64:/opt/cudnn-7.6.5.32-cuda-10.2/lib64\n",

            "2022-11-03 15:18:43.898080: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /usr/local/cuda-10.2/lib64:/opt/cudnn-7.6.5.32-cuda-10.2/lib64\n",

            "2022-11-03 15:18:43.898103: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly.\n",

            "2022-11-03 15:18:43.945918: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /usr/local/cuda-10.2/lib64:/opt/cudnn-7.6.5.32-cuda-10.2/lib64\n",

            "2022-11-03 15:18:43.946181: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /usr/local/cuda-10.2/lib64:/opt/cudnn-7.6.5.32-cuda-10.2/lib64\n",

            "2022-11-03 15:18:43.946208: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly.\n"

          ]

        },

        {

          "data": {

            "text/plain": [

              "<parallelformers.parallelize.parallelize at 0x14a2954f9f40>"

            ]

          },

          "execution_count": 16,

          "metadata": {},

          "output_type": "execute_result"

        }

      ],

      "source": [

        "# model = torch.nn.DataParallel(model)\n",

        "# model.to(device)\n",

        "from parallelformers import parallelize\n",

        "parallelize(model, num_gpus=4, fp16=True)"

      ]

    },

    {

      "cell_type": "markdown",

      "id": "a3a24048",

      "metadata": {

        "id": "a3a24048"

      },

      "source": [

        "Generate embeddings for snomed labels"

      ]

    },

    {

      "cell_type": "code",

      "execution_count": null,

      "id": "bb0b8655",

      "metadata": {

        "id": "bb0b8655"

      },

      "outputs": [],

      "source": [

        "# all_names1 = all_names[:100]"

      ]

    },

    {

      "cell_type": "code",

      "execution_count": null,

      "id": "5c5ff31c",

      "metadata": {

        "colab": {

          "base_uri": "https://localhost:8080/"

        },

        "id": "5c5ff31c",

        "outputId": "b229214d-80aa-4d6d-8ca7-7325c60944d6"

      },

      "outputs": [

        {

          "name": "stderr",

          "output_type": "stream",

          "text": [

            "100%|█████████████████████████████████████| 12260/12260 [17:08<00:00, 11.92it/s]\n"

          ]

        }

      ],

      "source": [

        "bs = 128\n",

        "all_reps = []\n",

        "for i in tqdm(np.arange(0, len(all_names), bs)):\n",

        "    toks = tokenizer.batch_encode_plus(all_names[i:i+bs],\n",

        "                                      padding=\"max_length\",\n",

        "                                      max_length=25,\n",

        "                                      truncation=True,\n",

        "                                      return_tensors=\"pt\")\n",

        "    toks = toks.to(device)\n",

        "    output = model(**toks)\n",

        "    cls_rep = output[0][:,0,:]\n",

        "    \n",

        "    all_reps.append(cls_rep.cpu().detach().numpy())\n",

        "all_reps_emb = np.concatenate(all_reps, axis=0)"

      ]

    },

    {

      "cell_type": "code",

      "execution_count": null,

      "id": "c1230654",

      "metadata": {

        "colab": {

          "base_uri": "https://localhost:8080/"

        },

        "id": "c1230654",

        "outputId": "a552bfff-4e03-4ac4-e3c4-34365c66c708"

      },

      "outputs": [

        {

          "name": "stdout",

          "output_type": "stream",

          "text": [

            "(1569232, 768)\n"

          ]

        }

      ],

      "source": [

        "print(all_reps_emb.shape)"

      ]

    },

    {

      "cell_type": "code",

      "execution_count": null,

      "id": "00a427c1",

      "metadata": {

        "id": "00a427c1"

      },

      "outputs": [],

      "source": [

        "all_reps_emb = all_reps_emb.astype(np.float32)"

      ]

    },

    {

      "cell_type": "code",

      "execution_count": null,

      "id": "3c884582",

      "metadata": {

        "id": "3c884582"

      },

      "outputs": [],

      "source": [

        "import faiss"

      ]

    },

    {

      "cell_type": "code",

      "execution_count": null,

      "id": "9d7d069d",

      "metadata": {

        "id": "9d7d069d",

        "outputId": "ff1b1498-4628-4c4a-edda-d98637fcace7"

      },

      "outputs": [

        {

          "name": "stdout",

          "output_type": "stream",

          "text": [

            "True\n"

          ]

        }

      ],

      "source": [

        "d = all_reps_emb.shape[1]\n",

        "index = faiss.IndexFlatL2(d)   # build the index\n",

        "print(index.is_trained)"

      ]

    },

    {

      "cell_type": "code",

      "execution_count": null,

      "id": "77b258e0",

      "metadata": {

        "id": "77b258e0",

        "outputId": "1bde8c9e-b1ed-420a-8bea-fc81b2090075"

      },

      "outputs": [

        {

          "name": "stdout",

          "output_type": "stream",

          "text": [

            "1569232\n"

          ]

        }

      ],

      "source": [

        "index.add(all_reps_emb)                  # add vectors to the index\n",

        "print(index.ntotal)"

      ]

    },

    {

      "cell_type": "markdown",

      "id": "40fe39a4",

      "metadata": {

        "id": "40fe39a4"

      },

      "source": [

        "Load ground truth data"

      ]

    },

    {

      "cell_type": "code",

      "execution_count": null,

      "id": "44851e30",

      "metadata": {

        "colab": {

          "base_uri": "https://localhost:8080/",

          "height": 337

        },

        "id": "44851e30",

        "outputId": "652b992e-de38-468a-e2f3-e5b48a60c143"

      },

      "outputs": [

        {

          "data": {

            "text/html": [

              "<div>\n",

              "<style scoped>\n",

              "    .dataframe tbody tr th:only-of-type {\n",

              "        vertical-align: middle;\n",

              "    }\n",

              "\n",

              "    .dataframe tbody tr th {\n",

              "        vertical-align: top;\n",

              "    }\n",

              "\n",

              "    .dataframe thead th {\n",

              "        text-align: right;\n",

              "    }\n",

              "</style>\n",

              "<table border=\"1\" class=\"dataframe\">\n",

              "  <thead>\n",

              "    <tr style=\"text-align: right;\">\n",

              "      <th></th>\n",

              "      <th>filename</th>\n",

              "      <th>mark</th>\n",

              "      <th>label</th>\n",

              "      <th>offset1</th>\n",

              "      <th>offset2</th>\n",

              "      <th>span</th>\n",

              "      <th>code</th>\n",

              "    </tr>\n",

              "  </thead>\n",

              "  <tbody>\n",

              "    <tr>\n",

              "      <th>0</th>\n",

              "      <td>es-S0212-71992007000100007-1</td>\n",

              "      <td>T1</td>\n",

              "      <td>ENFERMEDAD</td>\n",

              "      <td>40</td>\n",

              "      <td>61</td>\n",

              "      <td>arterial hypertension</td>\n",

              "      <td>38341003</td>\n",

              "    </tr>\n",

              "    <tr>\n",

              "      <th>1</th>\n",

              "      <td>es-S0212-71992007000100007-1</td>\n",

              "      <td>T2</td>\n",

              "      <td>ENFERMEDAD</td>\n",

              "      <td>66</td>\n",

              "      <td>79</td>\n",

              "      <td>polyarthrosis</td>\n",

              "      <td>36186002</td>\n",

              "    </tr>\n",

              "    <tr>\n",

              "      <th>2</th>\n",

              "      <td>es-S0212-71992007000100007-1</td>\n",

              "      <td>T3</td>\n",

              "      <td>ENFERMEDAD</td>\n",

              "      <td>1682</td>\n",

              "      <td>1698</td>\n",

              "      <td>pleural effusion</td>\n",

              "      <td>60046008</td>\n",

              "    </tr>\n",

              "    <tr>\n",

              "      <th>3</th>\n",

              "      <td>es-S0212-71992007000100007-1</td>\n",

              "      <td>T4</td>\n",

              "      <td>ENFERMEDAD</td>\n",

              "      <td>1859</td>\n",

              "      <td>1875</td>\n",

              "      <td>pleural effusion</td>\n",

              "      <td>60046008</td>\n",

              "    </tr>\n",

              "    <tr>\n",

              "      <th>4</th>\n",

              "      <td>es-S0212-71992007000100007-1</td>\n",

              "      <td>T5</td>\n",

              "      <td>ENFERMEDAD</td>\n",

              "      <td>1626</td>\n",

              "      <td>1648</td>\n",

              "      <td>lower lobe atelectasis</td>\n",

              "      <td>46621007</td>\n",

              "    </tr>\n",

              "  </tbody>\n",

              "</table>\n",

              "</div>"

            ],

            "text/plain": [

              "                       filename mark       label  offset1  offset2  \\\n",

              "0  es-S0212-71992007000100007-1   T1  ENFERMEDAD       40       61   \n",

              "1  es-S0212-71992007000100007-1   T2  ENFERMEDAD       66       79   \n",

              "2  es-S0212-71992007000100007-1   T3  ENFERMEDAD     1682     1698   \n",

              "3  es-S0212-71992007000100007-1   T4  ENFERMEDAD     1859     1875   \n",

              "4  es-S0212-71992007000100007-1   T5  ENFERMEDAD     1626     1648   \n",

              "\n",

              "                     span      code  \n",

              "0   arterial hypertension  38341003  \n",

              "1           polyarthrosis  36186002  \n",

              "2        pleural effusion  60046008  \n",

              "3        pleural effusion  60046008  \n",

              "4  lower lobe atelectasis  46621007  "

            ]

          },

          "execution_count": 27,

          "metadata": {},

          "output_type": "execute_result"

        }

      ],

      "source": [

        "entities = pd.read_csv(\"data/entities.tsv\", delimiter=\"\\t\")\n",

        "entities.head()"

      ]

    },

    {

      "cell_type": "code",

      "execution_count": null,

      "id": "8a009c68",

      "metadata": {

        "colab": {

          "base_uri": "https://localhost:8080/"

        },

        "id": "8a009c68",

        "outputId": "0b1bfb0f-0fa5-4487-d95c-2723a0c82f00",

        "scrolled": true

      },

      "outputs": [

        {

          "name": "stdout",

          "output_type": "stream",

          "text": [

            "['arterial hypertension', 'polyarthrosis', 'pleural effusion', 'pleural effusion', 'lower lobe atelectasis', 'infectious spondylodiscitis d10-d11', 'pleural effusion', 'brucellosis', 'orchiepididymitis', 'goitre']\n",

            "0     38341003\n",

            "1     36186002\n",

            "2     60046008\n",

            "3     60046008\n",

            "4     46621007\n",

            "5    302935008\n",

            "6     60046008\n",

            "7     75702008\n",

            "8    197983000\n",

            "9      3716002\n",

            "Name: code, dtype: object\n"

          ]

        }

      ],

      "source": [

        "inp_names = [i.lower() for i in entities['span']]\n",

        "inp_labels = entities['code']\n",

        "print(inp_names[:10])\n",

        "print(inp_labels[:10])"

      ]

    },

    {

      "cell_type": "code",

      "execution_count": null,

      "id": "90bbf268",

      "metadata": {

        "id": "90bbf268"

      },

      "outputs": [],

      "source": [

        "# c=0\n",

        "# for i in inp_label:\n",

        "# #     if type(i)!=float:\n",

        "#     try:\n",

        "#         [float(i)]\n",

        "#     except:\n",

        "#         c+=1\n",

        "# #         print(i.split('+'))\n",

        "# c"

      ]

    },

    {

      "cell_type": "code",

      "execution_count": null,

      "id": "49562d03",

      "metadata": {

        "id": "49562d03"

      },

      "outputs": [],

      "source": [

        "# inp_names1 = inp_names[:10]"

      ]

    },

    {

      "cell_type": "markdown",

      "id": "e6cf5d29",

      "metadata": {

        "id": "e6cf5d29"

      },

      "source": [

        "Generate embeddings for ground truth terms, get their closest snomedct embedding and list out its corresponding snomedct code"

      ]

    },

    {

      "cell_type": "code",

      "execution_count": null,

      "id": "049818b3",

      "metadata": {

        "id": "049818b3"

      },

      "outputs": [],

      "source": [

        "query_toks = tokenizer.batch_encode_plus(list(inp_names),\n",

        "                                        padding = \"max_length\",\n",

        "                                        max_length = 25,\n",

        "                                        truncation=True,\n",

        "                                        return_tensors=\"pt\")\n",

        "query_toks = query_toks.to(device)\n",