Log In Sign Up
Models:
Daniel Galbraith
DanielG/
plip
0
Downloads: 1
Diff of /reproducibility/Reproduce Linear Probing.ipynb [000000] .. [278d8a]

Switch to unified view

a b/reproducibility/Reproduce Linear Probing.ipynb 

  1
  
{





  
  

  2
  
 "cells": [





  
  

  3
  
  {





  
  

  4
  
   "cell_type": "code",





  
  

  5
  
   "execution_count": 2,





  
  

  6
  
   "id": "4d8f75cf",





  
  

  7
  
   "metadata": {},





  
  

  8
  
   "outputs": [],





  
  

  9
  
   "source": [





  
  

  10
  
    "import os\n",





  
  

  11
  
    "import torch\n",





  
  

  12
  
    "import clip\n",





  
  

  13
  
    "import pandas as pd\n",





  
  

  14
  
    "import numpy as np\n",





  
  

  15
  
    "from tqdm import tqdm\n",





  
  

  16
  
    "from PIL import Image\n",





  
  

  17
  
    "\n",





  
  

  18
  
    "from sklearn.linear_model import SGDClassifier\n",





  
  

  19
  
    "from torch.utils.data import Dataset, DataLoader\n",





  
  

  20
  
    "from metrics import eval_metrics\n",





  
  

  21
  
    "from sklearn.preprocessing import LabelEncoder\n",





  
  

  22
  
    "\n"





  
  

  23
  
   ]





  
  

  24
  
  },





  
  

  25
  
  {





  
  

  26
  
   "cell_type": "code",





  
  

  27
  
   "execution_count": 12,





  
  

  28
  
   "id": "e56d9ac9",





  
  

  29
  
   "metadata": {},





  
  

  30
  
   "outputs": [],





  
  

  31
  
   "source": [





  
  

  32
  
    "dataset = \"PanNuke\"\n",





  
  

  33
  
    "data_folder = \"/path/to/data\"\n",





  
  

  34
  
    "cache_dir = \".cache\"\n",





  
  

  35
  
    "model_name = \"plip\"\n",





  
  

  36
  
    "plip_path = \"/path/to/plip\"\n",





  
  

  37
  
    "device=\"cuda\"\n",





  
  

  38
  
    "\n",





  
  

  39
  
    "class CLIPImageDataset(Dataset):\n",





  
  

  40
  
    "    def __init__(self, list_of_images, preprocessing):\n",





  
  

  41
  
    "        self.images = list_of_images\n",





  
  

  42
  
    "        self.preprocessing = preprocessing\n",





  
  

  43
  
    "\n",





  
  

  44
  
    "    def __len__(self):\n",





  
  

  45
  
    "        return len(self.images)\n",





  
  

  46
  
    "\n",





  
  

  47
  
    "    def __getitem__(self, idx):\n",





  
  

  48
  
    "        images = self.preprocessing(Image.open(self.images[idx]))  # preprocess from clip.load\n",





  
  

  49
  
    "        return images\n",





  
  

  50
  
    "    \n",





  
  

  51
  
    "@torch.no_grad()\n",





  
  

  52
  
    "def get_embs(loader, model, model_name):\n",





  
  

  53
  
    "    all_embs = []\n",





  
  

  54
  
    "    for images in tqdm(loader):\n",





  
  

  55
  
    "        images = images.to(device)\n",





  
  

  56
  
    "        if model_name in [\"clip\", \"plip\"]:\n",





  
  

  57
  
    "            all_embs.append(model.encode_image(images).cpu().numpy())\n",





  
  

  58
  
    "        else:\n",





  
  

  59
  
    "            all_embs.append(model(images).squeeze().cpu().numpy())\n",





  
  

  60
  
    "    all_embs = np.concatenate(all_embs, axis=0)\n",





  
  

  61
  
    "    return all_embs\n",





  
  

  62
  
    "\n",





  
  

  63
  
    "\n",





  
  

  64
  
    "def run_classification(train_x, train_y, test_x, test_y, seed=1, alpha=0.1):\n",





  
  

  65
  
    "    classifier = SGDClassifier(random_state=seed, loss=\"log_loss\",\n",





  
  

  66
  
    "                               alpha=alpha, verbose=0,\n",





  
  

  67
  
    "                               penalty=\"l2\", max_iter=10000, class_weight=\"balanced\")\n",





  
  

  68
  
    "    \n",





  
  

  69
  
    "    le = LabelEncoder()\n",





  
  

  70
  
    "\n",





  
  

  71
  
    "    train_y = le.fit_transform(train_y)\n",





  
  

  72
  
    "    test_y = le.transform(test_y)\n",





  
  

  73
  
    "\n",





  
  

  74
  
    "    train_y = np.array(train_y)\n",





  
  

  75
  
    "    test_y = np.array(test_y)\n",





  
  

  76
  
    "\n",





  
  

  77
  
    "    classifier.fit(train_x, train_y)\n",





  
  

  78
  
    "    test_pred = classifier.predict(test_x)\n",





  
  

  79
  
    "    train_pred = classifier.predict(train_x)\n",





  
  

  80
  
    "    test_metrics = eval_metrics(test_y, test_pred, average_method=\"macro\")\n",





  
  

  81
  
    "    return test_metrics"





  
  

  82
  
   ]





  
  

  83
  
  },





  
  

  84
  
  {





  
  

  85
  
   "cell_type": "code",





  
  

  86
  
   "execution_count": null,





  
  

  87
  
   "id": "2ed88421",





  
  

  88
  
   "metadata": {},





  
  

  89
  
   "outputs": [],





  
  

  90
  
   "source": []





  
  

  91
  
  },





  
  

  92
  
  {





  
  

  93
  
   "cell_type": "code",





  
  

  94
  
   "execution_count": 13,





  
  

  95
  
   "id": "7bac69d2",





  
  

  96
  
   "metadata": {},





  
  

  97
  
   "outputs": [],





  
  

  98
  
   "source": [





  
  

  99
  
    "train_dataset_name = dataset + \"_train.csv\"\n",





  
  

  100
  
    "test_dataset_name = dataset + \"_test.csv\"\n",





  
  

  101
  
    "\n",





  
  

  102
  
    "train_dataset = pd.read_csv(os.path.join(data_folder, train_dataset_name))\n",





  
  

  103
  
    "test_dataset = pd.read_csv(os.path.join(data_folder, test_dataset_name))\n",





  
  

  104
  
    "\n",





  
  

  105
  
    "test_y = test_dataset[\"label\"].tolist()\n",





  
  

  106
  
    "train_y = train_dataset[\"label\"].tolist()"





  
  

  107
  
   ]





  
  

  108
  
  },





  
  

  109
  
  {





  
  

  110
  
   "cell_type": "code",





  
  

  111
  
   "execution_count": 14,





  
  

  112
  
   "id": "5f2d857b",





  
  

  113
  
   "metadata": {},





  
  

  114
  
   "outputs": [





  
  

  115
  
    {





  
  

  116
  
     "data": {





  
  

  117
  
      "text/plain": [





  
  

  118
  
       "<All keys matched successfully>"





  
  

  119
  
      ]





  
  

  120
  
     },





  
  

  121
  
     "execution_count": 14,





  
  

  122
  
     "metadata": {},





  
  

  123
  
     "output_type": "execute_result"





  
  

  124
  
    }





  
  

  125
  
   ],





  
  

  126
  
   "source": [





  
  

  127
  
    "model, preprocess = clip.load(\"ViT-B/32\", device=device, download_root=cache_dir)\n",





  
  

  128
  
    "model.load_state_dict(torch.load(plip_path))"





  
  

  129
  
   ]





  
  

  130
  
  },





  
  

  131
  
  {





  
  

  132
  
   "cell_type": "code",





  
  

  133
  
   "execution_count": 15,





  
  

  134
  
   "id": "efb1786b",





  
  

  135
  
   "metadata": {},





  
  

  136
  
   "outputs": [],





  
  

  137
  
   "source": [





  
  

  138
  
    "model = model.eval()"





  
  

  139
  
   ]





  
  

  140
  
  },





  
  

  141
  
  {





  
  

  142
  
   "cell_type": "code",





  
  

  143
  
   "execution_count": 16,





  
  

  144
  
   "id": "aabf4aca",





  
  

  145
  
   "metadata": {},





  
  

  146
  
   "outputs": [],





  
  

  147
  
   "source": [





  
  

  148
  
    "def run_study(model_name, cache_dir=cache_dir):\n",





  
  

  149
  
    "    if model_name == \"plip\":\n",





  
  

  150
  
    "        model, preprocess = clip.load(\"ViT-B/32\", device=device, download_root=cache_dir)\n",





  
  

  151
  
    "        model.load_state_dict(torch.load(plip_path))\n",





  
  

  152
  
    "    elif model_name == \"clip\":\n",





  
  

  153
  
    "        model, preprocess = clip.load(\"ViT-B/32\", device=device, download_root=cache_dir)\n",





  
  

  154
  
    "    elif model_name == \"mudipath\": \n",





  
  

  155
  
    "        from torchvision import transforms\n",





  
  

  156
  
    "        from embedders.mudipath import build_densenet\n",





  
  

  157
  
    "        \n",





  
  

  158
  
    "        model = build_densenet(download_dir=cache_dir,\n",





  
  

  159
  
    "                                      pretrained=\"mtdp\")\n",





  
  

  160
  
    "        model.num_feats = model.n_features()\n",





  
  

  161
  
    "        model.forward_type = \"image\"\n",





  
  

  162
  
    "        model = model.to(device)\n",





  
  

  163
  
    "        model.eval()\n",





  
  

  164
  
    "        preprocess = transforms.Compose([\n",





  
  

  165
  
    "            transforms.Resize(224),\n",





  
  

  166
  
    "            transforms.CenterCrop(224),\n",





  
  

  167
  
    "            transforms.ToTensor(),\n",





  
  

  168
  
    "            transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])  # ImageNet stats\n",





  
  

  169
  
    "        ])\n",





  
  

  170
  
    "    \n",





  
  

  171
  
    "    train_loader = DataLoader(CLIPImageDataset(train_dataset[\"image\"].tolist(), preprocess), batch_size=32)\n",





  
  

  172
  
    "    test_loader = DataLoader(CLIPImageDataset(test_dataset[\"image\"].tolist(), preprocess), batch_size=32)\n",





  
  

  173
  
    "\n",





  
  

  174
  
    "    train_embs = get_embs(train_loader, model, model_name)\n",





  
  

  175
  
    "    test_embs = get_embs(test_loader, model, model_name)\n",





  
  

  176
  
    "    \n",





  
  

  177
  
    "\n",





  
  

  178
  
    "    all_records = []\n",





  
  

  179
  
    "    for alpha in [1.0, 0.1, 0.01, 0.001]:\n",





  
  

  180
  
    "        metrics = run_classification(train_embs, train_y, test_embs, test_y, alpha=alpha)\n",





  
  

  181
  
    "        metrics[\"alpha\"] = alpha\n",





  
  

  182
  
    "        metrics[\"model_name\"] = model_name\n",





  
  

  183
  
    "        all_records.append(metrics)\n",





  
  

  184
  
    "    return all_records"





  
  

  185
  
   ]





  
  

  186
  
  },





  
  

  187
  
  {





  
  

  188
  
   "cell_type": "code",





  
  

  189
  
   "execution_count": 17,





  
  

  190
  
   "id": "e331473e",





  
  

  191
  
   "metadata": {},





  
  

  192
  
   "outputs": [





  
  

  193
  
    {





  
  

  194
  
     "name": "stderr",





  
  

  195
  
     "output_type": "stream",





  
  

  196
  
     "text": [





  
  

  197
  
      "100%|██████████| 136/136 [00:47<00:00,  2.86it/s]\n",





  
  

  198
  
      "100%|██████████| 59/59 [00:20<00:00,  2.85it/s]\n",





  
  

  199
  
      "100%|██████████| 136/136 [00:38<00:00,  3.49it/s]\n",





  
  

  200
  
      "100%|██████████| 59/59 [00:17<00:00,  3.37it/s]\n",





  
  

  201
  
      "100%|██████████| 136/136 [00:39<00:00,  3.48it/s]\n",





  
  

  202
  
      "100%|██████████| 59/59 [00:17<00:00,  3.46it/s]\n"





  
  

  203
  
     ]





  
  

  204
  
    }





  
  

  205
  
   ],





  
  

  206
  
   "source": [





  
  

  207
  
    "all_records = []\n",





  
  

  208
  
    "for model_name in [\"mudipath\", \"plip\", \"clip\"]:\n",





  
  

  209
  
    "    all_records.extend(run_study(model_name))"





  
  

  210
  
   ]





  
  

  211
  
  },





  
  

  212
  
  {





  
  

  213
  
   "cell_type": "code",





  
  

  214
  
   "execution_count": null,





  
  

  215
  
   "id": "fccfc443",





  
  

  216
  
   "metadata": {},





  
  

  217
  
   "outputs": [],





  
  

  218
  
   "source": []





  
  

  219
  
  },





  
  

  220
  
  {





  
  

  221
  
   "cell_type": "code",





  
  

  222
  
   "execution_count": 18,





  
  

  223
  
   "id": "80436432",





  
  

  224
  
   "metadata": {},





  
  

  225
  
   "outputs": [],





  
  

  226
  
   "source": [





  
  

  227
  
    "result_df = pd.DataFrame(all_records)"





  
  

  228
  
   ]





  
  

  229
  
  },





  
  

  230
  
  {





  
  

  231
  
   "cell_type": "code",





  
  

  232
  
   "execution_count": 19,





  
  

  233
  
   "id": "63d4ca50",





  
  

  234
  
   "metadata": {},





  
  

  235
  
   "outputs": [





  
  

  236
  
    {





  
  

  237
  
     "data": {





  
  

  238
  
      "text/plain": [





  
  

  239
  
       "<AxesSubplot:xlabel='model_name', ylabel='F1'>"





  
  

  240
  
      ]





  
  

  241
  
     },





  
  

  242
  
     "execution_count": 19,





  
  

  243
  
     "metadata": {},





  
  

  244
  
     "output_type": "execute_result"





  
  

  245
  
    },





  
  

  246
  
    {





  
  

  247
  
     "data": {





  
  

  248
  
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEHCAYAAAC0pdErAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAocklEQVR4nO3dd3hUddrG8e+TQOgkAUINAaSHliY2mtgQOzaau7qurghiAfdV0RVR1waKBWHdte2CYFsVQQWlCHaS0EPvRSFAQoeQ5Pf+kcGNGCCQTE4mc3+ui4vJmTMzdzDOnfObmeeYcw4REZFjhXgdQERESicVhIiIFEgFISIiBVJBiIhIgVQQIiJSoHJeBygutWrVco0bN/Y6hohIQElJSdnhnIsq6LoyUxCNGzcmOTnZ6xgiIgHFzDYc7zotMYmISIFUECIiUiAVhIiIFEgFISIiBVJBiIhIgVQQIiJSIBWEiIgUSAUh4kfOOeauSmdzxgGvo4icsjLzQTmR0mbFL3sZPnkp36/dSUyNynx6VyfCK5X3OpZIoekIQqSY7T5whOGTl9Lzpbmk/byHO7s1ZWvmQYa8t5DcXJ2gSwKHjiBEiklOruPdeZt4btpydh88Qt+zYhhyUUsiq4RRq2oFRkxJ4x9z1jKgW1Ovo4oUigpCpBikbNjFo5OXsmTLHjo2rsGjV8bSpn74r9ffcl5jUjZm8Ny05cQ1jOCcpjU9TCtSOCoIkSLYtucQT3++nI/mb6Fu9Yq82DuOKzvUx8x+s5+Z8cy17Vn28x7umjifzwZ3onb1ih6lFikcvQYhchoOZ+cw7us1dB85m6mLfmbg+U2ZMaQrV8U1+F05HFW1QjnG9U9k/+FsBr0znyM5uSWcWuTUqCBETtGs5dvpMXouT3++nHOa1uTL+7pw/yWtqFLh5AfkLepU46le7fhp/S6em7aiBNKKnD4tMYkU0vod+xkxJY2Zy7dzRq0qvHnLmZzfsvYp38/V8Q1I3rCL1+asJSEmkh5t6/ohrUjRqSBETmL/4WxembWa1+euo3yo8VDPVtx8bhPCyp3+Afgjl8eyePNu7n9/IS3rVqNJrSrFmFikeGiJSeQ4nHN8smAL3UfNZuzsNVzeoR6zhnbj9i5Ni1QOABXKhTKmXwKhocaA8SkczMopptQixUcFIVKAJVt2c/2477l70gJqV6vIhwPO5fkb4or1nUfRkZV54cY4VmzbyyOfLME5fYhOShctMYnks2t/FiOnr2DiTxupUTmMZ65tx/WJDQkJKfidSUV1fsva3HV+M16auZqkRpH07hjjl8cROR0qCBEgOyeXCT9uZNT0FezPyuHmcxtzz4UtSmR20t0XtmD+pkz+NnkpbRuE07ZB+MlvJFICtMQkQe/7NTu5/OVveNT3BP353Z159Io2JTZYLzTEGH1jHDWrhDFgQgq7DxwpkccVORkVhAStLZkHGfhOKn3++QN7D2Uzrn8CE/58Fi3qVCvxLDWrVmBMvwR+2X2IIe8v0FA/KRW0xCRB59CRHF6bs5ZXZ6/GObjnwub8pUtTKoWFeporISaSYT1bM/zTNMZ+vYaB5zfzNI+ICkKChnOO6WnbeGJqGpt2HeTStnUZdllroiMrex3tV388tzEpGzMZNX0F8TERnNu0lteRJIhpiUmCwurte/nDGz/xl/+kUKl8KBP+fBZj+yeWqnKAvKF+T/dqR5NaVRg8cT6/7D7kdSQJYn4tCDPrYWYrzGy1mT1QwPWNzGyGmS0ys9lmFu3bHmdm35vZUt91N/ozp5Rdew4d4YkpafQYPZcFmzJ59IpYpg7uzHnNSu9v5lV8Q/0OZOUw6J1UDfUTz/itIMwsFBgDXArEAn3MLPaY3UYC/3bOtQdGAE/5th8A/uCcawP0AEabWYS/skrZk5vreC95E91Hzub1b9dxXWI0s4d245bzmlA+tPQfODf3DfVL3pDBM58v9zqOBCl/vgbREVjtnFsLYGaTgKuAtHz7xAL3+S7PAj4GcM6tPLqDc26rmW0HooBMP+aVMmLBpkwenbyUhZsyiY+J4I2bz6R9dITXsU7ZVXENSNmQwb++WUdio0gubVfP60gSZPxZEA2ATfm+3gycdcw+C4FewIvANUA1M6vpnNt5dAcz6wiEAWv8mFXKgPS9h3n2i+W8n7KZqGoVeP6GDlwd18Bvn4IuCcMua83Czbu5/4NFtKxbjTOiqnodSYKI18faQ4GuZjYf6ApsAX6dWmZm9YD/ALc45363EGtmt5tZspklp6enl1RmKWWO5OTyr7lr6T5yNh8v2MJfupzBrKHd6JUQHdDlAHlD/V7tl0D5UOPOCaka6iclyp8FsQVomO/raN+2Xznntjrnejnn4oFhvm2ZAGZWHZgKDHPO/VDQAzjnXnPOJTnnkqKiovzwLUhpN3dVOpe+OJcnpi4joVEkX9zThQd7tqZqIU7eEygaRFRidO94Vmzby7CPF2uon5QYf/5fNA9obmZNyCuG3kDf/DuYWS1gl+/o4EHgDd/2MOAj8l7A/sCPGSVAbdx5gCempjE9bRsxNSrzrz8kcUHr2sc93Weg69oiisHdm/PijFUkNapB37M01E/8z28F4ZzLNrNBwDQgFHjDObfUzEYAyc65yUA34Ckzc8AcYKDv5jcAXYCaZnazb9vNzrkF/sorgeFgVg5jZ69m3Jy1hJpx/yUtubVTEyqW9/ZT0CVh8AXNSd2YwfDJS2nXIJx20RrqJ/5lZeVwNSkpySUnJ3sdQ/zEOcfUxT/z96nL2Lr7EFfF1eeBS1tRL7yS19FK1K79WVz+0lxCQowpd3UionKY15EkwJlZinMuqaDrvH6RWuSklv+yhz7//IFB78wnvHIY7/3lHF7sHR905QBQo0oYr/ZPZNueQ9z7rob6iX+pIKTUyjyQxaOfLKHni3NZ/stenri6LVPu6kTHJjW8juapuIYRPHJ5LLNWpPPq7NVex5EyrOy81UPKjJxcx6R5Gxk5bQW7Dx6h31mNGHJxCy2n5HPT2Y1IXp/B81+uJD4mslSPDpHApYKQUiV5/S4enbyUpVv30LFJDYZf0YbY+tW9jlXqmBlP9WpH2s97GDxxPlMHd6ZuePGdL1sEtMQkpcQvuw9xz6T5XDfue3btz+LlPvG8e/vZKocTyBvql8DBIzkM1FA/8QMVhHjqcHYOY2evofuo2Xy2+BcGnd+MGUO6ckWH+mX2Mw3FqVntajxzbXtSNmTw1Gca6ifFS0tM4pmZy7cx4tM01u88wEWxdXj4stY0qlnF61gB54oO9UnZkMEb3+YN9busvYb6SfFQQUiJW5u+j8enpDFrRTpnRFXh7T91pGsLjUopiod6tmbh5kz++sFCWtWrRlMN9ZNioCUmKTH7Dmfz9OfLuWT0HOatz2BYz9Z8cXcXlUMxCCsXwpi+CVQoH8qA8SkcyMr2OpKUASoI8TvnHB/N30z3kbMZ9/UaroprwMyhXbmtyxmEldOPYHGpH1GJF3vHsWr7PoZ9tERD/aTItMQkfrVky24enbyUlA0ZtI8OZ9xNiSTERHodq8zq3DyKey9swfNfriSxUST9z27kdSQJYCoI8Yud+w4zcvpKJs3bSI3KYTx7bXuuSwz88zMEgkHnNyN1YwYjPk2jfXR4QJ5NT0oHHd9LscrOyeWtb9dx/sjZvJe8iT+d14SZQ7txw5kNVQ4lJCTEeOGGOKKqVWDA+FQy9md5HUkClApCis13a3Zw2UvfMPzTNNpFh/PF3Z155PJYwiuV9zpa0ImsEsar/RJI33uYe9/TUD85PSoIKbItmQcZOCGVvv/8kf1Z2Yzrn8j4W8+ieZ1qXkcLah0aRvDIFbHMXpHOK7M01E9OnV6DkNN26EgO//h6LWO/znvyue+iFtze5YygOHlPoOh/Vgwp63fxwlcriY+JoHNzvaVYCk8FIafMOce0pdt4YmoamzMOclm7ejzYsxXRkZW9jibHMDP+7hvqd/ekBUy5qxP1I4LvPBpyerTEJKdk9fa93PT6T9wxPoUqYeV457azGNMvQeVQilUOK8fY/okc9g31y8rWUD8pHBWEFMqeQ0d4fEoaPUbPZdHmTIZfEcvUwZ04t6nOQxAImkZV5dnrOjB/YyZ//2yZ13EkQGiJSU4oN9fxQcpmnp22nJ37s+h9ZgxDL25BzaoVvI4mp+iy9vVI3tCYN79dT2KjSK7oUN/rSFLKqSDkuOZvzGD45KUs3LybhJgI3ry5I+2iw72OJUXw4KWtWbgpkwc+XETretVpVltD/eT4tMQkv7N97yGGvr+Qa179jp93H+KFGzvw4YBzVQ5lQFi5EMb0+99Qv/2HNdRPjk8FIb/Kys7ln3PW0n3k13yyYAt3dG3KzKHduCY+WifvKUPqhVfi5T7xrEnfx0MfLdZQPzkuLTEJAHNWpvPYp0tZk76fbi2j+NvlsZyhcwqUWec1q8V9F7Vg5PSVJDWK5KZzGnsdSUohFUSQ27jzAI9PTePLtG00rlmZN25OonurOl7HkhJwZ7dmpG7MZMSUNNpFRxDXMMLrSFLKaIkpSB3IymbktBVc+MLXfLt6B//XoxXT7u2icggiISHG8zd0oE71itw5PoVdGuonx1BBBBnnHJ8u3MoFo77mlVmr6dm2LjOHdGNAt6ZUKKcRGcEmonLeUL8d+7K4590F5Gion+SjgggiaVv3cONrP3DXxPlEVg7j/TvOYXTveOqGV/Q6mniofXQEj14Zy5yV6bw8c5XXcaQU0WsQQSDzQBajpq9kwo8bCK9UnievaUvvM2MI1fkZxKdvxxhS1mfw4oxVxMdE6jzhAqggyrScXMfEnzYycvoK9hw8wk1nN+Lei1oQUTnM62hSypgZT17TjqVb93DPpPlMGdyZBhrqF/S0xFRGzVu/iyte/oaHP15CyzrVmDq4M49d1VblIMdVKSyUsf0TOJLjGDhBQ/1EBVHm/LL7EHdPms/1474n40AWr/SNZ9LtZ9O6XnWvo0kAOCOqKs9e154FmzJ5cmqa13HEY1piKiMOZ+fwr7nrGDNrNdm5jru6N2NAt6ZUDtN/Yjk1PdvV49ZOTXj9m3UkNq7BlRrqF7T07BHgnHPMXL6dEVPS2LDzABfH1uHhy2KJqanzM8jpe+DSVv8b6le3mk4fG6S0xBTA1qbv45a35nHr28mUCzH+/aeOvPaHJJWDFFn50BBe6ZtA5bBQBkxI1VC/IKWCCED7Dmfz1OfLuGT0HJLXZ/DwZa354p4udNFbE6UY1Q2vyEt94lmbvo8H/quhfsFIS0wBJDfX8fGCLTz1+XLS9x7m+sRo7u/RktrV9EE38Y9zm9ZiyMUteW7aCpIaRfLHcxt7HUlKkAoiQCzevJtHJy8hdWMmHRpG8NpNicTHRHodS4LAgK5NSd2QwRNT02gXHU6Cfu6Chl+XmMysh5mtMLPVZvZAAdc3MrMZZrbIzGabWXS+674ws0wzm+LPjKXdzn2HefC/i7hyzDds3HWAZ69rz0cDzlU5SInJG+oXR93wigyckMrOfYe9jiQlxG8FYWahwBjgUiAW6GNmscfsNhL4t3OuPTACeCrfdc8BN/krX2mXnZPLm9+uo9vI2byfvJlbz2vCzKHduCGpISEakSElLLxyecb2S2Tnfg31Cyb+PILoCKx2zq11zmUBk4CrjtknFpjpuzwr//XOuRnAXj/mK7W+W72Dni/N5bFP04hrGMEX93Tm4ctjqV6xvNfRJIi1bRDOY1e2Ye6qHbw4Q0P9goE/X4NoAGzK9/Vm4Kxj9lkI9AJeBK4BqplZTefczsI8gJndDtwOEBMTU+TAXtuccYC/f7aMzxb/QnRkJf5xUyIXx9bR6T6l1Oh9ZkOS12fw8sxVJMRE0K1lba8jiR95/TbXoUBXM5sPdAW2ADmFvbFz7jXnXJJzLikqKnDf4nnoSA6jv1rJBaO+Zuby7Qy5qAVf3deVS9rUVTlIqWJmPHF1W1rWqcY97y5gc8YBryOJH/mzILYADfN9He3b9ivn3FbnXC/nXDwwzLct04+ZShXnHF8s+ZkLRn3N6K9WcWFsHWYM6cZdFzSnYnmdvEdKp7yhfonk+Ib6Hc4u9O90EmD8WRDzgOZm1sTMwoDewOT8O5hZLTM7muFB4A0/5ilVVm7bS//Xf+SO8alUq1iOibedzZi+CRqxLAGhSa0qPHd9exZu3s0TU5Z5HUf8xG+vQTjnss1sEDANCAXecM4tNbMRQLJzbjLQDXjKzBwwBxh49PZmNhdoBVQ1s83Arc65af7KW1J2HzzCi1+t4u3v11MlLJTHrmxDv7NiKBfq9WqfyKnp0bYet3Vuwj/nriOpcSRXxTXwOpIUMysrH59PSkpyycnJXsc4rtxcx/spm3j2ixXsOpBFn44xDL24JTWq6PwMEriO5OTS758/snjLbj4ZdB4tNNQv4JhZinMuqaDr9GtrCUjdmMHVr37L/324mCa1qvDpoE78/Zp2KgcJeHlD/eKpUqEcd4xPYZ+G+pUpKgg/2r73EEPeW0ivV7/jl92HGH1jHO/fcQ5tG4R7HU2k2NSuXpGX+8Szfsd+/u/DRRrqV4ZoFpMfZGXn8tZ363hpxmoOZ+cwoFtTBp7fjKoV9M8tZdM5TWty/yWteOaL5SQ1iuSW85p4HUmKgZ6xitnsFdsZ8Wkaa3fs54JWtXn48lia1KridSwRv7uj6xmkbMjgyanLaB8dTmKjGl5HkiLSElMx2bBzP39+O5mb35yHA968+Uxev/lMlYMEDTNj1A0dqB9RiYET5rNDQ/0CngqiiA5kZfPctOVc9Pwcvl+zgwcubcUX93Tm/FYaQSDBJ7xSeV7tl8CuA1ncPWm+hvoFOBXEaXLOMXnhVrqP/Joxs9ZwWft6zBzajTu6NqVCOX0KWoJX2wbhPH5VG75dvZPRX630Oo4UgV6DOA1pW/cwfPJSflq/i7YNqvNK33iSGmu9VeSoG8+M8Q31W01CTKSOqAOUCuIUZOzPYtSXK3jnx41EVA7jqV7tuCGpIaE6P4PI7zx+dVuWbN3DPe8uYMpdnWhYo7LXkeQUaYmpEHJyHf/5YQPnj5rNxJ828YdzGjNrSDf6dIxROYgcR8XyoYztl0BurmPgOxrqF4hUECfx49qdXP7yNzzy8RJa1a3G1MGdGH5lG8Ir6+Q9IifTuFYVRt3QgUWbdzPi0zSv48gp0hLTcfy8+yBPfbacyQu3Uj+8ImP6JtCznc7PIHKqLm5Tl790PYN/fL2WpMaRXBMfffIbSamggjjGoSM5vP7NOl6ZuZoc5xh8QXMGdG1KpTC9M0nkdN1/cUsWbMzkwf8uJrZeOC3raqhfINASk49zji/TtnHxC3N4btoKuraIYsZ9XbnvohYqB5EiKhcawst946lWsTwDxqew99ARryNJIagggDXp+7j5zXnc9u9kwsqF8J9bOzLupkS960KkGNWuVpFX+sSzYdcBDfULEEFfEGvS93HJC3NI3ZDBI5fH8vndnencPHDPby1Smp11Rk3+eklLPlv8C69/s87rOHISQf8aRNOoqjzUszVXdKhPVLUKXscRKfNu75I31O/pz5cT1zBCHzItxYL+CALgT52aqBxESoiZ8dz1HWgQWYmB76RqqF8ppoIQkRIXXqk8Y/slknngCIMnaqhfaaWCEBFPxNavzuNXt+W7NTt5/ssVXseRAqggRMQzNyQ15MakhoyZtYYZy7Z5HUeOoYIQEU89dlUbYutV5953F7Bp1wGv40g+p10QZtaqOIOISHCqWD6Ucf0TARgwIYVDRzTUr7QoyhHE9GJLISJBLaZmZZ6/IY4lW/bwmIb6lRon/ByEmb10vKuAiGJPIyJB68LYOgzo1pSxs9eQ1CiSaxM11M9rJ/ug3C3AEKCgNyr3Kf44IhLMhlzUggUbMxn28WLaNKhOq7rVvY4U1E62xDQPWOKce/vYP8DeEsgnIkGkXGgIL/WJp3rF8gwYn8oeDfXz1MkK4jpgQUFXOOeaFHsaEQl6UdUq8ErfBDbuOsBf39dQPy+drCCqOuf0vjMRKVEdm9TggR6t+GLpL/xrrob6eeVkBfHx0Qtm9qF/o4iI/M+fOzehR5u6PP3Fcn5at8vrOEHpZAWR//yaZ/gziIhIfmbGs9e3p2FkJQa9k8r2vYe8jhR0TlYQ7jiXRUT8rnrF8oztn8ieQ3lD/bJzcr2OFFROVhAdzGyPme0F2vsu7zGzvWa2pyQCikhwa12vOk9c3Y4f1u5i1JcrvY4TVE74OQjnnE7GLCKeuy4xmpQNuxg7ew0JMZFcFFvH60hBQcP6RCQgPHpFG9o2qM597y1g4069ubIkqCBEJCBULB/K2H6JhJhpqF8JUUGISMBoWKMyL9zYgaVb9zB88lKv45R5KggRCSjdW9Vh4PlNmTRvE+8nb/I6Tpnm14Iwsx5mtsLMVpvZAwVc38jMZpjZIjObbWbR+a77o5mt8v35oz9zikhgue+ilpzbtCYPf7yEtK16Q6W/+K0gzCwUGANcCsQCfcws9pjdRgL/ds61B0YAT/luWwN4FDgL6Ag8amaR/soqIoElNMR4qU88EZXLM2BCCrsPaqifP/jzCKIjsNo5t9Y5lwVMAq46Zp9YYKbv8qx8118CfOmc2+WcywC+BHr4MauIBJhaVSswpm8CWzIOcv/7CzXUzw/8WRANgPwLhJt92/JbCPTyXb4GqGZmNQt5W8zsdjNLNrPk9PT0YgsuIoEhqXENHri0FdPTtvHanLVexylzvH6ReijQ1czmA12BLUCh37vmnHvNOZfknEuKioryV0YRKcVu7dSEnu3q8uy0Ffy4dqfXccoUfxbEFqBhvq+jfdt+5Zzb6pzr5ZyLB4b5tmUW5rYiIpA31O+Za9vTqEZlBk2cz/Y9GupXXPxZEPOA5mbWxMzCgN7A5Pw7mFktMzua4UHgDd/lacDFZhbpe3H6Yt82EZHfqVaxPK/2T2DvoSMM0lC/YuO3gnDOZQODyHtiXwa855xbamYjzOxK327dgBVmthKoAzzpu+0u4HHySmYeMMK3TUSkQK3qVufv17Tjp3W7eG76Cq/jlAlWVl75T0pKcsnJyV7HEBGPDftoMRN+3MhrNyVycZu6Xscp9cwsxTmXVNB1Xr9ILSJSrP52RSzto8MZ8v5CNuzc73WcgKaCEJEypUK5UMb0TSDEjDvGp2qoXxGoIESkzGlYozKjb4xj2c97+NsnS7yOE7BUECJSJp3fqjZ3dW/Ge8mbeW+ehvqdDhWEiJRZ91zYgk7NavHIJ0tYunW313ECjgpCRMqs0BDjxd5xRFYOY8D4VA31O0UqCBEp02pWrcCYfglszTzIkPcWkptbNt7aXxJUECJS5iU2iuShnq35atk2/qGhfoWmghCRoHDLeY25rH09npu2nO/XaKhfYaggRCQoHB3q17hWFe7SUL9CUUGISNCoWqEc4/onsv9wNoPemc8RDfU7IRWEiASVFnWq8VSvdvy0fhfPTdNQvxNRQYhI0Lk6vgE3nd2I1+as5Yslv3gdp9RSQYhIUHr48tZ0aBjB/e8vZN0ODfUriApCRIJS3lC/eEJDjQHjUziYpaF+x1JBiEjQio7MG+q3YtteHvlkCWXl/DjFRQUhIkGtW8va3NW9OR+kbOZdDfX7DRWEiAS9uy9oTufmtfjb5KUs2aKhfkepIEQk6OUN9YunZpUw7hifwu4DGuoHKggREQBqVAljTL8Etu05xH3vLdBQP1QQIiK/SoiJZFjP1sxYvp2xX6/xOo7nVBAiIvn88dzGXNGhPqOmr+C7NTu8juMpFYSISD5mxtO92nFGVFUGT5zPL7uDd6ifCkJE5BhVKpRjXP8EDmTlMOid1KAd6qeCEBEpQLPa1Xj62vYkb8jgmc+Xex3HEyoIEZHjuLJDff54TiP+9c06Pl/8s9dxSpwKQkTkBIZdFktcwwju/2ARa9P3eR2nRKkgREROIKxcCGP6JVA+1LhzQmpQDfVTQYiInESDiEq82DueFdv2MuzjxUEz1E8FISJSCF1aRHH3Bc35b+oWJv4UHEP9VBAiIoU0uHtzurSIYvjkpSzanOl1HL9TQYiIFFJIiDH6xjhqVQ1jwPhUMg9keR3Jr1QQIiKnoEaVMF7tn8j2vYe4992yPdRPBSEicoriGkbwyOWxzFqRzquzV3sdx29UECIip+GmsxtxVVx9nv9yJd+uLptD/VQQIiKnwcx4qlc7mpbhoX4qCBGR01Q5rBxj+ydy6EgOA8vgUD8VhIhIETSrXZVnrmtPyoYMnvqsbA3182tBmFkPM1thZqvN7IECro8xs1lmNt/MFplZT9/2MDN708wWm9lCM+vmz5wiIkVxefv63HxuY974dh1TF5WdoX5+KwgzCwXGAJcCsUAfM4s9ZreHgfecc/FAb+BV3/bbAJxz7YCLgFFmpqMdESm1HurZmoSYCP76wULWlJGhfv580u0IrHbOrXXOZQGTgKuO2ccB1X2Xw4GtvsuxwEwA59x2IBNI8mNWEZEiOTrUr0L5UAaMT+FAVrbXkYrMnwXRAMg/sGSzb1t+w4H+ZrYZ+Ay4y7d9IXClmZUzsyZAItDw2Acws9vNLNnMktPT04s7v4jIKakXXokXe8exavs+hn20JOCH+nm9bNMHeMs5Fw30BP7jW0p6g7xCSQZGA98Bv5ux65x7zTmX5JxLioqKKrnUIiLH0bl5FPde2IKP5m9hwo8bvY5TJOX8eN9b+O1v/dG+bfndCvQAcM59b2YVgVq+ZaV7j+5kZt8BK/2YVUSk2Aw6vxmpGzMY8Wka7RqE06FhhNeRTos/jyDmAc3NrImZhZH3IvTkY/bZCFwAYGatgYpAuplVNrMqvu0XAdnOuTQ/ZhURKTYhIcYLN8QRVa0Cd05IJWN/YA7181tBOOeygUHANGAZee9WWmpmI8zsSt9uQ4DbzGwhMBG42eUt2tUGUs1sGfB/wE3+yiki4g+RVcJ4tV8C6XsPc+97gTnUzwL9RZSjkpKSXHJystcxRER+Y/wPG3j44yXcd1ELBl/Q3Os4v2NmKc65At8l6vWL1CIiZVq/s2K4Jr4BL3y1krmrAuvdlioIERE/MjOevKYtzWtX5e5JC9iaedDrSIWmghAR8bOjQ/2ysnMZ+E4qWdmBMdRPBSEiUgKaRlXl2evaM39jJn//bJnXcQpFBSEiUkJ6tqvHn85rwlvfrefThVtPfgOPqSBERErQgz1bkdgokgc+XMTq7aV7qJ8KQkSkBJUPDWFM3wQq+ob67T9ceof6qSBEREpY3fCKvNQnnjXp+3joo8WldqifCkJExAPnNavFfRe14JMFWxn/wwav4xRIBSEi4pE7uzWje6vajJiSxvyNGV7H+R0VhIiIR0JCjOdv6ECd6hUZOCGVXaVsqJ8KQkTEQxGVwxjbL5Ed+7K4590F5JSioX4qCBERj7WLDmf4lW2YszKdl2eu8jrOr1QQIiKlQJ+ODemV0IAXZ6zi65WlY6ifCkJEpBQwM568uh0t61Tjnknz2VIKhvqpIERESolKYaG82i+BIzmOgRO8H+qnghARKUXOiKrKyOvbs2BTJk9O9fZMyyoIEZFSpkfbevy5UxPe/n4Dkz0c6qeCEBEphf7v0lac2ThvqN+qbXs9yaCCEBEphcqHhvBK3wQqh4UyYEKqJ0P9VBAiIqVUnep5Q/3Wpu/jgf+W/FA/FYSISCl2btNaDLm4JZ8u3Mrb360v0cdWQYiIlHIDujblwta1efKzZaSW4FA/FYSISCkXEmKMuj6OuuF5Q/127jtcMo9bIo8iIiJFEl65PGP7JbJzf8kN9VNBiIgEiLYNwhlxZRvmrtrBizP8P9RPBSEiEkBuPLMh1yVG8/LMVcxesd2vj6WCEBEJIGbG41e1zRvq9+4CNmcc8NtjqSBERAJMpbBQxvVPJMc31O9wdo5fHkcFISISgBrXqsJz13dg4ebdPDFlmV8eQwUhIhKgerSty+1dziDHOXL98K6mcsV+jyIiUmIe6NGKkBDzy33rCEJEJID5qxxABSEiIsehghARkQKpIEREpEAqCBERKZAKQkRECqSCEBGRAqkgRESkQFbS5zj1FzNLBzYU4S5qATuKKY7IsfTzJf5UlJ+vRs65qIKuKDMFUVRmluycS/I6h5RN+vkSf/LXz5eWmEREpEAqCBERKZAK4n9e8zqAlGn6+RJ/8svPl16DEBGRAukIQkRECqSCEBGRAgVtQZjZcDMb6rs8wswuPM376WZm5+b7+i0zu664ckpgM7PZZpbku/yZmUV4HEnKiOJ6DjsRnVEOcM79rQg37wbsA74rnjRSVjnnenqdQcqmIj6HHVdAHUGYWWMzW+77LX2lmU0wswvN7FszW2VmHfO3qu82S8ysse/yMN/tvgFa5tvn19/6zWy9mT1rZovN7Ccza+bbfoWZ/Whm883sKzOr47vfO4B7zWyBmXX23WUXM/vOzNbqaCI45PvZnGBmy8zsAzOrfMw+682sVmH2FTmWmf3BzBaZ2UIz+88x1530Oex0BFRB+DQDRgGtfH/6Ap2AocBDx7uRmSUCvYE4oCdw5gkeY7dzrh3wCjDat+0b4GznXDwwCfirc249MA54wTkX55yb69u3ni/T5cDTp/wdSqBqCbzqnGsN7AHuLKZ9JciZWRvgYaC7c64DcPdJblLQc9gpC8SCWOecW+ycywWWAjNc3nt1FwONT3C7zsBHzrkDzrk9wOQT7Dsx39/n+C5HA9PMbDFwP9DmBLf/2DmX65xLA+qc9DuSsmKTc+5b3+Xx5P2SUBz7inQH3nfO7QBwzu06yf4FPYedskAsiMP5Lufm+zqXvNdUsvnt91XxNB7DFXD5ZeAVXyv/5ST3mz+j/84oLqXNsR8qOtGHjE5lX5FTVdBz2CkLxII4mfVAAoCZJQBNfNvnAFebWSUzqwZccYL7uDHf39/7LocDW3yX/5hv371AtaLHljIgxsyO/rbWl7xlyeLYV2QmcL2Z1QQwsxon2b+g57BTVhYL4kOghpktBQYBKwGcc6nAu8BC4HNg3gnuI9LMFpG3znevb9tw4H0zS+G3Y3U/Ba455kVqCU4rgIFmtgyIBMYW074S5JxzS4Enga/NbCHw/EluUtBz2CnTqI1jmNl6IOnoWp9IYfje0TbFOde2OPcVOVXF+RxWFo8gRESkGOgIQkRECqQjCBERKZAKQkRECqSCEBGRAqkgRESkQCoIkdNwdPBeUfcRKc1UECIiUiAVhASNQo6Lr2FmH/vGKv9gZu19t61pZtPNbKmZ/Yt8M7bMrL9vrPICM/uHmYUWMssyM/un7z6nm1kl33W3mdk831jnD4+OAvflHuvLtdbyTlb1hu9+3sp33xeb2fdmlmpm75tZ1eL+t5TgoIKQYHOycfGPAfOdc+19X//bd7tHgW+cc22Aj4AYADNrTd68m/Occ3FADtCvkFmaA2N895kJXOvb/l/n3Jm+sc7LgFvz3SaSvOmc95I3kfgF8iYLtzOzON+S1sPAhc65BCAZuK+QeUR+Q2eUk2Czzjm3GMA3r2uGc875xrg3Bhrhe6J2zs30HTlUB7oAvXzbp5pZhu/+LgASgXlmBlAJ2H4KWRb4Lqfwv3H1bc3sCSACqApMy3ebT/Pl3XbM99KYvLH0scC3vjxhFGFYmwQ3FYQEm5ONiz9yivdnwNvOuQeLmCWHvHIBeAu42jm30MxuJu+0tsfeJn/2o1+X893Pl865PqeRR+Q3tMQk8ltz8S0RmVk3YIfvBFNzyFuOwswuJW+pB2AGcJ2Z1fZdV8PMGhUxQzXgZzMrT+GXq476ATjP/neq3Cpm1qKIeSRI6QhC5LeGA2/4RiUf4H/n/ngMmOhbyvkO2AjgnEszs4eB6WYWQt4RyEBgQxEyPAL8CKT7/i70+Uacc+m+o46JZlbBt/lhfGPvRU6FhvWJiEiBtMQkIiIF0hKTiB/5ThE5o4CrLnDO7SzpPCKnQktMIiJSIC0xiYhIgVQQIiJSIBWEiIgUSAUhIiIF+n/ONt0tRIICBgAAAABJRU5ErkJggg==",





  
  

  249
  
      "text/plain": [





  
  

  250
  
       "<Figure size 432x288 with 1 Axes>"





  
  

  251
  
      ]





  
  

  252
  
     },





  
  

  253
  
     "metadata": {





  
  

  254
  
      "needs_background": "light"





  
  

  255
  
     },





  
  

  256
  
     "output_type": "display_data"





  
  

  257
  
    }





  
  

  258
  
   ],





  
  

  259
  
   "source": [





  
  

  260
  
    "# Plot the metrics you care about\n",





  
  

  261
  
    "import seaborn as sns\n",





  
  

  262
  
    "sns.lineplot(x=\"model_name\", y=\"F1\", data=result_df, estimator=\"max\", ci=None)"





  
  

  263
  
   ]





  
  

  264
  
  },





  
  

  265
  
  {





  
  

  266
  
   "cell_type": "code",





  
  

  267
  
   "execution_count": 46,





  
  

  268
  
   "id": "914757a5",





  
  

  269
  
   "metadata": {},





  
  

  270
  
   "outputs": [],





  
  

  271
  
   "source": []





  
  

  272
  
  },





  
  

  273
  
  {





  
  

  274
  
   "cell_type": "code",





  
  

  275
  
   "execution_count": null,





  
  

  276
  
   "id": "df649cab",





  
  

  277
  
   "metadata": {},





  
  

  278
  
   "outputs": [],





  
  

  279
  
   "source": []





  
  

  280
  
  }





  
  

  281
  
 ],





  
  

  282
  
 "metadata": {





  
  

  283
  
  "kernelspec": {





  
  

  284
  
   "display_name": "Python 3 (ipykernel)",





  
  

  285
  
   "language": "python",





  
  

  286
  
   "name": "python3"





  
  

  287
  
  },





  
  

  288
  
  "language_info": {





  
  

  289
  
   "codemirror_mode": {





  
  

  290
  
    "name": "ipython",





  
  

  291
  
    "version": 3





  
  

  292
  
   },





  
  

  293
  
   "file_extension": ".py",





  
  

  294
  
   "mimetype": "text/x-python",





  
  

  295
  
   "name": "python",





  
  

  296
  
   "nbconvert_exporter": "python",





  
  

  297
  
   "pygments_lexer": "ipython3",





  
  

  298
  
   "version": "3.9.0"





  
  

  299
  
  }





  
  

  300
  
 },





  
  

  301
  
 "nbformat": 4,





  
  

  302
  
 "nbformat_minor": 5





  
  

  303
  
}