--- a
+++ b/reproducibility/Reproduce Linear Probing.ipynb
@@ -0,0 +1,303 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "4d8f75cf",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "import torch\n",
+    "import clip\n",
+    "import pandas as pd\n",
+    "import numpy as np\n",
+    "from tqdm import tqdm\n",
+    "from PIL import Image\n",
+    "\n",
+    "from sklearn.linear_model import SGDClassifier\n",
+    "from torch.utils.data import Dataset, DataLoader\n",
+    "from metrics import eval_metrics\n",
+    "from sklearn.preprocessing import LabelEncoder\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "id": "e56d9ac9",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "dataset = \"PanNuke\"\n",
+    "data_folder = \"/path/to/data\"\n",
+    "cache_dir = \".cache\"\n",
+    "model_name = \"plip\"\n",
+    "plip_path = \"/path/to/plip\"\n",
+    "device=\"cuda\"\n",
+    "\n",
+    "class CLIPImageDataset(Dataset):\n",
+    "    def __init__(self, list_of_images, preprocessing):\n",
+    "        self.images = list_of_images\n",
+    "        self.preprocessing = preprocessing\n",
+    "\n",
+    "    def __len__(self):\n",
+    "        return len(self.images)\n",
+    "\n",
+    "    def __getitem__(self, idx):\n",
+    "        images = self.preprocessing(Image.open(self.images[idx]))  # preprocess from clip.load\n",
+    "        return images\n",
+    "    \n",
+    "@torch.no_grad()\n",
+    "def get_embs(loader, model, model_name):\n",
+    "    all_embs = []\n",
+    "    for images in tqdm(loader):\n",
+    "        images = images.to(device)\n",
+    "        if model_name in [\"clip\", \"plip\"]:\n",
+    "            all_embs.append(model.encode_image(images).cpu().numpy())\n",
+    "        else:\n",
+    "            all_embs.append(model(images).squeeze().cpu().numpy())\n",
+    "    all_embs = np.concatenate(all_embs, axis=0)\n",
+    "    return all_embs\n",
+    "\n",
+    "\n",
+    "def run_classification(train_x, train_y, test_x, test_y, seed=1, alpha=0.1):\n",
+    "    classifier = SGDClassifier(random_state=seed, loss=\"log_loss\",\n",
+    "                               alpha=alpha, verbose=0,\n",
+    "                               penalty=\"l2\", max_iter=10000, class_weight=\"balanced\")\n",
+    "    \n",
+    "    le = LabelEncoder()\n",
+    "\n",
+    "    train_y = le.fit_transform(train_y)\n",
+    "    test_y = le.transform(test_y)\n",
+    "\n",
+    "    train_y = np.array(train_y)\n",
+    "    test_y = np.array(test_y)\n",
+    "\n",
+    "    classifier.fit(train_x, train_y)\n",
+    "    test_pred = classifier.predict(test_x)\n",
+    "    train_pred = classifier.predict(train_x)\n",
+    "    test_metrics = eval_metrics(test_y, test_pred, average_method=\"macro\")\n",
+    "    return test_metrics"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "2ed88421",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "id": "7bac69d2",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "train_dataset_name = dataset + \"_train.csv\"\n",
+    "test_dataset_name = dataset + \"_test.csv\"\n",
+    "\n",
+    "train_dataset = pd.read_csv(os.path.join(data_folder, train_dataset_name))\n",
+    "test_dataset = pd.read_csv(os.path.join(data_folder, test_dataset_name))\n",
+    "\n",
+    "test_y = test_dataset[\"label\"].tolist()\n",
+    "train_y = train_dataset[\"label\"].tolist()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "id": "5f2d857b",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "<All keys matched successfully>"
+      ]
+     },
+     "execution_count": 14,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "model, preprocess = clip.load(\"ViT-B/32\", device=device, download_root=cache_dir)\n",
+    "model.load_state_dict(torch.load(plip_path))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "id": "efb1786b",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "model = model.eval()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "id": "aabf4aca",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def run_study(model_name, cache_dir=cache_dir):\n",
+    "    if model_name == \"plip\":\n",
+    "        model, preprocess = clip.load(\"ViT-B/32\", device=device, download_root=cache_dir)\n",
+    "        model.load_state_dict(torch.load(plip_path))\n",
+    "    elif model_name == \"clip\":\n",
+    "        model, preprocess = clip.load(\"ViT-B/32\", device=device, download_root=cache_dir)\n",
+    "    elif model_name == \"mudipath\": \n",
+    "        from torchvision import transforms\n",
+    "        from embedders.mudipath import build_densenet\n",
+    "        \n",
+    "        model = build_densenet(download_dir=cache_dir,\n",
+    "                                      pretrained=\"mtdp\")\n",
+    "        model.num_feats = model.n_features()\n",
+    "        model.forward_type = \"image\"\n",
+    "        model = model.to(device)\n",
+    "        model.eval()\n",
+    "        preprocess = transforms.Compose([\n",
+    "            transforms.Resize(224),\n",
+    "            transforms.CenterCrop(224),\n",
+    "            transforms.ToTensor(),\n",
+    "            transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])  # ImageNet stats\n",
+    "        ])\n",
+    "    \n",
+    "    train_loader = DataLoader(CLIPImageDataset(train_dataset[\"image\"].tolist(), preprocess), batch_size=32)\n",
+    "    test_loader = DataLoader(CLIPImageDataset(test_dataset[\"image\"].tolist(), preprocess), batch_size=32)\n",
+    "\n",
+    "    train_embs = get_embs(train_loader, model, model_name)\n",
+    "    test_embs = get_embs(test_loader, model, model_name)\n",
+    "    \n",
+    "\n",
+    "    all_records = []\n",
+    "    for alpha in [1.0, 0.1, 0.01, 0.001]:\n",
+    "        metrics = run_classification(train_embs, train_y, test_embs, test_y, alpha=alpha)\n",
+    "        metrics[\"alpha\"] = alpha\n",
+    "        metrics[\"model_name\"] = model_name\n",
+    "        all_records.append(metrics)\n",
+    "    return all_records"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 17,
+   "id": "e331473e",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "100%|██████████| 136/136 [00:47<00:00,  2.86it/s]\n",
+      "100%|██████████| 59/59 [00:20<00:00,  2.85it/s]\n",
+      "100%|██████████| 136/136 [00:38<00:00,  3.49it/s]\n",
+      "100%|██████████| 59/59 [00:17<00:00,  3.37it/s]\n",
+      "100%|██████████| 136/136 [00:39<00:00,  3.48it/s]\n",
+      "100%|██████████| 59/59 [00:17<00:00,  3.46it/s]\n"
+     ]
+    }
+   ],
+   "source": [
+    "all_records = []\n",
+    "for model_name in [\"mudipath\", \"plip\", \"clip\"]:\n",
+    "    all_records.extend(run_study(model_name))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "fccfc443",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 18,
+   "id": "80436432",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "result_df = pd.DataFrame(all_records)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 19,
+   "id": "63d4ca50",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "<AxesSubplot:xlabel='model_name', ylabel='F1'>"
+      ]
+     },
+     "execution_count": 19,
+     "metadata": {},
+     "output_type": "execute_result"
+    },
+    {
+     "data": {
+      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEHCAYAAAC0pdErAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAocklEQVR4nO3dd3hUddrG8e+TQOgkAUINAaSHliY2mtgQOzaau7qurghiAfdV0RVR1waKBWHdte2CYFsVQQWlCHaS0EPvRSFAQoeQ5Pf+kcGNGCCQTE4mc3+ui4vJmTMzdzDOnfObmeeYcw4REZFjhXgdQERESicVhIiIFEgFISIiBVJBiIhIgVQQIiJSoHJeBygutWrVco0bN/Y6hohIQElJSdnhnIsq6LoyUxCNGzcmOTnZ6xgiIgHFzDYc7zotMYmISIFUECIiUiAVhIiIFEgFISIiBVJBiIhIgVQQIiJSIBWEiIgUSAUh4kfOOeauSmdzxgGvo4icsjLzQTmR0mbFL3sZPnkp36/dSUyNynx6VyfCK5X3OpZIoekIQqSY7T5whOGTl9Lzpbmk/byHO7s1ZWvmQYa8t5DcXJ2gSwKHjiBEiklOruPdeZt4btpydh88Qt+zYhhyUUsiq4RRq2oFRkxJ4x9z1jKgW1Ovo4oUigpCpBikbNjFo5OXsmTLHjo2rsGjV8bSpn74r9ffcl5jUjZm8Ny05cQ1jOCcpjU9TCtSOCoIkSLYtucQT3++nI/mb6Fu9Yq82DuOKzvUx8x+s5+Z8cy17Vn28x7umjifzwZ3onb1ih6lFikcvQYhchoOZ+cw7us1dB85m6mLfmbg+U2ZMaQrV8U1+F05HFW1QjnG9U9k/+FsBr0znyM5uSWcWuTUqCBETtGs5dvpMXouT3++nHOa1uTL+7pw/yWtqFLh5AfkLepU46le7fhp/S6em7aiBNKKnD4tMYkU0vod+xkxJY2Zy7dzRq0qvHnLmZzfsvYp38/V8Q1I3rCL1+asJSEmkh5t6/ohrUjRqSBETmL/4WxembWa1+euo3yo8VDPVtx8bhPCyp3+Afgjl8eyePNu7n9/IS3rVqNJrSrFmFikeGiJSeQ4nHN8smAL3UfNZuzsNVzeoR6zhnbj9i5Ni1QOABXKhTKmXwKhocaA8SkczMopptQixUcFIVKAJVt2c/2477l70gJqV6vIhwPO5fkb4or1nUfRkZV54cY4VmzbyyOfLME5fYhOShctMYnks2t/FiOnr2DiTxupUTmMZ65tx/WJDQkJKfidSUV1fsva3HV+M16auZqkRpH07hjjl8cROR0qCBEgOyeXCT9uZNT0FezPyuHmcxtzz4UtSmR20t0XtmD+pkz+NnkpbRuE07ZB+MlvJFICtMQkQe/7NTu5/OVveNT3BP353Z159Io2JTZYLzTEGH1jHDWrhDFgQgq7DxwpkccVORkVhAStLZkHGfhOKn3++QN7D2Uzrn8CE/58Fi3qVCvxLDWrVmBMvwR+2X2IIe8v0FA/KRW0xCRB59CRHF6bs5ZXZ6/GObjnwub8pUtTKoWFeporISaSYT1bM/zTNMZ+vYaB5zfzNI+ICkKChnOO6WnbeGJqGpt2HeTStnUZdllroiMrex3tV388tzEpGzMZNX0F8TERnNu0lteRJIhpiUmCwurte/nDGz/xl/+kUKl8KBP+fBZj+yeWqnKAvKF+T/dqR5NaVRg8cT6/7D7kdSQJYn4tCDPrYWYrzGy1mT1QwPWNzGyGmS0ys9lmFu3bHmdm35vZUt91N/ozp5Rdew4d4YkpafQYPZcFmzJ59IpYpg7uzHnNSu9v5lV8Q/0OZOUw6J1UDfUTz/itIMwsFBgDXArEAn3MLPaY3UYC/3bOtQdGAE/5th8A/uCcawP0AEabWYS/skrZk5vreC95E91Hzub1b9dxXWI0s4d245bzmlA+tPQfODf3DfVL3pDBM58v9zqOBCl/vgbREVjtnFsLYGaTgKuAtHz7xAL3+S7PAj4GcM6tPLqDc26rmW0HooBMP+aVMmLBpkwenbyUhZsyiY+J4I2bz6R9dITXsU7ZVXENSNmQwb++WUdio0gubVfP60gSZPxZEA2ATfm+3gycdcw+C4FewIvANUA1M6vpnNt5dAcz6wiEAWv8mFXKgPS9h3n2i+W8n7KZqGoVeP6GDlwd18Bvn4IuCcMua83Czbu5/4NFtKxbjTOiqnodSYKI18faQ4GuZjYf6ApsAX6dWmZm9YD/ALc45363EGtmt5tZspklp6enl1RmKWWO5OTyr7lr6T5yNh8v2MJfupzBrKHd6JUQHdDlAHlD/V7tl0D5UOPOCaka6iclyp8FsQVomO/raN+2Xznntjrnejnn4oFhvm2ZAGZWHZgKDHPO/VDQAzjnXnPOJTnnkqKiovzwLUhpN3dVOpe+OJcnpi4joVEkX9zThQd7tqZqIU7eEygaRFRidO94Vmzby7CPF2uon5QYf/5fNA9obmZNyCuG3kDf/DuYWS1gl+/o4EHgDd/2MOAj8l7A/sCPGSVAbdx5gCempjE9bRsxNSrzrz8kcUHr2sc93Weg69oiisHdm/PijFUkNapB37M01E/8z28F4ZzLNrNBwDQgFHjDObfUzEYAyc65yUA34Ckzc8AcYKDv5jcAXYCaZnazb9vNzrkF/sorgeFgVg5jZ69m3Jy1hJpx/yUtubVTEyqW9/ZT0CVh8AXNSd2YwfDJS2nXIJx20RrqJ/5lZeVwNSkpySUnJ3sdQ/zEOcfUxT/z96nL2Lr7EFfF1eeBS1tRL7yS19FK1K79WVz+0lxCQowpd3UionKY15EkwJlZinMuqaDrvH6RWuSklv+yhz7//IFB78wnvHIY7/3lHF7sHR905QBQo0oYr/ZPZNueQ9z7rob6iX+pIKTUyjyQxaOfLKHni3NZ/stenri6LVPu6kTHJjW8juapuIYRPHJ5LLNWpPPq7NVex5EyrOy81UPKjJxcx6R5Gxk5bQW7Dx6h31mNGHJxCy2n5HPT2Y1IXp/B81+uJD4mslSPDpHApYKQUiV5/S4enbyUpVv30LFJDYZf0YbY+tW9jlXqmBlP9WpH2s97GDxxPlMHd6ZuePGdL1sEtMQkpcQvuw9xz6T5XDfue3btz+LlPvG8e/vZKocTyBvql8DBIzkM1FA/8QMVhHjqcHYOY2evofuo2Xy2+BcGnd+MGUO6ckWH+mX2Mw3FqVntajxzbXtSNmTw1Gca6ifFS0tM4pmZy7cx4tM01u88wEWxdXj4stY0qlnF61gB54oO9UnZkMEb3+YN9busvYb6SfFQQUiJW5u+j8enpDFrRTpnRFXh7T91pGsLjUopiod6tmbh5kz++sFCWtWrRlMN9ZNioCUmKTH7Dmfz9OfLuWT0HOatz2BYz9Z8cXcXlUMxCCsXwpi+CVQoH8qA8SkcyMr2OpKUASoI8TvnHB/N30z3kbMZ9/UaroprwMyhXbmtyxmEldOPYHGpH1GJF3vHsWr7PoZ9tERD/aTItMQkfrVky24enbyUlA0ZtI8OZ9xNiSTERHodq8zq3DyKey9swfNfriSxUST9z27kdSQJYCoI8Yud+w4zcvpKJs3bSI3KYTx7bXuuSwz88zMEgkHnNyN1YwYjPk2jfXR4QJ5NT0oHHd9LscrOyeWtb9dx/sjZvJe8iT+d14SZQ7txw5kNVQ4lJCTEeOGGOKKqVWDA+FQy9md5HUkClApCis13a3Zw2UvfMPzTNNpFh/PF3Z155PJYwiuV9zpa0ImsEsar/RJI33uYe9/TUD85PSoIKbItmQcZOCGVvv/8kf1Z2Yzrn8j4W8+ieZ1qXkcLah0aRvDIFbHMXpHOK7M01E9OnV6DkNN26EgO//h6LWO/znvyue+iFtze5YygOHlPoOh/Vgwp63fxwlcriY+JoHNzvaVYCk8FIafMOce0pdt4YmoamzMOclm7ejzYsxXRkZW9jibHMDP+7hvqd/ekBUy5qxP1I4LvPBpyerTEJKdk9fa93PT6T9wxPoUqYeV457azGNMvQeVQilUOK8fY/okc9g31y8rWUD8pHBWEFMqeQ0d4fEoaPUbPZdHmTIZfEcvUwZ04t6nOQxAImkZV5dnrOjB/YyZ//2yZ13EkQGiJSU4oN9fxQcpmnp22nJ37s+h9ZgxDL25BzaoVvI4mp+iy9vVI3tCYN79dT2KjSK7oUN/rSFLKqSDkuOZvzGD45KUs3LybhJgI3ry5I+2iw72OJUXw4KWtWbgpkwc+XETretVpVltD/eT4tMQkv7N97yGGvr+Qa179jp93H+KFGzvw4YBzVQ5lQFi5EMb0+99Qv/2HNdRPjk8FIb/Kys7ln3PW0n3k13yyYAt3dG3KzKHduCY+WifvKUPqhVfi5T7xrEnfx0MfLdZQPzkuLTEJAHNWpvPYp0tZk76fbi2j+NvlsZyhcwqUWec1q8V9F7Vg5PSVJDWK5KZzGnsdSUohFUSQ27jzAI9PTePLtG00rlmZN25OonurOl7HkhJwZ7dmpG7MZMSUNNpFRxDXMMLrSFLKaIkpSB3IymbktBVc+MLXfLt6B//XoxXT7u2icggiISHG8zd0oE71itw5PoVdGuonx1BBBBnnHJ8u3MoFo77mlVmr6dm2LjOHdGNAt6ZUKKcRGcEmonLeUL8d+7K4590F5Gion+SjgggiaVv3cONrP3DXxPlEVg7j/TvOYXTveOqGV/Q6mniofXQEj14Zy5yV6bw8c5XXcaQU0WsQQSDzQBajpq9kwo8bCK9UnievaUvvM2MI1fkZxKdvxxhS1mfw4oxVxMdE6jzhAqggyrScXMfEnzYycvoK9hw8wk1nN+Lei1oQUTnM62hSypgZT17TjqVb93DPpPlMGdyZBhrqF/S0xFRGzVu/iyte/oaHP15CyzrVmDq4M49d1VblIMdVKSyUsf0TOJLjGDhBQ/1EBVHm/LL7EHdPms/1474n40AWr/SNZ9LtZ9O6XnWvo0kAOCOqKs9e154FmzJ5cmqa13HEY1piKiMOZ+fwr7nrGDNrNdm5jru6N2NAt6ZUDtN/Yjk1PdvV49ZOTXj9m3UkNq7BlRrqF7T07BHgnHPMXL6dEVPS2LDzABfH1uHhy2KJqanzM8jpe+DSVv8b6le3mk4fG6S0xBTA1qbv45a35nHr28mUCzH+/aeOvPaHJJWDFFn50BBe6ZtA5bBQBkxI1VC/IKWCCED7Dmfz1OfLuGT0HJLXZ/DwZa354p4udNFbE6UY1Q2vyEt94lmbvo8H/quhfsFIS0wBJDfX8fGCLTz1+XLS9x7m+sRo7u/RktrV9EE38Y9zm9ZiyMUteW7aCpIaRfLHcxt7HUlKkAoiQCzevJtHJy8hdWMmHRpG8NpNicTHRHodS4LAgK5NSd2QwRNT02gXHU6Cfu6Chl+XmMysh5mtMLPVZvZAAdc3MrMZZrbIzGabWXS+674ws0wzm+LPjKXdzn2HefC/i7hyzDds3HWAZ69rz0cDzlU5SInJG+oXR93wigyckMrOfYe9jiQlxG8FYWahwBjgUiAW6GNmscfsNhL4t3OuPTACeCrfdc8BN/krX2mXnZPLm9+uo9vI2byfvJlbz2vCzKHduCGpISEakSElLLxyecb2S2Tnfg31Cyb+PILoCKx2zq11zmUBk4CrjtknFpjpuzwr//XOuRnAXj/mK7W+W72Dni/N5bFP04hrGMEX93Tm4ctjqV6xvNfRJIi1bRDOY1e2Ye6qHbw4Q0P9goE/X4NoAGzK9/Vm4Kxj9lkI9AJeBK4BqplZTefczsI8gJndDtwOEBMTU+TAXtuccYC/f7aMzxb/QnRkJf5xUyIXx9bR6T6l1Oh9ZkOS12fw8sxVJMRE0K1lba8jiR95/TbXoUBXM5sPdAW2ADmFvbFz7jXnXJJzLikqKnDf4nnoSA6jv1rJBaO+Zuby7Qy5qAVf3deVS9rUVTlIqWJmPHF1W1rWqcY97y5gc8YBryOJH/mzILYADfN9He3b9ivn3FbnXC/nXDwwzLct04+ZShXnHF8s+ZkLRn3N6K9WcWFsHWYM6cZdFzSnYnmdvEdKp7yhfonk+Ib6Hc4u9O90EmD8WRDzgOZm1sTMwoDewOT8O5hZLTM7muFB4A0/5ilVVm7bS//Xf+SO8alUq1iOibedzZi+CRqxLAGhSa0qPHd9exZu3s0TU5Z5HUf8xG+vQTjnss1sEDANCAXecM4tNbMRQLJzbjLQDXjKzBwwBxh49PZmNhdoBVQ1s83Arc65af7KW1J2HzzCi1+t4u3v11MlLJTHrmxDv7NiKBfq9WqfyKnp0bYet3Vuwj/nriOpcSRXxTXwOpIUMysrH59PSkpyycnJXsc4rtxcx/spm3j2ixXsOpBFn44xDL24JTWq6PwMEriO5OTS758/snjLbj4ZdB4tNNQv4JhZinMuqaDr9GtrCUjdmMHVr37L/324mCa1qvDpoE78/Zp2KgcJeHlD/eKpUqEcd4xPYZ+G+pUpKgg/2r73EEPeW0ivV7/jl92HGH1jHO/fcQ5tG4R7HU2k2NSuXpGX+8Szfsd+/u/DRRrqV4ZoFpMfZGXn8tZ363hpxmoOZ+cwoFtTBp7fjKoV9M8tZdM5TWty/yWteOaL5SQ1iuSW85p4HUmKgZ6xitnsFdsZ8Wkaa3fs54JWtXn48lia1KridSwRv7uj6xmkbMjgyanLaB8dTmKjGl5HkiLSElMx2bBzP39+O5mb35yHA968+Uxev/lMlYMEDTNj1A0dqB9RiYET5rNDQ/0CngqiiA5kZfPctOVc9Pwcvl+zgwcubcUX93Tm/FYaQSDBJ7xSeV7tl8CuA1ncPWm+hvoFOBXEaXLOMXnhVrqP/Joxs9ZwWft6zBzajTu6NqVCOX0KWoJX2wbhPH5VG75dvZPRX630Oo4UgV6DOA1pW/cwfPJSflq/i7YNqvNK33iSGmu9VeSoG8+M8Q31W01CTKSOqAOUCuIUZOzPYtSXK3jnx41EVA7jqV7tuCGpIaE6P4PI7zx+dVuWbN3DPe8uYMpdnWhYo7LXkeQUaYmpEHJyHf/5YQPnj5rNxJ828YdzGjNrSDf6dIxROYgcR8XyoYztl0BurmPgOxrqF4hUECfx49qdXP7yNzzy8RJa1a3G1MGdGH5lG8Ir6+Q9IifTuFYVRt3QgUWbdzPi0zSv48gp0hLTcfy8+yBPfbacyQu3Uj+8ImP6JtCznc7PIHKqLm5Tl790PYN/fL2WpMaRXBMfffIbSamggjjGoSM5vP7NOl6ZuZoc5xh8QXMGdG1KpTC9M0nkdN1/cUsWbMzkwf8uJrZeOC3raqhfINASk49zji/TtnHxC3N4btoKuraIYsZ9XbnvohYqB5EiKhcawst946lWsTwDxqew99ARryNJIagggDXp+7j5zXnc9u9kwsqF8J9bOzLupkS960KkGNWuVpFX+sSzYdcBDfULEEFfEGvS93HJC3NI3ZDBI5fH8vndnencPHDPby1Smp11Rk3+eklLPlv8C69/s87rOHISQf8aRNOoqjzUszVXdKhPVLUKXscRKfNu75I31O/pz5cT1zBCHzItxYL+CALgT52aqBxESoiZ8dz1HWgQWYmB76RqqF8ppoIQkRIXXqk8Y/slknngCIMnaqhfaaWCEBFPxNavzuNXt+W7NTt5/ssVXseRAqggRMQzNyQ15MakhoyZtYYZy7Z5HUeOoYIQEU89dlUbYutV5953F7Bp1wGv40g+p10QZtaqOIOISHCqWD6Ucf0TARgwIYVDRzTUr7QoyhHE9GJLISJBLaZmZZ6/IY4lW/bwmIb6lRon/ByEmb10vKuAiGJPIyJB68LYOgzo1pSxs9eQ1CiSaxM11M9rJ/ug3C3AEKCgNyr3Kf44IhLMhlzUggUbMxn28WLaNKhOq7rVvY4U1E62xDQPWOKce/vYP8DeEsgnIkGkXGgIL/WJp3rF8gwYn8oeDfXz1MkK4jpgQUFXOOeaFHsaEQl6UdUq8ErfBDbuOsBf39dQPy+drCCqOuf0vjMRKVEdm9TggR6t+GLpL/xrrob6eeVkBfHx0Qtm9qF/o4iI/M+fOzehR5u6PP3Fcn5at8vrOEHpZAWR//yaZ/gziIhIfmbGs9e3p2FkJQa9k8r2vYe8jhR0TlYQ7jiXRUT8rnrF8oztn8ieQ3lD/bJzcr2OFFROVhAdzGyPme0F2vsu7zGzvWa2pyQCikhwa12vOk9c3Y4f1u5i1JcrvY4TVE74OQjnnE7GLCKeuy4xmpQNuxg7ew0JMZFcFFvH60hBQcP6RCQgPHpFG9o2qM597y1g4069ubIkqCBEJCBULB/K2H6JhJhpqF8JUUGISMBoWKMyL9zYgaVb9zB88lKv45R5KggRCSjdW9Vh4PlNmTRvE+8nb/I6Tpnm14Iwsx5mtsLMVpvZAwVc38jMZpjZIjObbWbR+a77o5mt8v35oz9zikhgue+ilpzbtCYPf7yEtK16Q6W/+K0gzCwUGANcCsQCfcws9pjdRgL/ds61B0YAT/luWwN4FDgL6Ag8amaR/soqIoElNMR4qU88EZXLM2BCCrsPaqifP/jzCKIjsNo5t9Y5lwVMAq46Zp9YYKbv8qx8118CfOmc2+WcywC+BHr4MauIBJhaVSswpm8CWzIOcv/7CzXUzw/8WRANgPwLhJt92/JbCPTyXb4GqGZmNQt5W8zsdjNLNrPk9PT0YgsuIoEhqXENHri0FdPTtvHanLVexylzvH6ReijQ1czmA12BLUCh37vmnHvNOZfknEuKioryV0YRKcVu7dSEnu3q8uy0Ffy4dqfXccoUfxbEFqBhvq+jfdt+5Zzb6pzr5ZyLB4b5tmUW5rYiIpA31O+Za9vTqEZlBk2cz/Y9GupXXPxZEPOA5mbWxMzCgN7A5Pw7mFktMzua4UHgDd/lacDFZhbpe3H6Yt82EZHfqVaxPK/2T2DvoSMM0lC/YuO3gnDOZQODyHtiXwa855xbamYjzOxK327dgBVmthKoAzzpu+0u4HHySmYeMMK3TUSkQK3qVufv17Tjp3W7eG76Cq/jlAlWVl75T0pKcsnJyV7HEBGPDftoMRN+3MhrNyVycZu6Xscp9cwsxTmXVNB1Xr9ILSJSrP52RSzto8MZ8v5CNuzc73WcgKaCEJEypUK5UMb0TSDEjDvGp2qoXxGoIESkzGlYozKjb4xj2c97+NsnS7yOE7BUECJSJp3fqjZ3dW/Ge8mbeW+ehvqdDhWEiJRZ91zYgk7NavHIJ0tYunW313ECjgpCRMqs0BDjxd5xRFYOY8D4VA31O0UqCBEp02pWrcCYfglszTzIkPcWkptbNt7aXxJUECJS5iU2iuShnq35atk2/qGhfoWmghCRoHDLeY25rH09npu2nO/XaKhfYaggRCQoHB3q17hWFe7SUL9CUUGISNCoWqEc4/onsv9wNoPemc8RDfU7IRWEiASVFnWq8VSvdvy0fhfPTdNQvxNRQYhI0Lk6vgE3nd2I1+as5Yslv3gdp9RSQYhIUHr48tZ0aBjB/e8vZN0ODfUriApCRIJS3lC/eEJDjQHjUziYpaF+x1JBiEjQio7MG+q3YtteHvlkCWXl/DjFRQUhIkGtW8va3NW9OR+kbOZdDfX7DRWEiAS9uy9oTufmtfjb5KUs2aKhfkepIEQk6OUN9YunZpUw7hifwu4DGuoHKggREQBqVAljTL8Etu05xH3vLdBQP1QQIiK/SoiJZFjP1sxYvp2xX6/xOo7nVBAiIvn88dzGXNGhPqOmr+C7NTu8juMpFYSISD5mxtO92nFGVFUGT5zPL7uDd6ifCkJE5BhVKpRjXP8EDmTlMOid1KAd6qeCEBEpQLPa1Xj62vYkb8jgmc+Xex3HEyoIEZHjuLJDff54TiP+9c06Pl/8s9dxSpwKQkTkBIZdFktcwwju/2ARa9P3eR2nRKkgREROIKxcCGP6JVA+1LhzQmpQDfVTQYiInESDiEq82DueFdv2MuzjxUEz1E8FISJSCF1aRHH3Bc35b+oWJv4UHEP9VBAiIoU0uHtzurSIYvjkpSzanOl1HL9TQYiIFFJIiDH6xjhqVQ1jwPhUMg9keR3Jr1QQIiKnoEaVMF7tn8j2vYe4992yPdRPBSEicoriGkbwyOWxzFqRzquzV3sdx29UECIip+GmsxtxVVx9nv9yJd+uLptD/VQQIiKnwcx4qlc7mpbhoX4qCBGR01Q5rBxj+ydy6EgOA8vgUD8VhIhIETSrXZVnrmtPyoYMnvqsbA3182tBmFkPM1thZqvN7IECro8xs1lmNt/MFplZT9/2MDN708wWm9lCM+vmz5wiIkVxefv63HxuY974dh1TF5WdoX5+KwgzCwXGAJcCsUAfM4s9ZreHgfecc/FAb+BV3/bbAJxz7YCLgFFmpqMdESm1HurZmoSYCP76wULWlJGhfv580u0IrHbOrXXOZQGTgKuO2ccB1X2Xw4GtvsuxwEwA59x2IBNI8mNWEZEiOTrUr0L5UAaMT+FAVrbXkYrMnwXRAMg/sGSzb1t+w4H+ZrYZ+Ay4y7d9IXClmZUzsyZAItDw2Acws9vNLNnMktPT04s7v4jIKakXXokXe8exavs+hn20JOCH+nm9bNMHeMs5Fw30BP7jW0p6g7xCSQZGA98Bv5ux65x7zTmX5JxLioqKKrnUIiLH0bl5FPde2IKP5m9hwo8bvY5TJOX8eN9b+O1v/dG+bfndCvQAcM59b2YVgVq+ZaV7j+5kZt8BK/2YVUSk2Aw6vxmpGzMY8Wka7RqE06FhhNeRTos/jyDmAc3NrImZhZH3IvTkY/bZCFwAYGatgYpAuplVNrMqvu0XAdnOuTQ/ZhURKTYhIcYLN8QRVa0Cd05IJWN/YA7181tBOOeygUHANGAZee9WWmpmI8zsSt9uQ4DbzGwhMBG42eUt2tUGUs1sGfB/wE3+yiki4g+RVcJ4tV8C6XsPc+97gTnUzwL9RZSjkpKSXHJystcxRER+Y/wPG3j44yXcd1ELBl/Q3Os4v2NmKc65At8l6vWL1CIiZVq/s2K4Jr4BL3y1krmrAuvdlioIERE/MjOevKYtzWtX5e5JC9iaedDrSIWmghAR8bOjQ/2ysnMZ+E4qWdmBMdRPBSEiUgKaRlXl2evaM39jJn//bJnXcQpFBSEiUkJ6tqvHn85rwlvfrefThVtPfgOPqSBERErQgz1bkdgokgc+XMTq7aV7qJ8KQkSkBJUPDWFM3wQq+ob67T9ceof6qSBEREpY3fCKvNQnnjXp+3joo8WldqifCkJExAPnNavFfRe14JMFWxn/wwav4xRIBSEi4pE7uzWje6vajJiSxvyNGV7H+R0VhIiIR0JCjOdv6ECd6hUZOCGVXaVsqJ8KQkTEQxGVwxjbL5Ed+7K4590F5JSioX4qCBERj7WLDmf4lW2YszKdl2eu8jrOr1QQIiKlQJ+ODemV0IAXZ6zi65WlY6ifCkJEpBQwM568uh0t61Tjnknz2VIKhvqpIERESolKYaG82i+BIzmOgRO8H+qnghARKUXOiKrKyOvbs2BTJk9O9fZMyyoIEZFSpkfbevy5UxPe/n4Dkz0c6qeCEBEphf7v0lac2ThvqN+qbXs9yaCCEBEphcqHhvBK3wQqh4UyYEKqJ0P9VBAiIqVUnep5Q/3Wpu/jgf+W/FA/FYSISCl2btNaDLm4JZ8u3Mrb360v0cdWQYiIlHIDujblwta1efKzZaSW4FA/FYSISCkXEmKMuj6OuuF5Q/127jtcMo9bIo8iIiJFEl65PGP7JbJzf8kN9VNBiIgEiLYNwhlxZRvmrtrBizP8P9RPBSEiEkBuPLMh1yVG8/LMVcxesd2vj6WCEBEJIGbG41e1zRvq9+4CNmcc8NtjqSBERAJMpbBQxvVPJMc31O9wdo5fHkcFISISgBrXqsJz13dg4ebdPDFlmV8eQwUhIhKgerSty+1dziDHOXL98K6mcsV+jyIiUmIe6NGKkBDzy33rCEJEJID5qxxABSEiIsehghARkQKpIEREpEAqCBERKZAKQkRECqSCEBGRAqkgRESkQFbS5zj1FzNLBzYU4S5qATuKKY7IsfTzJf5UlJ+vRs65qIKuKDMFUVRmluycS/I6h5RN+vkSf/LXz5eWmEREpEAqCBERKZAK4n9e8zqAlGn6+RJ/8svPl16DEBGRAukIQkRECqSCEBGRAgVtQZjZcDMb6rs8wswuPM376WZm5+b7+i0zu664ckpgM7PZZpbku/yZmUV4HEnKiOJ6DjsRnVEOcM79rQg37wbsA74rnjRSVjnnenqdQcqmIj6HHVdAHUGYWWMzW+77LX2lmU0wswvN7FszW2VmHfO3qu82S8ysse/yMN/tvgFa5tvn19/6zWy9mT1rZovN7Ccza+bbfoWZ/Whm883sKzOr47vfO4B7zWyBmXX23WUXM/vOzNbqaCI45PvZnGBmy8zsAzOrfMw+682sVmH2FTmWmf3BzBaZ2UIz+88x1530Oex0BFRB+DQDRgGtfH/6Ap2AocBDx7uRmSUCvYE4oCdw5gkeY7dzrh3wCjDat+0b4GznXDwwCfirc249MA54wTkX55yb69u3ni/T5cDTp/wdSqBqCbzqnGsN7AHuLKZ9JciZWRvgYaC7c64DcPdJblLQc9gpC8SCWOecW+ycywWWAjNc3nt1FwONT3C7zsBHzrkDzrk9wOQT7Dsx39/n+C5HA9PMbDFwP9DmBLf/2DmX65xLA+qc9DuSsmKTc+5b3+Xx5P2SUBz7inQH3nfO7QBwzu06yf4FPYedskAsiMP5Lufm+zqXvNdUsvnt91XxNB7DFXD5ZeAVXyv/5ST3mz+j/84oLqXNsR8qOtGHjE5lX5FTVdBz2CkLxII4mfVAAoCZJQBNfNvnAFebWSUzqwZccYL7uDHf39/7LocDW3yX/5hv371AtaLHljIgxsyO/rbWl7xlyeLYV2QmcL2Z1QQwsxon2b+g57BTVhYL4kOghpktBQYBKwGcc6nAu8BC4HNg3gnuI9LMFpG3znevb9tw4H0zS+G3Y3U/Ba455kVqCU4rgIFmtgyIBMYW074S5JxzS4Enga/NbCHw/EluUtBz2CnTqI1jmNl6IOnoWp9IYfje0TbFOde2OPcVOVXF+RxWFo8gRESkGOgIQkRECqQjCBERKZAKQkRECqSCEBGRAqkgRESkQCoIkdNwdPBeUfcRKc1UECIiUiAVhASNQo6Lr2FmH/vGKv9gZu19t61pZtPNbKmZ/Yt8M7bMrL9vrPICM/uHmYUWMssyM/un7z6nm1kl33W3mdk831jnD4+OAvflHuvLtdbyTlb1hu9+3sp33xeb2fdmlmpm75tZ1eL+t5TgoIKQYHOycfGPAfOdc+19X//bd7tHgW+cc22Aj4AYADNrTd68m/Occ3FADtCvkFmaA2N895kJXOvb/l/n3Jm+sc7LgFvz3SaSvOmc95I3kfgF8iYLtzOzON+S1sPAhc65BCAZuK+QeUR+Q2eUk2Czzjm3GMA3r2uGc875xrg3Bhrhe6J2zs30HTlUB7oAvXzbp5pZhu/+LgASgXlmBlAJ2H4KWRb4Lqfwv3H1bc3sCSACqApMy3ebT/Pl3XbM99KYvLH0scC3vjxhFGFYmwQ3FYQEm5ONiz9yivdnwNvOuQeLmCWHvHIBeAu42jm30MxuJu+0tsfeJn/2o1+X893Pl865PqeRR+Q3tMQk8ltz8S0RmVk3YIfvBFNzyFuOwswuJW+pB2AGcJ2Z1fZdV8PMGhUxQzXgZzMrT+GXq476ATjP/neq3Cpm1qKIeSRI6QhC5LeGA2/4RiUf4H/n/ngMmOhbyvkO2AjgnEszs4eB6WYWQt4RyEBgQxEyPAL8CKT7/i70+Uacc+m+o46JZlbBt/lhfGPvRU6FhvWJiEiBtMQkIiIF0hKTiB/5ThE5o4CrLnDO7SzpPCKnQktMIiJSIC0xiYhIgVQQIiJSIBWEiIgUSAUhIiIF+n/ONt0tRIICBgAAAABJRU5ErkJggg==",
+      "text/plain": [
+       "<Figure size 432x288 with 1 Axes>"
+      ]
+     },
+     "metadata": {
+      "needs_background": "light"
+     },
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "# Plot the metrics you care about\n",
+    "import seaborn as sns\n",
+    "sns.lineplot(x=\"model_name\", y=\"F1\", data=result_df, estimator=\"max\", ci=None)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 46,
+   "id": "914757a5",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "df649cab",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.0"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}