{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.preprocessing import binarize\n",
    "\n",
    "from sklearn.metrics import roc_curve, auc, precision_recall_curve, average_precision_score, plot_precision_recall_curve, f1_score, confusion_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "performances = pd.read_csv('performances.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ground_truth</th>\n",
       "      <th>probability</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0.99</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>0.98</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>0.97</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>0.96</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>0.95</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   ground_truth  probability\n",
       "0             1         0.99\n",
       "1             1         0.98\n",
       "2             1         0.97\n",
       "3             1         0.96\n",
       "4             1         0.95"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "performances.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_auc(t_y, p_y):\n",
    "    fig, c_ax = plt.subplots(1,1, figsize = (9, 9))\n",
    "    fpr, tpr, thresholds = roc_curve(t_y, p_y)\n",
    "    c_ax.plot(fpr, tpr, label = '%s (AUC:%0.2f)'  % ('Pneumonia', auc(fpr, tpr)))\n",
    "    c_ax.legend()\n",
    "    c_ax.set_xlabel('False Positive Rate')\n",
    "    c_ax.set_ylabel('True Positive Rate')\n",
    "    \n",
    "def plot_pr(t_y, p_y):\n",
    "    fig, c_ax = plt.subplots(1,1, figsize = (9, 9))\n",
    "    precision, recall, thresholds = precision_recall_curve(t_y, p_y)\n",
    "    c_ax.plot(precision, recall, label = '%s (AP Score:%0.2f)'  % ('Pneumonia', average_precision_score(t_y,p_y)))\n",
    "    c_ax.legend()\n",
    "    c_ax.set_xlabel('Recall')\n",
    "    c_ax.set_ylabel('Precision')\n",
    "\n",
    "def  calc_f1(prec,recall):\n",
    "\n",
    "    return 2*(prec*recall)/(prec+recall)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAikAAAIWCAYAAACSvx1mAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8GearUAAAgAElEQVR4nO3deXxU9b3G8edLCIQl7MiOIKuyChGKisXdupRrq3VpraLWi61Lr9VbW7vYvb1aa61WSq3g7q1WLVrq1qpY0LKL7ASIEPaELRAIWb73jxm4MQQyQM6cMzOf9+uVV3Jmzpx5ckTm4XfO+R1zdwEAAERNg7ADAAAA1IaSAgAAIomSAgAAIomSAgAAIomSAgAAIomSAgAAIqlh2AGOVLt27bxHjx5hxwAAAPVgzpw5Re7evrbnUq6k9OjRQ7Nnzw47BgAAqAdm9smhnuNwDwAAiCRKCgAAiCRKCgAAiKSUOyelNuXl5SosLNTevXvDjoI0kZOTo65duyo7OzvsKACQsdKipBQWFio3N1c9evSQmYUdBynO3VVcXKzCwkL17Nkz7DgAkLHS4nDP3r171bZtWwoK6oWZqW3btozMAUDI0qKkSKKgoF7x5wkAwpc2JQUAAKQXSko9ycrK0tChQzVw4EBdfvnlKi0tDTtSQmbPnq3bbrvtiF6zYcMGXXzxxZ967Pbbb1eXLl1UVVV14LF7771X999//6fW69Gjh4qKiiRJGzdu1JVXXqlevXrppJNO0oUXXqjly5cf9H5z5szRoEGD1Lt3b912221y94PWKS8v17XXXqtBgwbpxBNP1C9+8YsDzz333HMaNGiQBg8erAsuuODA+z/88MOaNGnSEf3uAIDkoaTUkyZNmmj+/PlauHChGjVqpAkTJoQdKSF5eXl66KGHjug1DzzwgL72ta8dWK6qqtLLL7+sbt26adq0aQltw9116aWXasyYMVq5cqUWL16sn//859q0adNB6958882aOHGiVqxYoRUrVuj1118/aJ0XXnhBZWVl+vjjjzVnzhz94Q9/UEFBgSoqKnT77bfrnXfe0YIFCzR48GA9/PDDkqTrr7/+iH93AEDypMXVPdX96NVFWrx+Z71u86TOLfTDSwYkvP7o0aO1YMECvfvuu7r33nvVrl07LVy4UMOHD9fTTz8tM9OcOXN0xx13aNeuXWrXrp0mT56sTp06acyYMbr//vuVl5enoqIi5eXlqaCgQJMnT9Yrr7yiyspKLVy4UN/61re0b98+PfXUU2rcuLGmTp2qNm3aaP78+Ro/frxKS0vVq1cvPf7442rdurXGjBmjkSNH6p133tH27dv1pz/9SaNHj9a7776r+++/X6+99ppmzpypb37zm9qzZ4+aNGmiSZMmqV+/fgf9fn/5y1/005/+9MDyO++8o4EDB+qKK67Qc889pzFjxtS5j9555x1lZ2dr/PjxBx4bOnToQett2LBBO3fu1KhRoyRJX/3qV/XKK6/oc5/73KfWMzPt3r1bFRUV2rNnjxo1aqQWLVrI3eXu2r17t9q2baudO3eqd+/ekqSmTZuqR48emjlzpkaMGJHQf1sAQPIwklLPKioq9Pe//12DBg2SJM2bN08PPvigFi9erFWrVmn69OkqLy/XrbfeqhdffFFz5szR9ddfr3vuuafObS9cuFDPPvusZs6cqXvuuUdNmzbVvHnzNGrUKD355JOSYh/iv/rVr7RgwQINGjRIP/rRjz6VbebMmXrwwQc/9fh+/fv317Rp0zRv3jz9+Mc/1ne/+92D1lm9erVat26txo0bH3jsueee01VXXaVLL71Ur732msrLyxP6XYYPH37I5/cXlnXr1qlr164HHu/atavWrVt30PqXXXaZmjVrpk6dOql79+6688471aZNG2VnZ+vRRx/VoEGD1LlzZy1evFg33HDDgdfl5eXp/fffrzMvACD50m4k5UhGPOrTnj17Dnywjh49WjfccINmzJihESNGHPiQHTp0qAoKCtSqVSstXLhQ5557riSpsrJSnTp1qvM9zjzzTOXm5io3N1ctW7bUJZdcIkkaNGiQFixYoB07dmj79u367Gc/K0m69tprdfnllx94/Re+8AVJ0vDhw1VQUHDQ9nfs2KFrr71WK1askJnVWjY2bNig9u3//2aV+/bt09SpU/Wb3/xGubm5GjlypN58801ddNFFh7xCJpErZ+bPny9JtZ5/UtvrZ86cqaysLK1fv17btm3T6NGjdc4556hbt2569NFHNW/ePJ1wwgm69dZb9Ytf/ELf+973JEnHHXecli5dWmceAEDypV1JCcv+c1Jqqj7ikJWVpYqKCrm7BgwYoA8++OCg9Rs2bHjg5NOa83RU31aDBg0OLDdo0EAVFRV1Zty//v4cNX3/+9/XmWeeqZdfflkFBQW1HrZp0qTJp3K9/vrr2rFjx4GRo9LSUjVt2lQXXXSR2rZtqw0bNnzq9SUlJWrVqpUGDBigF198sc7MXbt2VWFh4YHlwsJCde7c+aD1nn32WV1wwQXKzs7Wcccdp9NOO02zZ89WcXGxJKlXr16SpC996Uv65S9/eeB1e/fuVZMmTerMAQBIPg73hKBfv37asmXLgZJSXl6uRYsWSYpd/TJnzhxJSuhDvLqWLVuqdevWBw5fPPXUUwdGVRKxY8cOdenSRZI0efLkWtfp27fvp0ZhnnvuOT322GMqKChQQUGBVq9erTfffFOlpaU644wzNGXKFJWUlEiSXnrpJQ0ZMkRZWVk666yzVFZWpj/+8Y8HtjVr1iy99957n3q/Tp06KTc3Vx9++KHcXU8++aTGjh17UK7u3bvrn//854HzTz788EP1799fXbp00eLFi7VlyxZJ0ltvvaUTTzzxwOuWL1+ugQMHJryPAADJQ0kJQaNGjfTiiy/q29/+toYMGaKhQ4dqxowZkqQ777xTjz76qE499dQDl8oeiSeeeEJ33XWXBg8erPnz5+sHP/hBwq/97//+b33nO9/RaaedpsrKylrXadasmXr16qX8/HyVlpbqjTfe0EUXXfSp508//XS9+uqrGjx4sG655RadfvrpGjp0qCZMmKDHHntMUuyQzcsvv6y33npLvXr10oABA3TvvfceGCWpfhLto48+qhtvvFG9e/dWr169Dpw0O2XKlAO/3ze+8Q3t2rVLAwcO1CmnnKJx48Zp8ODB6ty5s374wx/qjDPOOLBPqp9rM336dJ1zzjkJ7yMAQPJYbcf862XDZo9LuljSZnc/6J+qFjux4LeSLpRUKuk6d59b13bz8vJ89uzZn3psyZIln/rXMYL18ssva86cOZ+6wicVzZs3Tw888ICeeuqpWp/nzxUABM/M5rh7Xm3PBTmSMlnSBYd5/nOS+sS/bpL0aIBZUI8uvfRS9ejRI+wYx6yoqEg/+clPwo4BADiEwE6cdfdpZtbjMKuMlfSkx4ZyPjSzVmbWyd03HOY1iIgbb7wx7AjHbP/VVQCAaArz6p4uktZWWy6MP3ZUJcXduSkc6k1Qh0EBREPRrjJdP3mW8jfvCjtKyvjNFUN1/oCOSX3PMEtKbY2i1k8GM7tJsUNC6t69+0HP5+TkqLi4WG3btqWo4Ji5u4qLi5WTkxN2FAAB2LOvUjc8MVvLN5Xo6hHHK4tLSBLSvU3TpL9nmCWlUFK3astdJa2vbUV3nyhpohQ7cbbm8/vn0th/mSlwrHJycj410y2A9FBZ5brt+XlaULhdf/jKcJ2X5JEBHJkwS8oUSbeY2fOSRkracbTno2RnZ6tnz571Gg4AkF7cXT9+dZHeWrxJP/r8AApKCgispJjZc5LGSGpnZoWSfigpW5LcfYKkqYpdfpyv2CXI44LKAgDAY++v1hMffKKbzjhB157aI+w4SECQV/dcVcfzLukbQb0/AAD7/W3BBv1s6hJdNKiT7r6gf9hxkCBOFwIApLVZBVv1X3+er1N6tNavvzREDRpwgUWqoKQAANLWyi279LUnZ6tr6yb641fzlJOdFXYkHAFKCgAgLW0pKdN1k2aqYQPT5OtGqFXTRmFHwhEK8+oeAAACUbqvQjc+MUtbSsr0vzeNUve2yZ/jA8eOkRQAQFqprHLd9tw8fbxuhx6+apiGdGsVdiQcJUZSAABpw91175RFenvJZv1k7ACdc1KHsCPhGDCSAgBIGxOnrdJTH36i//zsCbpmVI+w4+AYUVIAAGnh1Y/W6xd/X6qLB3fSt89nLpR0QEkBAKS8mau36lt//kgjerbR/ZczF0q6oKQAAFJa/uYSfe3J2erWpokmXjOcuVDSCCUFAJCyNpfs1XWTZik7q4Emj2MulHRDSQEApKTSfRW6YfJsFe/ap8evy1O3NsyFkm4oKQCAlFNRWaVbn52nRet36JEvn6zBXZkLJR0xTwoAIKW4u344ZZH+sXSzfnbpQJ3Vn7lQ0hUjKQCAlDLhvVV65t9rdPOYXvryyOPDjoMAUVIAACnjr/PX6VevL9Xnh3TWXef1CzsOAkZJAQCkhA9XFeuuFxZoZM82uu/ywcyFkgEoKQCAyFuxqUQ3PTlb3ds21cRr8tS4IXOhZAJKCgAg0jbvjM2F0jg7S5PHnaKWTbPDjoQkoaQAACJrd1mFrn9ilraV7tOk605R19bMhZJJuAQZABBJFZVVuuXZuVqyoUSPXZungV1ahh0JSUZJAQIy9pHp+mjt9rBjACnv55cO0pn9jgs7BkJASQECsmJTiU7u3kqj+7QPOwqQsvp3zNWFgzqFHQMhoaQAAco7vrXuOLdv2DEAICVx4iwAAIgkSgoQEPewEwBAaqOkAAGYPH219pRXqkurJmFHAYCURUkB6tkbizbqR68t1rknddA1o3qEHQcAUhYlBahHc9ds023PzdPgrq300JUnK4t7iwDAUePqHuAoubuqqp13smZrqW58YrY6tMjRn67NU5NG3FsEAI4FJQU4CiV7y3Xt4zM1d82nJ2tr3TRbk8edonbNG4eUDADSByUFOELllVX6+jNz9VHhDo3/bC81jY+YmKTzB3bUCe2bhxsQANIEJQU4Au6ue17+WO+vKNL/XDZYX8rrFnYkAEhbnDgLHIHf/TNff55dqNvP7kNBAYCAUVKABL04p1APvLVcXxzWVd88p0/YcQAg7VFSgAT8a0WR7v7LAp3eu51+8YVBMuPSYgAIGiUFqMOSDTs1/uk56n1cc/3+K8PUqCH/2wBAMvC3LXAYG3bs0bhJs9S8cUNNGneKWuRkhx0JADIGV/cAh7Bzb7nGTZqlXWUVemH8KHVqyX14ACCZGEkBalFeWaWvPz1X+Zt36dGvDNOJnVqEHQkAMg4jKUAN7q7vvPSx/pVfpPsuG6zRfdqHHQkAMhIjKUAND769Qi/OKdR/ndNXlzMXCgCEhpICVPPn2Wv123+s0OXDu+q2s3uHHQcAMholBYibtnyLvvvSxxrdp51+zlwoABA6SgogafH6nfr6M3PVp0Oufv/lYcrO4n8NAAgbfxMj463fvkfjJs9Ubk5DTbruFOUyFwoARAIlBRlt/1wopWWVmjTuFHVsmRN2JABAHJcgI2Ptq6jSzU/P0cotu/TE9SPUvyNzoQBAlFBSkJHcXXe/tEDT84v168uH6LTe7cKOBACogcM9yEi/eWu5Xpq7Tt86t6++OLxr2HEAALWgpCDj/O+sNXron/m6Iq+bbjmLuVAAIKooKcgo7y3fou++vFBn9G2vn146kLlQACDCKCnIGAvX7dDXn56jfsyFAgApgb+lkRHWbd+j6yfPUssm2Zo07hQ1b8w54wAQdfxNjbS3Y0+5xk2aqT3llfrLzaeqQwvmQgGAVMBICtJaWUWlxj81R6uLdusPXxmuvh1yw44EAEgQIylIW+6ub7+4QB+sKtZvrhiiU5kLBQBSCiMpSFsvzV2nV+av113n99OlJzMXCgCkGkoK0ta/8ovUrnljfX1Mr7CjAACOAiUFaWvemm0a1r0Vc6EAQIqipCAtbd29TwXFpTq5e+uwowAAjhIlBWlp/tptkqSTu7cKOQkA4GhRUpCW5q3ZrgYmDe7aMuwoAICjRElBWpq3Zrv6d2yhpo24yh4AUhUlBWmnsso1f+12DTueQz0AkMooKUg7K7fs0q6yCp3cjZNmASCVUVKQduat4aRZAEgHlBSknXlrtqtlk2z1bNcs7CgAgGNASUHambdmu05mEjcASHmUFKSVkr3lWr65hPNRACANUFKQVhYU7pA756MAQDqgpCCtzP0kdtLskG6UFABIdZQUpJV5a7er93HN1bJJdthRAADHiJKCtOHumrdmm05mFAUA0gIlBWnjk+JSbSst587HAJAmKClIG3+ZWyhJTIcPAGmCkoK08PK8Qv3un/n6j6Gd1a9DbthxAAD1gJKClDcjv0j//eICjTqhrf7nsiFM4gYAaYKSgpS2bGOJ/vPpOerZrpkmXDNcjRryRxoA0kXDsAMA++0tr9Tv312pol1lCb/mnaWb1SQ7S5PGjeCyYwBIM4GWFDO7QNJvJWVJeszdf1nj+ZaSnpbUPZ7lfnefFGQmRFNVletbL3ykvy3YoHbNGyf8ulZNs/XgFUPVpVWTANMBAMIQWEkxsyxJj0g6V1KhpFlmNsXdF1db7RuSFrv7JWbWXtIyM3vG3fcFlQvR9KvXl+pvCzbongtP1NfOOCHsOACACAjyAP4ISfnuvipeOp6XNLbGOi4p12JnOjaXtFVSRYCZEEFPflCgP0xbpWtHHa8bR/cMOw4AICKCLCldJK2ttlwYf6y6hyWdKGm9pI8l3e7uVQFmQsS8uWij7p2ySOec2EE/uGQAV+YAAA4IsqTU9mnjNZbPlzRfUmdJQyU9bGYtDtqQ2U1mNtvMZm/ZsqX+kyIU89du123Pz9Ogrq30u6tOVlYDCgoA4P8FWVIKJXWrttxVsRGT6sZJeslj8iWtltS/5obcfaK757l7Xvv27QMLjOT5pHi3bpg8S+1zG+tP1+apSaOssCMBACImyJIyS1IfM+tpZo0kXSlpSo111kg6W5LMrIOkfpJWBZgJEbB19z5dN2mWKt01edyII7qaBwCQOQK7usfdK8zsFklvKHYJ8uPuvsjMxsefnyDpJ5Imm9nHih0e+ra7FwWVCeHbW16prz05W+u279GzN45Ur/bNw44EAIioQOdJcfepkqbWeGxCtZ/XSzovyAyIjqoq1x1/nq+5a7bpkauHKa9Hm7AjAQAijDnEkTQ/n7pEUz/eqHsuPFEXDuoUdhwAQMRRUpAUk6av1mP/Wq3rTu2hG05nLhQAQN0oKQjc6ws36sevLdZ5J3XQ9y8+iblQAAAJoaQgUHPXbNPtz8/TkK6t9NsrmQsFAJA4SgoCU1ZRqZufnqOOLXOYCwUAcMQCvboHmW3K/PXatLNMT90wQm2ZCwUAcIQYSUEg3F2PTy9Q3w7NdXrvdmHHAQCkIEoKAvHhqq1asmGnrj+tJyfKAgCOCiUFgXh8+mq1bpqt/zi55o2vAQBIDCUF9e6T4t16e8kmfXnk8crJ5mRZAMDRoaSg3k2eUaCGDUzXjDo+7CgAgBRGSUG9KtlbrhdmF+riwZ3VoUVO2HEAACmMkoJ69b+z1mpXWYWuP42p7wEAx4aSgnr10tx1Gta9lQZ1bRl2FABAiqOkoF7t3leh7m2ahh0DAJAGKCkAACCSKCkAACCSKCkAACCSKCkAACCSKCkAACCSKCkAACCSGoYdAOGqqKxS4bY98nraXnlFVT1tCQCQ6SgpGe6nf1uiyTMK6nWbjRtyU0EAwLGjpGSwNcWlevrDT3TBgI66YGDHetvuqF5t621bAIDMRUnJYA++vVwNs0w/GjuAmwECACKHE2cz1LKNJXp5/jpde2oPCgoAIJIoKRnqgbeWqXmjhhp/Rq+wowAAUCtKSgYq2Vuut5ds1tUju6t1s0ZhxwEAoFaUlAw0c/VWVVa5Ptu3fdhRAAA4JEpKBpqeX6zGDRto2PGtw44CAMAhUVIy0PT8Ip3So41yspnPBAAQXZSUDLOlpEzLNpXo1N7MZQIAiDZKSoaZsbJIknRar3YhJwEA4PAoKRlmRn6xWuQ01MAuLcOOAgDAYVFSMoi761/5RRrVq62yGljYcQAAOCxKSgZZs7VU67bv0Wm9OdQDAIg+SkoGmZ5fLEmUFABASqCkZJDpK4vUsUWOTmjXLOwoAADUiZKSIaqqXDPyi3Rq77Yy43wUAED0UVIyxJKNO7WttFync6gHAJAiKCkZYgbnowAAUgwlJUNMX1mkXu2bqUOLnLCjAACQEEpKBthXUaV/r9rKoR4AQEqhpGSA+Wu3a095pU6lpAAAUgglJQP8K79IDUz6zAncVBAAkDooKWmudF+Fnv33Go3q1VYtm2SHHQcAgIRRUtLcpOkFKtpVpjvO7Rd2FAAAjgglJY3tKC3XhPdW6pwTj9Pw41uHHQcAgCNCSUljE6at1K6yCn3rPEZRAACph5KSprbu3qfJ0wv0+SGddWKnFmHHAQDgiFFS0tTkGQXaU16pW8/qHXYUAACOCiUlDe0uq9ATMwp07kkd1Pu43LDjAABwVCgpaej5WWu1Y0+5bh7TK+woAAAcNUpKmtlXUaXH3l+lkT3baFh3rugBAKQuSkqa+ev8ddqwY6/GM4oCAEhxlJQ0UlXl+sO0VerfMVdj+rYPOw4AAMeEkpJG3l6ySfmbd+nmMb1kZmHHAQDgmFBS0oS769H3Vqpbmya6aFCnsOMAAHDMKClpYubqrZq3ZrtuGn2CGmbxnxUAkPr4NEsTj763Um2bNdLled3CjgIAQL2gpKSBJRt26t1lWzTutB7Kyc4KOw4AAPWCkpIGJry3Us0aZemaz/QIOwoAAPWGkpLiCreV6rUFG3T1yO5q2TQ77DgAANQbSkqKe3PRJlVWOaMoAIC0Q0lJcTNWFun4tk3VvW3TsKMAAFCvKCkprKKySh+u2qrTercLOwoAAPWOkpLCPircoV1lFTqtFyUFAJB+KCkpbEZ+kSRpVK+2IScBAKD+UVJS2PSVRTqpUwu1adYo7CgAANQ7SkqK2rOvUnM/2a7T+3CoBwCQnigpKWpWwVbtq6zSqRzqAQCkKUpKivpXfpGys0wjerYJOwoAAIGgpKSgHaXlen7mGn2273Fq2qhh2HEAAAgEJSUFTXx/pXburdAd5/YNOwoAAIGhpKSYLSVlevxfBfr8kM46qXOLsOMAABAYSkqKeeSdfO2rrNJ/MYoCAEhzlJQUsqusQs/+e40uG9ZVPds1CzsOAACBoqSkkJmri7WvskqfH9o57CgAAASOkpJCpucXq1HDBhp+fOuwowAAEDhKSgqZnl+kU3q0Vk52VthRAAAIXKAlxcwuMLNlZpZvZncfYp0xZjbfzBaZ2XtB5kllRbvKtHRjiU7ljscAgAwR2ExgZpYl6RFJ50oqlDTLzKa4++Jq67SS9HtJF7j7GjM7Lqg8qW7GymJJ0mm9KSkAgMwQ5EjKCEn57r7K3fdJel7S2BrrXC3pJXdfI0nuvjnAPCltRn6RcnMaalCXlmFHAQAgKYIsKV0kra22XBh/rLq+klqb2btmNsfMvhpgnpT2r/wijTqhrbIaWNhRAABIiiBLSm2fpl5juaGk4ZIuknS+pO+b2UGzlJnZTWY228xmb9mypf6TRtya4lIVbtvDoR4AQEZJuKSY2ZHOHlYoqVu15a6S1teyzuvuvtvdiyRNkzSk5obcfaK757l7Xvv27Y8wRup7dUFst53eh5ICAMgcdZYUMzvVzBZLWhJfHmJmv09g27Mk9TGznmbWSNKVkqbUWOevkkabWUMzaypp5P73QcyOPeX6w3srdWa/9urVvnnYcQAASJpEru75jWKHYqZIkrt/ZGZn1PUid68ws1skvSEpS9Lj7r7IzMbHn5/g7kvM7HVJCyRVSXrM3Rce5e+SliZOi93x+M7z+4UdBQCApEroEmR3X2v2qVNMKhN83VRJU2s8NqHG8n2S7ktke5lm/x2PLxnSWQM6c1UPACCzJFJS1prZqZI8ftjmNnFIJin23/H4Du54DADIQImcODte0jcUu3y4UNJQSV8PMhSkwm2leubfn+hLedzxGACQmRIZSenn7l+u/oCZnSZpejCRIEkPvr1CZqbbzu4TdhQAAEKRyEjK7xJ8DPUkf3OJXppbqK9+5nh1atkk7DgAAITikCMpZjZK0qmS2pvZHdWeaqHY1ToIyK/fXK4m2Vm6eUyvsKMAABCawx3uaSSpeXyd3GqP75R0WZChMtmCwu36+8KNuv3sPmrbvHHYcQAACM0hS4q7vyfpPTOb7O6fJDFTRrvvjWVq3TRbN47uGXYUAABClciJs6Vmdp+kAZJy9j/o7mcFlipDfbCyWO+vKNI9F56o3JzssOMAABCqRE6cfUbSUkk9Jf1IUoFiU96jHrm77ntjqTq2yNE1o44POw4AAKFLpKS0dfc/SSp39/fc/XpJnwk4V8b5x5LNmrtmu247u49ysjkvGQCARA73lMe/bzCzixS7k3HX4CJlnqoq1/1vLlOPtk11eR67FgAAKbGS8lMzaynpW4rNj9JC0jcDTZVhXl2wXks3lui3Vw5VdlYig1sAAKS/OkuKu78W/3GHpDOlAzPOoh6UV1bpgbeWq3/HXF0yuHPYcQAAiIxD/rPdzLLM7Cozu9PMBsYfu9jMZkh6OGkJ09yfZ6/VJ8Wluuv8fmrQwOp+AQAAGeJwIyl/ktRN0kxJD5nZJ5JGSbrb3V9JRrh0t7e8Ug/9Y4WGH99aZ/U/Luw4AABEyuFKSp6kwe5eZWY5kook9Xb3jcmJlv6e/KBAm3aW6bdXniwzRlEAAKjucGdp7nP3Kkly972SllNQ6k/J3nL9/t2VOqNve33mhLZhxwEAIHION5LS38wWxH82Sb3iyybJ3X1w4OnS2B/fX63tpeW667x+YUcBACCSDldSTkxaigxTvKtMf3p/lS4c1FGDurYMOw4AAJF0uBsMclPBgPz+3ZXaU16pO87tG3YUAAAii5nDkmz99j166sNP9MVhXdX7uNyw4wAAEFmUlCR76B8rJJduP6dP2FEAAIi0hEqKmTUxM87wPEartuzSC3MKdfXI7uraumnYcQAAiLQ6S4qZXSJpvqTX48tDzWxK0JDvvzEAABlCSURBVMHS0QNvLVfjhg30jTN7hx0FAIDIS2Qk5V5JIyRtlyR3ny+pR3CR0tPCdTv02oINuv60nmqf2zjsOAAARF4iJaXC3XcEniTN/frNZWrZJFtfO+OEsKMAAJASEikpC83saklZZtbHzH4naUbAudLKrIKtemfZFo3/bC+1bJIddhwAAFJCIiXlVkkDJJVJelbSDknfDDJUOnF3/c/rS9U+t7GuO7VH2HEAAEgZh5txdr9+7n6PpHuCDpOO3l2+RbMKtuknYweoSaOssOMAAJAyEhlJecDMlprZT8xsQOCJ0khVlev+N5apW5smuuKU7mHHAQAgpdRZUtz9TEljJG2RNNHMPjaz7wUdLB1MXbhBi9bv1H+d01eNGjJvHgAARyKhT0533+juD0kar9icKT8INFUaqKis0gNvLlffDs01dmiXsOMAAJByEpnM7UQzu9fMFkp6WLEre7oGnizF/WVuoVYV7dad5/VTVgMLOw4AACknkRNnJ0l6TtJ57r4+4DxpYW95pR58e4WGdmulc0/qEHYcAABSUp0lxd0/k4wg6eSZf6/Rhh179evLh8iMURQAAI7GIUuKmf3Z3b9kZh9L8upPSXJ3Hxx4uhS0q6xCj7yTr9N6t9WpvduFHQcAgJR1uJGU2+PfL05GkHTx5AcF2rp7n+46v3/YUQAASGmHPHHW3TfEf/y6u39S/UvS15MTL/XMyC/WwC4tNLRbq7CjAACQ0hK5BPncWh77XH0HSRdLN5aof8cWYccAACDlHe6clJsVGzE5wcwWVHsqV9L0oIOlouJdZSraVab+HXPDjgIAQMo73Dkpz0r6u6RfSLq72uMl7r410FQpatnGEklSP0oKAADH7HAlxd29wMy+UfMJM2tDUTnYUkoKAAD1pq6RlIslzVHsEuTqE364pBMCzJWSlm8qUZtmjdS+eeOwowAAkPIOWVLc/eL4957Ji5Palm4sUb8OuUzgBgBAPUjk3j2nmVmz+M9fMbMHzKx78NFSS1WVa/mmEg71AABQTxK5BPlRSaVmNkTSf0v6RNJTgaZKQYXb9qh0XyVX9gAAUE8SKSkV7u6Sxkr6rbv/VrHLkFHN0o07JUl9KSkAANSLRO6CXGJm35F0jaTRZpYlKTvYWKln/+XHfTtQUgAAqA+JjKRcIalM0vXuvlFSF0n3BZoqBS3dVKJubZqoeeNEeh8AAKhLnSUlXkyekdTSzC6WtNfdnww8WYpZvrFE/TowHT4AAPUlkat7viRppqTLJX1J0r/N7LKgg6WSsopKrSrazUmzAADUo0SOTdwj6RR33yxJZtZe0tuSXgwyWCpZuXm3Kqucy48BAKhHiZyT0mB/QYkrTvB1GWPZptiVPZQUAADqTyIjKa+b2RuSnosvXyFpanCRUs/SjSXKzjL1bNcs7CgAAKSNOkuKu99lZl+QdLpi9++Z6O4vB54shSzbWKJe7ZsrO4sBJgAA6sshS4qZ9ZF0v6Rekj6WdKe7r0tWsFSybGOJRvZsE3YMAADSyuH+6f+4pNckfVGxOyH/LimJUsyOPeXasGOv+nXk8mMAAOrT4Q735Lr7H+M/LzOzuckIlGqWb4rNNMvlxwAA1K/DlZQcMztZsfNQJKlJ9WV3p7QodtKsxJU9AADUt8OVlA2SHqi2vLHasks6K6hQqWTZxp3KzWmoTi1zwo4CAEBaOWRJcfczkxkkVS3bWKJ+HXJlZnWvDAAAEsY1s8fA3bV0YwmHegAACAAl5Rhs2LFXJXsrOGkWAIAAUFKOwbJN+0+a5fJjAADqWyJ3QTYz+4qZ/SC+3N3MRgQfLfqW7b+ypwMjKQAA1LdERlJ+L2mUpKviyyWSHgksUQpZtrFEnVrmqGXT7LCjAACQdhK5weBIdx9mZvMkyd23mVmjgHOlBE6aBQAgOImUlHIzy1JsbhSZWXtJVYGmiri1W0s1c/VWrdy8S2f0bRd2HAAA0lIiJeUhSS9LOs7MfibpMknfCzRVxP3o1cV6e8kmSdKw7q1DTgMAQHqqs6S4+zNmNkfS2YpNif8f7r4k8GQRtnV3mfKOb63fXX2yOrVsEnYcAADSUp0lxcy6SyqV9Gr1x9x9TZDBoqxkb4V6tW9OQQEAIECJHO75m2Lno5ikHEk9JS2TNCDAXJFWsrdCuTmJ7DoAAHC0EjncM6j6spkNk/SfgSVKASV7y5Wbw2XHAAAE6YhnnHX3uZJOCSBLSqiscu3eV8lICgAAAUvknJQ7qi02kDRM0pbAEkXcrr0VkkRJAQAgYIl80lafraxCsXNU/hJMnOjbubdcktSCwz0AAATqsCUlPolbc3e/K0l5Iq+EkRQAAJLikOekmFlDd69U7PAO4kriIymcOAsAQLAONxwwU7GCMt/Mpkh6QdLu/U+6+0sBZ4ukXWWMpAAAkAyJfNK2kVQs6Sz9/3wpLikjSwqHewAASI7DXYJ8XPzKnoWSPo5/XxT/vjCRjZvZBWa2zMzyzezuw6x3iplVmtllR5A9FBzuAQAgOQ43HJAlqbliIyc1eV0bjp90+4ikcyUVSpplZlPcfXEt6/1K0huJhg7LJ8W79f2/LpLESAoAAEE73CftBnf/8TFse4SkfHdfJUlm9ryksZIW11jvVsUuaY78BHELCndIki4f3lU52VkhpwEAIL0d7nBPbSMoR6KLpLXVlgvjj/3/G5h1kXSppAnH+F5JUbyrTJL0nQtPDDkJAADp73Al5exj3HYih4kelPTt+KXOh96Q2U1mNtvMZm/ZEt5kt0W79qmBSa2acD4KAABBO+ThHnffeozbLpTUrdpyV0nra6yTJ+l5M5OkdpIuNLMKd3+lRpaJkiZKUl5eXp3nwwSleHeZ2jRrrAYNjnWQCQAA1CXIsz9nSepjZj0lrZN0paSrq6/g7j33/2xmkyW9VrOgRMmWkn1q17xR2DEAAMgIgZUUd68ws1sUu2onS9Lj7r7IzMbHn0+J81CqK95dpnbNG4cdAwCAjBDodbTuPlXS1BqP1VpO3P26ILPUh6JdZerevWnYMQAAyAiHO3EWNRTv2sdICgAASUJJSVDJ3nKV7qtUW85JAQAgKSgpCXpnWezS56HdWoWcBACAzEBJSdCU+evUsUWORvZsG3YUAAAyAiUlAdtL9+m95Vt08eBOymKOFAAAkoKSkoC/L9yo8krX2KFd6l4ZAADUC0pKAt5dtlnd2jTRwC4two4CAEDGoKQkYOnGEg3q0lLx6fsBAEASUFLqULqvQmu2lqpfB0ZRAABIJkpKHVZs2iV3qV/H5mFHAQAgo1BS6rBg3Q5JUr+OjKQAAJBMlJTDKK+s0mPvr9KJnVqoR1vu2QMAQDJRUg7jz7PX6pPiUt11fl9OmgUAIMkoKYewt7xSD/1jhfKOb60z+x0XdhwAADIOJeUQnvygQJt2lumu8/sxigIAQAgoKYfwwuxCjezZRiNP4F49AACEgZJSi7KKSq0q2q1TerQJOwoAABmLklKLlZt3q7LK1a9jbthRAADIWJSUWizbtFOSKCkAAISoYdgBoqRkb7ku+d2/tGHHXmVnmXq2axZ2JAAAMhYlpZotJWUqKC7V6D7t9PkhnZWdxUATAABhoaTU4rLhXTV2aJewYwAAkNEYKgAAAJFESQEAAJFESQEAAJFESQEAAJFESQEAAJFESQEAAJFESQEAAJFESQEAAJFESQEAAJFESQEAAJFESQEAAJFESQEAAJFESQEAAJFESQEAAJFESQEAAJFESQEAAJFESQEAAJFESQEAAJFESQEAAJFESQEAAJFESQEAAJFESQEAAJFESQEAAJFESQEAAJFESammyj3sCAAAII6SUs2araWSpM6tmoScBAAAUFKqWbZxlySpb4fckJMAAABKSjXLNu5U55Y5atkkO+woAABkPEpKNUs3lqhfR0ZRAACIAkpKXHlllVZu2aW+lBQAACKBkhK3umi3yitd/SkpAABEAiUlbunGEklSvw4tQk4CAAAkSsoByzeWKKuBqddxzcKOAgAAREk5YPmmEvVs10yNG2aFHQUAAIiScsCe8krl5jQMOwYAAIijpAAAgEiipAAAgEiipAAAgEiipAAAgEiipAAAgEiipAAAgEiipAAAgEiipAAAgEiipAAAgEiipAAAgEiipAAAgEiipAAAgEiipAAAgEiipAAAgEiipAAAgEiipAAAgEiipAAAgEiipAAAgEiipAAAgEiipAAAgEiipAAAgEiipAAAgEiipAAAgEgKtKSY2QVmtszM8s3s7lqe/7KZLYh/zTCzIUHmAQAAqSOwkmJmWZIekfQ5SSdJusrMTqqx2mpJn3X3wZJ+ImliUHkAAEBqCXIkZYSkfHdf5e77JD0vaWz1Fdx9hrtviy9+KKlrgHkAAEAKCbKkdJG0ttpyYfyxQ7lB0t8DzAMAAFJIwwC3bbU85rWuaHamYiXl9EM8f5OkmySpe/fu9ZUPAABEWJAjKYWSulVb7ippfc2VzGywpMckjXX34to25O4T3T3P3fPat28fSFgAABAtQZaUWZL6mFlPM2sk6UpJU6qvYGbdJb0k6Rp3Xx5gFgAAkGICO9zj7hVmdoukNyRlSXrc3ReZ2fj48xMk/UBSW0m/NzNJqnD3vKAyAQCA1BHkOSly96mSptZ4bEK1n2+UdGOQGQAAQGpixlkAABBJlBQAABBJlBQAABBJlBQAABBJlBQAABBJlBQAABBJlBQAABBJlBQAABBJlBQAABBJlBQAABBJlBQAABBJlBQAABBJlBQAABBJlBQAABBJlBQAABBJlBQAABBJlBQAABBJlBQAABBJlBQAABBJlBQAABBJlBQAABBJlBQAABBJlBQAABBJlBQAABBJlBQAABBJlBQAABBJlBQAABBJlBQAABBJlBQAABBJlBQAABBJlBQAABBJlBQAABBJlBQAABBJlBQAABBJlBQAABBJlBQAABBJlBQAABBJlBQAABBJlBQAABBJlBQAABBJlBQAABBJlBQAABBJlBQAABBJlBQAABBJlBQAABBJlBQAABBJlBQAABBJlBQAABBJlBQAABBJlBQAABBJlBQAABBJlBQAABBJlBQAABBJlBQAABBJlBQAABBJlBQAABBJlBQAABBJlBQAABBJlBQAABBJlBQAABBJlBQAABBJlBQAABBJlBQAABBJlBQAABBJlBQAABBJlBQAABBJlBQAABBJlBQAABBJlBQAABBJlBQAABBJlBQAABBJlBQAABBJlBQAABBJlBQAABBJlBQAABBJlBQAABBJlBQAABBJlBQAABBJlBQAABBJlBQAABBJgZYUM7vAzJaZWb6Z3V3L82ZmD8WfX2Bmw4LMAwAAUkdgJcXMsiQ9Iulzkk6SdJWZnVRjtc9J6hP/uknSo0HlAQAAqSXIkZQRkvLdfZW775P0vKSxNdYZK+lJj/lQUisz6xRgJgAAkCKCLCldJK2ttlwYf+xI1wEAABmoYYDbtloe86NYR2Z2k2KHg9S9e/djT1aLb57TR+WVB701AAAISZAlpVBSt2rLXSWtP4p15O4TJU2UpLy8vECaxPDj2wSxWQAAcJSCPNwzS1IfM+tpZo0kXSlpSo11pkj6avwqn89I2uHuGwLMBAAAUkRgIynuXmFmt0h6Q1KWpMfdfZGZjY8/P0HSVEkXSsqXVCppXFB5AABAagnycI/cfapiRaT6YxOq/eySvhFkBgAAkJqYcRYAAEQSJQUAAEQSJQUAAEQSJQUAAEQSJQUAAEQSJQUAAEQSJQUAAEQSJQUAAEQSJQUAAEQSJQUAAEQSJQUAAEQSJQUAAEQSJQUAAEQSJQUAAEQSJQUAAEQSJQUAAEQSJQUAAESSuXvYGY6ImW2R9ElAm28nqSigbePT2NfJxf5OHvZ18rCvkyuo/X28u7ev7YmUKylBMrPZ7p4Xdo5MwL5OLvZ38rCvk4d9nVxh7G8O9wAAgEiipAAAgEiipHzaxLADZBD2dXKxv5OHfZ087OvkSvr+5pwUAAAQSYykAACASMrIkmJmF5jZMjPLN7O7a3nezOyh+PMLzGxYGDnTQQL7+svxfbzAzGaY2ZAwcqaDuvZ1tfVOMbNKM7ssmfnSTSL728zGmNl8M1tkZu8lO2O6SODvkZZm9qqZfRTf1+PCyJkOzOxxM9tsZgsP8XxyPx/dPaO+JGVJWinpBEmNJH0k6aQa61wo6e+STNJnJP077Nyp+JXgvj5VUuv4z59jXwe3r6ut909JUyVdFnbuVP1K8M92K0mLJXWPLx8Xdu5U/EpwX39X0q/iP7eXtFVSo7Czp+KXpDMkDZO08BDPJ/XzMRNHUkZIynf3Ve6+T9LzksbWWGespCc95kNJrcysU7KDpoE697W7z3D3bfHFDyV1TXLGdJHIn2tJulXSXyRtTma4NJTI/r5a0kvuvkaS3J19fnQS2dcuKdfMTFJzxUpKRXJjpgd3n6bY/juUpH4+ZmJJ6SJpbbXlwvhjR7oO6nak+/EGxRo6jlyd+9rMuki6VNKEJOZKV4n82e4rqbWZvWtmc8zsq0lLl14S2dcPSzpR0npJH0u63d2rkhMv4yT187FhUBuOMKvlsZqXOCWyDuqW8H40szMVKymnB5oofSWyrx+U9G13r4z9gxPHIJH93VDScElnS2oi6QMz+9DdlwcdLs0ksq/PlzRf0lmSekl6y8zed/edQYfLQEn9fMzEklIoqVu15a6Kte8jXQd1S2g/mtlgSY9J+py7FycpW7pJZF/nSXo+XlDaSbrQzCrc/ZXkREwrif49UuTuuyXtNrNpkoZIoqQcmUT29ThJv/TYSRP5ZrZaUn9JM5MTMaMk9fMxEw/3zJLUx8x6mlkjSVdKmlJjnSmSvho/i/kzkna4+4ZkB00Dde5rM+su6SVJ1/AvzGNS5752957u3sPde0h6UdLXKShHLZG/R/4qabSZNTSzppJGSlqS5JzpIJF9vUaxESuZWQdJ/SStSmrKzJHUz8eMG0lx9wozu0XSG4qdNf64uy8ys/Hx5ycoduXDhZLyJZUq1tJxhBLc1z+Q1FbS7+P/wq9wbhh2xBLc16gniexvd19iZq9LWiCpStJj7l7rZZ04tAT/bP9E0mQz+1ixwxHfdnfujnwUzOw5SWMktTOzQkk/lJQthfP5yIyzAAAgkjLxcA8AAEgBlBQAABBJlBQAABBJlBQAABBJlBQAABBJlBQgA8Tvejy/2lePw6y7qx7eb7KZrY6/11wzG3UU23jMzE6K//zdGs/NONaM8e3s3y8L43fRbVXH+kPN7ML6eG8AdeMSZCADmNkud29e3+seZhuTJb3m7i+a2XmS7nf3wcewvWPOVNd2zewJScvd/WeHWf86SXnufkt9ZwFwMEZSgAxkZs3N7B/xUY6PzeygOyabWSczm1ZtpGF0/PHzzOyD+GtfMLO6ysM0Sb3jr70jvq2FZvbN+GPNzOxvZvZR/PEr4o+/a2Z5ZvZLSU3iOZ6JP7cr/v1/q49sxEdwvmhmWWZ2n5nNMrMFZvafCeyWDxS/UZqZjTCzGWY2L/69X3y20x9LuiKe5Yp49sfj7zOvtv0I4Ohl3IyzQIZqYmbz4z+vlnS5pEvdfaeZtZP0oZlN8U8PrV4t6Q13/5mZZUlqGl/3e5LOcffdZvZtSXco9uF9KJdI+tjMhis2O+VIxWYF/beZvSfpBEnr3f0iSTKzltVf7O53m9kt7j60lm0/L+kKSVPjJeJsSTcrdrPKHe5+ipk1ljTdzN5099W1BYz/fmdL+lP8oaWSzojPdnqOpJ+7+xfN7AeqNpJiZj+X9E93vz5+qGimmb0dv18PgGNESQEyw57qH/Jmli3p52Z2hmJTtneR1EHSxmqvmSXp8fi6r7j7fDP7rKSTFPvQl6RGio1A1OY+M/uepC2KlYazJb28/wPczF6SNFrS65LuN7NfKXaI6P0j+L3+LumheBG5QNI0d98TP8Q02Mwui6/XUlIfxQpadfvLWw9JcyS9VW39J8ysj2J3eM0+xPufJ+nzZnZnfDlHUndxjx6gXlBSgMz0ZUntJQ1393IzK1DsA/YAd58WLzEXSXrKzO6TtE3SW+5+VQLvcZe7v7h/IT4icRB3Xx4fZblQ0i/iIx6HG5mp/tq9ZvaupPMVG1F5bv/bSbrV3d+oYxN73H1ofPTmNUnfkPSQYveCecfdL42fZPzuIV5vkr7o7ssSyQvgyHBOCpCZWkraHC8oZ0o6vuYKZnZ8fJ0/KnYYZJikDyWdZmb7zzFpamZ9E3zPaZL+I/6aZpIulfS+mXWWVOruT0u6P/4+NZXHR3Rq87xih5FGK3YTOsW/37z/NWbWN/6etXL3HZJuk3Rn/DUtJa2LP31dtVVLJOVWW35D0q0WH1Yys5MP9R4AjhwlBchMz0jKM7PZio2qLK1lnTGS5pvZPElflPRbd9+i2If2c2a2QLHS0j+RN3T3uZImS5op6d+K3RV4nqRBip3LMV/SPZJ+WsvLJ0pasP/E2RrelHSGpLfdfV/8scckLZY018wWSvqD6hg5jmf5SNKVkv5HsVGd6YrdeXe/dySdtP/EWcVGXLLj2RbGlwHUEy5BBgAAkcRICgAAiCRKCgAAiCRKCgAAiCRKCgAAiCRKCgAAiCRKCgAAiCRKCgAAiCRKCgAAiKT/A6M/yf1dcxe7AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 648x648 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_auc(performances['ground_truth'],performances['probability'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAikAAAIWCAYAAACSvx1mAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8GearUAAAgAElEQVR4nOzdeXgV5f3+8fuTfSGBQMKWBBL2nSABRFFQtC6tC9QNW3drrWt3be23ra3dvl9trfsuaFWsO/qjihbBBRQSWWSRLQQIICQhYc/+/P7IIQYIECAnMyd5v64rFzlz5sy5wwRy55lnZsw5JwAAAL8J8zoAAABAQygpAADAlygpAADAlygpAADAlygpAADAlygpAADAlyK8DnC0kpOTXUZGhtcxAABAE8jNzS1yzqU09FzIlZSMjAzl5OR4HQMAADQBM1t3qOc43AMAAHyJkgIAAHyJkgIAAHwp5OakAAC+UVlZqYKCApWVlXkdBTismJgYpaWlKTIystGvoaQAQAgrKChQQkKCMjIyZGZexwEa5JxTcXGxCgoKlJmZ2ejXcbgHAEJYWVmZOnToQEGBr5mZOnTocNQjfpQUAAhxFBSEgmP5PqWkAAAAX6KkAACOS3h4uLKysjRo0CBdfPHF2rNnj9eRGiUnJ0e33XbbUb1m8+bN+s53vrPfsttvv12pqamqqampWzZ58mSlpKQoKytLAwYM0JNPPnnQtvbs2aPvfe97Gjx4sAYNGqQxY8Zo165dx/bFHIO//OUv6tWrl/r27av33nvvkOs9+OCD6tu3rwYOHKhf/vKXkqQXXnhBWVlZdR9hYWFauHChJOmMM85QSUlJ04R0zoXUx/Dhwx0AoNayZcu8juDi4+PrPr/88svdfffd52Ga4Pr5z3/u3nzzzbrH1dXVLj093Y0aNcp9+OGHdcufffZZd/PNNzvnnNuyZYtLTk52X3/99X7b+vOf/+x+8pOf1D3+6quvXFlZ2XHlq6ysbNR6S5cudUOGDHFlZWUuLy/P9ejRw1VVVR203syZM9348ePrcm3ZsuWgdRYvXuwyMzPrHk+ePNndc889Db5vQ9+vknLcIX7mc3YPALQQd7+9VMs27WjSbQ7omqjfnTew0eufcsopWrx4sWbNmqXf//73Sk5O1pIlSzR8+HD961//kpkpNzdXP/3pT7Vr1y4lJydr8uTJ6tKli8aNG6d7771X2dnZKioqUnZ2tvLz8zV58mS9+eabqq6u1pIlS/Szn/1MFRUVev755xUdHa3p06erffv2WrhwoW688Ubt2bNHPXv21DPPPKOkpCSNGzdOo0aN0ocffqjS0lI9/fTTOuWUUzRr1izde++9eueddzRv3jz9+Mc/1t69exUbG6tnn31Wffv2Pejre+2113TPPffUPf7www81aNAgXXrppXrppZc0bty4g17TsWNH9ezZU+vWrVOnTp3qlm/evFndu3eve1z//Z577jnde++9MjMNGTJEzz//vNatW6drr71WhYWFSklJ0bPPPqtu3brp6quvVvv27bVgwQKdcMIJuummm3TzzTersLBQcXFxevLJJ9WvX7/9Mr311lu67LLLFB0drczMTPXq1Uvz5s3T6NGj91vv0Ucf1Z133qno6Oi6r+VAL730kiZNmlT3+Pzzz9cpp5yiu+6661DfJo3G4R4AQJOoqqrSf/7zHw0ePFiStGDBAt1///1atmyZ8vLy9Omnn6qyslK33nqrXn31VeXm5uraa69t1A+zJUuW6MUXX9S8efN01113KS4uTgsWLNDo0aP13HPPSZKuvPJK/e1vf9PixYs1ePBg3X333ftlmzdvnu6///79lu/Tr18/ffTRR1qwYIH+8Ic/6Ne//vVB66xdu1ZJSUl1P7Clb35AT5gwQe+8844qKysPel1eXp7y8vLUq1ev/ZZfe+21+tvf/qbRo0frN7/5jVatWiVJWrp0qf70pz9p5syZWrRokf75z39Kkm655RZdeeWVWrx4sb73ve/td6hq5cqV+uCDD3Tffffphhtu0IMPPqjc3Fzde++9uummmyRJ06ZN029/+1tJ0saNG5Wenl73+rS0NG3cuPGg7CtXrtTHH3+sUaNGaezYsZo/f/5B67z88sv7lZSkpCSVl5eruLj4oHWPFiMpANBCHM2IR1Pau3evsrKyJNWOpFx33XWaM2eORo4cqbS0NElSVlaW8vPz1a5dOy1ZskRnnnmmJKm6ulpdunQ54nucdtppSkhIUEJCgtq2bavzzjtPkjR48GAtXrxY27dvV2lpqcaOHStJuuqqq3TxxRfXvX7ixImSpOHDhys/P/+g7W/fvl1XXXWVVq1aJTNrsGxs3rxZKSnf3Ky3oqJC06dP1z/+8Q8lJCRo1KhRmjFjhr797W9Lqv3h/cknnyg6OlqPP/642rdvv9/2srKylJeXpxkzZuiDDz7QiBEjNHfuXM2cOVMXXXSRkpOTJanudXPnztXrr78uSbriiivq5odI0sUXX6zw8HDt2rVLc+bM2e9rLy8vl1Q7wnH++edLqp3qcaCGzr6pqqpSSUmJPvvsM82fP1+XXHKJ8vLy6tb9/PPPFRcXp0GDBu33uo4dO2rTpk3q0KHDQds8GpQUAMBxiY2NrZs0WV/9EYfw8HBVVVXJOaeBAwdq7ty5B60fERFRN/n0wOtp1N9WWFhY3eOwsDBVVVUdMeO+9fflOND//M//6LTTTtMbb7yh/Pz8Bg/bxMbG7pfr3Xff1fbt2+tGjvbs2aO4uLi6knLppZfqoYceOmyuNm3aaOLEiZo4caLCwsI0ffp0RUZGNup03frrxMfHS5JqamrUrl27BvdHfWlpadqwYUPd44KCAnXt2rXB9SZOnCgz08iRIxUWFqaioqK6sjZ16tT9RlH2KSsrU2xs7BG/hiPhcA8AoNn07dtXhYWFdSWlsrJSS5culSRlZGQoNzdXkvTqq68e1Xbbtm2rpKQkffzxx5Kk559/vm5UpTG2b9+u1NRUSbVn5jSkT58++43CvPTSS3rqqaeUn5+v/Px8rV27VjNmzGj02U2ffvpp3VkwFRUVWrZsmbp3767x48fr3//+d93hkm3btkmSTjrpJE2dOlVS7dk1Y8aMOWibiYmJyszM1CuvvCKpdsRk0aJFB613/vnna+rUqSovL9fatWu1atUqjRw58qD1LrzwQs2cOVNS7aGfioqKuhGempoavfLKK7rsssv2e41zTl9//bUyMjIa9fdwOJQUAECziYqK0quvvqo77rhDQ4cOVVZWlubMmSNJ+vnPf65HH31UJ510koqKio5621OmTNEvfvELDRkyRAsXLqybf9EYv/zlL/WrX/1KJ598sqqrqxtcJz4+Xj179tTq1au1Z88evffee3WjJvueHzNmjN5+++1GveeaNWs0duxYDR48WMOGDVN2dra++93vauDAgbrrrrs0duxYDR06VD/96U8lSQ888ICeffbZuom0++aqHOiFF17Q008/raFDh2rgwIF66623JO0/J2XgwIG65JJLNGDAAJ199tl6+OGHFR4eLkm6/vrrlZOTI6l23kxeXp4GDRqkyy67TFOmTKkbwfnoo4+UlpamHj167Pf+ubm5OvHEExURcfwHa6yh41JNwcyekfQdSVudc4MaeN4k/VPSuZL2SLraOffFkbabnZ3t9v3lAUBrt3z5cvXv39/rGK3GG2+8odzc3P3O8MH+br/9dp1//vkaP378Qc819P1qZrnOueyGthXMkZTJks4+zPPnSOod+LhB0qNBzAIAwHGbMGFCkxzGaMkGDRrUYEE5FkErKc65jyRtO8wqF0h6LnAtl88ktTOzI0/xBgDAQ9dff73XEXztBz/4QZNty8s5KamSNtR7XBBY5onrp+To8ic/8+rtAeCYBeuwPdCUjuX71MuS0tD5VQ1+BWZ2g5nlmFlOYWFhUMKUV1Vrb2XDk6UAwK9iYmJUXFxMUYGvOedUXFysmJiYo3qdl9dJKZCUXu9xmqRNDa3onHtC0hNS7cTZYAXi3ziAUJOWlqaCggIF6xc4oKnExMTUXdyvsbwsKdMk3WJmUyWNkrTdObfZqzBm1vAwDgD4WGRkpDIzM72OAQRF0EqKmb0kaZykZDMrkPQ7SZGS5Jx7TNJ01Z5+vFq1pyBfE6wsjWESQykAAPhI0EqKc+7g6+Tu/7yTdHOw3v9omR1iQgwAAPAEV5wNMDGQAgCAn1BSAmrnpNBSAADwC0pKACMpAAD4CyUlwIySAgCAn1BS6nAKMgAAfkJJCagdSaGmAADgF5SUgIau0Q8AALxDSQlgTgoAAP5CSQkwcQoyAAB+QkkJYCQFAAB/oaQEcFl8AAD8hZISYDLO7gEAwEcoKfswkgIAgK9QUgJMoqUAAOAjlJSA2hsMAgAAv6CkBNTeYJCaAgCAX1BSAji7BwAAf6GkBNSOpHidAgAA7ENJCaidk0JLAQDALygpAYykAADgL5SUfbgsPgAAvhLhdQC/iAgzbSzdq4w7/5/XUVqsbw3opCeuzPY6BgAgRFBSAn5wSg91bhvrdYyWyTk9NjtPNQxVAQCOAiUloHenBP30zASvY7RIq7fu1AMzV+u0fh29jgIACCHMSUHQzfxqqyRpXF9KCgCg8SgpCLqZX21Vv84JSm3H4TQAQONRUhB0xbsqlBDDkUUAwNGhpCDovjs8TfPzS/TV1zu8jgIACCGUFATdZSPSFRMZpilz8r2OAgAIIZQUBF27uChNGJaqNxZsVPGuclXXOFXXOO46DQA4LCYKoFlcdVKGXpq3QcPv+aBuWVZ6O73+o5MUFmYeJgMA+BUlBc2iX+dE3X9pltZv2yNJ2liyVy/nbNAHy7foWwM7e5wOAOBHlBQ0mwuHpdZ9XlVdozl5RXp41hqdOaCTzBhNAQDsjzkp8EREeJh+eGpPLdpQqrlrir2OAwDwIUoKPHPR8DSlJETrkVlrvI4CAPAhSgo8ExMZruvHZOqT1UX6Yn2J13EAAD5DSYGnvn9id7WPj9ID/13ldRQAgM9QUuCp+OgIXX9KpmatKNTCDaVexwEA+AglBZ67cnSG2sVFMpoCANgPJQWeaxMdoR+c0kMzv9qqRYymAAACKCnwhatOylBSXKTunbHC6ygAAJ+gpMAX2kRH6EfjeurjVUX6PI/rpgAAKCnwkStHZ6hjQrTunbGCmw8CACgp8I+YyHDdenovzc8v0eyVhV7HAQB4jJICX7l0RDelJcXqf99doZoaRlMAoDWjpMBXoiLC9Iuz+mrZ5h16a9FGr+MAADxESYHvnDekqwalJure91aqrLLa6zgAAI9QUuA7YWGmX53TXxtL9+r5ueu8jgMA8AglBb50cq9kje2TogdnrlLJ7gqv4wAAPEBJgW/9+tz+2lVepX9yuXwAaJUoKfCtvp0TdPmobnr+s3VatWWn13EAAM2MkgJf+8kZfRQXFa57/t9yr6MAAJoZJQW+1qFNtG4f31uzVxbqw6+2eh0HANCMKCnwvStHZ6hHcrzufnupyqs4JRkAWgtKCnwvKiJMvz9/oPKL9+jJj/K8jgMAaCaUFISEU/uk6JxBnfXQh6tVULLH6zgAgGZASUHI+M13Bshk+uM7y7yOAgBoBpQUhIzUdrG65fReem/pFi3dtN3rOACAIKOkIKT06thGklRT43EQAEDQUVIQUrbuLJckdUyM9jgJACDYKCkIKVt3lCnMpA7xUV5HAQAEGSUFIWXrjnJ1aBOtiHC+dQGgpeN/eoSUrTvL1IlDPQDQKlBSEFK27ChXx4QYr2MAAJoBJQUhZevOckZSAKCVoKQgZFRV16h4d7lSGEkBgFaBkoKQUbSrQs6JkRQAaCUoKQgZW3eWSRJzUgCglaCkIGRs2VF7ITdGUgCgdaCkIGQwkgIArQslBSFjy45ymUnJbbjaLAC0BpQUhIzCnWXqEM/VZgGgteB/e4SMrTvK1TGB+SgA0FpQUhAytnBJfABoVSgpCBlbuSQ+ALQqlBSEBOecSvdWysl5HQUA0EwoKQgJZqYJWal67YuNWrJxu9dxAADNgJKCkPHrc/urfXyUfvnqYlVW13gdBwAQZJQUhIy2cZH64wWDtGzzDj35cZ7XcQAAQRbhdQDgaJw9qLPOHdxZ93+wShu27ZXZ0W+jfVyUbjqtp+Ki+PYHAD8L6v/SZna2pH9KCpf0lHPurwc831bSvyR1C2S51zn3bDAzIfT9/vyBKijZq/eXbTmm1xfvLtfqrbv0yPdOUFjYMbQcAECzCFpJMbNwSQ9LOlNSgaT5ZjbNObes3mo3S1rmnDvPzFIkrTCzF5xzFcHKhdDXMSFG024Zc8yvf/qTtfrjO8v09/dX6udn9W3CZACAphTMOSkjJa12zuUFSsdUSRccsI6TlGBmJqmNpG2SqoKYCdC1J2foshHpeujD1XpzwUav4wAADiGYJSVV0oZ6jwsCy+p7SFJ/SZskfSnpduccp20gqMxMf7hgkEZlttcvX1us3HUlXkcCADQgmCWloYP9B16J6yxJCyV1lZQl6SEzSzxoQ2Y3mFmOmeUUFhY2fVK0OlERYXrs+8PVpW2MbnguR+uKd3sdCQBwgGCWlAJJ6fUep6l2xKS+ayS97mqtlrRWUr8DN+Sce8I5l+2cy05JSQlaYLQuSfFReubqEap2Tlc/O1/Fu8q9jgQAqCeYJWW+pN5mlmlmUZIukzTtgHXWSxovSWbWSVJfSVwAA82mZ0obPX1VtjaV7tV1U3K0t6La60gAgICglRTnXJWkWyS9J2m5pH8755aa2Y1mdmNgtT9KOsnMvpT0X0l3OOeKgpUJaMjw7u31wKRhWlxQqltf+kJVXM0WAHzBnAutG7ZlZ2e7nJwcr2OgBXr+s3X6nzeXaNLIdP15wmDZsVwpDgBwVMws1zmX3dBzXHITCLjixO76evtePfzhGiXGRurOs/tRVADAQ5QUoJ6ff6uvduyt0uOz85QQHaFbTu/tdSQAaLUoKUA9Zqa7zx+o3eVVunfGSsVHR+iakzO9jgUArRIlBThAWJjpfy8aot0VVbr77WWKj47QJdnpR34hAKBJBfMUZCBkRYSH6YFJw3RK72Td+dpi5RXu8joSALQ6lBTgEKIjwvW/Fw1RjZPeW3psd1wGABw7SgpwGF3axmpQaqLeX/a111EAoNWhpABHcGb/zlqwoVSFO7lsPgA0J0oKcARnDOgo56SZX3HIBwCaEyUFOIIBXRKV2i5W7y/b6nUUAGhVKCnAEZiZzhzQSZ+sLuQGhADQjCgpQCOc0b+Tyipr9PGqQq+jAECrQUkBGmFUj/ZKiInQB8uZlwIAzYWSAjRCZHiYhndP0ry127yOAgCtBiUFaITd5VXKyS/R8O7tvY4CAK0GJQVohHcWb9Ku8ipdPop7+ABAc6GkAI3w4rwN6tOpjU7oluR1FABoNSgpwBEs3bRdizaUatLIbjIzr+MAQKtBSQGO4MXP1ys6IkwTh6V5HQUAWhVKCnAYu8qr9OaCjfrOkK5qGxfpdRwAaFUoKcBhTFu4SbsrqnX5qG5eRwGAVifC6wCAXznn9MLn65SWFKukuEitLdrtdSRJkknq1j5OYWHMjwHQslFSgEMo2lWhpZt2SJJOv2+2x2n2d0b/jnriimyKCoAWjZICHEJymyg9d+1Ibdtd4XWU/Xz19U49NnuNHp29Rjef1svrOAAQNJQU4BDMTKf2SfE6xkGcc9pUulf3zVihYd3a6aSeyV5HAoCgYOIsEGLMTH+ZOFiZyfG67aWF2rqjzOtIABAUlBQgBMVHR+jR7w/X7vIq3T51oZxzXkcCgCZHSQFCVJ9OCbptfG/NzSvW14ymAGiBKClACBuRUXsvoWWBs5AAoCWhpAAhrF+XREnS8s2UFAAtDyUFCGFtoiPUrX2cllFSALRAlBQgxA3okqjlm3d6HQMAmhwlBQhx/bskKr94t3aXV3kdBQCaFCUFCHH9uyTIudor0QJAS0JJAULcgK5MngXQMlFSgBCX2i5WiTERTJ4F0OJQUoAQZ2bq3yWx7o7NANBSUFKAFmBwalst37xDldU1XkcBgCZDSQFagMFpbVVRVaNVW3Z5HQUAmgwlBWgBBqW2lSQt2bjd4yQA0HQoKUALkNkhXvFR4VqyiZICoOWgpAAtQFiYaWDXtvqSkRQALQglBWghBgUmz1YxeRZAC0FJAVqIwWmJKqus0epCJs8CaBkoKUALMTgweTZ3XYnHSQCgaVBSgBYiM7mN+nZK0F//85VWcB8fAC0AJQVoIcLDTM9cM0KxkeG6+tl52rx9r9eRAOC4UFKAFiS1XawmXzNSO8uqdPUz87V9b6XXkQDgmFFSgBZmQNdEPfb94VpTuEs/fD5H5VXVXkcCgGNCSQFaoDG9k/V/Fw/RZ3nb9KvXv5RzzutIAHDUIrwOACA4JgxL0/rivfrHByvVt1OCfji2p9eRAOCoUFKAFuy28b20cutO/fXdr9SrYxuN79/J60gA0Ggc7gFaMDPTvRcN1cCuibrtpQWcmgwgpFBSgBYuNipcT16ZrfjoCF3/3Hxt213hdSQAaBRKCtAKdGkbqyeuzNaWHeW68V+5qqji/j4A/I+SArQSWent9H8XDdG8tdv027eWcMYPAN9j4izQilyQlaqVW3bq4Q/XqG/nBF1zcqbXkQDgkBhJAVqZn53ZV98a0El/fGeZPlpZ6HUcADgkSgrQyoSFmf5xaZb6dErQzS9+oTWFu7yOBAANoqQArVB8dISeuipbUeFhun5Kjkr3cMYPAP+hpACtVFpSnB6/Yrg2luzVzS9+ocpqzvgB4C+UFKAVy85orz9NGKRPVxfrnneWeR0HAPbD2T1AK3dxdrpWbd2lJz7K06DUtro4O93rSAAgiZEUAJLuOLufBqe21eQ5+V5HAYA6lBQACg8znTe0i5Zu2qGCkj1exwEASZQUAAFnDugsSXp/2RaPkwBALUoKAElSZnK8+nRqoxlLKSkA/IGSAqDOtwZ01rz8bSrhTskAfICSAqDOmQM6qbrGaeZXW72OAgCUFADfGJzaVp0TYzRj2ddeRwEASgqAb4SFmc4c0EkfrSxSWWW113EAtHKUFAD7+dbATtpbWa1pizZ5HQVAK0dJAbCfE3t00ODUtrrztcWa/Olar+MAaMUoKQD2Exkepqk3nKjT+3XS799ept++tURV3HwQgAcoKQAOEh8docevGK4fnJKp5+au07VTcrSjrNLrWABaGUoKgAaFh5nu+vYA/WXiYM1ZXaTvPjJHG7ZxyXwAzYeSAuCwJo3spinXjtSWHWW69aUFXscB0IpQUgAc0cm9knXb+N5auKFUK7fs9DoOgFaCkgKgUSYMS1VEmOmVnA1eRwHQSlBSADRKhzbROr1fR72xYKMqOdsHQDMIakkxs7PNbIWZrTazOw+xzjgzW2hmS81sdjDzADg+l2Snq2hXhT7k3j4AmkHQSoqZhUt6WNI5kgZImmRmAw5Yp52kRySd75wbKOniYOUBcPzG9U1RcptovZJb4HUUAK1AMEdSRkpa7ZzLc85VSJoq6YID1rlc0uvOufWS5Jzj1zPAxyLCwzTxhFR9+NVWFe0q9zoOgBYumCUlVVL9GXYFgWX19ZGUZGazzCzXzK4MYh4ATeDi4WmqqnF6c8FGr6MAaOGCWVKsgWXugMcRkoZL+raksyT9j5n1OWhDZjeYWY6Z5RQWFjZ9UgCN1rtTgoamt9MrOQVy7sB/0gDQdIJZUgokpdd7nCbpwNuqFkh61zm32zlXJOkjSUMP3JBz7gnnXLZzLjslJSVogQE0zmUj0rViy079/JXF2l1e5XUcAC1UMEvKfEm9zSzTzKIkXSZp2gHrvCXpFDOLMLM4SaMkLQ9iJgBN4JLsdN12ei+9vqBA5z30iZZt2uF1JAAtUNBKinOuStItkt5TbfH4t3NuqZndaGY3BtZZLuldSYslzZP0lHNuSbAyAWga4WGmn36rr164bpR2llXpwkc+1fNz8zn8A6BJWaj9p5Kdne1ycnK8jgEgoGhXuX7270WavbJQ5wzqrL9OHKK2cZFexwIQIsws1zmX3dBzXHEWwHFJbhOtZ68eoV+f20/vL9uicx/4WJ/nFXsdC0ALQEkBcNzCwkw3nNpTr/7oJEWEmy578jPd884ylVVWex0NQAijpABoMlnp7TT9tlP0/VHd9dQna/WdBz/R4oJSr2MBCFGUFABNKj46Qn+8cJCeu3akdpVVacIjc/T391dyU0IAR42SAiAoTu2Tovd+cqouGNpVD/x3lS58+FOt+Hqn17EAhBBKCoCgaRsbqb9fmqXHvj9cX28v03ce/Fj3zVjBXBUAjUJJARB0Zw/qrBk/OVXfGdJVD85crXP++bHmrCnyOhYAn6OkAGgWHdpE6x+XZun560aqusbp8ic/1y9eWaSS3RVeRwPgU5QUAM3qlN4peu/Hp+pH43rq9QUbNf7vs/XGAm5WCOBglBQAzS42Klx3nN1P79w6Rt3ax+knLy/SFU/P0+qtTKwF8A1KCgDP9O+SqNd+dJLuPn+gFhWU6qz7P9bdby/V9r2VXkcD4AOUFACeCg8zXXVShmb9fJwuHZGuyXPyddq9s/Ti5+tVXcMhIKA1o6QA8IUObaL15wmD9c6tY9SrYxv9+o0vdd6Dn3AfIKAVo6QA8JWBXdvq5RtO1EOXD1Ppngpd+sRnuvnFL1RQssfraACaWYTXAQDgQGam7wzpqvH9Ounxj9bosdlr9P7SLbpidHfdfFovtY+P8joigGZgoXbaX3Z2tsvJyfE6BoBmtKl0r+7/YKVezS1QXFSEbji1h64bk6n4aH7PAkKdmeU657IbfI6SAiBUrN66U//33gq9t3SLkttE6dbTe2vSyG6KiuDINRCqDldS+JcNIGT06pigx6/I1hs3naReHdvod9OWavzfZ+nNBRtVw5lAQItDSQEQcoZ1S9JLPzhRU64dqcSYSP345YU694GPNf3LzZQVoAWhpAAISWamsX1S9PYtY/TApGGqqKrRTS98obPu/0hvLdyoquoaryMCOE6NmpNiZidL+r2k7qo9I8gkOedcj6CmawBzUgA0pLrGafqXm/XQzNVasWWnMjrE6abTemnCsFRFhvP7GOBXxz1x1ruvU/8AAB19SURBVMy+kvQTSbmSqvctd841+1WWKCkADqemxmnGsi166MNVWrJxh1LbxepH43rq4uw0RUeEex0PwAGaoqR87pwb1eTJjgElBUBjOOc0a0WhHpi5SgvWl6pTYrR+eGpPXTYyXXFRnLoM+EVTlJS/SgqX9Lqk8n3LnXNfNFXIxqKkADgazjnNWVOsB/67Sp+v3aa2sZG6fFQ3XTU6Q53bxngdD2j1mqKkfNjAYuecO/14wx0tSgqAY5W7bpue/mSt3l3ytcLM9O0hXXTdmEwNSWvndTSg1TpcSWnUmKdz7rSmjQQAzW949/Ya3r29Nmzbo8lz8vXy/A16a+Emjcxor2vHZOrMAZ0UHmZexwQQ0NiRlLaSfifp1MCi2ZL+4JzbHsRsDWIkBUBT2VlWqZfnb9Czn+ZrY+ledWsfp2tOztDF2elqwyX3gWbRFId7XpO0RNKUwKIrJA11zk1sspSNREkB0NSqqms0Y9kWPfVxnr5YX6qEmAhdNDxN3xvVTb06JngdD2jRmqKkLHTOZR1pWXOgpAAIpgXrS/TMp/l6d8lmVVY7jcxor0mj0nXOoC6KieQUZqCpHfecFEl7zWyMc+6TwAZPlrS3qQICgF8M65akB7slqWjXAL2WW6CX5q3XT15epLvfXqbvnpCmSSO7qVfHNl7HBFqFxo6kZKn2UE9b1V5tdpukq51zi4Ib72CMpABoTjU1TnPzivXi5+v13tKvVVXjNCqzvS4f1U1nD+rMBeKA43Tch3vqbShRkpxzO5oo21GjpADwSuHOcr0aGF1Zv22PkuIiddHwNF06Ip25K8AxOuaSYmbfd879y8x+2tDzzrm/N1HGRqOkAPBaTU3tBeJenLdOM5ZuUVWN0+DUtpowLFXnDe2qlIRoryMCIeN45qTEB/7kVwQACAgLM43pnawxvZNVuLNc0xZt0psLNuoP7yzTn6Yv1ym9kzVhWKq+NaCzYqM4HAQcq6M63OMHjKQA8KtVW3bqjQUb9dbCTdpYulfxUeE6a1BnTRyWptE9O3ChOKABTXEK8v9Kuke1Z/S8K2mopB875/7VlEEbg5ICwO9qapzm5W/TG19s1PQvN2tneZU6JUbrgqxUTRiWqv5dEr2OCPhGk10nxcwmSLpQ0k8kfeicG9q0UY+MkgIglJRVVuu/y7fqjQUbNWvFVlXVOA1Na6sp145Uu7gor+MBnjtcSQlr5DYiA3+eK+kl59y2JkkGAC1cTGS4vj2ki566Klvz7jpDvztvgJZu2qE//b/lXkcDfK+xJeVtM/tKUrak/5pZiqSy4MUCgJanfXyUrjk5Uz84tYdeyS3QnNVFXkcCfK1RJcU5d6ek0ZKynXOVknZLuiCYwQCgpbp9fG9ldIjTr974UmWV1V7HAXzrsCXFzE4P/DlR0mmSLgh8frakk4IfDwBanpjIcP154mCtK96j+z9Y5XUcwLeOdJ2UsZJmSjqvgeecpNebPBEAtAIn9UzWpdnpevLjPJ07uLOGpLXzOhLgO1wnBQA8sn1vpc76x0dKiInQ27eO4S7LaJWO++weM/uzmbWr9zjJzO5pqoAA0Bq1jY3UX787WKu27tI/PljpdRzAdxp7ds85zrnSfQ+ccyWqPR0ZAHAcxvXtqEkj0/XkR3nKXVfidRzAVxpbUsLNrO6OWWYWK4k7aAFAE/j1uf3VpW2sfvHKIu2pqPI6DuAbjS0p/1Lt9VGuM7NrJb0vaUrwYgFA65EQE6l7Lx6qtcW79du3lnodB/CNxl4nZd+9e/pLGijpj4FlAIAmMLpnB916Wi+9mlugNxYUeB0H8IUjnYJc33JJVc65D8wszswSnHM7gxUMAFqb28b31md52/SbN5YoKz1JmcnxXkcCPNXYs3t+IOlVSY8HFqVKejNYoQCgNYoID9P9l2UpMiJMt7z4hcqruBotWrfGzkm5WdLJknZIknNulaSOwQoFAK1V13ax+r+Lhmrpph36y/SvvI4DeKqxJaXcOVex74GZRaj2irMAgCZ25oBOuubkDE2ek6/3l23xOg7gmcaWlNlm9mtJsWZ2pqRXJL0dvFgA0LrdeU4/DUpN1C9eXaRNpXu9jgN4orEl5Q5JhZK+lPRDSdMl/SZYoQCgtYuOCNeDk05QZVWNbntpgaqqa7yOBDS7I5YUMwuT9KVz7knn3MXOuYsCn3O4BwCCKDM5Xn+aMFg560r09/e5bD5anyOWFOdcjaRFZtatGfIAAOq5cFiqJo1M1yOz1mj6l5u9jgM0q8ZeJ6WLpKVmNk/S7n0LnXPnByUVAKDO788fqBVf79TP/r1Imcnx6t8l0etIQLOwxhy1MbOxDS13zs1u8kRHkJ2d7XJycpr7bQHAU1t3lOm8hz5RVESYpt08RknxUV5HApqEmeU657Ibeu6wh3vMLMbMfizpYkn9JH3qnJu97yMIWQEADeiYGKPHvj9cW7aX65aXvmAiLVqFI81JmSIpW7Vn9Zwj6b6gJwIANGhYtyT9acIgfbq6WL9/e6k4fwEt3ZHmpAxwzg2WJDN7WtK84EcCABzKxdnpWl24S4/PzlNGh3hdf0oPryMBQXOkklK57xPnXJWZBTkOAOBI7jirn9YX79Gfpi9Xevs4nTWws9eRgKA40uGeoWa2I/CxU9KQfZ+b2Y7mCAgA2F9YmOkfl2ZpaFo73T51gRYXlHodCQiKw5YU51y4cy4x8JHgnIuo9znnwAGAR2Iiw/XkldlKbhOtayfnaH3xHq8jAU2usZfFBwD4TEpCtJ69eoQqq2t0xTOfq3BnudeRgCZFSQGAENa7U4KeuXqEtu4o11XPzNOOssojvwgIEZQUAAhxw7sn6dHvn6CVW3bqB1NyVFZZ7XUkoEk09rL4AAAfG9e3o+67ZKhun7pQN73whb49uMtRbyM2KlxnDuikyHB+f4U/UFIAoIW4ICtVpXsq9btpSzXzq63HtI0p147U2D4pTZwMODaUFABoQa46KUPnDu6ivRVHd8hn1dadum5KzlG/DggmSgoAtDApCdFH/Zo9lVWSpBoutQ8f4cAjAEDhgSuKV9VQUuAflBQAgMLDaktKDSUFPkJJAQDUlZRqSgp8hJICAKCkwJcoKQCAb0oKE2fhI5QUAEBdSWHiLPyEkgIAqDu7h4mz8BNKCgBAEWG1Pw4YSYGfUFIAAAp0FEZS4CtBLSlmdraZrTCz1WZ252HWG2Fm1WZ2UTDzAAAaxpwU+FHQLotvZuGSHpZ0pqQCSfPNbJpzblkD6/1N0nvBygIAOLx9JeXhD1drypz8Q65z9/kDdcaATs2YDK1ZMO/dM1LSaudcniSZ2VRJF0hadsB6t0p6TdKIIGYBABxGdES47ji7n/KLdh9ynQ9XbNUTH+VRUtBsgllSUiVtqPe4QNKo+iuYWaqkCZJOFyUFADz1o3E9D/v8o7PW6G/vfqW8wl3qkdKmmVKhNQvmnBRrYNmBBzvvl3SHc+6w9wY3sxvMLMfMcgoLC5ssIACg8b57QqrCw0yv5BZ4HQWtRDBLSoGk9HqP0yRtOmCdbElTzSxf0kWSHjGzCw/ckHPuCedctnMuOyUlJVh5AQCH0TExRqf1TdFruQWqqq7xOg5agWCWlPmSeptZpplFSbpM0rT6KzjnMp1zGc65DEmvSrrJOfdmEDMBAI7Dxdnp2rqzXB+tYlQbwRe0kuKcq5J0i2rP2lku6d/OuaVmdqOZ3Ris9wUABM/p/ToquU2UXp6/4cgrA8cpmBNn5ZybLmn6AcseO8S6VwczCwDg+EWGh2nCsFQ9+2m+inaVK7lNtNeR0IJxxVkAwFG5JDtdVTVOby7Y6HUUtHBBHUkBALQ8vTslaFi3dnpk1hptKi3TuL4pGpnZXjGR4V5HQwtDSQEAHLU/nD9I/zdjhf71+To98+laxUSGaXSPDhrXt6PG9klRRnK81xHRAphzoXWfhuzsbJeTk+N1DACApL0V1fpsbbFmryjU7JWFWhu4Ym1GhziN7ZOicX076sQeHRQbxSgLGmZmuc657Aafo6QAAJrKuuLdmr2yULNWFGrOmiKVVdYoKiJMozLb142y9EyJl1lD1/tEa0RJAQA0u7LKas3P36bZKwo1a2WhVm/dJUlKS4rV2D4punBYqkZktPc4JbxGSQEAeK6gZM83oyyri7S7olrfHtJFd53bX13bxXodDx6hpAAAfKWsslqPz87TI7NWK8xMN5/WU9ef0oMzhFqhw5UUrpMCAGh2MZHhuv2M3vrgp2M1tk+K7p2xUmfd/5H+u3yL19HgI5QUAIBn0tvH6bErhuv560YqIsx03ZQcXfPsvLqzhNC6cbgHAOALFVU1mjInX//87yqVV1WrT6cEZSTHK7NDvDKS45XRIU4ZyfHqEB/F2UEtCHNSAAAhY+uOMj396Vqt+Hqn8ot2a0PJXlXXfPOzKiE6ora07CsugRKTmRyvpLhICkyIOVxJ4YqzAABf6ZgYo1+d07/ucWV1jQpK9iq/aLfyi3crv2i31hbv0aINpfp/izepXn9RYkygwNQVl0CJ6RCvpPgoD74aHA9KCgDA1yLDw5QZGCk5UEVVjTaU7NG64t1aW7Snrsh8sb5E7xxQYNrGRgYOH8XtX2Q6xKttXGQzfkVoLEoKACBkRUWEqWdKG/VMaXPQc+VV1dqw7ZsRmLWBP+fnl+itRZtUf7ZDUlzkN8WlQ7zG9E7WCd3acejIY8xJAQC0OmWV1dqwbU9dcdk3CrOueLc2bS+TJA3okqgrRnfXBVldFRfF7/TBwsRZAAAaaVd5laYt3KTn5ubrq693KiE6Qt8dnqbvn9hdvToePGKD40NJAQDgKDnn9MX6Ej0/d52mf/m1KqprdFLPDrrixO46Y0AnRYZzqbGmQEkBAOA4FO0q179zNuiFz9ZrY+ledUqM1qSR3TRpZDd1SozxOl5Io6QAANAEqmucZq3Yquc/W6fZKwtlktpEN998lUtHpOuubw9otvdrDlwnBQCAJhAeZhrfv5PG9++kdcW79foXG7V9b2WzvPcLn6/T1zvKm+W9/IKSAgDAMejeIV4/ObNPs7zX1h1lmjwnX1np7Zrl/fyCWT8AAPjcwg2lkqSs9LYeJ2lelBQAAHxuUUGpIsJMA7tSUgAAgI8s3FCqfl0SFBMZ7nWUZkVJAQDAx2pqnBZv2K6haa1rPopESQEAwNfyinZrZ3mVhraySbMSJQUAAF/bN2l2GCUFAAD4yaINpWoTHaEeDdzpuaWjpAAA4GOLCko1JK2twsPM6yjNjpICAIBPlVVWa/nmHa1yPopESQEAwLeWbd6hymrXKs/skSgpAAD41sL1gUmz3SgpAADARxYVlKpzYow6JcZ4HcUTlBQAAHxq0YZSDW1l9+upj5ICAIAPleyuUH7xHmWlJ3kdxTOUFAAAfGhRQe18FEZSAACAryzcUCozaXAqJQUAAPjIZ3nFGtAlUQkxkV5H8QwlBQAAnymrrNYX60s1ukcHr6N4ipICAIDPfLG+RBVVNRrdk5ICAAB8ZO6aYoWZNCKzvddRPEVJAQDAZ+auKdbg1LZKbMXzUSRKCgAAvrKnokoLN5RqdM9kr6N4jpICAICPzM8vUVWNa/XzUSRKCgAAvjJ3TbEiwkzZ3VvvlWb3oaQAAOAjC9aXKCzM9KfpyzX9y80q2V3hdSTPRHgdAAAAfOOHY3vohc/Wa9rCTXrx8/UykwZ2TdTJvZJ1cs9kjchor9iocK9jNgtzznmd4ahkZ2e7nJwcr2MAABBUldU1WlxQqk9WFevTNUVasL5EldVOUeFhGt49SSf36qCTeyVrcGpbRYSH7oERM8t1zmU3+BwlBQAA/9tTUaV5a7fp09VF+nR1sZZt3iFJSoiJ0Ik9OmhMr2Sd3KuDeqa0kZl5nLbxDldSONwDAEAIiIuK0Li+HTWub0dJUvGucs1ZU6w5a4r0yeoivb9siySpU2K0Tu6VrBN7dNDw7knqkRwfUqWlPkZSAABoAdYX79GngcIyd02xtgUm3CbFRWp49yQN795e2RlJGpzaVjGR/pnTwkgKAAAtXLcOcerWoZsmjeymmhqnvKJdyskvUe662o8Plm+VJEWGmwalttXwbknKzkjSCd2T1DEhxuP0DWMkBQCAVmDb7oq6wpK7bpsWFWxXRVWNJKlb+7jAaEttcendMUHhYc1ziIiJswAAYD8VVTVasmm7cgOjLTnrSlS0q1ySlBAdoWHdk+pGW4amt1Ob6OAcfKGkAACAw3LOacO2vcpZt61uxGXFlp1yTooKD9OrPxqtIWntmvx9mZMCAAAOy8wC81riNPGENEnSjrJK/efLzbrjtS+1sWRvUErK4YTu1V8AAEBQJcZEamh68xaT+igpAADAlygpAADAlygpAADAlygpAADAlygpAADAlygpAADAlygpAADAlygpAADAlygpAADAlygpAADAlygpAADAlygpAADAlygpAADAlygpAADAlygpAADAlygpAADAlygpAADAlygpAADAlygpAADAlygpAADAlygpAADAlygpAADAl4JaUszsbDNbYWarzezOBp7/npktDnzMMbOhwcwDAABCR9BKipmFS3pY0jmSBkiaZGYDDlhtraSxzrkhkv4o6Ylg5QEAAKElmCMpIyWtds7lOecqJE2VdEH9FZxzc5xzJYGHn0lKC2IeAAAQQoJZUlIlbaj3uCCw7FCuk/SfIOYBAAAhJCKI27YGlrkGVzQ7TbUlZcwhnr9B0g2S1K1bt6bKBwAAfCyYIykFktLrPU6TtOnAlcxsiKSnJF3gnCtuaEPOuSecc9nOueyUlJSghAUAAP4SzJIyX1JvM8s0syhJl0maVn8FM+sm6XVJVzjnVgYxCwAACDFBO9zjnKsys1skvScpXNIzzrmlZnZj4PnHJP1WUgdJj5iZJFU557KDlQkAAISOYM5JkXNuuqTpByx7rN7n10u6PpgZAABAaOKKswAAwJcoKQAAwJcoKQAAwJcoKQAAwJcoKQAAwJcoKQAAwJcoKQAAwJcoKQAAwJcoKQAAwJcoKQAAwJcoKQAAwJcoKQAAwJcoKQAAwJcoKQAAwJcoKQAAwJcoKQAAwJcoKQAAwJcoKQAAwJcoKQAAwJcoKQAAwJcoKQAAwJcoKQAAwJcoKQAAwJcoKQAAwJcoKQAAwJcoKQAAwJcoKQAAwJcoKQAAwJcoKQAAwJcoKQAAwJcoKQAAwJcoKQAAwJcoKQAAwJcoKQAAwJcoKQAAwJcoKQAAwJcoKQAAwJcoKQAAwJcoKQAAwJcoKQAAwJcoKQAAwJcoKQAAwJcoKQAAwJcoKQAAwJcoKQAAwJcoKQAAwJcoKQAAwJcoKQAAwJcoKQAAwJcoKQAAwJcoKQAAwJcoKQAAwJcoKQAAwJcoKQAAwJcoKQAAwJcoKQAAwJcoKQAAwJcoKQAAwJcoKQAAwJcoKQAAwJcoKQAAwJcoKQAAwJcoKQAAwJcoKQAAwJcoKQAAwJcoKQAAwJcoKQAAwJcoKQAAwJcoKQAAwJcoKQAAwJcoKQAAwJcoKQAAwJcoKQAAwJcoKQAAwJcoKQAAwJcoKQAAwJcoKQAAwJcoKQAAwJcoKQAAwJcoKQAAwJeCWlLM7GwzW2Fmq83szgaeNzN7IPD8YjM7IZh5AABA6AhaSTGzcEkPSzpH0gBJk8xswAGrnSOpd+DjBkmPBisPAAAILcEcSRkpabVzLs85VyFpqqQLDljnAknPuVqfSWpnZl2CmAkAAISIYJaUVEkb6j0uCCw72nUAAEArFMySYg0sc8ewjszsBjPLMbOcwsLCJgkHAACOrGu7WD1+xXCd0D2p2d87mCWlQFJ6vcdpkjYdwzpyzj3hnMt2zmWnpKQ0eVAAANCwxJhInTWwszolxjT7ewezpMyX1NvMMs0sStJlkqYdsM40SVcGzvI5UdJ259zmIGYCAAAhIiJYG3bOVZnZLZLekxQu6Rnn3FIzuzHw/GOSpks6V9JqSXskXROsPAAAILQEraRIknNuumqLSP1lj9X73Em6OZgZAABAaOKKswAAwJcoKQAAwJcoKQAAwJcoKQAAwJcoKQAAwJcoKQAAwJcoKQAAwJcoKQAAwJcoKQAAwJcoKQAAwJcoKQAAwJcoKQAAwJcoKQAAwJcoKQAAwJcoKQAAwJcoKQAAwJcoKQAAwJfMOed1hqNiZoWS1nmd4zCSJRV5HQJHjf0WmthvoYn9FpqCtd+6O+dSGnoi5EqK35lZjnMu2+scODrst9DEfgtN7LfQ5MV+43APAADwJUoKAADwJUpK03vC6wA4Juy30MR+C03st9DU7PuNOSkAAMCXGEkBAAC+REk5BmZ2tpmtMLPVZnZnA89/z8wWBz7mmNlQL3Jif0fab/XWG2Fm1WZ2UXPmQ8Mas9/MbJyZLTSzpWY2u7kz4mCN+H+yrZm9bWaLAvvtGi9yYn9m9oyZbTWzJYd43szsgcB+XWxmJwQ1D4d7jo6ZhUtaKelMSQWS5kua5JxbVm+dkyQtd86VmNk5kn7vnBvlSWBIatx+q7fe+5LKJD3jnHu1ubPiG43899ZO0hxJZzvn1ptZR+fcVk8CQ1Kj99uvJbV1zt1hZimSVkjq7Jyr8CIzapnZqZJ2SXrOOTeogefPlXSrpHMljZL0z2D+fGMk5eiNlLTaOZcX+Mc0VdIF9Vdwzs1xzpUEHn4mKa2ZM+JgR9xvAbdKek0SP+T8oTH77XJJrzvn1ksSBcUXGrPfnKQEMzNJbSRtk1TVvDFxIOfcR6rdF4dygWoLjHPOfSapnZl1CVYeSsrRS5W0od7jgsCyQ7lO0n+CmgiNccT9ZmapkiZIeqwZc+HwGvPvrY+kJDObZWa5ZnZls6XDoTRmvz0kqb+kTZK+lHS7c66meeLhOBztz8DjEhGsDbdg1sCyBo+Zmdlpqi0pY4KaCI3RmP12v6Q7nHPVtb/cwQcas98iJA2XNF5SrKS5ZvaZc25lsMPhkBqz386StFDS6ZJ6SnrfzD52zu0Idjgcl0b/DGwKlJSjVyApvd7jNNX+JrAfMxsi6SlJ5zjnipspGw6tMfstW9LUQEFJlnSumVU5595snohoQGP2W4GkIufcbkm7zewjSUNVOycC3mjMfrtG0l9d7cTI1Wa2VlI/SfOaJyKOUaN+BjYVDvccvfmSeptZpplFSbpM0rT6K5hZN0mvS7qC3+Z844j7zTmX6ZzLcM5lSHpV0k0UFM8dcb9JekvSKWYWYWZxqp3Mt7yZc2J/jdlv61U7+iUz6ySpr6S8Zk2JYzFN0pWBs3xOlLTdObc5WG/GSMpRcs5Vmdktkt6TFK7aM0CWmtmNgecfk/RbSR0kPRL4rbyKm2l5q5H7DT7TmP3mnFtuZu9KWiypRtJTzrkGT59E82jkv7c/SppsZl+q9hDCHc457ozsMTN7SdI4SclmViDpd5Iipbr9Nl21Z/aslrRHtSNiwcvDKcgAAMCPONwDAAB8iZICAAB8iZICAAB8iZICAAB8iZICAAB8iZICIKgCd5ReaGZLAne9bdfE2883s+TA57uactsAvEVJARBse51zWYE7qm6TdLPXgQCEBkoKgOY0V4GbkZlZTzN7N3BTwI/NrF9geScze8PMFgU+TgosfzOw7lIzu8HDrwFAM+GKswCahZmFq/Yy6E8HFj0h6Ubn3CozGyXpEdXebO4BSbOdcxMCr2kTWP9a59w2M4uVNN/MXuO+WEDLRkkBEGyxZrZQUoakXNXe7baNpJMkvVLvjtPRgT9Pl3SlJDnnqiVtDyy/zcwmBD5Pl9RbEiUFaMEoKQCCba9zLsvM2kp6R7VzUiZLKnXOZTVmA2Y2TtIZkkY75/aY2f9v7w5RMgrCAIreQYNJtyEYrFZXYBIEo8mixcVZBKNF5BfMLkFMxmfwbcHHhHPiMGGmXYaB77k6+J/jArPwJwXYxLIs39Vd9VD9VJ9jjMuqdaLq6br1qbpd1/fGGIfVUfW1Bspxdbb5BYDNiRRgM8uyvFW76qq6rm7GGLvqo7pYt91X5+t03NfqpHqs9scY7/1Nz33Z+uzA9kxBBgCm5CUFAJiSSAEApiRSAIApiRQAYEoiBQCYkkgBAKYkUgCAKYkUAGBKvxzwLwaPCUbSAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 648x648 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_pr(performances['ground_truth'],performances['probability'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "I'm going to calculate F1 score for two different scenarios here, one where I choose a threshold that favors precision, and one with a threshold that favors recall: "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "precision, recall, thresholds = precision_recall_curve(performances['ground_truth'],performances['probability'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision is: 0.8043478260869565\n",
      "Recall is: 0.37373737373737376\n",
      "Threshold is: 0.79\n",
      "F1 Score is: 0.5103448275862069\n"
     ]
    }
   ],
   "source": [
    "# Look at the threshold where precision is 0.8\n",
    "precision_value = 0.8\n",
    "idx = (np.abs(precision - precision_value)).argmin() \n",
    "print('Precision is: '+ str(precision[idx]))\n",
    "print('Recall is: '+ str(recall[idx]))\n",
    "print('Threshold is: '+ str(thresholds[idx]))\n",
    "print('F1 Score is: ' + str(calc_f1(precision[idx],recall[idx])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision is: 0.398989898989899\n",
      "Recall is: 0.797979797979798\n",
      "Threshold is: 0.39\n",
      "F1 Score is: 0.531986531986532\n"
     ]
    }
   ],
   "source": [
    "# Look at the threshold where recall is 0.8\n",
    "recall_value = 0.8\n",
    "idx = (np.abs(recall - recall_value)).argmin() \n",
    "print('Precision is: '+ str(precision[idx]))\n",
    "print('Recall is: '+ str(recall[idx]))\n",
    "print('Threshold is: '+ str(thresholds[idx]))\n",
    "print('F1 Score is: ' + str(calc_f1(precision[idx],recall[idx])))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "As we can see, the thresholds are really different for the two cases. Let's look at what the _accuracy_ of our model would be using those two values, and think about why accuracy really isn't a great performance statistic when evaluating and interpreting the utility of our models. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "probs = performances['probability']\n",
    "t1 = (probs > 0.79)\n",
    "t2 = (probs > 0.39)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "t1 = t1.replace(True,1).replace(False,0)\n",
    "t2 = t2.replace(True,1).replace(False,0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "compare_t1 = (t1 == performances['ground_truth'])\n",
    "compare_t2 = (t2 == performances['ground_truth'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('Accuracy at threshold 1: ' + str(len(compare_t1[compare_t1])/len(performances)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('Accuracy at threshold 2: ' + str(len(compare_t2[compare_t2])/len(performances)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}