{

 "cells": [

  {

   "cell_type": "code",

   "execution_count": 24,

   "metadata": {},

   "outputs": [],

   "source": [

    "import json\n",

    "import numpy as np\n",

    "\n",

    "from pylab import rcParams\n",

    "from sklearn.metrics import accuracy_score, confusion_matrix, precision_score, recall_score, balanced_accuracy_score, f1_score\n",

    "import matplotlib.pyplot as plt\n",

    "import seaborn as sns\n",

    "from matplotlib.colors import ListedColormap\n",

    "\n",

    "rcParams['figure.figsize'] = 8, 8"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": 2,

   "metadata": {},

   "outputs": [],

   "source": [

    "preds = np.loadtxt('../experiments/EcgResNet34/results/predictions.txt').astype(int)\n",

    "mapping = json.load(open('../data/class-mapper.json'))\n",

    "gt = [mapping[i['label']] for i in json.load(open('../data/val.json'))]"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": 44,

   "metadata": {},

   "outputs": [

    {

     "data": {

      "text/plain": [

       "0.9938449701865744"

      ]

     },

     "execution_count": 44,

     "metadata": {},

     "output_type": "execute_result"

    }

   ],

   "source": [

    "accuracy_score(gt, preds)"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": 45,

   "metadata": {},

   "outputs": [

    {

     "data": {

      "text/plain": [

       "0.9938449701865744"

      ]

     },

     "execution_count": 45,

     "metadata": {},

     "output_type": "execute_result"

    }

   ],

   "source": [

    "recall_score(gt, preds, average='micro')"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": 53,

   "metadata": {},

   "outputs": [

    {

     "data": {

      "text/plain": [

       "0.9938449701865744"

      ]

     },

     "execution_count": 53,

     "metadata": {},

     "output_type": "execute_result"

    }

   ],

   "source": [

    "precision_score(gt, preds, average='micro')"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": 46,

   "metadata": {},

   "outputs": [

    {

     "data": {

      "text/plain": [

       "0.9715227374777318"

      ]

     },

     "execution_count": 46,

     "metadata": {},

     "output_type": "execute_result"

    }

   ],

   "source": [

    "balanced_accuracy_score(gt, preds)"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": 47,

   "metadata": {},

   "outputs": [

    {

     "data": {

      "text/plain": [

       "0.9715227374777318"

      ]

     },

     "execution_count": 47,

     "metadata": {},

     "output_type": "execute_result"

    }

   ],

   "source": [

    "np.mean(recall_score(gt, preds, average=None))"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": 49,

   "metadata": {},

   "outputs": [

    {

     "data": {

      "text/plain": [

       "0.9715227374777318"

      ]

     },

     "execution_count": 49,

     "metadata": {},

     "output_type": "execute_result"

    }

   ],

   "source": [

    "recall_score(gt, preds, average='macro')"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": 54,

   "metadata": {},

   "outputs": [

    {

     "data": {

      "text/plain": [

       "0.9787018706519426"

      ]

     },

     "execution_count": 54,

     "metadata": {},

     "output_type": "execute_result"

    }

   ],

   "source": [

    "f1_score(gt, preds, average='macro')"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": 22,

   "metadata": {},

   "outputs": [

    {

     "data": {

      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAeEAAAHVCAYAAADGoUO1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3Hl4VPXZxvH7RxKEAAHBLGqi7FVBXEAURdlEQNkCqOD2WhUEdy0VKBVxBUWtbVUkCFq1CLigUiW4sEVQKaAV6o4KhZpFEUFCIJn83j/AFBDCdiZP5vj9XFcumTNz5tyPZ2ZuzpkTnPdeAACg4lWxDgAAwK8VJQwAgBFKGAAAI5QwAABGKGEAAIxQwgAAGKGEAQAwQgkDAGCEEgYAwEh8tDfgOqeH+p/k2pz9uXUEAEAlUy0u0e3L4zgSBgDACCUMAIARShgAACOUMAAARihhAACMUMIAABihhAEAMEIJAwBghBIGAMAIJQwAgBFKGAAAI5QwAABGKGEAAIxQwgAAGKGEAQAwQgkDAGCEEgYAwAglDACAEUoYAAAjlDAAAEYoYQAAjFDCAAAYqfQl3DS9oT54fHbZz48vf6IbM68su/+WfoPk31yjekmHSpLatWij9S9/XPb42y65qeyxN2ReqeVZb2nFxLd3eo5YMGrkaLVv21F9evazjhK4b77+RhdkXlj2c/opbfXs03+3jhWohTkL1fPc3urepacmTZxsHSdwYZtvd++3N7LfVGaPvjqx2cn694p/G6YLVtj23a4q+3yVvoQ/X/OVThrcRScN7qKW13RT4ZbNmrEwW5KUnny4zml5llblrdlpnZzli8vWuevZhyVJzer/RgO7DVDr67vrhKvPUffTzlajI+pX9DgHrFdmD43PetQ6RlTUb1Bf02dM0/QZ0/TcC1NUrVo1dezUwTpWYCKRiO69e6wem/CIZsx8UdmvZ2vllyutYwUmjPPt7v3WuEkj/ekvD6plq5ONUgUvjPtuR7EwX6Uv4R11OqmtVn67Sqvz10qS/jR4tG6deI+893td99ijGuv9Tz/U5i1FipRGNP+j99SnbbdoRw5My1YtlVS7tnWMqHv/vcXKOCpdRxx5hHWUwKxYvkIZR2UoPSNdCVUT1LVbF82bM886VmDCON/u3m8NGzVU/Qb1TfJESxj33Y5iYb748u50zo0q527vvb8r4Dzl6t++p56b+4okqWebc7T2+1x99NUnv3hcm+Na6sPH39B/v8/T0Ky79PGqz7Xim890z2+HqW6tOtq8tUjntu6oJZ9/VJHxsQ+yX5+trud2tY4RqPy8fKWlpZbdTklL1fKPVhgmClbY5wuzsO+7WJiv3BKWtGk3yxIlXSWpnqTdlrBzbpCkQZKkY+pI6TUOIuI2CfEJ6tnmHI2YNFbVD6mmPwy4XucMv+gXj1v25XIdffGp2lRUqG6tO+rlOyap6eVn6tPVX+q+aY/pjbFTtKmoUB+u/LcipZGDzoXgFG8t1vy583XjzddbRwGAClHu6Wjv/YM//0jKklRd0hWSpkpqWM56Wd77Vt77VkEUsCR1O6WDln25XPnrv1Ojw+urQVqG/jXhDX39zLtKTz5cy8ZnK/XQZG0s/EmbigolSbMWz1FCXHzZRVuTs6eq1bXnqt3v+umHn37U52u+CiQbgvFOzjs65rhjVO+wetZRApWSmqLc3Lyy2/m5eUpNSTZMFKywzxdmYd93sTDfXr8Tds7Vdc7dLekjbTtyPtl7P8x7nx/1dDsY0KFX2anoFd98qtQLTlSDS9uowaVttKbgW508pKvyfihQ6qH/+x98ym9OVJUqVfT9hh8kScl1tn24ZyQfoT5ndNOUOS9X5AjYi1mvZ6tbyE5FS1Kz5s20etVqrVmzVsVbi5U9a7badWhvHSswYZ8vzMK+72Jhvr19JzxOUh9tOwo+3nv/U4Wk2kViterq3PIsXf3w8L0+tt9Z52lI90tVEolo89Yi9b/nmrL7XhyVpXpJh6q4pETXPjJSP27aEM3YgRo2dLiWLF6q9evXq3OHLhpy3WD16ZtpHSswhYWb9d6i93Xb6D9aRwlcfHy8RowcpiEDr1Fpaal6Z/ZS4yaNrGMFJozz7e79Vrt2bY295z79sO4HXTfkBv3mmN/o8YmPWUc9KGHcdzuKhflceVcWO+dKJW2RVCJpxwc6bbswK2mvG+icvvdLl2PY5uzPrSMAACqZanGJbl8eV+6RsPc+pn6FCQCAWELJAgBghBIGAMAIJQwAgBFKGAAAI5QwAABGKGEAAIxQwgAAGKGEAQAwQgkDAGCEEgYAwAglDACAEUoYAAAjlDAAAEYoYQAAjFDCAAAYoYQBADBCCQMAYIQSBgDACCUMAIARShgAACOUMAAARpz3PqobKIoURncDxn7cus46QtTUrlrXOgIAxKRqcYluXx7HkTAAAEYoYQAAjFDCAAAYoYQBADBCCQMAYIQSBgDACCUMAIARShgAACOUMAAARihhAACMUMIAABihhAEAMEIJAwBghBIGAMAIJQwAgBFKGAAAI5QwAABGKGEAAIxQwgAAGKGEAQAwQgkDAGCEEgYAwEhoSnjLli266MJLdH7mBcrs0VeP/XW8daQDMv2ZF3RZnyv0f32v1B3D79aWLVv137Xf6upLrtWAHpfq9lvvUnFxsSTpledn6v/6XaUrLhikay+/Ud+s/MY2/EFYmLNQPc/tre5demrSxMnWcQLHfLEtzPONGjla7dt2VJ+e/ayjREVl33ehKeGqVavqiclZen7GdE1/aaoWvrNIH/3rI+tY+6Ugr0AvPDdDE6eM199enKTSSKnmZM/RhIcn6oJL+uq5mc+oVlJNvTZjliTp7G4d9bcXntDk6VkacPmFeuTBx40nODCRSET33j1Wj014RDNmvqjs17O18suV1rECw3yxLezz9crsofFZj1rHiIpY2HehKWHnnBJrJEqSSkpKVFJSIsnZhjoAkUhEW7ZsUUlJREVFRaqXXE/L/vmB2p3dTpLUtcc5ypm7UJJUo2aNsvWKNhfJxd64kqQVy1co46gMpWekK6Fqgrp266J5c+ZZxwoM88W2sM/XslVLJdWubR0jKmJh3x1wCTvnbgoySBAikYguyLxQHdp20mmnn6YWJxxvHWm/JKcmq/9l5+v8rgOU2fl81ahZU02PbaqatWoqPj6u7DHf5X9Xts5LU19W/+6XaPzDWbrh1uusoh+U/Lx8paWllt1OSUtVXn6BYaJgMV9sC/t8YRYL++5gjoRv2dMdzrlBzrklzrklFXkOPi4uTtNnTNMbc2drxfIV+uKLLyts20HYuGGj3pm3SNNe+7tmvDFdRZs3a/GixeWu06d/b039x7MafONAPT3x2QpKCgAIQvxBrLvHk5/e+yxJWZJUFCn0B7GNA5KUVEuntG6lRTmL1KRJ44re/AFb8t4yHX5kmurUrSNJOqvTmVr+4b/108afVFISUXx8nAryCnRYymG/WLdT1w566N4/V3TkQKSkpig3N6/sdn5unlJTkg0TBYv5YlvY5wuzWNh3B3MkXOHlWp5169Zpw4aNkqSioiK9t+h91W9Y3zbUfko9PEUff/SJijYXyXuvpe8vU/2GR+ukVidq/lvzJUnZM99Q2/anS5L+s2pN2brv5ryn9KOONMl9sJo1b6bVq1ZrzZq1Kt5arOxZs9WuQ3vrWIFhvtgW9vnCLBb2XblHws65jdp92TpJ1aOS6AB9V/Cd/jhilEpLS1VaWqpzunZWu/ZnWcfaL8cdf6zan32WrhowWHFxcWpyTGP16Hue2px5qkYPu1tPPPqkmvymsc7L7CZp2/fBS99fpvj4eNVKqqk/3DnMeIIDEx8frxEjh2nIwGtUWlqq3pm91LhJI+tYgWG+2Bb2+YYNHa4li5dq/fr16tyhi4ZcN1h9+mZaxwpELOw75310D2gtTkdXpB+3rrOOEDW1q9a1jgAAMalaXOI+/b5KaH5FCQCAWEMJAwBghBIGAMAIJQwAgBFKGAAAI5QwAABGKGEAAIxQwgAAGKGEAQAwQgkDAGCEEgYAwAglDACAEUoYAAAjlDAAAEYoYQAAjFDCAAAYoYQBADBCCQMAYIQSBgDACCUMAIARShgAACOUMAAARuKtA8S62lXrWkeImo3FP1pHiKpaCbWtIwD4leNIGAAAI5QwAABGKGEAAIxQwgAAGKGEAQAwQgkDAGCEEgYAwAglDACAEUoYAAAjlDAAAEYoYQAAjFDCAAAYoYQBADBCCQMAYIQSBgDACCUMAIARShgAACOUMAAARihhAACMUMIAABihhAEAMEIJAwBghBIGAMBIvHWAgzFq5GgtmL9AdevW1UuvviBJemjcnzR/3gIlJCQoPSNdd95zh5KSahknPXi7mzUWbdywUffdMU5fffm1nHMacccwHVLtED1w90PaXLhZaUek6fYxf1SNmjX0xmtvasrfppatu/LzrzR5apaaHNPEcIIDszBnoe4bM06lkVJl9uutKwdeYR0pUGGeLyzvvT0J+3yV/bW5xyNh59yjzrkzKjLM/uqV2UPjsx7dadlpp5+mF195Xi+8PF1H1z9akyZONkoXrN3NGov+fP8jOvWM1pryyjN66vlJOrrBUbrvjnEafOMgPf3ikzqr45ma8tS24j3nvM56avokPTV9km67Z6QOP/LwmCzgSCSie+8eq8cmPKIZM19U9uvZWvnlSutYgQn7fGF57+1JmOeLhddmeaejP5f0gHPuG+fc/c65kyoq1L5q2aqlkmrX3mnZ6We0UXz8tgP8Ficcr/zcPItogdvdrLHmp40/6V9L/6XumedJkhISElQrqZb+s2qNTmx5giTplDatNP/tBb9Y961Zb6tT144VmjcoK5avUMZRGUrPSFdC1QR17dZF8+bMs44VmLDPF4b3XnnCPF8svDb3WMLe+z9779tIaifpe0mTnXOfOudud841rbCEB+Hll17RGWdW6oP5X5Vv136rOofW0b2jxuq3F1ylsaPv1+bCzWrQqL5y5r4jSZr7xjzl5eb/Yt23Z89V5xgt4fy8fKWlpZbdTklLVV5+gWGiYIV9PsSuWHht7vXCLO/9Ku/9fd77kyQNkNRb0iflreOcG+ScW+KcW2J1Onji408oLi5O5/U412T7+KVIJKLPP/1cvc/vpSenP6Fq1avr2clTNOKOWzVj2iu6ov8gFRYWKiEhYaf1/v3Rx6pW7RA1bNLQKDkARMdeL8xyzsVL6iapv6ROkuZJGl3eOt77LElZklQUKfQHG3J/vTLjVS2Yv0BZkyfIOVfRm8ceJKcmKzk1Wc1aHCdJ6tC5nZ6dPEUDr7tSf5rwgCRp9Tf/0bsL3ttpvbdnz9HZ3TpVeN6gpKSmKHeHr0Xyc/OUmpJsmChYYZ8PsSsWXpvlXZjV2Tk3WdIaSQMlvSapkfe+v/f+lYoKuL8W5izUU5Oe0p8ffVjVq1e3joMd1DusnlJSU7T6m9WSpCXvL1X9hkfrh+9/kCSVlpbqbxOfUa/ze5atU1paqjmz58Xs98GS1Kx5M61etVpr1qxV8dZiZc+arXYd2lvHCkzY50PsioXXpvN+9weqzrk5kqZIetF7/8OBbiCaR8LDhg7XksVLtX79etWtV1dDrhusyVlPamvxVtXZfqHB8Sccr9tG/zFaESrM7mbt0zczqtvcWPxj4M/5xadfaOwd41RSXKIj0g/XiDuHK3vmbL009WVJUrtOZ2rwjYPKzmAs++cHevzPWcp6dnzgWWolVNzFKDnzc3T/2AdUWlqq3pm9NHDwVRW27YoQ5vks3nsVKezzWb02q8Ul7tNp2D2WcFAsTkcjGNEo4cqkIksYwK/LvpYw/2IWAABGKGEAAIxQwgAAGKGEAQAwQgkDAGCEEgYAwAglDACAEUoYAAAjlDAAAEYoYQAAjFDCAAAYoYQBADBCCQMAYIQSBgDACCUMAIARShgAACOUMAAARihhAACMUMIAABihhAEAMEIJAwBghBIGAMBIvHUAVF61EmpbR4iqLZEi6whRdUhcNesIAPaCI2EAAIxQwgAAGKGEAQAwQgkDAGCEEgYAwAglDACAEUoYAAAjlDAAAEYoYQAAjFDCAAAYoYQBADBCCQMAYIQSBgDACCUMAIARShgAACOUMAAARihhAACMUMIAABihhAEAMEIJAwBghBIGAMAIJQwAgJFQlfDCnIXqeW5vde/SU5MmTraOE6gwzyaFY74tW7bo8gFX6KK+l+rC3hcp69GJkiTvvR77y+Pq2/0CXdCzv6b9fbokKfsfs3VRn0s0IPNiXXnJQH3+2ReW8Q9KGPZfecI8X5hnkyr/fPEHspJzzkk6zXv/bsB5DlgkEtG9d4/VhCfGKzU1VRddeLHad2inRo0bWUc7aGGeTQrPfFWrVtVjkx5RYmKiSopLNPD/rlabtm30zVffKC83T8+/OlVVqlTRuu/XSZKOSD9cjz/5mJJqJ2lRzrsac8dYPTllkvEU+y8s+29PwjxfmGeTYmO+fT4Sds6lOecud85Nk7RM0qXRi7X/VixfoYyjMpSeka6Eqgnq2q2L5s2ZZx0rEGGeTQrPfM45JSYmSpJKSkpUUlIi55xenP6Srhp8hapU2fZ2q1uvriSpxYktlFQ7SZLUvEUz5efl2wQ/SGHZf3sS5vnCPJsUG/PtsYSdc3HOubOcc2Occ+9JelJSHUm3e+9P8t5fU2Ep90F+Xr7S0lLLbqekpSovv8AwUXDCPJsUrvkikYgu7neZurQ7V61Pa63mLZppzX/W6s3st3XZhb/VjYNv1upV//nFeq/OmKk2bdsYJD54Ydp/uxPm+cI8mxQb85V3JLxYUj9JOZI6eO+7ee8f9t5/urcndc4Ncs4tcc4tqYzn4IFoiYuL099feFr/eOsVfbziY638YqWKtxbrkEOq6ulpT6p3v166a9Q9O62zZPFSvfrSTF1387VGqQFY2eN3wt77lgf6pN77LElZklQUKfQH+jz7IyU1Rbm5eWW383PzlJqSXBGbjrowzyaFc75aSbXU8pST9e7C95SSmqz2ndpLktp3aqc7b7u77HFffPal7rl9jB4e/5Dq1KltlPbghHH/7SjM84V5Nik25gvN1dHNmjfT6lWrtWbNWhVvLVb2rNlq16G9daxAhHk2KTzz/bDuB23csFGSVFRUpPff+6eObnC02nVsp6X/XCpJWrbkAx119FGSpNxvczXs5uG6Y8woHV3/KLPcByss+29PwjxfmGeTYmO+A7o6ujKKj4/XiJHDNGTgNSotLVXvzF5q3KTyXAF3MMI8mxSe+b4r+F53/PFOlUZKVeq9zj6no85s11YnnnSCRg0freeenqrqiYkaeccISdITj0/Wj+s36L67H5C07VT209OetBzhgIRl/+1JmOcL82xSbMznvI/u2eKKOh0N7K8tkSLrCFF1SFw16wjAr1a1uES3L48LzeloAABiDSUMAIARShgAACOUMAAARihhAACMUMIAABihhAEAMEIJAwBghBIGAMAIJQwAgBFKGAAAI5QwAABGKGEAAIxQwgAAGKGEAQAwQgkDAGCEEgYAwAglDACAEUoYAAAjlDAAAEYoYQAAjFDCAAAYibcOAFg5JK6adYSo2lC83jpCVCUl1LGOABw0joQBADBCCQMAYIQSBgDACCUMAIARShgAACOUMAAARihhAACMUMIAABihhAEAMEIJAwBghBIGAMAIJQwAgBFKGAAAI5QwAABGKGEAAIxQwgAAGKGEAQAwQgkDAGCEEgYAwAglDACAEUoYAAAjlDAAAEbirQMEZdTI0Vowf4Hq1q2rl159wTpO4BbmLNR9Y8apNFKqzH69deXAK6wjBYr5Kr/V36zW7bfeWXb7v2u+1ZXX/FYF+d9p0fxFik9I0JHpR2jEncNUK6mm/vnuEj3+5yyVFJcoPiFe19w8WC1PPdlwggOzZcsW/fayK1W8datKSiLqfM7Zuub6IdaxAhOG12Z5Kvt8znsf1Q0URQqju4Htli5ZqsTERI0cflvoSjgSiajnub014YnxSk1N1UUXXqyx48aoUeNG1tECwXzRsaF4fdSeOxKJqE/n8zXh2ce0+pv/6OTWJys+Pk7j/zRBkjTk5qv1+SdfqG69Q3VYymH66ouv9bsht2rGW88HliEpoU5gz1Ue7702F25WYo1EFRcX6/JLrtCwP/xeLU5oUSHbjybee9FTLS7R7cvjQnM6umWrlkqqXds6RlSsWL5CGUdlKD0jXQlVE9S1WxfNmzPPOlZgmC/2LH1/mY7IOEJpR6Sp9emnKD4+TpLUrMVxKsgvkCQ1PbaJDks5TJLUoHF9bdmyRVu3bjXLfKCcc0qskShJKikpUUlJiaR9+nyt9ML42txRLMx3QCXsnKvinLs46DDYvfy8fKWlpZbdTklLVd72D7owYL7Y83b2HJ3dtdMvlr/28iydesapv1g+760FanpsE1WtWrUi4gUuEonogswL1aFtJ512+mlqccLx1pECEcbX5o5iYb5yS9g5l+ScG+Gce8Q5d47b5npJX0m6oGIiAqhMiouLtXD+InU4p91Oy5+e+Kzi4uJ0znln77T86y+/1uMPZ+n3t91SkTEDFRcXp+kzpumNubO1YvkKffHFl9aREBJ7OxJ+RtJvJC2XdJWkuZL6Sertve+1p5Wcc4Occ0ucc0smTZwcWNhfq5TUFOXm5pXdzs/NU2pKsmGiYDFfbHnvnffV9Jimqluvbtmy11/J1qIF72rUmJFy7n+navPzCvSHm0dp5N3DdWTGkRZxA5WUVEuntG6lRTmLrKMEImyvzV3Fwnx7K+GG3vvLvfcTJA2QdJykLt77D8tbyXuf5b1v5b1vVdmuRItFzZo30+pVq7VmzVoVby1W9qzZatehvXWswDBfbHlr1hx16tax7Pb7CxdrylNTNebP96ha9Wplyzdu+Em3Xjdcg28cqBYnxe7p23Xr1mnDho2SpKKiIr236H3Vb1jfNlRAwvba3FUszFfu1dHOuWXe+5P3dHtfVNTV0cOGDteSxUu1fv161a1XV0OuG6w+fTMrYtMVImd+ju4f+4BKS0vVO7OXBg6+yjpSoJgveNG4Onpz4Wb169pf0177u2rWqilJ6t/9YhVvLVZSnSRJUrPjj9PQ227R37Ke0bOTpij96P8dAT80fpwOrXdoIFkq6urozz/7XH8cMUqlpaUqLS3VOV07a/A1V1fItisC773o2Nero/dWwhFJm36+Kam6pMLtf/be+6S9baCiShjAzqL5K0qVQUWVMHAg9rWEy/3HOrz3ccHEAQAAuwrN7wkDABBrKGEAAIxQwgAAGKGEAQAwQgkDAGCEEgYAwAglDACAEUoYAAAjlDAAAEYoYQAAjFDCAAAYoYQBADBCCQMAYIQSBgDACCUMAIARShgAACOUMAAARihhAACMUMIAABihhAEAMEIJAwBgJN46AIDoSEqoYx0hqkpKi60jRFV8lQTrCKgAHAkDAGCEEgYAwAglDACAEUoYAAAjlDAAAEYoYQAAjFDCAAAYoYQBADBCCQMAYIQSBgDACCUMAIARShgAACOUMAAARihhAACMUMIAABihhAEAMEIJAwBghBIGAMAIJQwAgBFKGAAAI5QwAABGKGEAAIxQwgAAGIm3DnCgcr/N1cgRt2ndd99LzqnfBX118aUX6fe3DNOqr7+RJG3cuFG1atXS9BnTbMMGYGHOQt03ZpxKI6XK7NdbVw68wjpSoJgvdu3pvRhLcr/N06g/jNa679fJOSmzX6YuurS/Hn7gL1owP0cJ8QlKzzhSo+8epVpJtSRJX3z2he65c4w2/bRJrkoVPTP1KR1yyCHGk+y/ML82pco/n/PeR3UDRZHCqGygoKBA3xV8p2OPO1abNm1S/34X6eG/PqRGjRuVPeaB+x5UzVo1Nfiaq6MRocJEIhH1PLe3JjwxXqmpqbrowos1dtyYnWaNZcwX2/blvRgNJaXFgT1XQcF322c4Rps2bdIlF1ymB/8yTnm5+Trl1FaKj4/XXx76qyTphluuV0lJiS4+/zLdNWa0mh7TVOvXr1etWrUUFxcXWKb4KgmBPdeehP21aTlftbhEty+P2+/T0c65ts65R/c/UrCSk5N17HHHSpJq1Kihhg0bKD+/oOx+773emP2mup3b1SpiYFYsX6GMozKUnpGuhKoJ6tqti+bNmWcdKzDMF9v29l6MBcnJh+nY446RtG2GBg0bKD+vQG3OOE3x8dtOGDZv0Vx5efmSpPcWva8mTRur6TFNJUl16tQJtIArSthfm7Ew3z6VsHPuJOfcOOfcN5LukvRpVFPtp7Vr/6tPP/lMx7doXrZs2dJlqlevro6uf7RhsmDk5+UrLS217HZKWqryYuxDrjzMFx67ey/Gmv9un6F5i2Y7LX91xkyd0fZ0SdLqVavlnNO1g67XRedfqr9Nftoi6kEL+2szFubbYwk755o65253zn0q6a+SVmvb6esO3vu/lvekzrlBzrklzrklkyZODjjyzgo3Fep3Nw7V70cMVc2aNcuWz3otW11DcBQMxIo9vRdjSWFhoX5/83ANHXbLTjNMmjBZcXFx6tZ922dKSUlEH37woe6+7y5Nenqi5r49T4vfW2wVGzGsvAuzPpWUI6m79/5LSXLO3bwvT+q9z5KUJUXvO2FJKi4u1i03DdW53bvp7M6dypaXlJTo7bfmaOrzU6K16QqVkpqi3Ny8stv5uXlKTUk2TBQs5ot9e3ovxpLi4hL9/qZh6nZeF3Xs3KFs+asv/0M5C97R+Ccek3PbvuZLTU3RSS1P0qGH1pEknXHmGfr048/U+rTWJtkPVNhfm7EwX3mno/tI+lbSXOfcROdcJ0n79EVzRfDea/Rtd6hhwwa67PJLd7rv/XffV4MG9ZW6w2mIWNaseTOtXrVaa9asVfHWYmXPmq12HdpbxwoM88W28t6LscJ7r7tG3aUGDRvokv+7uGz5onfe1dOTn9Gf/vqgqlevVra8zRmn6csvVmrz5iKVlJRo2ZJlatCogUX0gxL212YszLfXq6OdczUk9ZI0QFJHSU9LmuG9f2NfNhCtI+FlSz/Qby+9Qk2aNlGV7X87vf6m63RmuzN12x9G6fgWx+uC/udHY9Mmcua8dm3vAAALhUlEQVTn6P6xD6i0tFS9M3tp4OCrrCMFivliV3nvxWgK8uroD5Z9qKsuG6TGTRqrSpVtM1x74zUaN+ZBFW/dqtp1akuSjm/RXH+4fYQk6fWZs/TkE0/JOaczzjxdN/7uhsDySBVzdbQU7temZDffvl4dvV+/ouScO1TS+ZIu9N7v0zmnaJ6OBvDrFWQJV0YVVcKIjqiU8IGghAFEAyWMyixqvycMAACCQQkDAGCEEgYAwAglDACAEUoYAAAjlDAAAEYoYQAAjFDCAAAYoYQBADBCCQMAYIQSBgDACCUMAIARShgAACOUMAAARihhAACMUMIAABihhAEAMEIJAwBghBIGAMAIJQwAgBFKGAAAI/HWAQDgQMRXSbCOEFVe3jpC1Dg56wiVBkfCAAAYoYQBADBCCQMAYIQSBgDACCUMAIARShgAACOUMAAARihhAACMUMIAABihhAEAMEIJAwBghBIGAMAIJQwAgBFKGAAAI5QwAABGKGEAAIxQwgAAGKGEAQAwQgkDAGCEEgYAwAglDACAEUoYAAAjoSnhUSNHq33bjurTs591lKhYmLNQPc/tre5demrSxMnWcQKV+22urrx8oDK791Fmj776+zNTrCMFLsz7T2K+WBSJRHRhnwG6fsgNOy2/75771ablGUapglfZ911oSrhXZg+Nz3rUOkZURCIR3Xv3WD024RHNmPmisl/P1sovV1rHCkxcfJyG3nqLZvzjJT079WlNnTItVPOFff8xX2ya8sxzatCowU7L/r3iY23YsMEoUfBiYd+FpoRbtmqppNq1rWNExYrlK5RxVIbSM9KVUDVBXbt10bw586xjBSY5OVnHHnesJKlGjRpq2LCB8vMLjFMFJ+z7j/liT15unnLm56hP395lyyKRiP70wMO6aeiNhsmCFQv7LjQlHGb5eflKS0stu52Slqq8EJXUjtau/a8+/eQzHd+iuXWUwIR9/zFf7Bk39gHdNPRGuSr/q4CpU6apXYezlJycbJgsWLGw7/Zaws65W3b92eG+S/awziDn3BLn3JLKeA4elVPhpkL97sah+v2IoapZs6Z1HCCUFsxboEPr1tVxzY4rW5afX6A3Z7+lARf3N0z26xS/D4+pVc59NXa30HufJSlLkooihf4AcmEHKakpys3NK7udn5un1JTw/G1VkoqLi3XLTUN1bvduOrtzJ+s4gQr7/mO+2PLhsn9p/tz5emfBO9q6Zas2bdqkvj37qWpCVfXo2kuSVFRUpB5demrm7FeN0x6cWNh3ez0S9t7fsevPDvdNiG48SFKz5s20etVqrVmzVsVbi5U9a7badWhvHSsw3nuNvu0ONWzYQJddfql1nMCFff8xX2y54Zbr9cbcbM166zWNfXCMTjm1lXLem6+3c97UrLde06y3XlO1atVivoCl2Nh3+3IkHBOGDR2uJYuXav369ercoYuGXDdYffpmWscKRHx8vEaMHKYhA69RaWmpemf2UuMmjaxjBeaDZR/qH6++piZNm+iCzAslSdffdJ3ObHemcbJghH3/MR8qq1jYd8776J4t5nQ0AOw/r/B+dDo56whRVy0ucZ+G5OpoAACMUMIAABihhAEAMEIJAwBghBIGAMAIJQwAgBFKGAAAI5QwAABGKGEAAIxQwgAAGKGEAQAwQgkDAGCEEgYAwAglDACAEUoYAAAjlDAAAEYoYQAAjFDCAAAYoYQBADBCCQMAYIQSBgDACCUMAICReOsAAIBfcnLWEVABOBIGAMAIJQwAgBFKGAAAI5QwAABGKGEAAIxQwgAAGKGEAQAwQgkDAGCEEgYAwAglDACAEUoYAAAjlDAAAEYoYQAAjFDCAAAYoYQBADBCCQMAYIQSBgDACCUMAIARShgAACOUMAAARihhAACMUMIAABgJVQkvzFmonuf2VvcuPTVp4mTrOIEK82ySNGrkaLVv21F9evazjhIVYd9/zBe7wjjb7j5Pflz/o66+crB6dO2pq68crA0/bjBM+D+hKeFIJKJ77x6rxyY8ohkzX1T269la+eVK61iBCPNsP+uV2UPjsx61jhEVYd9/zBe7wjrb7j5PJj/xpFqf1lozs19V69Naa9ITTxql21loSnjF8hXKOCpD6RnpSqiaoK7dumjenHnWsQIR5tl+1rJVSyXVrm0dIyrCvv+YL3aFdbbdfZ7MnTNPPXv3kCT17N1Dc9+eaxHtF0JTwvl5+UpLSy27nZKWqrz8AsNEwQnzbL8GYd9/zBe7wjzbrtZ9/72Sk5MlSYcddpjWff+9caJtyi1h59ytO/z5/F3uuzdaoQAAiBbnnOScdQxJez8S7r/Dn0fscl/XPa3knBvknFvinFtSUV/0p6SmKDc3r+x2fm6eUlOSK2Tb0Rbm2X4Nwr7/mC92hXm2XdWtV08FBduO8gsKClS3bl3jRNvsrYTdHv68u9tlvPdZ3vtW3vtWVw684oDD7Y9mzZtp9arVWrNmrYq3Fit71my169C+QrYdbWGe7dcg7PuP+WJXmGfbVfsO7fTqyzMlSa++PFMdOrY3zfOz+L3c7/fw593dNhUfH68RI4dpyMBrVFpaqt6ZvdS4SSPrWIEI82w/GzZ0uJYsXqr169erc4cuGnLdYPXpm2kdKxBh33/MF7vCOtvuPk+uGPhb/f7mYXr5xZd1+BGHa9xD91vHlCQ57/fcpc65iKRN2nbUW11S4c93SarmvU/Y2waKIoWVqqwBAIi2anGJ+/Slc7lHwt77uGDiAACAXYXmV5QAAIg1lDAAAEYoYQAAjFDCAAAYoYQBADBCCQMAYIQSBgDACCUMAIARShgAACOUMAAARihhAACMUMIAABihhAEAMEIJAwBghBIGAMAIJQwAgBFKGAAAI5QwAABGKGEAAIxQwgAAGKGEAQAwQgkDAGDEee+tMwTKOTfIe59lnSNamC+2hXm+MM8mMV+sq6zzhfFIeJB1gChjvtgW5vnCPJvEfLGuUs4XxhIGACAmUMIAABgJYwlXunP+AWO+2Bbm+cI8m8R8sa5Szhe6C7MAAIgVYTwSBgAgJlDCAAAYCUUJO+e8c+7BHW4Pdc6NNowUOOfcT9YZosE5N9c512WXZTc558ZbZYoG59wy51yCdY5ocM5FnHMfOudWOOdmOufqWGcKmnOu9/bPmWOss0SDc260c26odY5o2OH1+fPPcOtMOwpFCUvaIqmPc+4w6yDYb89J6r/Lsv7bl4fJIklnWIeIks3e+xO9980lrZN0rXWgKBgg6Z3t/0Vs+fn1+fPPWOtAOwpLCZdo25VvN1sHwX57QdJ5zrmqkuScqy/pCEk5hpmiYZakrtYhKsC7ko60DhEk51xNSW0lXalf/oUROChhKWFJelTSxc652tZBsO+89+skLZbUbfui/pKm+/Bdtj9XUgfrENHknIuT1EnSq9ZZAtZLUrb3/nNJ3zvnWloHwn6pvsvp6AutA+0oNCXsvd8g6WlJN1hnwX7b8ZR0GE9Fy3tfKGm9c+4I6yxRUN0596GkXEmpkt40zhO0AZKmbv/zVHFKOtbsejp6mnWgHYXi94Sdcz9572s65+pKWibpSW2bbbRtsuD8PKN1jmjYfrrvK207XTvVe9/UOFJUOOdukbTeez/ZOkuQdnj/JUqaLel57/1frHMFYftnyhpJBZK8pLjt/z06hGdrQqmyf3aG5khYKju1OV3bvrtBjPDe/6Rtp2snK4RHwTsI9ffC24/2b5D0O+dcvHWegPST9Iz3/mjvfX3vfYakryWdaZwLIRGqEt7uQUlhvEo60Tm3ZoefW6wDBew5SScoxCXsvf9EUtPt352Gkvf+A0kfKTynbAdImrHLshcVnvkkSc65wc65y6xzRMmu3wlXqqujQ3E6GogVzrnHte3IaqF1FgD2KGEAAIyE8XQ0AAAxgRIGAMAIJQwAgBFKGAAAI5QwAABGKGEAAIz8Px9EftLXOWaAAAAAAElFTkSuQmCC\n",

      "text/plain": [

       "<Figure size 576x576 with 1 Axes>"

      ]

     },

     "metadata": {

      "needs_background": "light"

     },

     "output_type": "display_data"

    }

   ],

   "source": [

    "ax = sns.heatmap(confusion_matrix(gt, preds), annot=True, cmap=\"Greens\", fmt='g', \n",

    "                 xticklabels=mapping.keys(), yticklabels=mapping.keys(), cbar=False, square=False)"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": 55,

   "metadata": {},

   "outputs": [

    {

     "name": "stdout",

     "output_type": "stream",

     "text": [

      "N - 0.998262, L - 0.996283, V - 0.983146, \\ - 1.000000, R - 0.997245, A - 0.882353, ! - 0.914894, E - 1.000000, "

     ]

    }

   ],

   "source": [

    "for value, label in zip(recall_score(gt, preds, average=None), mapping.keys()):\n",

    "    print (\"{} - {:4f}\".format(label, value), end=', ')"

   ]

  }

 ],

 "metadata": {

  "kernelspec": {

   "display_name": "Python 3",

   "language": "python",

   "name": "python3"

  },

  "language_info": {

   "codemirror_mode": {

    "name": "ipython",

    "version": 3

   },

   "file_extension": ".py",

   "mimetype": "text/x-python",

   "name": "python",

   "nbconvert_exporter": "python",

   "pygments_lexer": "ipython3",

   "version": "3.7.5"

  }

 },

 "nbformat": 4,

 "nbformat_minor": 2

}