{

 "cells": [

  {

   "cell_type": "code",

   "execution_count": 42,

   "metadata": {},

   "outputs": [],

   "source": [

    "import json\n",

    "import numpy as np\n",

    "import os\n",

    "import sys\n",

    "sys.path.append(\"../../../ecg\")\n",

    "\n",

    "import load\n",

    "import util\n",

    "\n",

    "def fleiss_kappa(ratings):\n",

    "    \"\"\"\n",

    "    Args:\n",

    "        ratings: An N x R numpy array. N is the number of\n",

    "            samples and R is the number of reviewers. Each\n",

    "            entry (n, r) is the category assigned to example\n",

    "            n by reviewer r.\n",

    "    Returns:\n",

    "        Fleiss' kappa score.\n",

    "    https://en.wikipedia.org/wiki/Fleiss%27_kappa\n",

    "    \"\"\"\n",

    "    N, R = ratings.shape\n",

    "    NR =  N * R\n",

    "    categories = set(ratings.ravel().tolist())\n",

    "    P_example = -np.full(N, R)\n",

    "    p_class = 0.0\n",

    "    for c in categories:\n",

    "        c_sum = np.sum(ratings == c, axis=1)\n",

    "        P_example += c_sum**2\n",

    "        p_class += (np.sum(c_sum) / float(NR)) ** 2\n",

    "    P_example = np.sum(P_example) / float(NR * (R-1))\n",

    "    k = (P_example - p_class) / (1 - p_class)\n",

    "    return k\n",

    "\n",

    "def average_pairwise_agreement(revs):\n",

    "    \"\"\"\n",

    "    Here, we use the same method as the diabetic\n",

    "    retinopathy paper. The number of pair-wise\n",

    "    agreements over the total number of pairwise\n",

    "    comparisons.\n",

    "    \"\"\"\n",

    "    corr = 0\n",

    "    tot = 0\n",

    "    n_revs = len(revs)\n",

    "    for i in range(n_revs):\n",

    "        for j in range(i+1, n_revs):\n",

    "            c = np.sum(revs[i] == revs[j])\n",

    "            t = revs[i].size\n",

    "            corr += c\n",

    "            tot += t\n",

    "    return corr / float(tot)\n"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": 10,

   "metadata": {},

   "outputs": [],

   "source": [

    "model_path = \"/deep/group/awni/ecg_models/default/1527627404-9/0.337-0.880-012-0.255-0.906.hdf5\"\n",

    "preproc = util.load(os.path.dirname(model_path))\n",

    "\n",

    "revs = []\n",

    "for i in range(6):\n",

    "    with open(\"../test_rev{}.json\".format(i), 'r') as fid:\n",

    "        revs.append([json.loads(l)['labels'] for l in fid])\n",

    "revs = [np.argmax(preproc.process_y(r), axis=2) for r in revs]\n"

   ]

  },

  {

   "cell_type": "markdown",

   "metadata": {},

   "source": [

    "### Sequence Level Agreements"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": 66,

   "metadata": {},

   "outputs": [

    {

     "name": "stdout",

     "output_type": "stream",

     "text": [

      "\t     Fleiss' kappa  \t Avg Pairwise\n",

      "AF           0.613 \t\t 0.904\n",

      "AVB          0.707 \t\t 0.950\n",

      "BIGEMINY     0.796 \t\t 0.989\n",

      "EAR          0.407 \t\t 0.977\n",

      "IVR          0.535 \t\t 0.978\n",

      "JUNCTIONAL   0.610 \t\t 0.956\n",

      "NOISE        0.729 \t\t 0.962\n",

      "SINUS        0.678 \t\t 0.841\n",

      "SVT          0.398 \t\t 0.961\n",

      "TRIGEMINY    0.783 \t\t 0.987\n",

      "VT           0.500 \t\t 0.992\n",

      "WENCKEBACH   0.496 \t\t 0.962\n",

      "\n",

      "All          0.645 \t\t 0.730\n"

     ]

    }

   ],

   "source": [

    "print \"\\t     Fleiss' kappa  \\t Avg Pairwise\"\n",

    "for e, c in enumerate(preproc.classes):\n",

    "    binary_revs = [np.reshape(r == e, -1) for r in revs]\n",

    "    print \"{:<10}   {:.3f} \\t\\t {:.3f}\".format(\n",

    "        c, fleiss_kappa(np.stack(binary_revs, axis=1)),\n",

    "        average_pairwise_agreement(binary_revs)) \n",

    "print\n",

    "ratings = np.hstack([r.reshape(-1, 1) for r in revs])\n",

    "print \"{:<10}   {:.3f} \\t\\t {:.3f}\".format(\n",

    "        \"All\", fleiss_kappa(ratings),\n",

    "        average_pairwise_agreement(revs))"

   ]

  },

  {

   "cell_type": "markdown",

   "metadata": {},

   "source": [

    "### Set-level agreements (can only compute for a given rhythm)"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": 71,

   "metadata": {},

   "outputs": [

    {

     "name": "stdout",

     "output_type": "stream",

     "text": [

      "\t     Fleiss' kappa  \t Avg Pairwise\n",

      "AF           0.591 \t\t 0.873\n",

      "AVB          0.703 \t\t 0.929\n",

      "BIGEMINY     0.791 \t\t 0.976\n",

      "EAR          0.415 \t\t 0.950\n",

      "IVR          0.645 \t\t 0.944\n",

      "JUNCTIONAL   0.607 \t\t 0.928\n",

      "NOISE        0.625 \t\t 0.924\n",

      "SINUS        0.666 \t\t 0.858\n",

      "SVT          0.485 \t\t 0.921\n",

      "TRIGEMINY    0.732 \t\t 0.971\n",

      "VT           0.677 \t\t 0.967\n",

      "WENCKEBACH   0.609 \t\t 0.947\n"

     ]

    }

   ],

   "source": [

    "print \"\\t     Fleiss' kappa  \\t Avg Pairwise\"\n",

    "for e, c in enumerate(preproc.classes):\n",

    "    binary_revs = [np.any(r == e, axis=1) for r in revs]\n",

    "    print \"{:<10}   {:.3f} \\t\\t {:.3f}\".format(\n",

    "        c, fleiss_kappa(np.stack(binary_revs, axis=1)),\n",

    "        average_pairwise_agreement(binary_revs))   "

   ]

  },

  {

   "cell_type": "markdown",

   "metadata": {},

   "source": [

    "### Confusions between \"AFIB\" and \"AFL\" and \"AVB type 2 second degree\" and \"AVB third degree\""

   ]

  },

  {

   "cell_type": "code",

   "execution_count": null,

   "metadata": {},

   "outputs": [],

   "source": []

  }

 ],

 "metadata": {

  "kernelspec": {

   "display_name": "Python 2",

   "language": "python",

   "name": "python2"

  },

  "language_info": {

   "codemirror_mode": {

    "name": "ipython",

    "version": 2

   },

   "file_extension": ".py",

   "mimetype": "text/x-python",

   "name": "python",

   "nbconvert_exporter": "python",

   "pygments_lexer": "ipython2",

   "version": "2.7.12"

  }

 },

 "nbformat": 4,

 "nbformat_minor": 2

}