--- a
+++ b/table_2.ipynb
@@ -0,0 +1,538 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "#### Importing python libraries"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Using TensorFlow backend.\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Python version:\n",
+      "3.7.3 (default, Mar 27 2019, 22:11:17) \n",
+      "[GCC 7.3.0]\n",
+      "\n",
+      "matplotlib version: 3.1.3\n",
+      "pandas version: 0.24.2\n",
+      "numpy version: 1.18.3\n",
+      "sklearn version: 0.22.1\n",
+      "keras version: 2.2.4\n"
+     ]
+    }
+   ],
+   "source": [
+    "import warnings\n",
+    "warnings.filterwarnings('ignore',category=FutureWarning)\n",
+    "import warnings\n",
+    "warnings.filterwarnings(\"ignore\")\n",
+    "\n",
+    "import sys\n",
+    "#import os\n",
+    "import matplotlib\n",
+    "import matplotlib.pyplot as plt\n",
+    "import pandas as pd\n",
+    "import numpy as np\n",
+    "import h5py as h5\n",
+    "import sklearn\n",
+    "from sklearn.multioutput import MultiOutputClassifier\n",
+    "from sklearn import metrics\n",
+    "from sklearn import preprocessing\n",
+    "from sklearn.ensemble import RandomForestClassifier\n",
+    "from sklearn.metrics import confusion_matrix, classification_report, average_precision_score, precision_recall_curve, accuracy_score, confusion_matrix  \n",
+    "from sklearn.metrics import average_precision_score\n",
+    "import pickle\n",
+    "import keras\n",
+    "from keras.models import load_model\n",
+    "\n",
+    "import util\n",
+    "\n",
+    "print(\"Python version:\\n{}\\n\".format(sys.version))\n",
+    "print(\"matplotlib version: {}\".format(matplotlib.__version__))\n",
+    "print(\"pandas version: {}\".format(pd.__version__))\n",
+    "print(\"numpy version: {}\".format(np.__version__))\n",
+    "print(\"sklearn version: {}\".format(sklearn.__version__))\n",
+    "print(\"keras version: {}\".format(keras.__version__))\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Loading data: Held out test set"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "(17617, 800, 1)\n",
+      "(17617, 3)\n",
+      "(17617, 2)\n",
+      "(17617, 3)\n"
+     ]
+    }
+   ],
+   "source": [
+    "path = '/path/to/data/'\n",
+    "\n",
+    "data_test = np.load(path + 'test.npz', allow_pickle=True)\n",
+    "test_x = data_test['signal']\n",
+    "test_qa = data_test['qa_label']\n",
+    "test_r = data_test['rhythm']\n",
+    "test_p = pd.DataFrame(data_test['parameters'])\n",
+    "print(test_x.shape)\n",
+    "print(test_qa.shape)\n",
+    "print(test_r.shape)\n",
+    "print(test_p.shape)\n",
+    "test_p.rename(index=str, columns={0:'timestamp', \n",
+    "                                  1:'stream', \n",
+    "                                  2:'ID'}, inplace=True)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## VGG model : single-task\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "WARNING:tensorflow:From /home/users/jntorres/jessica/anaconda3/envs/tf_gpu/lib/python3.7/site-packages/tensorflow/python/framework/op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
+      "Instructions for updating:\n",
+      "Colocations handled automatically by placer.\n",
+      "WARNING:tensorflow:From /home/users/jntorres/jessica/anaconda3/envs/tf_gpu/lib/python3.7/site-packages/tensorflow/python/ops/math_ops.py:3066: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
+      "Instructions for updating:\n",
+      "Use tf.cast instead.\n"
+     ]
+    }
+   ],
+   "source": [
+    "path = '/path/to/model/'\n",
+    "vgg = load_model(path + 'VGG_singletask.h5')\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "#### VGG Single-task \n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "TPR: 0.92\n",
+      "TNR: 0.71\n",
+      "FPR: 0.29\n",
+      "FNR: 0.08\n",
+      "PPV: 0.50\n",
+      "NPV: 0.96\n",
+      "F1: 0.64\n"
+     ]
+    }
+   ],
+   "source": [
+    "# weighted macro-average across all indivduals\n",
+    "\n",
+    "test_metrics = util.collecting_individual_metrics_singletask(vgg, test_x, test_p, test_r, out_message=False)\n",
+    "test_metrics = pd.DataFrame.from_dict(test_metrics).T.rename(columns={0:'TPR', 1:'TNR', 2:'FPR', 3:'FNR', 4:\"total_samples\"})\n",
+    "\n",
+    "for m in ['TPR', 'TNR', 'FPR', 'FNR']:\n",
+    "    metric_wmu = np.average(test_metrics[m][~test_metrics[m].isna()], weights=test_metrics['total_samples'][~test_metrics[m].isna()])\n",
+    "    print('%s: %0.2f' % (m, metric_wmu))\n",
+    "    \n",
+    "# PPV, NPV and F1\n",
+    "\n",
+    "episode_m = util.episode_metrics_singletask(vgg, test_x, test_p, test_r, out_message=False)\n",
+    "episode_metrics = pd.DataFrame(episode_m).T\n",
+    "episode_metrics.rename(columns={0:'TPR', 1:'TNR', 2:'PPV', 3:'NPV', 4:\"FPR\", 5:'FNR', 6:'F1', 7:'total_samples'}, inplace=True)\n",
+    "for m in ['PPV', 'NPV', 'F1']:\n",
+    "    print('%s: %0.2f' % (m, episode_metrics[m]))\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## DeepBeat Single task (no pretrain CDAE + AF task)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "#### Loading non pretrained DeepBeat single task model"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "WARNING:tensorflow:From /home/users/jntorres/.local/lib/python3.7/site-packages/keras/backend/tensorflow_backend.py:3445: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n",
+      "Instructions for updating:\n",
+      "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n"
+     ]
+    }
+   ],
+   "source": [
+    "path = '/path/to/model/'\n",
+    "model_reint = load_model(path + 'deepbeat_singletask_nopretrain.h5')\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "#### DeepBeat Single task"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "TPR: 0.49\n",
+      "TNR: 0.90\n",
+      "FPR: 0.10\n",
+      "FNR: 0.51\n",
+      "PPV: 0.60\n",
+      "NPV: 0.85\n",
+      "F1: 0.54\n"
+     ]
+    }
+   ],
+   "source": [
+    "# weighted macro-average across all indivduals\n",
+    "\n",
+    "test_metrics = util.collecting_individual_metrics_singletask(model_reint, test_x, test_p, test_r, out_message=False)\n",
+    "test_metrics = pd.DataFrame.from_dict(test_metrics).T.rename(columns={0:'TPR', 1:'TNR', 2:'FPR', 3:'FNR', 4:\"total_samples\"})\n",
+    "\n",
+    "\n",
+    "for m in ['TPR', 'TNR', 'FPR', 'FNR']:\n",
+    "    metric_wmu = np.average(test_metrics[m][~test_metrics[m].isna()], weights=test_metrics['total_samples'][~test_metrics[m].isna()])\n",
+    "    print('%s: %0.2f' % (m, metric_wmu))\n",
+    "    \n",
+    "# PPV, NPV and F1\n",
+    "\n",
+    "episode_m = util.episode_metrics_singletask(model_reint, test_x, test_p, test_r, out_message=False)\n",
+    "episode_metrics = pd.DataFrame(episode_m).T\n",
+    "episode_metrics.rename(columns={0:'TPR', 1:'TNR', 2:'PPV', 3:'NPV', 4:\"FPR\", 5:'FNR', 6:'F1', 7:'total_samples'}, inplace=True)\n",
+    "for m in ['PPV', 'NPV', 'F1']:\n",
+    "    print('%s: %0.2f' % (m, episode_metrics[m]))\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## DeepBeat Single task (pretrain with CDAE + AF)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "#### Loading pretrained DeepBeat single task model"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "path_to_model ='/path/to/model/'\n",
+    "model_name = 'deepbeat_singletask_pretrained.h5'\n",
+    "deepbeat_st = load_model(path_to_model + model_name) "
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "#### DeepBeat Single task"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "TPR: 0.52\n",
+      "TNR: 0.88\n",
+      "FPR: 0.12\n",
+      "FNR: 0.48\n",
+      "PPV: 0.59\n",
+      "NPV: 0.85\n",
+      "F1: 0.56\n"
+     ]
+    }
+   ],
+   "source": [
+    "# weighted macro-average across all indivduals\n",
+    "\n",
+    "test_metrics = util.collecting_individual_metrics_singletask(deepbeat_st, test_x, test_p, test_r, out_message=False)\n",
+    "test_metrics = pd.DataFrame.from_dict(test_metrics).T.rename(columns={0:'TPR', 1:'TNR', 2:'FPR', 3:'FNR', 4:\"total_samples\"})\n",
+    "\n",
+    "\n",
+    "for m in ['TPR', 'TNR', 'FPR', 'FNR']:\n",
+    "    metric_wmu = np.average(test_metrics[m][~test_metrics[m].isna()], weights=test_metrics['total_samples'][~test_metrics[m].isna()])\n",
+    "    print('%s: %0.2f' % (m, metric_wmu))\n",
+    "    \n",
+    "# PPV, NPV and F1\n",
+    "\n",
+    "episode_m = util.episode_metrics_singletask(deepbeat_st, test_x, test_p, test_r, out_message=False)\n",
+    "episode_metrics = pd.DataFrame(episode_m).T\n",
+    "episode_metrics.rename(columns={0:'TPR', 1:'TNR', 2:'PPV', 3:'NPV', 4:\"FPR\", 5:'FNR', 6:'F1', 7:'total_samples'}, inplace=True)\n",
+    "for m in ['PPV', 'NPV', 'F1']:\n",
+    "    print('%s: %0.2f' % (m, episode_metrics[m]))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## DeepBeat Multi-task (no pretrain CDAE + AF + Excellent QA)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "#### Loading non pretrained DeepBeat model"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "path_to_model ='/path/to/model/'\n",
+    "model_reint = load_model(path_to_model + 'deepbeat_no_pretraining.h5')"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "#### DeepBeat Multi-task (no pretrain CDAE + AF + Excellent QA) "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "TPR: 0.97\n",
+      "TNR: 0.89\n",
+      "FPR: 0.11\n",
+      "FNR: 0.03\n",
+      "PPV: 0.55\n",
+      "NPV: 1.00\n",
+      "F1: 0.71\n"
+     ]
+    }
+   ],
+   "source": [
+    "## QA results\n",
+    "\n",
+    "predictions_qa, predictions_r = model_reint.predict(test_x)\n",
+    "predictions_QA = np.argmax(predictions_qa, axis=1)\n",
+    "\n",
+    "#print(classification_report(np.argmax(test_qa, axis=1), predictions_QA))\n",
+    "\n",
+    "\n",
+    "excellent_qa_indx = np.where(predictions_QA==2)[0]\n",
+    "x_test_excellent = test_x[excellent_qa_indx,:]\n",
+    "p_test_excellent = test_p.iloc[excellent_qa_indx,:]\n",
+    "rhythm_test_excellent = test_r[excellent_qa_indx,:]\n",
+    "quality_assessment_test_excellent = test_qa[excellent_qa_indx,:]\n",
+    "\n",
+    "\n",
+    "# weighted macro-average across all indivduals\n",
+    "\n",
+    "test_metrics = util.collecting_individual_metrics(model_reint, x_test_excellent, p_test_excellent, rhythm_test_excellent, out_message=False)\n",
+    "test_metrics = pd.DataFrame.from_dict(test_metrics).T.rename(columns={0:'TPR', 1:'TNR', 2:'FPR', 3:'FNR', 4:\"total_samples\"})\n",
+    "\n",
+    "\n",
+    "for m in ['TPR', 'TNR', 'FPR', 'FNR']:\n",
+    "    metric_wmu = np.average(test_metrics[m][~test_metrics[m].isna()], weights=test_metrics['total_samples'][~test_metrics[m].isna()])\n",
+    "    print('%s: %0.2f' % (m, metric_wmu))\n",
+    "    \n",
+    "# PPV, NPV and F1\n",
+    "\n",
+    "episode_m = util.episode_metrics(model_reint, x_test_excellent, p_test_excellent, rhythm_test_excellent, out_message=False)\n",
+    "episode_metrics = pd.DataFrame(episode_m).T\n",
+    "episode_metrics.rename(columns={0:'TPR', 1:'TNR', 2:'PPV', 3:'NPV', 4:\"FPR\", 5:'FNR', 6:'F1', 7:'total_samples'}, inplace=True)\n",
+    "for m in ['PPV', 'NPV', 'F1']:\n",
+    "    print('%s: %0.2f' % (m, episode_metrics[m]))\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## DeepBeat Multi-task (pretrained with CDAE + AF + Excellent QA)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "#### Loading pretrained DeepBeat model"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "path_to_model ='/path/to/model/'\n",
+    "model_name = 'deepbeat.h5'\n",
+    "deepbeat = load_model(path_to_model + model_name) "
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "#### DeepBeat Multi-task (pretrain CDAE + AF + Excellent QA) \n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "TPR: 0.98\n",
+      "TNR: 0.99\n",
+      "FPR: 0.01\n",
+      "FNR: 0.02\n",
+      "PPV: 0.94\n",
+      "NPV: 1.00\n",
+      "F1: 0.96\n"
+     ]
+    }
+   ],
+   "source": [
+    "## QA results\n",
+    "\n",
+    "predictions_qa, predictions_r = deepbeat.predict(test_x)\n",
+    "predictions_QA = np.argmax(predictions_qa, axis=1)\n",
+    "\n",
+    "#print(classification_report(np.argmax(test_qa, axis=1), predictions_QA))\n",
+    "\n",
+    "\n",
+    "excellent_qa_indx = np.where(predictions_QA==2)[0]\n",
+    "x_test_excellent = test_x[excellent_qa_indx,:]\n",
+    "p_test_excellent = test_p.iloc[excellent_qa_indx,:]\n",
+    "rhythm_test_excellent = test_r[excellent_qa_indx,:]\n",
+    "quality_assessment_test_excellent = test_qa[excellent_qa_indx,:]\n",
+    "\n",
+    "\n",
+    "# weighted macro-average across all indivduals\n",
+    "\n",
+    "test_metrics = util.collecting_individual_metrics(deepbeat, x_test_excellent, p_test_excellent, rhythm_test_excellent, out_message=False)\n",
+    "test_metrics = pd.DataFrame.from_dict(test_metrics).T.rename(columns={0:'TPR', 1:'TNR', 2:'FPR', 3:'FNR', 4:\"total_samples\"})\n",
+    "\n",
+    "\n",
+    "for m in ['TPR', 'TNR', 'FPR', 'FNR']:\n",
+    "    metric_wmu = np.average(test_metrics[m][~test_metrics[m].isna()], weights=test_metrics['total_samples'][~test_metrics[m].isna()])\n",
+    "    print('%s: %0.2f' % (m, metric_wmu))\n",
+    "    \n",
+    "# PPV, NPV and F1\n",
+    "\n",
+    "episode_m = util.episode_metrics(deepbeat, x_test_excellent, p_test_excellent, rhythm_test_excellent, out_message=False)\n",
+    "episode_metrics = pd.DataFrame(episode_m).T\n",
+    "episode_metrics.rename(columns={0:'TPR', 1:'TNR', 2:'PPV', 3:'NPV', 4:\"FPR\", 5:'FNR', 6:'F1', 7:'total_samples'}, inplace=True)\n",
+    "for m in ['PPV', 'NPV', 'F1']:\n",
+    "    print('%s: %0.2f' % (m, episode_metrics[m]))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.7.3"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}