539 lines (538 with data), 16.1 kB

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Importing python libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Python version:\n",
      "3.7.3 (default, Mar 27 2019, 22:11:17) \n",
      "[GCC 7.3.0]\n",
      "\n",
      "matplotlib version: 3.1.3\n",
      "pandas version: 0.24.2\n",
      "numpy version: 1.18.3\n",
      "sklearn version: 0.22.1\n",
      "keras version: 2.2.4\n"
     ]
    }
   ],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings('ignore',category=FutureWarning)\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "\n",
    "import sys\n",
    "#import os\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import h5py as h5\n",
    "import sklearn\n",
    "from sklearn.multioutput import MultiOutputClassifier\n",
    "from sklearn import metrics\n",
    "from sklearn import preprocessing\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.metrics import confusion_matrix, classification_report, average_precision_score, precision_recall_curve, accuracy_score, confusion_matrix  \n",
    "from sklearn.metrics import average_precision_score\n",
    "import pickle\n",
    "import keras\n",
    "from keras.models import load_model\n",
    "\n",
    "import util\n",
    "\n",
    "print(\"Python version:\\n{}\\n\".format(sys.version))\n",
    "print(\"matplotlib version: {}\".format(matplotlib.__version__))\n",
    "print(\"pandas version: {}\".format(pd.__version__))\n",
    "print(\"numpy version: {}\".format(np.__version__))\n",
    "print(\"sklearn version: {}\".format(sklearn.__version__))\n",
    "print(\"keras version: {}\".format(keras.__version__))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Loading data: Held out test set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(17617, 800, 1)\n",
      "(17617, 3)\n",
      "(17617, 2)\n",
      "(17617, 3)\n"
     ]
    }
   ],
   "source": [
    "path = '/path/to/data/'\n",
    "\n",
    "data_test = np.load(path + 'test.npz', allow_pickle=True)\n",
    "test_x = data_test['signal']\n",
    "test_qa = data_test['qa_label']\n",
    "test_r = data_test['rhythm']\n",
    "test_p = pd.DataFrame(data_test['parameters'])\n",
    "print(test_x.shape)\n",
    "print(test_qa.shape)\n",
    "print(test_r.shape)\n",
    "print(test_p.shape)\n",
    "test_p.rename(index=str, columns={0:'timestamp', \n",
    "                                  1:'stream', \n",
    "                                  2:'ID'}, inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## VGG model : single-task\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/users/jntorres/jessica/anaconda3/envs/tf_gpu/lib/python3.7/site-packages/tensorflow/python/framework/op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Colocations handled automatically by placer.\n",
      "WARNING:tensorflow:From /home/users/jntorres/jessica/anaconda3/envs/tf_gpu/lib/python3.7/site-packages/tensorflow/python/ops/math_ops.py:3066: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.cast instead.\n"
     ]
    }
   ],
   "source": [
    "path = '/path/to/model/'\n",
    "vgg = load_model(path + 'VGG_singletask.h5')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### VGG Single-task \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TPR: 0.92\n",
      "TNR: 0.71\n",
      "FPR: 0.29\n",
      "FNR: 0.08\n",
      "PPV: 0.50\n",
      "NPV: 0.96\n",
      "F1: 0.64\n"
     ]
    }
   ],
   "source": [
    "# weighted macro-average across all indivduals\n",
    "\n",
    "test_metrics = util.collecting_individual_metrics_singletask(vgg, test_x, test_p, test_r, out_message=False)\n",
    "test_metrics = pd.DataFrame.from_dict(test_metrics).T.rename(columns={0:'TPR', 1:'TNR', 2:'FPR', 3:'FNR', 4:\"total_samples\"})\n",
    "\n",
    "for m in ['TPR', 'TNR', 'FPR', 'FNR']:\n",
    "    metric_wmu = np.average(test_metrics[m][~test_metrics[m].isna()], weights=test_metrics['total_samples'][~test_metrics[m].isna()])\n",
    "    print('%s: %0.2f' % (m, metric_wmu))\n",
    "    \n",
    "# PPV, NPV and F1\n",
    "\n",
    "episode_m = util.episode_metrics_singletask(vgg, test_x, test_p, test_r, out_message=False)\n",
    "episode_metrics = pd.DataFrame(episode_m).T\n",
    "episode_metrics.rename(columns={0:'TPR', 1:'TNR', 2:'PPV', 3:'NPV', 4:\"FPR\", 5:'FNR', 6:'F1', 7:'total_samples'}, inplace=True)\n",
    "for m in ['PPV', 'NPV', 'F1']:\n",
    "    print('%s: %0.2f' % (m, episode_metrics[m]))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## DeepBeat Single task (no pretrain CDAE + AF task)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Loading non pretrained DeepBeat single task model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/users/jntorres/.local/lib/python3.7/site-packages/keras/backend/tensorflow_backend.py:3445: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n"
     ]
    }
   ],
   "source": [
    "path = '/path/to/model/'\n",
    "model_reint = load_model(path + 'deepbeat_singletask_nopretrain.h5')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### DeepBeat Single task"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TPR: 0.49\n",
      "TNR: 0.90\n",
      "FPR: 0.10\n",
      "FNR: 0.51\n",
      "PPV: 0.60\n",
      "NPV: 0.85\n",
      "F1: 0.54\n"
     ]
    }
   ],
   "source": [
    "# weighted macro-average across all indivduals\n",
    "\n",
    "test_metrics = util.collecting_individual_metrics_singletask(model_reint, test_x, test_p, test_r, out_message=False)\n",
    "test_metrics = pd.DataFrame.from_dict(test_metrics).T.rename(columns={0:'TPR', 1:'TNR', 2:'FPR', 3:'FNR', 4:\"total_samples\"})\n",
    "\n",
    "\n",
    "for m in ['TPR', 'TNR', 'FPR', 'FNR']:\n",
    "    metric_wmu = np.average(test_metrics[m][~test_metrics[m].isna()], weights=test_metrics['total_samples'][~test_metrics[m].isna()])\n",
    "    print('%s: %0.2f' % (m, metric_wmu))\n",
    "    \n",
    "# PPV, NPV and F1\n",
    "\n",
    "episode_m = util.episode_metrics_singletask(model_reint, test_x, test_p, test_r, out_message=False)\n",
    "episode_metrics = pd.DataFrame(episode_m).T\n",
    "episode_metrics.rename(columns={0:'TPR', 1:'TNR', 2:'PPV', 3:'NPV', 4:\"FPR\", 5:'FNR', 6:'F1', 7:'total_samples'}, inplace=True)\n",
    "for m in ['PPV', 'NPV', 'F1']:\n",
    "    print('%s: %0.2f' % (m, episode_metrics[m]))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## DeepBeat Single task (pretrain with CDAE + AF)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Loading pretrained DeepBeat single task model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "path_to_model ='/path/to/model/'\n",
    "model_name = 'deepbeat_singletask_pretrained.h5'\n",
    "deepbeat_st = load_model(path_to_model + model_name) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### DeepBeat Single task"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TPR: 0.52\n",
      "TNR: 0.88\n",
      "FPR: 0.12\n",
      "FNR: 0.48\n",
      "PPV: 0.59\n",
      "NPV: 0.85\n",
      "F1: 0.56\n"
     ]
    }
   ],
   "source": [
    "# weighted macro-average across all indivduals\n",
    "\n",
    "test_metrics = util.collecting_individual_metrics_singletask(deepbeat_st, test_x, test_p, test_r, out_message=False)\n",
    "test_metrics = pd.DataFrame.from_dict(test_metrics).T.rename(columns={0:'TPR', 1:'TNR', 2:'FPR', 3:'FNR', 4:\"total_samples\"})\n",
    "\n",
    "\n",
    "for m in ['TPR', 'TNR', 'FPR', 'FNR']:\n",
    "    metric_wmu = np.average(test_metrics[m][~test_metrics[m].isna()], weights=test_metrics['total_samples'][~test_metrics[m].isna()])\n",
    "    print('%s: %0.2f' % (m, metric_wmu))\n",
    "    \n",
    "# PPV, NPV and F1\n",
    "\n",
    "episode_m = util.episode_metrics_singletask(deepbeat_st, test_x, test_p, test_r, out_message=False)\n",
    "episode_metrics = pd.DataFrame(episode_m).T\n",
    "episode_metrics.rename(columns={0:'TPR', 1:'TNR', 2:'PPV', 3:'NPV', 4:\"FPR\", 5:'FNR', 6:'F1', 7:'total_samples'}, inplace=True)\n",
    "for m in ['PPV', 'NPV', 'F1']:\n",
    "    print('%s: %0.2f' % (m, episode_metrics[m]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## DeepBeat Multi-task (no pretrain CDAE + AF + Excellent QA)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Loading non pretrained DeepBeat model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "path_to_model ='/path/to/model/'\n",
    "model_reint = load_model(path_to_model + 'deepbeat_no_pretraining.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### DeepBeat Multi-task (no pretrain CDAE + AF + Excellent QA) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TPR: 0.97\n",
      "TNR: 0.89\n",
      "FPR: 0.11\n",
      "FNR: 0.03\n",
      "PPV: 0.55\n",
      "NPV: 1.00\n",
      "F1: 0.71\n"
     ]
    }
   ],
   "source": [
    "## QA results\n",
    "\n",
    "predictions_qa, predictions_r = model_reint.predict(test_x)\n",
    "predictions_QA = np.argmax(predictions_qa, axis=1)\n",
    "\n",
    "#print(classification_report(np.argmax(test_qa, axis=1), predictions_QA))\n",
    "\n",
    "\n",
    "excellent_qa_indx = np.where(predictions_QA==2)[0]\n",
    "x_test_excellent = test_x[excellent_qa_indx,:]\n",
    "p_test_excellent = test_p.iloc[excellent_qa_indx,:]\n",
    "rhythm_test_excellent = test_r[excellent_qa_indx,:]\n",
    "quality_assessment_test_excellent = test_qa[excellent_qa_indx,:]\n",
    "\n",
    "\n",
    "# weighted macro-average across all indivduals\n",
    "\n",
    "test_metrics = util.collecting_individual_metrics(model_reint, x_test_excellent, p_test_excellent, rhythm_test_excellent, out_message=False)\n",
    "test_metrics = pd.DataFrame.from_dict(test_metrics).T.rename(columns={0:'TPR', 1:'TNR', 2:'FPR', 3:'FNR', 4:\"total_samples\"})\n",
    "\n",
    "\n",
    "for m in ['TPR', 'TNR', 'FPR', 'FNR']:\n",
    "    metric_wmu = np.average(test_metrics[m][~test_metrics[m].isna()], weights=test_metrics['total_samples'][~test_metrics[m].isna()])\n",
    "    print('%s: %0.2f' % (m, metric_wmu))\n",
    "    \n",
    "# PPV, NPV and F1\n",
    "\n",
    "episode_m = util.episode_metrics(model_reint, x_test_excellent, p_test_excellent, rhythm_test_excellent, out_message=False)\n",
    "episode_metrics = pd.DataFrame(episode_m).T\n",
    "episode_metrics.rename(columns={0:'TPR', 1:'TNR', 2:'PPV', 3:'NPV', 4:\"FPR\", 5:'FNR', 6:'F1', 7:'total_samples'}, inplace=True)\n",
    "for m in ['PPV', 'NPV', 'F1']:\n",
    "    print('%s: %0.2f' % (m, episode_metrics[m]))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## DeepBeat Multi-task (pretrained with CDAE + AF + Excellent QA)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Loading pretrained DeepBeat model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "path_to_model ='/path/to/model/'\n",
    "model_name = 'deepbeat.h5'\n",
    "deepbeat = load_model(path_to_model + model_name) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### DeepBeat Multi-task (pretrain CDAE + AF + Excellent QA) \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TPR: 0.98\n",
      "TNR: 0.99\n",
      "FPR: 0.01\n",
      "FNR: 0.02\n",
      "PPV: 0.94\n",
      "NPV: 1.00\n",
      "F1: 0.96\n"
     ]
    }
   ],
   "source": [
    "## QA results\n",
    "\n",
    "predictions_qa, predictions_r = deepbeat.predict(test_x)\n",
    "predictions_QA = np.argmax(predictions_qa, axis=1)\n",
    "\n",
    "#print(classification_report(np.argmax(test_qa, axis=1), predictions_QA))\n",
    "\n",
    "\n",
    "excellent_qa_indx = np.where(predictions_QA==2)[0]\n",
    "x_test_excellent = test_x[excellent_qa_indx,:]\n",
    "p_test_excellent = test_p.iloc[excellent_qa_indx,:]\n",
    "rhythm_test_excellent = test_r[excellent_qa_indx,:]\n",
    "quality_assessment_test_excellent = test_qa[excellent_qa_indx,:]\n",
    "\n",
    "\n",
    "# weighted macro-average across all indivduals\n",
    "\n",
    "test_metrics = util.collecting_individual_metrics(deepbeat, x_test_excellent, p_test_excellent, rhythm_test_excellent, out_message=False)\n",
    "test_metrics = pd.DataFrame.from_dict(test_metrics).T.rename(columns={0:'TPR', 1:'TNR', 2:'FPR', 3:'FNR', 4:\"total_samples\"})\n",
    "\n",
    "\n",
    "for m in ['TPR', 'TNR', 'FPR', 'FNR']:\n",
    "    metric_wmu = np.average(test_metrics[m][~test_metrics[m].isna()], weights=test_metrics['total_samples'][~test_metrics[m].isna()])\n",
    "    print('%s: %0.2f' % (m, metric_wmu))\n",
    "    \n",
    "# PPV, NPV and F1\n",
    "\n",
    "episode_m = util.episode_metrics(deepbeat, x_test_excellent, p_test_excellent, rhythm_test_excellent, out_message=False)\n",
    "episode_metrics = pd.DataFrame(episode_m).T\n",
    "episode_metrics.rename(columns={0:'TPR', 1:'TNR', 2:'PPV', 3:'NPV', 4:\"FPR\", 5:'FNR', 6:'F1', 7:'total_samples'}, inplace=True)\n",
    "for m in ['PPV', 'NPV', 'F1']:\n",
    "    print('%s: %0.2f' % (m, episode_metrics[m]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}