Datasets
Models
Downloads: 1
Diff of
/table_2.ipynb
[000000]
..
[090445]
Switch to side-by-side view
--- a +++ b/table_2.ipynb @@ -0,0 +1,538 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Importing python libraries" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Using TensorFlow backend.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Python version:\n", + "3.7.3 (default, Mar 27 2019, 22:11:17) \n", + "[GCC 7.3.0]\n", + "\n", + "matplotlib version: 3.1.3\n", + "pandas version: 0.24.2\n", + "numpy version: 1.18.3\n", + "sklearn version: 0.22.1\n", + "keras version: 2.2.4\n" + ] + } + ], + "source": [ + "import warnings\n", + "warnings.filterwarnings('ignore',category=FutureWarning)\n", + "import warnings\n", + "warnings.filterwarnings(\"ignore\")\n", + "\n", + "import sys\n", + "#import os\n", + "import matplotlib\n", + "import matplotlib.pyplot as plt\n", + "import pandas as pd\n", + "import numpy as np\n", + "import h5py as h5\n", + "import sklearn\n", + "from sklearn.multioutput import MultiOutputClassifier\n", + "from sklearn import metrics\n", + "from sklearn import preprocessing\n", + "from sklearn.ensemble import RandomForestClassifier\n", + "from sklearn.metrics import confusion_matrix, classification_report, average_precision_score, precision_recall_curve, accuracy_score, confusion_matrix \n", + "from sklearn.metrics import average_precision_score\n", + "import pickle\n", + "import keras\n", + "from keras.models import load_model\n", + "\n", + "import util\n", + "\n", + "print(\"Python version:\\n{}\\n\".format(sys.version))\n", + "print(\"matplotlib version: {}\".format(matplotlib.__version__))\n", + "print(\"pandas version: {}\".format(pd.__version__))\n", + "print(\"numpy version: {}\".format(np.__version__))\n", + "print(\"sklearn version: {}\".format(sklearn.__version__))\n", + "print(\"keras version: {}\".format(keras.__version__))\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Loading data: Held out test set" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "(17617, 800, 1)\n", + "(17617, 3)\n", + "(17617, 2)\n", + "(17617, 3)\n" + ] + } + ], + "source": [ + "path = '/path/to/data/'\n", + "\n", + "data_test = np.load(path + 'test.npz', allow_pickle=True)\n", + "test_x = data_test['signal']\n", + "test_qa = data_test['qa_label']\n", + "test_r = data_test['rhythm']\n", + "test_p = pd.DataFrame(data_test['parameters'])\n", + "print(test_x.shape)\n", + "print(test_qa.shape)\n", + "print(test_r.shape)\n", + "print(test_p.shape)\n", + "test_p.rename(index=str, columns={0:'timestamp', \n", + " 1:'stream', \n", + " 2:'ID'}, inplace=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## VGG model : single-task\n" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "WARNING:tensorflow:From /home/users/jntorres/jessica/anaconda3/envs/tf_gpu/lib/python3.7/site-packages/tensorflow/python/framework/op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n", + "Instructions for updating:\n", + "Colocations handled automatically by placer.\n", + "WARNING:tensorflow:From /home/users/jntorres/jessica/anaconda3/envs/tf_gpu/lib/python3.7/site-packages/tensorflow/python/ops/math_ops.py:3066: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n", + "Instructions for updating:\n", + "Use tf.cast instead.\n" + ] + } + ], + "source": [ + "path = '/path/to/model/'\n", + "vgg = load_model(path + 'VGG_singletask.h5')\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### VGG Single-task \n" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "TPR: 0.92\n", + "TNR: 0.71\n", + "FPR: 0.29\n", + "FNR: 0.08\n", + "PPV: 0.50\n", + "NPV: 0.96\n", + "F1: 0.64\n" + ] + } + ], + "source": [ + "# weighted macro-average across all indivduals\n", + "\n", + "test_metrics = util.collecting_individual_metrics_singletask(vgg, test_x, test_p, test_r, out_message=False)\n", + "test_metrics = pd.DataFrame.from_dict(test_metrics).T.rename(columns={0:'TPR', 1:'TNR', 2:'FPR', 3:'FNR', 4:\"total_samples\"})\n", + "\n", + "for m in ['TPR', 'TNR', 'FPR', 'FNR']:\n", + " metric_wmu = np.average(test_metrics[m][~test_metrics[m].isna()], weights=test_metrics['total_samples'][~test_metrics[m].isna()])\n", + " print('%s: %0.2f' % (m, metric_wmu))\n", + " \n", + "# PPV, NPV and F1\n", + "\n", + "episode_m = util.episode_metrics_singletask(vgg, test_x, test_p, test_r, out_message=False)\n", + "episode_metrics = pd.DataFrame(episode_m).T\n", + "episode_metrics.rename(columns={0:'TPR', 1:'TNR', 2:'PPV', 3:'NPV', 4:\"FPR\", 5:'FNR', 6:'F1', 7:'total_samples'}, inplace=True)\n", + "for m in ['PPV', 'NPV', 'F1']:\n", + " print('%s: %0.2f' % (m, episode_metrics[m]))\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## DeepBeat Single task (no pretrain CDAE + AF task)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Loading non pretrained DeepBeat single task model" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "WARNING:tensorflow:From /home/users/jntorres/.local/lib/python3.7/site-packages/keras/backend/tensorflow_backend.py:3445: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n", + "Instructions for updating:\n", + "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n" + ] + } + ], + "source": [ + "path = '/path/to/model/'\n", + "model_reint = load_model(path + 'deepbeat_singletask_nopretrain.h5')\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### DeepBeat Single task" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "TPR: 0.49\n", + "TNR: 0.90\n", + "FPR: 0.10\n", + "FNR: 0.51\n", + "PPV: 0.60\n", + "NPV: 0.85\n", + "F1: 0.54\n" + ] + } + ], + "source": [ + "# weighted macro-average across all indivduals\n", + "\n", + "test_metrics = util.collecting_individual_metrics_singletask(model_reint, test_x, test_p, test_r, out_message=False)\n", + "test_metrics = pd.DataFrame.from_dict(test_metrics).T.rename(columns={0:'TPR', 1:'TNR', 2:'FPR', 3:'FNR', 4:\"total_samples\"})\n", + "\n", + "\n", + "for m in ['TPR', 'TNR', 'FPR', 'FNR']:\n", + " metric_wmu = np.average(test_metrics[m][~test_metrics[m].isna()], weights=test_metrics['total_samples'][~test_metrics[m].isna()])\n", + " print('%s: %0.2f' % (m, metric_wmu))\n", + " \n", + "# PPV, NPV and F1\n", + "\n", + "episode_m = util.episode_metrics_singletask(model_reint, test_x, test_p, test_r, out_message=False)\n", + "episode_metrics = pd.DataFrame(episode_m).T\n", + "episode_metrics.rename(columns={0:'TPR', 1:'TNR', 2:'PPV', 3:'NPV', 4:\"FPR\", 5:'FNR', 6:'F1', 7:'total_samples'}, inplace=True)\n", + "for m in ['PPV', 'NPV', 'F1']:\n", + " print('%s: %0.2f' % (m, episode_metrics[m]))\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## DeepBeat Single task (pretrain with CDAE + AF)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Loading pretrained DeepBeat single task model" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "path_to_model ='/path/to/model/'\n", + "model_name = 'deepbeat_singletask_pretrained.h5'\n", + "deepbeat_st = load_model(path_to_model + model_name) " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### DeepBeat Single task" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "TPR: 0.52\n", + "TNR: 0.88\n", + "FPR: 0.12\n", + "FNR: 0.48\n", + "PPV: 0.59\n", + "NPV: 0.85\n", + "F1: 0.56\n" + ] + } + ], + "source": [ + "# weighted macro-average across all indivduals\n", + "\n", + "test_metrics = util.collecting_individual_metrics_singletask(deepbeat_st, test_x, test_p, test_r, out_message=False)\n", + "test_metrics = pd.DataFrame.from_dict(test_metrics).T.rename(columns={0:'TPR', 1:'TNR', 2:'FPR', 3:'FNR', 4:\"total_samples\"})\n", + "\n", + "\n", + "for m in ['TPR', 'TNR', 'FPR', 'FNR']:\n", + " metric_wmu = np.average(test_metrics[m][~test_metrics[m].isna()], weights=test_metrics['total_samples'][~test_metrics[m].isna()])\n", + " print('%s: %0.2f' % (m, metric_wmu))\n", + " \n", + "# PPV, NPV and F1\n", + "\n", + "episode_m = util.episode_metrics_singletask(deepbeat_st, test_x, test_p, test_r, out_message=False)\n", + "episode_metrics = pd.DataFrame(episode_m).T\n", + "episode_metrics.rename(columns={0:'TPR', 1:'TNR', 2:'PPV', 3:'NPV', 4:\"FPR\", 5:'FNR', 6:'F1', 7:'total_samples'}, inplace=True)\n", + "for m in ['PPV', 'NPV', 'F1']:\n", + " print('%s: %0.2f' % (m, episode_metrics[m]))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## DeepBeat Multi-task (no pretrain CDAE + AF + Excellent QA)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Loading non pretrained DeepBeat model" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "path_to_model ='/path/to/model/'\n", + "model_reint = load_model(path_to_model + 'deepbeat_no_pretraining.h5')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### DeepBeat Multi-task (no pretrain CDAE + AF + Excellent QA) " + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "TPR: 0.97\n", + "TNR: 0.89\n", + "FPR: 0.11\n", + "FNR: 0.03\n", + "PPV: 0.55\n", + "NPV: 1.00\n", + "F1: 0.71\n" + ] + } + ], + "source": [ + "## QA results\n", + "\n", + "predictions_qa, predictions_r = model_reint.predict(test_x)\n", + "predictions_QA = np.argmax(predictions_qa, axis=1)\n", + "\n", + "#print(classification_report(np.argmax(test_qa, axis=1), predictions_QA))\n", + "\n", + "\n", + "excellent_qa_indx = np.where(predictions_QA==2)[0]\n", + "x_test_excellent = test_x[excellent_qa_indx,:]\n", + "p_test_excellent = test_p.iloc[excellent_qa_indx,:]\n", + "rhythm_test_excellent = test_r[excellent_qa_indx,:]\n", + "quality_assessment_test_excellent = test_qa[excellent_qa_indx,:]\n", + "\n", + "\n", + "# weighted macro-average across all indivduals\n", + "\n", + "test_metrics = util.collecting_individual_metrics(model_reint, x_test_excellent, p_test_excellent, rhythm_test_excellent, out_message=False)\n", + "test_metrics = pd.DataFrame.from_dict(test_metrics).T.rename(columns={0:'TPR', 1:'TNR', 2:'FPR', 3:'FNR', 4:\"total_samples\"})\n", + "\n", + "\n", + "for m in ['TPR', 'TNR', 'FPR', 'FNR']:\n", + " metric_wmu = np.average(test_metrics[m][~test_metrics[m].isna()], weights=test_metrics['total_samples'][~test_metrics[m].isna()])\n", + " print('%s: %0.2f' % (m, metric_wmu))\n", + " \n", + "# PPV, NPV and F1\n", + "\n", + "episode_m = util.episode_metrics(model_reint, x_test_excellent, p_test_excellent, rhythm_test_excellent, out_message=False)\n", + "episode_metrics = pd.DataFrame(episode_m).T\n", + "episode_metrics.rename(columns={0:'TPR', 1:'TNR', 2:'PPV', 3:'NPV', 4:\"FPR\", 5:'FNR', 6:'F1', 7:'total_samples'}, inplace=True)\n", + "for m in ['PPV', 'NPV', 'F1']:\n", + " print('%s: %0.2f' % (m, episode_metrics[m]))\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## DeepBeat Multi-task (pretrained with CDAE + AF + Excellent QA)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Loading pretrained DeepBeat model" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "path_to_model ='/path/to/model/'\n", + "model_name = 'deepbeat.h5'\n", + "deepbeat = load_model(path_to_model + model_name) " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### DeepBeat Multi-task (pretrain CDAE + AF + Excellent QA) \n" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "TPR: 0.98\n", + "TNR: 0.99\n", + "FPR: 0.01\n", + "FNR: 0.02\n", + "PPV: 0.94\n", + "NPV: 1.00\n", + "F1: 0.96\n" + ] + } + ], + "source": [ + "## QA results\n", + "\n", + "predictions_qa, predictions_r = deepbeat.predict(test_x)\n", + "predictions_QA = np.argmax(predictions_qa, axis=1)\n", + "\n", + "#print(classification_report(np.argmax(test_qa, axis=1), predictions_QA))\n", + "\n", + "\n", + "excellent_qa_indx = np.where(predictions_QA==2)[0]\n", + "x_test_excellent = test_x[excellent_qa_indx,:]\n", + "p_test_excellent = test_p.iloc[excellent_qa_indx,:]\n", + "rhythm_test_excellent = test_r[excellent_qa_indx,:]\n", + "quality_assessment_test_excellent = test_qa[excellent_qa_indx,:]\n", + "\n", + "\n", + "# weighted macro-average across all indivduals\n", + "\n", + "test_metrics = util.collecting_individual_metrics(deepbeat, x_test_excellent, p_test_excellent, rhythm_test_excellent, out_message=False)\n", + "test_metrics = pd.DataFrame.from_dict(test_metrics).T.rename(columns={0:'TPR', 1:'TNR', 2:'FPR', 3:'FNR', 4:\"total_samples\"})\n", + "\n", + "\n", + "for m in ['TPR', 'TNR', 'FPR', 'FNR']:\n", + " metric_wmu = np.average(test_metrics[m][~test_metrics[m].isna()], weights=test_metrics['total_samples'][~test_metrics[m].isna()])\n", + " print('%s: %0.2f' % (m, metric_wmu))\n", + " \n", + "# PPV, NPV and F1\n", + "\n", + "episode_m = util.episode_metrics(deepbeat, x_test_excellent, p_test_excellent, rhythm_test_excellent, out_message=False)\n", + "episode_metrics = pd.DataFrame(episode_m).T\n", + "episode_metrics.rename(columns={0:'TPR', 1:'TNR', 2:'PPV', 3:'NPV', 4:\"FPR\", 5:'FNR', 6:'F1', 7:'total_samples'}, inplace=True)\n", + "for m in ['PPV', 'NPV', 'F1']:\n", + " print('%s: %0.2f' % (m, episode_metrics[m]))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +}