{

 "cells": [

  {

   "cell_type": "markdown",

   "metadata": {},

   "source": [

    "# What this notebook is for\n",

    "\n",

    "This notebook aids the process of evaluating accuracy measures and predicting class outputs of already trained models."

   ]

  },

  {

   "cell_type": "code",

   "execution_count": null,

   "metadata": {},

   "outputs": [],

   "source": [

    "import tensorflow as tf\n",

    "import numpy as np\n",

    "import h5py as h5"

   ]

  },

  {

   "cell_type": "markdown",

   "metadata": {},

   "source": [

    "## Reading data\n",

    "\n",

    "Throughout this project we've used these functions in order to read the processed data."

   ]

  },

  {

   "cell_type": "code",

   "execution_count": null,

   "metadata": {},

   "outputs": [],

   "source": [

    "def make_dimensions_compatible(arr):\n",

    "    \n",

    "    return arr.reshape(arr.shape[0],-1,1)"

   ]

  },

  {

   "cell_type": "markdown",

   "metadata": {},

   "source": [

    "<blockquote>\n",

    "    <b>Note:</b> We're using Python <b>global variables</b> in the load_dataset method. They'll be available throughout this notebook.\n",

    "    </blockquote>"

   ]

  },

  {

   "cell_type": "markdown",

   "metadata": {},

   "source": [

    "<blockquote>\n",

    "    <b>Important:</b> This method uses default folder names, prefixes and suffixes (herd-coded) as chosen by the author. If you've not changed anything these names in other files, it should work without any issues.\n",

    "</blockquote>"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": null,

   "metadata": {},

   "outputs": [],

   "source": [

    "def load_dataset(dataset_iter=1, window_size=512):\n",

    "    global X_train, Y_train, X_dev, Y_dev, X_test, Y_test\n",

    "    \n",

    "    dataset_relative_path = 'dataset/random-iter-%d/' % dataset_iter\n",

    "    \n",

    "    datafile = dataset_relative_path + 'datafile%d.h5' % window_size\n",

    "\n",

    "    with h5.File(datafile, 'r') as datafile:\n",

    "        X_train = np.array(datafile['X_train'])\n",

    "        Y_train = np.array(datafile['Y_train'])\n",

    "\n",

    "        X_dev = np.array(datafile['X_dev'])\n",

    "        Y_dev = np.array(datafile['Y_dev'])\n",

    "\n",

    "        X_test = np.array(datafile['X_test'])\n",

    "        Y_test = np.array(datafile['Y_test'])\n",

    "        \n",

    "        # setting the rank of the data to be compatible with 1d convolution functions\n",

    "        # defined in tensorflow\n",

    "        X_train = make_dimensions_compatible(X_train)\n",

    "        X_dev = make_dimensions_compatible(X_dev)\n",

    "        X_test = make_dimensions_compatible(X_test)\n",

    "        \n",

    "        # normalization\n",

    "        X_train = X_train / 1000\n",

    "        X_dev = X_dev / 1000\n",

    "        X_test = X_test / 1000"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": null,

   "metadata": {},

   "outputs": [],

   "source": [

    "def get_session_path(dataset_iter=1, window_size=512, model_num=1, model_prefix='cnn', model_suffix='_lr-0.00002_mbs-128'):\n",

    "    return ('train/dataset-%d-%d/' + model_prefix + '%d' + model_suffix + '/') % (window_size, dataset_iter, model_num)\n",

    "\n"

   ]

  },

  {

   "cell_type": "markdown",

   "metadata": {},

   "source": [

    "## Prediction"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": null,

   "metadata": {},

   "outputs": [],

   "source": [

    "def predict(X_test, session_path, model_file, Y_test_onehot=None):\n",

    "\n",

    "    tf.reset_default_graph()\n",

    "\n",

    "    checkpoint_path = session_path\n",

    "    model_path = session_path + model_file\n",

    "\n",

    "    with tf.Session() as sess:\n",

    "        loader = tf.train.import_meta_graph(model_path)\n",

    "        loader.restore(sess, tf.train.latest_checkpoint(checkpoint_path))\n",

    "\n",

    "        graph = tf.get_default_graph()\n",

    "\n",

    "        X = graph.get_tensor_by_name('X:0')\n",

    "        Y = graph.get_tensor_by_name('Y:0')\n",

    "        is_train = graph.get_tensor_by_name('is_train:0')\n",

    "        \n",

    "#         epoch_counter = graph.get_tensor_by_name('epoch_counter:0')\n",

    "#         print(epoch_counter.eval())\n",

    "\n",

    "        Y_hat = graph.get_tensor_by_name('softmax_output:0')\n",

    "\n",

    "        predict_op = tf.argmax(Y_hat, 1)\n",

    "\n",

    "        y_hat_test = predict_op.eval({X: X_test, is_train: False})\n",

    "        \n",

    "        # print the accuracy of the test set if the labels are provided\n",

    "        if (Y_test_onehot is not None):\n",

    "            y_test = np.argmax(Y_test_onehot, 1)\n",

    "            acc = (y_hat_test == y_test).mean()\n",

    "        \n",

    "\n",

    "    return y_hat_test, acc\n"

   ]

  },

  {

   "cell_type": "markdown",

   "metadata": {},

   "source": [

    "## Prediction with majority voting on ensemble network"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": null,

   "metadata": {},

   "outputs": [],

   "source": [

    "def predict_voting(X_test_voting, session_path, model_file):\n",

    "\n",

    "    tf.reset_default_graph()\n",

    "\n",

    "    checkpoint_path = session_path\n",

    "    model_path = session_path + model_file\n",

    "    \n",

    "    y_hat_test_voting = []\n",

    "\n",

    "    with tf.Session() as sess:\n",

    "        loader = tf.train.import_meta_graph(model_path)\n",

    "        loader.restore(sess, tf.train.latest_checkpoint(checkpoint_path))\n",

    "\n",

    "        graph = tf.get_default_graph()\n",

    "\n",

    "        X = graph.get_tensor_by_name('X:0')\n",

    "        is_train = graph.get_tensor_by_name('is_train:0')\n",

    "\n",

    "        Y_hat = graph.get_tensor_by_name('softmax_output:0')\n",

    "\n",

    "        predict_op = tf.argmax(Y_hat, 1)\n",

    "        \n",

    "        classname, idx, counts = tf.unique_with_counts(predict_op)\n",

    "        predict_voting_op = tf.gather(classname, tf.argmax(counts))\n",

    "\n",

    "        # no. of training examples with the original feature size\n",

    "        m = X_test_voting.shape[0]\n",

    "        \n",

    "        # no. of split training examples of each original example\n",

    "        m_each = X_test_voting.shape[1]\n",

    "        \n",

    "        for ex in range(m):\n",

    "            x_test_voting = make_dimensions_compatible(X_test_voting[ex])\n",

    "            pred = predict_voting_op.eval({X: x_test_voting, is_train: False})\n",

    "            \n",

    "            y_hat_test_voting.append(pred)\n",

    "\n",

    "    return y_hat_test_voting"

   ]

  },

  {

   "cell_type": "markdown",

   "metadata": {},

   "source": [

    "### Accuracy measure\n",

    "\n",

    "Change the values of `model_num`, `window_size`, and `dataset_iters` to test on the different models (cnn1, cnn2, and so on... as described in the literature) you've trained on. If you've generated multiple datasets for k-fold cross validation use `dataset_iters`. In the literature, we've used window sizes of 512 and 1024."

   ]

  },

  {

   "cell_type": "code",

   "execution_count": null,

   "metadata": {

    "scrolled": false

   },

   "outputs": [],

   "source": [

    "# Run the following code to call the above define predict method and check accuracy of the models\n",

    "\n",

    "model_num = 8\n",

    "window_size = 1024\n",

    "dataset_iters = (1, 2, 3, 4, 5, 6, 7, 8)\n",

    "\n",

    "accuracies = np.array([])\n",

    "\n",

    "for dataset_iter in dataset_iters:\n",

    "    load_dataset(dataset_iter=dataset_iter, window_size=window_size)\n",

    "    predictions, acc = predict(X_test, get_session_path(dataset_iter=dataset_iter, window_size=window_size, model_num=model_num), 'model.meta', Y_test_onehot=Y_test)\n",

    "    accuracies = np.append(accuracies, acc)\n",

    "\n",

    "accuracies = accuracies * 100\n",

    "print(accuracies)\n",

    "\n",

    "print(\"Mean accuracy: %f\" % accuracies.mean())"

   ]

  },

  {

   "cell_type": "markdown",

   "metadata": {},

   "source": [

    "### Single model accuracy with voting measure"

   ]

  },

  {

   "cell_type": "markdown",

   "metadata": {},

   "source": [

    "<blockquote>\n",

    "    <b>Note:</b> This method uses default folder names and suffixes chosen by the author.\n",

    "</blockquote>\n",

    "\n",

    "Calculate accuracy with voting, F-score, and other relevant measures. Change the values of `model_num`, `window_size`, and `dataset_iters` to try on different models and random permuations (random-iter-) of dataset."

   ]

  },

  {

   "cell_type": "code",

   "execution_count": null,

   "metadata": {},

   "outputs": [],

   "source": [

    "model_num = 8\n",

    "window_size = 1024\n",

    "dataset_iters = (1, 2, 3, 4, 5, 6, 7, 8)\n",

    "\n",

    "accuracies_voting = np.array([])\n",

    "sensitivities_voting = np.array([])\n",

    "specificities_voting = np.array([])\n",

    "precisions_voting = np.array([])\n",

    "\n",

    "for dataset_iter in dataset_iters:\n",

    "    dataset_relative_path = 'dataset/random-iter-%d/' % dataset_iter\n",

    "    testfile = (dataset_relative_path + 'testset_voting_%d.h5') % window_size\n",

    "    session_path = get_session_path(dataset_iter=dataset_iter, window_size=window_size, model_num=model_num)\n",

    "    model_file = 'model.meta'\n",

    "\n",

    "    with h5.File(testfile, 'r') as testfile:\n",

    "        X_test_voting = testfile['X']\n",

    "        X_test_voting = np.array(X_test_voting) / 1000\n",

    "        y_test_voting = np.array(testfile['Y'])\n",

    "        \n",

    "        no_of_classes = 3\n",

    "\n",

    "        y_hat_test_voting = np.array(predict_voting(X_test_voting, session_path, model_file))\n",

    "\n",

    "        acc_voting_model = (y_test_voting == y_hat_test_voting).mean()\n",

    "        accuracies_voting = np.append(accuracies_voting, acc_voting_model)\n",

    "        \n",

    "        # specificity and sensitivity\n",

    "        specificity = 0\n",

    "        sensitivity = 0\n",

    "        precision = 0\n",

    "        for i in range(no_of_classes):\n",

    "            specificity_vector = ((y_hat_test_voting != i) == (y_test_voting != i))[y_test_voting != i]\n",

    "            sensitivity_vector = ((y_hat_test_voting == i) == (y_test_voting == i))[y_test_voting == i]\n",

    "            precision_vector = ((y_hat_test_voting == i) == (y_test_voting == i))[y_hat_test_voting == i]\n",

    "            \n",

    "            specificity = specificity + specificity_vector.sum() / len(specificity_vector)\n",

    "            sensitivity = sensitivity + sensitivity_vector.sum() / len(sensitivity_vector)\n",

    "            precision = precision + precision_vector.sum() / len(precision_vector)\n",

    "        \n",

    "        specificity = specificity / no_of_classes\n",

    "        sensitivity = sensitivity / no_of_classes\n",

    "        precision = precision / no_of_classes\n",

    "        \n",

    "        specificities_voting = np.append(specificities_voting, specificity)\n",

    "        sensitivities_voting = np.append(sensitivities_voting, sensitivity)\n",

    "        precisions_voting = np.append(precisions_voting, precision)\n",

    "\n",

    "\n",

    "# Computing F-Score\n",

    "Pre = precisions_voting.mean()\n",

    "Sen = sensitivities_voting.mean()\n",

    "F_Score = (2 * Pre * Sen) / (Pre + Sen)\n",

    "        \n",

    "accuracies_voting = accuracies_voting * 100\n",

    "print(\"Accuracy with voting: \", accuracies_voting)\n",

    "print(\"Mean accuracy with voting: %f\" % accuracies_voting.mean())\n",

    "print(\"\\n\\n\")\n",

    "print(\"Specifities: \", specificities_voting)\n",

    "print(\"Mean specificity with voting: %f\" % specificities_voting.mean())\n",

    "print(\"\\n\\n\")\n",

    "print(\"Sensitivities: \", sensitivities_voting)\n",

    "print(\"Mean sensitivity with voting: %f\" % sensitivities_voting.mean())\n",

    "# print(\"Mean precision with voting: %f\" % Pre)\n",

    "print(\"Mean F-Score with voting: %f\" % F_Score)\n",

    "print(\"Standard deviation: %f\" % accuracies_voting.std())\n",

    "\n",

    "# print(\"Actual labels:\\n\", y_test_voting)\n",

    "# print(\"\\n\\nPredictions:\\n\", y_hat_test_voting)\n"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": null,

   "metadata": {},

   "outputs": [],

   "source": []

  }

 ],

 "metadata": {

  "kernelspec": {

   "display_name": "Python 3",

   "language": "python",

   "name": "python3"

  },

  "language_info": {

   "codemirror_mode": {

    "name": "ipython",

    "version": 3

   },

   "file_extension": ".py",

   "mimetype": "text/x-python",

   "name": "python",

   "nbconvert_exporter": "python",

   "pygments_lexer": "ipython3",

   "version": "3.6.8"

  }

 },

 "nbformat": 4,

 "nbformat_minor": 2

}