{

 "cells": [

  {

   "cell_type": "markdown",

   "metadata": {

    "deletable": true,

    "editable": true

   },

   "source": [

    "## Initialise"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": 20,

   "metadata": {

    "collapsed": false,

    "deletable": true,

    "editable": true

   },

   "outputs": [

    {

     "name": "stdout",

     "output_type": "stream",

     "text": [

      "The autoreload extension is already loaded. To reload it, use:\n",

      "  %reload_ext autoreload\n"

     ]

    }

   ],

   "source": [

    "# Reload modules \n",

    "# It is an optional step. It is useful to run when external Python modules are being modified\n",

    "# It is reloading all modules (except those excluded by %aimport) every time before executing the Python code typed.\n",

    "# Note: It may conflict with serialisation, when external modules are being modified\n",

    "\n",

    "%load_ext autoreload \n",

    "%autoreload 2"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": 21,

   "metadata": {

    "collapsed": true,

    "deletable": true,

    "editable": true

   },

   "outputs": [],

   "source": [

    "# Import Python libraries\n",

    "import logging\n",

    "import os\n",

    "import sys\n",

    "import gc\n",

    "import pandas as pd\n",

    "from IPython.display import display, HTML\n",

    "import numpy as np\n",

    "import matplotlib.pyplot as plt"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": 22,

   "metadata": {

    "collapsed": false,

    "deletable": true,

    "editable": true

   },

   "outputs": [],

   "source": [

    "# Import local Python modules\n",

    "from Configs.CONSTANTS import CONSTANTS\n",

    "from Configs.Logger import Logger\n",

    "from ReadersWriters.ReadersWriters import ReadersWriters\n",

    "from Stats.Plots import Plots"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": 23,

   "metadata": {

    "collapsed": false,

    "deletable": true,

    "editable": true

   },

   "outputs": [

    {

     "name": "stdout",

     "output_type": "stream",

     "text": [

      "\n",

      "Make sure the correct Python interpreter is used!\n",

      "3.5.3 (v3.5.3:1880cb95a742, Jan 16 2017, 16:02:32) [MSC v.1900 64 bit (AMD64)]\n",

      "\n",

      "Make sure sys.path of the Python interpreter is correct!\n",

      "C:\\Users\\eagle\\Documents\\GitHub\\Analytics_UoW\\TCARER\n"

     ]

    }

   ],

   "source": [

    "# Check the interpreter\n",

    "print(\"\\nMake sure the correct Python interpreter is used!\")\n",

    "print(sys.version)\n",

    "print(\"\\nMake sure sys.path of the Python interpreter is correct!\")\n",

    "print(os.getcwd())"

   ]

  },

  {

   "cell_type": "markdown",

   "metadata": {

    "deletable": true,

    "editable": true

   },

   "source": [

    "Set constants"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": 24,

   "metadata": {

    "collapsed": false,

    "deletable": true,

    "editable": true

   },

   "outputs": [

    {

     "name": "stdout",

     "output_type": "stream",

     "text": [

      "\n",

      " The full path of the configuration file: \n",

      "\t C:\\Users\\eagle\\Documents\\GitHub\\Analytics_UoW\\TCARER\\ConfigInputs\\CONFIGURATIONS.ini \n",

      " The full path of the output folder: \n",

      "\t C:\\Users\\eagle\\Documents\\GitHub\\tmp\\TCARER\\scores_adhoc \n",

      " The application name (the suffix of the outputs file name): \n",

      "\t tcarerPlots\n"

     ]

    }

   ],

   "source": [

    "config_path = os.path.abspath(\"ConfigInputs/CONFIGURATIONS.ini\")\n",

    "io_path = os.path.abspath(\"../../tmp/TCARER/scores_adhoc\")\n",

    "app_name = 'tcarerPlots'\n",

    "\n",

    "print(\"\\n The full path of the configuration file: \\n\\t\", config_path,\n",

    "      \"\\n The full path of the output folder: \\n\\t\", io_path,\n",

    "      \"\\n The application name (the suffix of the outputs file name): \\n\\t\", app_name)"

   ]

  },

  {

   "cell_type": "markdown",

   "metadata": {

    "deletable": true,

    "editable": true

   },

   "source": [

    "<br/>"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": 25,

   "metadata": {

    "collapsed": false,

    "deletable": true,

    "editable": true

   },

   "outputs": [

    {

     "name": "stderr",

     "output_type": "stream",

     "text": [

      "2017-10-29 12:29:33,032 - tcarerPlots - INFO - Creating 'C:\\Users\\eagle\\Documents\\GitHub\\tmp\\TCARER\\scores_adhoc\\tcarerPlots.log' File.\n",

      "2017-10-29 12:29:33,032 - tcarerPlots - INFO - Creating 'C:\\Users\\eagle\\Documents\\GitHub\\tmp\\TCARER\\scores_adhoc\\tcarerPlots.log' File.\n",

      "2017-10-29 12:29:33,032 - tcarerPlots - INFO - Creating 'C:\\Users\\eagle\\Documents\\GitHub\\tmp\\TCARER\\scores_adhoc\\tcarerPlots.log' File.\n"

     ]

    }

   ],

   "source": [

    "if not os.path.exists(io_path):\n",

    "    os.makedirs(io_path, exist_ok=True)\n",

    "\n",

    "logger = Logger(path=io_path, app_name=app_name, ext=\"log\")\n",

    "logger = logging.getLogger(app_name)"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": 26,

   "metadata": {

    "collapsed": true,

    "deletable": true,

    "editable": true

   },

   "outputs": [],

   "source": [

    "# Initialise constants        \n",

    "CONSTANTS.set(io_path, app_name)"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": 27,

   "metadata": {

    "collapsed": true,

    "deletable": true,

    "editable": true

   },

   "outputs": [],

   "source": [

    "# Initialise other classes\n",

    "readers_writers = ReadersWriters()\n",

    "plots = Plots()"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": 35,

   "metadata": {

    "collapsed": true,

    "deletable": true,

    "editable": true

   },

   "outputs": [],

   "source": [

    "samples = [\"Basic_prototype\"]\n",

    "targets = [\"365\"]\n",

    "submodels = [\"rfc\", \"wdnn\"]\n",

    "paths = [os.path.abspath(\"../../tmp/TCARER/Basic_prototype\")]\n",

    "file_names = [[\"Step_09_Model_rfc_label365\", \"Step_09_Model_wdnn_label365\"]]"

   ]

  },

  {

   "cell_type": "markdown",

   "metadata": {

    "deletable": true,

    "editable": true

   },

   "source": [

    "<br/><br/>"

   ]

  },

  {

   "cell_type": "markdown",

   "metadata": {

    "deletable": true,

    "editable": true

   },

   "source": [

    "## Load Predictions"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": 37,

   "metadata": {

    "collapsed": false,

    "deletable": true,

    "editable": true,

    "scrolled": true

   },

   "outputs": [],

   "source": [

    "models = dict()\n",

    "for i in range(len(samples)):\n",

    "    models[samples[i]] = dict()\n",

    "    for j in range(len(targets)):\n",

    "        models[samples[i]][targets[j]] = dict()\n",

    "        for k in range(len(submodels)):\n",

    "            models[samples[i]][targets[j]][submodels[k]] = \\\n",

    "                readers_writers.load_serialised(path=paths[i], title=file_names[j][k])"

   ]

  },

  {

   "cell_type": "markdown",

   "metadata": {

    "deletable": true,

    "editable": true

   },

   "source": [

    "hack - correction of dictionaries"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": 40,

   "metadata": {

    "collapsed": false,

    "deletable": true,

    "editable": true

   },

   "outputs": [

    {

     "ename": "KeyError",

     "evalue": "'wdnn'",

     "output_type": "error",

     "traceback": [

      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",

      "\u001b[0;31mKeyError\u001b[0m                                  Traceback (most recent call last)",

      "\u001b[0;32m<ipython-input-40-a1f912f3713b>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msamples\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m     \u001b[1;32mfor\u001b[0m \u001b[0mj\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtargets\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m         \u001b[0mmodels\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0msamples\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mtargets\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mj\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"wdnn\"\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m{\u001b[0m\u001b[1;34m'model_predict'\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mmodels\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0msamples\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mtargets\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mj\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"wdnn\"\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m}\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",

      "\u001b[0;31mKeyError\u001b[0m: 'wdnn'"

     ]

    }

   ],

   "source": [

    "for i in range(len(samples)):\n",

    "    for j in range(len(targets)):\n",

    "        models[samples[i]][targets[j]][\"wdnn\"] = {'model_predict': models[samples[i]][targets[j]][\"wdnn\"]}"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": 39,

   "metadata": {

    "collapsed": false,

    "deletable": true,

    "editable": true

   },

   "outputs": [

    {

     "ename": "KeyError",

     "evalue": "'sample-1'",

     "output_type": "error",

     "traceback": [

      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",

      "\u001b[0;31mKeyError\u001b[0m                                  Traceback (most recent call last)",

      "\u001b[0;32m<ipython-input-39-0533bcf87ed4>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mmodels\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"sample-1\"\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"30\"\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"wdnn\"\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'model_predict'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'test'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'score'\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0marray\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mmodels\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"sample-1\"\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"30\"\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"wdnn\"\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'model_predict'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'test'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'predict_proba'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0mmodels\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"sample-1\"\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"365\"\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"wdnn\"\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'model_predict'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'test'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'score'\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0marray\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mmodels\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"sample-1\"\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"365\"\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"wdnn\"\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'model_predict'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'test'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'predict_proba'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0mmodels\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"sample-2\"\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"30\"\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"wdnn\"\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'model_predict'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'test'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'score'\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0marray\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mmodels\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"sample-2\"\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"30\"\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"wdnn\"\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'model_predict'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'test'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'predict_proba'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0mmodels\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"sample-2\"\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"365\"\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"wdnn\"\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'model_predict'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'test'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'score'\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0marray\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mmodels\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"sample-2\"\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"365\"\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"wdnn\"\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'model_predict'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'test'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'predict_proba'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",

      "\u001b[0;31mKeyError\u001b[0m: 'sample-1'"

     ]

    }

   ],

   "source": [

    "models[\"sample-1\"][\"30\"][\"wdnn\"]['model_predict']['test']['score'] = np.array([i[1] for i in models[\"sample-1\"][\"30\"][\"wdnn\"]['model_predict']['test']['predict_proba']])\n",

    "models[\"sample-1\"][\"365\"][\"wdnn\"]['model_predict']['test']['score'] = np.array([i[1] for i in models[\"sample-1\"][\"365\"][\"wdnn\"]['model_predict']['test']['predict_proba']])\n",

    "models[\"sample-2\"][\"30\"][\"wdnn\"]['model_predict']['test']['score'] = np.array([i[1] for i in models[\"sample-2\"][\"30\"][\"wdnn\"]['model_predict']['test']['predict_proba']])\n",

    "models[\"sample-2\"][\"365\"][\"wdnn\"]['model_predict']['test']['score'] = np.array([i[1] for i in models[\"sample-2\"][\"365\"][\"wdnn\"]['model_predict']['test']['predict_proba']])"

   ]

  },

  {

   "cell_type": "markdown",

   "metadata": {

    "deletable": true,

    "editable": true

   },

   "source": [

    "<br/><br/>\n",

    "temporary solution - Load samples"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": null,

   "metadata": {

    "collapsed": true,

    "deletable": true,

    "editable": true

   },

   "outputs": [],

   "source": [

    "file_names = [\"Step_07_Features\", \"Step_07_Features\"]"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": null,

   "metadata": {

    "collapsed": true,

    "deletable": true,

    "editable": true,

    "scrolled": true

   },

   "outputs": [],

   "source": [

    "target_labels = dict()\n",

    "for i in range(len(samples)):\n",

    "    target_labels[samples[i]] = readers_writers.load_serialised(path=paths[i], title=file_names[i])[\"test_target\"]\n",

    "    \n",

    "for i in range(len(samples)):\n",

    "    target_labels[samples[i]] = {\"30\": target_labels[samples[i]][\"label30\"], \n",

    "                                 \"365\": target_labels[samples[i]][\"label365\"]}\n",

    "    target_labels[\"label30\"] = None\n",

    "    target_labels[\"label365\"] = None\n",

    "\n",

    "gc.collect()"

   ]

  },

  {

   "cell_type": "markdown",

   "metadata": {

    "collapsed": true,

    "deletable": true,

    "editable": true

   },

   "source": [

    "temporary solution - correction of wdnn labels"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": null,

   "metadata": {

    "collapsed": true,

    "deletable": true,

    "editable": true

   },

   "outputs": [],

   "source": [

    "for sample in samples:\n",

    "    for target in targets:\n",

    "        print(len(target_labels[sample][target]), len(models[sample][target][\"wdnn\"]['model_predict']['test']['score']))\n",

    "        target_labels[sample][target] = np.append(target_labels[sample][target], target_labels[sample][target])\n",

    "        target_labels[sample][target] = target_labels[sample][target][0:len(models[sample][target][\"wdnn\"]['model_predict']['test']['score'])]"

   ]

  },

  {

   "cell_type": "markdown",

   "metadata": {

    "deletable": true,

    "editable": true

   },

   "source": [

    "<br/><br/><br/>"

   ]

  },

  {

   "cell_type": "markdown",

   "metadata": {

    "deletable": true,

    "editable": true

   },

   "source": [

    "## Save"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": null,

   "metadata": {

    "collapsed": true,

    "deletable": true,

    "editable": true

   },

   "outputs": [],

   "source": [

    "file_name = \"saved_models\"\n",

    "readers_writers.save_serialised(path=io_path, title=file_name, objects=models)\n",

    "\n",

    "file_name = \"saved_target_labels\"\n",

    "readers_writers.save_serialised(path=io_path, title=file_name, objects=target_labels)"

   ]

  },

  {

   "cell_type": "markdown",

   "metadata": {

    "deletable": true,

    "editable": true

   },

   "source": [

    "## Load"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": null,

   "metadata": {

    "collapsed": false,

    "deletable": true,

    "editable": true

   },

   "outputs": [],

   "source": [

    "file_name = \"saved_models\"\n",

    "models = readers_writers.load_serialised(path=io_path, title=file_name)\n",

    "\n",

    "file_name = \"saved_target_labels\"\n",

    "target_labels = readers_writers.load_serialised(path=io_path, title=file_name)"

   ]

  },

  {

   "cell_type": "markdown",

   "metadata": {

    "deletable": true,

    "editable": true

   },

   "source": [

    "<br/><br/>"

   ]

  },

  {

   "cell_type": "markdown",

   "metadata": {

    "deletable": true,

    "editable": true

   },

   "source": [

    "## Plot"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": null,

   "metadata": {

    "collapsed": true,

    "deletable": true,

    "editable": true

   },

   "outputs": [],

   "source": [

    "file_name = \"Combined_plots\"\n",

    "\n",

    "#%pylab inline\n",

    "#pylab.rcParams['figure.figsize'] = (10, 10)"

   ]

  },

  {

   "cell_type": "markdown",

   "metadata": {

    "deletable": true,

    "editable": true

   },

   "source": [

    "Combine"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": null,

   "metadata": {

    "collapsed": true,

    "deletable": true,

    "editable": true

   },

   "outputs": [],

   "source": [

    "markers = {'.': 'point', ',': 'pixel', 'o': 'circle', 'v': 'triangle_down', '^': 'triangle_up', '<': 'triangle_left', '>': 'triangle_right', '1': 'tri_down', '2': 'tri_up', '3': 'tri_left', '4': 'tri_right', '8': 'octagon', 's': 'square', 'p': 'pentagon', '*': 'star', 'h': 'hexagon1', 'H': 'hexagon2', '+': 'plus', 'x': 'x', 'D': 'diamond', 'd': 'thin_diamond', '|': 'vline', '_': 'hline', 'P': 'plus_filled', 'X': 'x_filled', 0: 'tickleft', 1: 'tickright', 2: 'tickup', 3: 'tickdown', 4: 'caretleft', 5: 'caretright', 6: 'caretup', 7: 'caretdown', 8: 'caretleftbase', 9: 'caretrightbase', 10: 'caretupbase', 11: 'caretdownbase', 'None': 'nothing', None: 'nothing', ' ': 'nothing', '': 'nothing'}\n",

    "scores_list = []\n",

    "target_labels_list = []\n",

    "label_list = []\n",

    "marker_list = ['.', 'o', 'v', '2', 'p', 's', '*', '+', '|'] #['.', 'o', 'v', '2', 'p', 's', '*', '+', '|']\n",

    "linestyle_list = [':', ':', ':', ':', ':', ':', ':', ':', ':'] #[':', '-.', '--', '-', ':', '-.', '--', '-', ':'] \n",

    "color_list = ['grey', 'firebrick', 'coral', 'darkgreen', 'coral', 'mediumblue', 'lightseagreen', 'blueviolet', 'darkgray']\n",

    "\n",

    "for sample in samples:\n",

    "    for target in targets:\n",

    "        for submodel in submodels:\n",

    "            scores_list.append(models[sample][target][submodel]['model_predict']['test']['score'])\n",

    "            target_labels_list.append(np.array(target_labels[sample][target][\n",

    "                0:len(models[sample][target][submodel]['model_predict']['test']['score'])]))\n",

    "            label_list.append(sample + \", \" + submodel + \"-\" + target)"

   ]

  },

  {

   "cell_type": "markdown",

   "metadata": {

    "deletable": true,

    "editable": true

   },

   "source": [

    "### ROC Plot"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": null,

   "metadata": {

    "collapsed": false,

    "deletable": true,

    "editable": true

   },

   "outputs": [],

   "source": [

    "plt.clf()\n",

    "fig, o_summaries = plots.roc_multiple(scores_list, target_labels_list, \n",

    "                                      label_list=label_list,  marker_list=marker_list, \n",

    "                                      linestyle_list=linestyle_list, color_list=color_list, \n",

    "                                      lw=2, markersize=8, legend_prop=15, legend_markerscale=1)"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": null,

   "metadata": {

    "collapsed": false,

    "deletable": true,

    "editable": true

   },

   "outputs": [],

   "source": [

    "display(fig)"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": null,

   "metadata": {

    "collapsed": true,

    "deletable": true,

    "editable": true

   },

   "outputs": [],

   "source": [

    "# save\n",

    "fig.savefig(os.path.join(io_path, file_name + '_roc.png'), \n",

    "            dpi=1500, facecolor='w', edgecolor='w', orientation='portrait', papertype=None, format='png',\n",

    "            transparent=False, bbox_inches=None, pad_inches=0.1, frameon=None)\n",

    "plt.clf()"

   ]

  },

  {

   "cell_type": "markdown",

   "metadata": {

    "deletable": true,

    "editable": true

   },

   "source": [

    "### Precision-Recall Plot"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": null,

   "metadata": {

    "collapsed": false,

    "deletable": true,

    "editable": true

   },

   "outputs": [],

   "source": [

    "plt.clf()\n",

    "fig, o_summaries = plots.precision_recall_multiple(scores_list, target_labels_list, \n",

    "                                                   label_list=label_list,  marker_list=marker_list, \n",

    "                                                   linestyle_list=linestyle_list, color_list=color_list,\n",

    "                                                   lw=2, markersize=8, legend_prop=10, legend_markerscale=1)"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": null,

   "metadata": {

    "collapsed": false,

    "deletable": true,

    "editable": true

   },

   "outputs": [],

   "source": [

    "display(fig)"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": null,

   "metadata": {

    "collapsed": true,

    "deletable": true,

    "editable": true

   },

   "outputs": [],

   "source": [

    "# save\n",

    "fig.savefig(os.path.join(io_path, file_name + '_precision_recall.png'), \n",

    "            dpi=1500, facecolor='w', edgecolor='w', orientation='portrait', papertype=None, format='png',\n",

    "            transparent=False, bbox_inches=None, pad_inches=0.1, frameon=None)\n",

    "plt.clf()"

   ]

  },

  {

   "cell_type": "markdown",

   "metadata": {

    "deletable": true,

    "editable": true

   },

   "source": [

    "Fin!"

   ]

  }

 ],

 "metadata": {

  "kernelspec": {

   "display_name": "Python 3",

   "language": "python",

   "name": "python3"

  },

  "language_info": {

   "codemirror_mode": {

    "name": "ipython",

    "version": 3

   },

   "file_extension": ".py",

   "mimetype": "text/x-python",

   "name": "python",

   "nbconvert_exporter": "python",

   "pygments_lexer": "ipython3",

   "version": "3.5.3"

  }

 },

 "nbformat": 4,

 "nbformat_minor": 2

}