--- a
+++ b/TCARER_plots.ipynb
@@ -0,0 +1,700 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "deletable": true,
+    "editable": true
+   },
+   "source": [
+    "## Initialise"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 20,
+   "metadata": {
+    "collapsed": false,
+    "deletable": true,
+    "editable": true
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "The autoreload extension is already loaded. To reload it, use:\n",
+      "  %reload_ext autoreload\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Reload modules \n",
+    "# It is an optional step. It is useful to run when external Python modules are being modified\n",
+    "# It is reloading all modules (except those excluded by %aimport) every time before executing the Python code typed.\n",
+    "# Note: It may conflict with serialisation, when external modules are being modified\n",
+    "\n",
+    "%load_ext autoreload \n",
+    "%autoreload 2"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 21,
+   "metadata": {
+    "collapsed": true,
+    "deletable": true,
+    "editable": true
+   },
+   "outputs": [],
+   "source": [
+    "# Import Python libraries\n",
+    "import logging\n",
+    "import os\n",
+    "import sys\n",
+    "import gc\n",
+    "import pandas as pd\n",
+    "from IPython.display import display, HTML\n",
+    "import numpy as np\n",
+    "import matplotlib.pyplot as plt"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 22,
+   "metadata": {
+    "collapsed": false,
+    "deletable": true,
+    "editable": true
+   },
+   "outputs": [],
+   "source": [
+    "# Import local Python modules\n",
+    "from Configs.CONSTANTS import CONSTANTS\n",
+    "from Configs.Logger import Logger\n",
+    "from ReadersWriters.ReadersWriters import ReadersWriters\n",
+    "from Stats.Plots import Plots"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 23,
+   "metadata": {
+    "collapsed": false,
+    "deletable": true,
+    "editable": true
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "Make sure the correct Python interpreter is used!\n",
+      "3.5.3 (v3.5.3:1880cb95a742, Jan 16 2017, 16:02:32) [MSC v.1900 64 bit (AMD64)]\n",
+      "\n",
+      "Make sure sys.path of the Python interpreter is correct!\n",
+      "C:\\Users\\eagle\\Documents\\GitHub\\Analytics_UoW\\TCARER\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Check the interpreter\n",
+    "print(\"\\nMake sure the correct Python interpreter is used!\")\n",
+    "print(sys.version)\n",
+    "print(\"\\nMake sure sys.path of the Python interpreter is correct!\")\n",
+    "print(os.getcwd())"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "deletable": true,
+    "editable": true
+   },
+   "source": [
+    "Set constants"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 24,
+   "metadata": {
+    "collapsed": false,
+    "deletable": true,
+    "editable": true
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      " The full path of the configuration file: \n",
+      "\t C:\\Users\\eagle\\Documents\\GitHub\\Analytics_UoW\\TCARER\\ConfigInputs\\CONFIGURATIONS.ini \n",
+      " The full path of the output folder: \n",
+      "\t C:\\Users\\eagle\\Documents\\GitHub\\tmp\\TCARER\\scores_adhoc \n",
+      " The application name (the suffix of the outputs file name): \n",
+      "\t tcarerPlots\n"
+     ]
+    }
+   ],
+   "source": [
+    "config_path = os.path.abspath(\"ConfigInputs/CONFIGURATIONS.ini\")\n",
+    "io_path = os.path.abspath(\"../../tmp/TCARER/scores_adhoc\")\n",
+    "app_name = 'tcarerPlots'\n",
+    "\n",
+    "print(\"\\n The full path of the configuration file: \\n\\t\", config_path,\n",
+    "      \"\\n The full path of the output folder: \\n\\t\", io_path,\n",
+    "      \"\\n The application name (the suffix of the outputs file name): \\n\\t\", app_name)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "deletable": true,
+    "editable": true
+   },
+   "source": [
+    "<br/>"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 25,
+   "metadata": {
+    "collapsed": false,
+    "deletable": true,
+    "editable": true
+   },
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "2017-10-29 12:29:33,032 - tcarerPlots - INFO - Creating 'C:\\Users\\eagle\\Documents\\GitHub\\tmp\\TCARER\\scores_adhoc\\tcarerPlots.log' File.\n",
+      "2017-10-29 12:29:33,032 - tcarerPlots - INFO - Creating 'C:\\Users\\eagle\\Documents\\GitHub\\tmp\\TCARER\\scores_adhoc\\tcarerPlots.log' File.\n",
+      "2017-10-29 12:29:33,032 - tcarerPlots - INFO - Creating 'C:\\Users\\eagle\\Documents\\GitHub\\tmp\\TCARER\\scores_adhoc\\tcarerPlots.log' File.\n"
+     ]
+    }
+   ],
+   "source": [
+    "if not os.path.exists(io_path):\n",
+    "    os.makedirs(io_path, exist_ok=True)\n",
+    "\n",
+    "logger = Logger(path=io_path, app_name=app_name, ext=\"log\")\n",
+    "logger = logging.getLogger(app_name)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 26,
+   "metadata": {
+    "collapsed": true,
+    "deletable": true,
+    "editable": true
+   },
+   "outputs": [],
+   "source": [
+    "# Initialise constants        \n",
+    "CONSTANTS.set(io_path, app_name)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 27,
+   "metadata": {
+    "collapsed": true,
+    "deletable": true,
+    "editable": true
+   },
+   "outputs": [],
+   "source": [
+    "# Initialise other classes\n",
+    "readers_writers = ReadersWriters()\n",
+    "plots = Plots()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 35,
+   "metadata": {
+    "collapsed": true,
+    "deletable": true,
+    "editable": true
+   },
+   "outputs": [],
+   "source": [
+    "samples = [\"Basic_prototype\"]\n",
+    "targets = [\"365\"]\n",
+    "submodels = [\"rfc\", \"wdnn\"]\n",
+    "paths = [os.path.abspath(\"../../tmp/TCARER/Basic_prototype\")]\n",
+    "file_names = [[\"Step_09_Model_rfc_label365\", \"Step_09_Model_wdnn_label365\"]]"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "deletable": true,
+    "editable": true
+   },
+   "source": [
+    "<br/><br/>"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "deletable": true,
+    "editable": true
+   },
+   "source": [
+    "## Load Predictions"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 37,
+   "metadata": {
+    "collapsed": false,
+    "deletable": true,
+    "editable": true,
+    "scrolled": true
+   },
+   "outputs": [],
+   "source": [
+    "models = dict()\n",
+    "for i in range(len(samples)):\n",
+    "    models[samples[i]] = dict()\n",
+    "    for j in range(len(targets)):\n",
+    "        models[samples[i]][targets[j]] = dict()\n",
+    "        for k in range(len(submodels)):\n",
+    "            models[samples[i]][targets[j]][submodels[k]] = \\\n",
+    "                readers_writers.load_serialised(path=paths[i], title=file_names[j][k])"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "deletable": true,
+    "editable": true
+   },
+   "source": [
+    "hack - correction of dictionaries"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 40,
+   "metadata": {
+    "collapsed": false,
+    "deletable": true,
+    "editable": true
+   },
+   "outputs": [
+    {
+     "ename": "KeyError",
+     "evalue": "'wdnn'",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[0;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
+      "\u001b[0;32m<ipython-input-40-a1f912f3713b>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msamples\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m     \u001b[1;32mfor\u001b[0m \u001b[0mj\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtargets\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m         \u001b[0mmodels\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0msamples\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mtargets\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mj\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"wdnn\"\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m{\u001b[0m\u001b[1;34m'model_predict'\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mmodels\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0msamples\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mtargets\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mj\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"wdnn\"\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m}\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
+      "\u001b[0;31mKeyError\u001b[0m: 'wdnn'"
+     ]
+    }
+   ],
+   "source": [
+    "for i in range(len(samples)):\n",
+    "    for j in range(len(targets)):\n",
+    "        models[samples[i]][targets[j]][\"wdnn\"] = {'model_predict': models[samples[i]][targets[j]][\"wdnn\"]}"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 39,
+   "metadata": {
+    "collapsed": false,
+    "deletable": true,
+    "editable": true
+   },
+   "outputs": [
+    {
+     "ename": "KeyError",
+     "evalue": "'sample-1'",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[0;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
+      "\u001b[0;32m<ipython-input-39-0533bcf87ed4>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mmodels\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"sample-1\"\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"30\"\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"wdnn\"\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'model_predict'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'test'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'score'\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0marray\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mmodels\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"sample-1\"\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"30\"\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"wdnn\"\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'model_predict'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'test'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'predict_proba'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0mmodels\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"sample-1\"\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"365\"\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"wdnn\"\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'model_predict'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'test'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'score'\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0marray\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mmodels\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"sample-1\"\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"365\"\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"wdnn\"\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'model_predict'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'test'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'predict_proba'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0mmodels\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"sample-2\"\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"30\"\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"wdnn\"\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'model_predict'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'test'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'score'\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0marray\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mmodels\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"sample-2\"\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"30\"\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"wdnn\"\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'model_predict'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'test'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'predict_proba'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0mmodels\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"sample-2\"\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"365\"\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"wdnn\"\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'model_predict'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'test'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'score'\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0marray\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mmodels\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"sample-2\"\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"365\"\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"wdnn\"\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'model_predict'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'test'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'predict_proba'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
+      "\u001b[0;31mKeyError\u001b[0m: 'sample-1'"
+     ]
+    }
+   ],
+   "source": [
+    "models[\"sample-1\"][\"30\"][\"wdnn\"]['model_predict']['test']['score'] = np.array([i[1] for i in models[\"sample-1\"][\"30\"][\"wdnn\"]['model_predict']['test']['predict_proba']])\n",
+    "models[\"sample-1\"][\"365\"][\"wdnn\"]['model_predict']['test']['score'] = np.array([i[1] for i in models[\"sample-1\"][\"365\"][\"wdnn\"]['model_predict']['test']['predict_proba']])\n",
+    "models[\"sample-2\"][\"30\"][\"wdnn\"]['model_predict']['test']['score'] = np.array([i[1] for i in models[\"sample-2\"][\"30\"][\"wdnn\"]['model_predict']['test']['predict_proba']])\n",
+    "models[\"sample-2\"][\"365\"][\"wdnn\"]['model_predict']['test']['score'] = np.array([i[1] for i in models[\"sample-2\"][\"365\"][\"wdnn\"]['model_predict']['test']['predict_proba']])"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "deletable": true,
+    "editable": true
+   },
+   "source": [
+    "<br/><br/>\n",
+    "temporary solution - Load samples"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": true,
+    "deletable": true,
+    "editable": true
+   },
+   "outputs": [],
+   "source": [
+    "file_names = [\"Step_07_Features\", \"Step_07_Features\"]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": true,
+    "deletable": true,
+    "editable": true,
+    "scrolled": true
+   },
+   "outputs": [],
+   "source": [
+    "target_labels = dict()\n",
+    "for i in range(len(samples)):\n",
+    "    target_labels[samples[i]] = readers_writers.load_serialised(path=paths[i], title=file_names[i])[\"test_target\"]\n",
+    "    \n",
+    "for i in range(len(samples)):\n",
+    "    target_labels[samples[i]] = {\"30\": target_labels[samples[i]][\"label30\"], \n",
+    "                                 \"365\": target_labels[samples[i]][\"label365\"]}\n",
+    "    target_labels[\"label30\"] = None\n",
+    "    target_labels[\"label365\"] = None\n",
+    "\n",
+    "gc.collect()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "collapsed": true,
+    "deletable": true,
+    "editable": true
+   },
+   "source": [
+    "temporary solution - correction of wdnn labels"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": true,
+    "deletable": true,
+    "editable": true
+   },
+   "outputs": [],
+   "source": [
+    "for sample in samples:\n",
+    "    for target in targets:\n",
+    "        print(len(target_labels[sample][target]), len(models[sample][target][\"wdnn\"]['model_predict']['test']['score']))\n",
+    "        target_labels[sample][target] = np.append(target_labels[sample][target], target_labels[sample][target])\n",
+    "        target_labels[sample][target] = target_labels[sample][target][0:len(models[sample][target][\"wdnn\"]['model_predict']['test']['score'])]"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "deletable": true,
+    "editable": true
+   },
+   "source": [
+    "<br/><br/><br/>"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "deletable": true,
+    "editable": true
+   },
+   "source": [
+    "## Save"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": true,
+    "deletable": true,
+    "editable": true
+   },
+   "outputs": [],
+   "source": [
+    "file_name = \"saved_models\"\n",
+    "readers_writers.save_serialised(path=io_path, title=file_name, objects=models)\n",
+    "\n",
+    "file_name = \"saved_target_labels\"\n",
+    "readers_writers.save_serialised(path=io_path, title=file_name, objects=target_labels)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "deletable": true,
+    "editable": true
+   },
+   "source": [
+    "## Load"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": false,
+    "deletable": true,
+    "editable": true
+   },
+   "outputs": [],
+   "source": [
+    "file_name = \"saved_models\"\n",
+    "models = readers_writers.load_serialised(path=io_path, title=file_name)\n",
+    "\n",
+    "file_name = \"saved_target_labels\"\n",
+    "target_labels = readers_writers.load_serialised(path=io_path, title=file_name)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "deletable": true,
+    "editable": true
+   },
+   "source": [
+    "<br/><br/>"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "deletable": true,
+    "editable": true
+   },
+   "source": [
+    "## Plot"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": true,
+    "deletable": true,
+    "editable": true
+   },
+   "outputs": [],
+   "source": [
+    "file_name = \"Combined_plots\"\n",
+    "\n",
+    "#%pylab inline\n",
+    "#pylab.rcParams['figure.figsize'] = (10, 10)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "deletable": true,
+    "editable": true
+   },
+   "source": [
+    "Combine"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": true,
+    "deletable": true,
+    "editable": true
+   },
+   "outputs": [],
+   "source": [
+    "markers = {'.': 'point', ',': 'pixel', 'o': 'circle', 'v': 'triangle_down', '^': 'triangle_up', '<': 'triangle_left', '>': 'triangle_right', '1': 'tri_down', '2': 'tri_up', '3': 'tri_left', '4': 'tri_right', '8': 'octagon', 's': 'square', 'p': 'pentagon', '*': 'star', 'h': 'hexagon1', 'H': 'hexagon2', '+': 'plus', 'x': 'x', 'D': 'diamond', 'd': 'thin_diamond', '|': 'vline', '_': 'hline', 'P': 'plus_filled', 'X': 'x_filled', 0: 'tickleft', 1: 'tickright', 2: 'tickup', 3: 'tickdown', 4: 'caretleft', 5: 'caretright', 6: 'caretup', 7: 'caretdown', 8: 'caretleftbase', 9: 'caretrightbase', 10: 'caretupbase', 11: 'caretdownbase', 'None': 'nothing', None: 'nothing', ' ': 'nothing', '': 'nothing'}\n",
+    "scores_list = []\n",
+    "target_labels_list = []\n",
+    "label_list = []\n",
+    "marker_list = ['.', 'o', 'v', '2', 'p', 's', '*', '+', '|'] #['.', 'o', 'v', '2', 'p', 's', '*', '+', '|']\n",
+    "linestyle_list = [':', ':', ':', ':', ':', ':', ':', ':', ':'] #[':', '-.', '--', '-', ':', '-.', '--', '-', ':'] \n",
+    "color_list = ['grey', 'firebrick', 'coral', 'darkgreen', 'coral', 'mediumblue', 'lightseagreen', 'blueviolet', 'darkgray']\n",
+    "\n",
+    "for sample in samples:\n",
+    "    for target in targets:\n",
+    "        for submodel in submodels:\n",
+    "            scores_list.append(models[sample][target][submodel]['model_predict']['test']['score'])\n",
+    "            target_labels_list.append(np.array(target_labels[sample][target][\n",
+    "                0:len(models[sample][target][submodel]['model_predict']['test']['score'])]))\n",
+    "            label_list.append(sample + \", \" + submodel + \"-\" + target)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "deletable": true,
+    "editable": true
+   },
+   "source": [
+    "### ROC Plot"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": false,
+    "deletable": true,
+    "editable": true
+   },
+   "outputs": [],
+   "source": [
+    "plt.clf()\n",
+    "fig, o_summaries = plots.roc_multiple(scores_list, target_labels_list, \n",
+    "                                      label_list=label_list,  marker_list=marker_list, \n",
+    "                                      linestyle_list=linestyle_list, color_list=color_list, \n",
+    "                                      lw=2, markersize=8, legend_prop=15, legend_markerscale=1)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": false,
+    "deletable": true,
+    "editable": true
+   },
+   "outputs": [],
+   "source": [
+    "display(fig)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": true,
+    "deletable": true,
+    "editable": true
+   },
+   "outputs": [],
+   "source": [
+    "# save\n",
+    "fig.savefig(os.path.join(io_path, file_name + '_roc.png'), \n",
+    "            dpi=1500, facecolor='w', edgecolor='w', orientation='portrait', papertype=None, format='png',\n",
+    "            transparent=False, bbox_inches=None, pad_inches=0.1, frameon=None)\n",
+    "plt.clf()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "deletable": true,
+    "editable": true
+   },
+   "source": [
+    "### Precision-Recall Plot"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": false,
+    "deletable": true,
+    "editable": true
+   },
+   "outputs": [],
+   "source": [
+    "plt.clf()\n",
+    "fig, o_summaries = plots.precision_recall_multiple(scores_list, target_labels_list, \n",
+    "                                                   label_list=label_list,  marker_list=marker_list, \n",
+    "                                                   linestyle_list=linestyle_list, color_list=color_list,\n",
+    "                                                   lw=2, markersize=8, legend_prop=10, legend_markerscale=1)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": false,
+    "deletable": true,
+    "editable": true
+   },
+   "outputs": [],
+   "source": [
+    "display(fig)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": true,
+    "deletable": true,
+    "editable": true
+   },
+   "outputs": [],
+   "source": [
+    "# save\n",
+    "fig.savefig(os.path.join(io_path, file_name + '_precision_recall.png'), \n",
+    "            dpi=1500, facecolor='w', edgecolor='w', orientation='portrait', papertype=None, format='png',\n",
+    "            transparent=False, bbox_inches=None, pad_inches=0.1, frameon=None)\n",
+    "plt.clf()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "deletable": true,
+    "editable": true
+   },
+   "source": [
+    "Fin!"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.5.3"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}