--- a
+++ b/SWELL-KW/SWELL-KW_FastGRNN.ipynb
@@ -0,0 +1,978 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# SWELL-KW FastGRNN"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Adapted from Microsoft's notebooks, available at https://github.com/microsoft/EdgeML authored by Dennis et al."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pandas as pd\n",
+    "import numpy as np\n",
+    "from tabulate import tabulate\n",
+    "import os\n",
+    "import datetime as datetime\n",
+    "import pickle as pkl\n",
+    "from sklearn.model_selection import train_test_split\n",
+    "import pathlib\n",
+    "from os import mkdir"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def loadData(dirname):\n",
+    "    x_train = np.load(dirname + '/' + 'x_train.npy')\n",
+    "    y_train = np.load(dirname + '/' + 'y_train.npy')\n",
+    "    x_test = np.load(dirname + '/' + 'x_test.npy')\n",
+    "    y_test = np.load(dirname + '/' + 'y_test.npy')\n",
+    "    x_val = np.load(dirname + '/' + 'x_val.npy')\n",
+    "    y_val = np.load(dirname + '/' + 'y_val.npy')\n",
+    "    return x_train, y_train, x_test, y_test, x_val, y_val\n",
+    "def makeEMIData(subinstanceLen, subinstanceStride, sourceDir, outDir):\n",
+    "    x_train, y_train, x_test, y_test, x_val, y_val = loadData(sourceDir)\n",
+    "    x, y = bagData(x_train, y_train, subinstanceLen, subinstanceStride)\n",
+    "    np.save(outDir + '/x_train.npy', x)\n",
+    "    np.save(outDir + '/y_train.npy', y)\n",
+    "    print('Num train %d' % len(x))\n",
+    "    x, y = bagData(x_test, y_test, subinstanceLen, subinstanceStride)\n",
+    "    np.save(outDir + '/x_test.npy', x)\n",
+    "    np.save(outDir + '/y_test.npy', y)\n",
+    "    print('Num test %d' % len(x))\n",
+    "    x, y = bagData(x_val, y_val, subinstanceLen, subinstanceStride)\n",
+    "    np.save(outDir + '/x_val.npy', x)\n",
+    "    np.save(outDir + '/y_val.npy', y)\n",
+    "    print('Num val %d' % len(x))\n",
+    "def bagData(X, Y, subinstanceLen, subinstanceStride):\n",
+    "    numClass = 2\n",
+    "    numSteps = 20\n",
+    "    numFeats = 22\n",
+    "    assert X.ndim == 3\n",
+    "    print(X.shape)\n",
+    "    assert X.shape[1] == numSteps\n",
+    "    assert X.shape[2] == numFeats\n",
+    "    assert subinstanceLen <= numSteps\n",
+    "    assert subinstanceLen > 0\n",
+    "    assert subinstanceStride <= numSteps\n",
+    "    assert subinstanceStride >= 0\n",
+    "    assert len(X) == len(Y)\n",
+    "    assert Y.ndim == 2\n",
+    "    assert Y.shape[1] == numClass\n",
+    "    x_bagged = []\n",
+    "    y_bagged = []\n",
+    "    for i, point in enumerate(X[:, :, :]):\n",
+    "        instanceList = []\n",
+    "        start = 0\n",
+    "        end = subinstanceLen\n",
+    "        while True:\n",
+    "            x = point[start:end, :]\n",
+    "            if len(x) < subinstanceLen:\n",
+    "                x_ = np.zeros([subinstanceLen, x.shape[1]])\n",
+    "                x_[:len(x), :] = x[:, :]\n",
+    "                x = x_\n",
+    "            instanceList.append(x)\n",
+    "            if end >= numSteps:\n",
+    "                break\n",
+    "            start += subinstanceStride\n",
+    "            end += subinstanceStride\n",
+    "        bag = np.array(instanceList)\n",
+    "        numSubinstance = bag.shape[0]\n",
+    "        label = Y[i]\n",
+    "        label = np.argmax(label)\n",
+    "        labelBag = np.zeros([numSubinstance, numClass])\n",
+    "        labelBag[:, label] = 1\n",
+    "        x_bagged.append(bag)\n",
+    "        label = np.array(labelBag)\n",
+    "        y_bagged.append(label)\n",
+    "    return np.array(x_bagged), np.array(y_bagged)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "(3679, 20, 22)\n",
+      "Num train 3679\n",
+      "(1022, 20, 22)\n",
+      "Num test 1022\n",
+      "(409, 20, 22)\n",
+      "Num val 409\n"
+     ]
+    }
+   ],
+   "source": [
+    "subinstanceLen=8\n",
+    "subinstanceStride=3\n",
+    "extractedDir = '/home/sf/data/SWELL-KW/'\n",
+    "#mkdir('/home/sf/data/SWELL-KW/FG_8_3')\n",
+    "rawDir = extractedDir + '/RAW'\n",
+    "sourceDir = rawDir\n",
+    "outDir = extractedDir + '/FG_%d_%d/' % (subinstanceLen, subinstanceStride)\n",
+    "makeEMIData(subinstanceLen, subinstanceStride, sourceDir, outDir)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2018-08-19T12:39:06.272261Z",
+     "start_time": "2018-08-19T12:39:05.330668Z"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "from __future__ import print_function\n",
+    "import os\n",
+    "import sys\n",
+    "import tensorflow as tf\n",
+    "import numpy as np\n",
+    "os.environ['CUDA_VISIBLE_DEVICES'] ='0'\n",
+    "\n",
+    "# FastGRNN and FastRNN imports\n",
+    "from edgeml.graph.rnn import EMI_DataPipeline\n",
+    "from edgeml.graph.rnn import EMI_FastGRNN\n",
+    "from edgeml.graph.rnn import EMI_FastRNN\n",
+    "from edgeml.trainer.emirnnTrainer import EMI_Trainer, EMI_Driver\n",
+    "import edgeml.utils"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2018-08-19T12:39:06.292205Z",
+     "start_time": "2018-08-19T12:39:06.274254Z"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "# Network parameters for our FastGRNN + FC Layer\n",
+    "NUM_HIDDEN = 128\n",
+    "NUM_TIMESTEPS = 8\n",
+    "NUM_FEATS = 22\n",
+    "FORGET_BIAS = 1.0\n",
+    "NUM_OUTPUT = 2\n",
+    "USE_DROPOUT = False\n",
+    "KEEP_PROB = 0.9\n",
+    "\n",
+    "# Non-linearities can be chosen among \"tanh, sigmoid, relu, quantTanh, quantSigm\"\n",
+    "UPDATE_NL = \"quantTanh\"\n",
+    "GATE_NL = \"quantSigm\"\n",
+    "\n",
+    "# Ranks of Parameter matrices for low-rank parameterisation to compress models.\n",
+    "WRANK = 5\n",
+    "URANK = 6\n",
+    "\n",
+    "# For dataset API\n",
+    "PREFETCH_NUM = 5\n",
+    "BATCH_SIZE = 32\n",
+    "\n",
+    "# Number of epochs in *one iteration*\n",
+    "NUM_EPOCHS = 3\n",
+    "# Number of iterations in *one round*. After each iteration,\n",
+    "# the model is dumped to disk. At the end of the current\n",
+    "# round, the best model among all the dumped models in the\n",
+    "# current round is picked up..\n",
+    "NUM_ITER = 4\n",
+    "# A round consists of multiple training iterations and a belief\n",
+    "# update step using the best model from all of these iterations\n",
+    "NUM_ROUNDS = 30\n",
+    "\n",
+    "# A staging direcory to store models\n",
+    "MODEL_PREFIX = '/home/sf/data/SWELL-KW/FG_8_13/model-fgrnn'"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Loading Data"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2018-08-19T12:39:06.410372Z",
+     "start_time": "2018-08-19T12:39:06.294014Z"
+    }
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "x_train shape is: (3679, 5, 8, 22)\n",
+      "y_train shape is: (3679, 5, 2)\n",
+      "x_test shape is: (409, 5, 8, 22)\n",
+      "y_test shape is: (409, 5, 2)\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Loading the data\n",
+    "path='/home/sf/data/SWELL-KW/FG_8_3/'\n",
+    "x_train, y_train = np.load(path + 'x_train.npy'), np.load(path + 'y_train.npy')\n",
+    "x_test, y_test = np.load(path + 'x_test.npy'), np.load(path + 'y_test.npy')\n",
+    "x_val, y_val = np.load(path + 'x_val.npy'), np.load(path + 'y_val.npy')\n",
+    "\n",
+    "# BAG_TEST, BAG_TRAIN, BAG_VAL represent bag_level labels. These are used for the label update\n",
+    "# step of EMI/MI RNN\n",
+    "BAG_TEST = np.argmax(y_test[:, 0, :], axis=1)\n",
+    "BAG_TRAIN = np.argmax(y_train[:, 0, :], axis=1)\n",
+    "BAG_VAL = np.argmax(y_val[:, 0, :], axis=1)\n",
+    "NUM_SUBINSTANCE = x_train.shape[1]\n",
+    "print(\"x_train shape is:\", x_train.shape)\n",
+    "print(\"y_train shape is:\", y_train.shape)\n",
+    "print(\"x_test shape is:\", x_val.shape)\n",
+    "print(\"y_test shape is:\", y_val.shape)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Computation Graph"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2018-08-19T12:39:06.653612Z",
+     "start_time": "2018-08-19T12:39:06.412290Z"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "# Define the linear secondary classifier\n",
+    "def createExtendedGraph(self, baseOutput, *args, **kwargs):\n",
+    "    W1 = tf.Variable(np.random.normal(size=[NUM_HIDDEN, NUM_OUTPUT]).astype('float32'), name='W1')\n",
+    "    B1 = tf.Variable(np.random.normal(size=[NUM_OUTPUT]).astype('float32'), name='B1')\n",
+    "    y_cap = tf.add(tf.tensordot(baseOutput, W1, axes=1), B1, name='y_cap_tata')\n",
+    "    self.output = y_cap\n",
+    "    self.graphCreated = True\n",
+    "\n",
+    "def restoreExtendedGraph(self, graph, *args, **kwargs):\n",
+    "    y_cap = graph.get_tensor_by_name('y_cap_tata:0')\n",
+    "    self.output = y_cap\n",
+    "    self.graphCreated = True\n",
+    "    \n",
+    "def feedDictFunc(self, keep_prob=None, inference=False, **kwargs):\n",
+    "    if inference is False:\n",
+    "        feedDict = {self._emiGraph.keep_prob: keep_prob}\n",
+    "    else:\n",
+    "        feedDict = {self._emiGraph.keep_prob: 1.0}\n",
+    "    return feedDict\n",
+    "\n",
+    "    \n",
+    "EMI_FastGRNN._createExtendedGraph = createExtendedGraph\n",
+    "EMI_FastGRNN._restoreExtendedGraph = restoreExtendedGraph\n",
+    "if USE_DROPOUT is True:\n",
+    "    EMI_FastGRNN.feedDictFunc = feedDictFunc"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2018-08-19T12:39:06.701740Z",
+     "start_time": "2018-08-19T12:39:06.655328Z"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "inputPipeline = EMI_DataPipeline(NUM_SUBINSTANCE, NUM_TIMESTEPS, NUM_FEATS, NUM_OUTPUT)\n",
+    "emiFastGRNN = EMI_FastGRNN(NUM_SUBINSTANCE, NUM_HIDDEN, NUM_TIMESTEPS, NUM_FEATS, wRank=WRANK, uRank=URANK, \n",
+    "                           gate_non_linearity=GATE_NL, update_non_linearity=UPDATE_NL, useDropout=USE_DROPOUT)\n",
+    "emiTrainer = EMI_Trainer(NUM_TIMESTEPS, NUM_OUTPUT, lossType='xentropy')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "x_train shape is: (3679, 5, 8, 22)\n",
+      "y_train shape is: (3679, 5, 2)\n",
+      "x_test shape is: (409, 5, 8, 22)\n",
+      "y_test shape is: (409, 5, 2)\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(\"x_train shape is:\", x_train.shape)\n",
+    "print(\"y_train shape is:\", y_train.shape)\n",
+    "print(\"x_test shape is:\", x_val.shape)\n",
+    "print(\"y_test shape is:\", y_val.shape)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2018-08-19T12:39:14.187456Z",
+     "start_time": "2018-08-19T12:39:06.703481Z"
+    },
+    "scrolled": true
+   },
+   "outputs": [],
+   "source": [
+    "tf.reset_default_graph()\n",
+    "g1 = tf.Graph()    \n",
+    "with g1.as_default():\n",
+    "    # Obtain the iterators to each batch of the data\n",
+    "    x_batch, y_batch = inputPipeline()\n",
+    "    # Create the forward computation graph based on the iterators\n",
+    "    y_cap = emiFastGRNN(x_batch)\n",
+    "    # Create loss graphs and training routines\n",
+    "    emiTrainer(y_cap, y_batch)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# EMI Driver "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2018-08-19T12:51:45.803360Z",
+     "start_time": "2018-08-19T12:39:14.189648Z"
+    },
+    "scrolled": true
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Update policy: top-k\n",
+      "Training with MI-RNN loss for 15 rounds\n",
+      "Round: 0\n",
+      "Epoch   2 Batch   100 (  330) Loss 0.09010 Acc 0.49375 | Val acc 0.65037 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1000\n",
+      "Epoch   2 Batch   100 (  330) Loss 0.08737 Acc 0.53750 | Val acc 0.66748 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1001\n",
+      "Epoch   2 Batch   100 (  330) Loss 0.08466 Acc 0.50625 | Val acc 0.68215 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1002\n",
+      "Epoch   2 Batch   100 (  330) Loss 0.08016 Acc 0.57500 | Val acc 0.69193 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1003\n",
+      "INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn-1003\n",
+      "Round: 1\n",
+      "Epoch   2 Batch   100 (  330) Loss 0.07575 Acc 0.64375 | Val acc 0.70416 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1004\n",
+      "Epoch   2 Batch   100 (  330) Loss 0.07176 Acc 0.71875 | Val acc 0.71883 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1005\n",
+      "Epoch   2 Batch   100 (  330) Loss 0.06881 Acc 0.72500 | Val acc 0.71883 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1006\n",
+      "Epoch   2 Batch   100 (  330) Loss 0.06622 Acc 0.72500 | Val acc 0.72372 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1007\n",
+      "INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn-1007\n",
+      "Round: 2\n",
+      "Epoch   2 Batch   100 (  330) Loss 0.06392 Acc 0.72500 | Val acc 0.72616 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1008\n",
+      "Epoch   2 Batch   100 (  330) Loss 0.06182 Acc 0.73125 | Val acc 0.74083 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1009\n",
+      "Epoch   2 Batch   100 (  330) Loss 0.06007 Acc 0.74375 | Val acc 0.75061 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1010\n",
+      "Epoch   2 Batch   100 (  330) Loss 0.05848 Acc 0.76250 | Val acc 0.76528 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1011\n",
+      "INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn-1011\n",
+      "Round: 3\n",
+      "Epoch   2 Batch   100 (  330) Loss 0.05694 Acc 0.75000 | Val acc 0.76039 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1012\n",
+      "Epoch   2 Batch   100 (  330) Loss 0.05462 Acc 0.73125 | Val acc 0.76528 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1013\n",
+      "Epoch   2 Batch   100 (  330) Loss 0.05265 Acc 0.78750 | Val acc 0.77751 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1014\n",
+      "Epoch   2 Batch   100 (  330) Loss 0.05050 Acc 0.78750 | Val acc 0.77262 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1015\n",
+      "INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn-1014\n",
+      "Round: 4\n",
+      "Epoch   2 Batch   100 (  330) Loss 0.05050 Acc 0.78750 | Val acc 0.77262 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1016\n",
+      "Epoch   2 Batch   100 (  330) Loss 0.04971 Acc 0.77500 | Val acc 0.78240 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1017\n",
+      "Epoch   2 Batch   100 (  330) Loss 0.04890 Acc 0.76250 | Val acc 0.77751 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1018\n",
+      "Epoch   2 Batch   100 (  330) Loss 0.04847 Acc 0.76250 | Val acc 0.78484 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1019\n",
+      "INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn-1019\n",
+      "Round: 5\n",
+      "Epoch   2 Batch   100 (  330) Loss 0.04872 Acc 0.79375 | Val acc 0.78973 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1020\n",
+      "Epoch   2 Batch   100 (  330) Loss 0.04798 Acc 0.79375 | Val acc 0.78973 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1021\n",
+      "Epoch   2 Batch   100 (  330) Loss 0.04730 Acc 0.79375 | Val acc 0.79462 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1022\n",
+      "Epoch   2 Batch   100 (  330) Loss 0.04622 Acc 0.80000 | Val acc 0.79707 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1023\n",
+      "INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn-1023\n",
+      "Round: 6\n",
+      "Epoch   2 Batch   100 (  330) Loss 0.04515 Acc 0.80000 | Val acc 0.80685 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1024\n",
+      "Epoch   2 Batch   100 (  330) Loss 0.04360 Acc 0.80625 | Val acc 0.80929 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1025\n",
+      "Epoch   2 Batch   100 (  330) Loss 0.04234 Acc 0.81875 | Val acc 0.80440 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1026\n",
+      "Epoch   2 Batch   100 (  330) Loss 0.04157 Acc 0.83750 | Val acc 0.81907 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1027\n",
+      "INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn-1027\n",
+      "Round: 7\n",
+      "Epoch   2 Batch   100 (  330) Loss 0.04077 Acc 0.85000 | Val acc 0.81174 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1028\n",
+      "Epoch   2 Batch   100 (  330) Loss 0.04000 Acc 0.85000 | Val acc 0.81418 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1029\n",
+      "Epoch   2 Batch   100 (  330) Loss 0.03823 Acc 0.85000 | Val acc 0.81418 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1030\n",
+      "Epoch   2 Batch   100 (  330) Loss 0.03676 Acc 0.85625 | Val acc 0.83619 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1031\n",
+      "INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn-1031\n",
+      "Round: 8\n",
+      "Epoch   2 Batch   100 (  330) Loss 0.03610 Acc 0.85625 | Val acc 0.83863 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1032\n",
+      "Epoch   2 Batch   100 (  330) Loss 0.03498 Acc 0.86250 | Val acc 0.84108 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1033\n",
+      "Epoch   2 Batch   100 (  330) Loss 0.03424 Acc 0.87500 | Val acc 0.84108 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1034\n",
+      "Epoch   2 Batch   100 (  330) Loss 0.03375 Acc 0.88125 | Val acc 0.83863 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1035\n",
+      "INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn-1033\n",
+      "Round: 9\n",
+      "Epoch   2 Batch   100 (  330) Loss 0.03424 Acc 0.87500 | Val acc 0.84108 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1036\n",
+      "Epoch   2 Batch   100 (  330) Loss 0.03375 Acc 0.88125 | Val acc 0.83863 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1037\n",
+      "Epoch   2 Batch   100 (  330) Loss 0.03238 Acc 0.87500 | Val acc 0.84108 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1038\n",
+      "Epoch   2 Batch   100 (  330) Loss 0.02963 Acc 0.90000 | Val acc 0.83619 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1039\n",
+      "INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn-1036\n",
+      "Round: 10\n",
+      "Epoch   2 Batch   100 (  330) Loss 0.03375 Acc 0.88125 | Val acc 0.83863 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1040\n",
+      "Epoch   2 Batch   100 (  330) Loss 0.03238 Acc 0.87500 | Val acc 0.84108 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1041\n",
+      "Epoch   2 Batch   100 (  330) Loss 0.02963 Acc 0.90000 | Val acc 0.83619 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1042\n",
+      "Epoch   2 Batch   100 (  330) Loss 0.03045 Acc 0.89375 | Val acc 0.84108 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1043\n",
+      "INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn-1041\n",
+      "Round: 11\n",
+      "Epoch   2 Batch   100 (  330) Loss 0.02963 Acc 0.90000 | Val acc 0.83619 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1044\n",
+      "Epoch   2 Batch   100 (  330) Loss 0.03045 Acc 0.89375 | Val acc 0.84108 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1045\n",
+      "Epoch   2 Batch   100 (  330) Loss 0.03176 Acc 0.88750 | Val acc 0.84352 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1046\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Epoch   2 Batch   100 (  330) Loss 0.03206 Acc 0.88750 | Val acc 0.83863 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1047\n",
+      "INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn-1046\n",
+      "Round: 12\n",
+      "Epoch   2 Batch   100 (  330) Loss 0.03206 Acc 0.88750 | Val acc 0.83863 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1048\n",
+      "Epoch   2 Batch   100 (  330) Loss 0.03282 Acc 0.89375 | Val acc 0.83863 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1049\n",
+      "Epoch   2 Batch   100 (  330) Loss 0.03429 Acc 0.89375 | Val acc 0.83619 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1050\n",
+      "Epoch   2 Batch   100 (  330) Loss 0.03082 Acc 0.92500 | Val acc 0.84108 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1051\n",
+      "INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn-1051\n",
+      "Round: 13\n",
+      "Epoch   2 Batch   100 (  330) Loss 0.02883 Acc 0.94375 | Val acc 0.84108 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1052\n",
+      "Epoch   2 Batch   100 (  330) Loss 0.03036 Acc 0.93125 | Val acc 0.83863 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1053\n",
+      "Epoch   2 Batch   100 (  330) Loss 0.02961 Acc 0.91875 | Val acc 0.84108 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1054\n",
+      "Epoch   2 Batch   100 (  330) Loss 0.02976 Acc 0.93125 | Val acc 0.83619 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1055\n",
+      "INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn-1052\n",
+      "Round: 14\n",
+      "Epoch   2 Batch   100 (  330) Loss 0.03036 Acc 0.93125 | Val acc 0.83863 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1056\n",
+      "Epoch   2 Batch   100 (  330) Loss 0.02961 Acc 0.91875 | Val acc 0.84108 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1057\n",
+      "Epoch   2 Batch   100 (  330) Loss 0.02976 Acc 0.93125 | Val acc 0.83619 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1058\n",
+      "Epoch   2 Batch   100 (  330) Loss 0.03076 Acc 0.91250 | Val acc 0.84108 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1059\n",
+      "INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn-1057\n",
+      "Round: 15\n",
+      "Switching to EMI-Loss function\n",
+      "Epoch   2 Batch   100 (  330) Loss 0.35416 Acc 0.88125 | Val acc 0.80685 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1060\n",
+      "Epoch   2 Batch   100 (  330) Loss 0.33687 Acc 0.86875 | Val acc 0.81174 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1061\n",
+      "Epoch   2 Batch   100 (  330) Loss 0.33655 Acc 0.87500 | Val acc 0.81418 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1062\n",
+      "Epoch   2 Batch   100 (  330) Loss 0.33672 Acc 0.88750 | Val acc 0.81907 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1063\n",
+      "INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn-1063\n",
+      "Round: 16\n",
+      "Epoch   2 Batch   100 (  330) Loss 0.32793 Acc 0.88750 | Val acc 0.81418 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1064\n",
+      "Epoch   2 Batch   100 (  330) Loss 0.32143 Acc 0.86250 | Val acc 0.82152 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1065\n",
+      "Epoch   2 Batch   100 (  330) Loss 0.31822 Acc 0.86250 | Val acc 0.81663 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1066\n",
+      "Epoch   2 Batch   100 (  330) Loss 0.31108 Acc 0.87500 | Val acc 0.82396 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1067\n",
+      "INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn-1067\n",
+      "Round: 17\n",
+      "Epoch   2 Batch   100 (  330) Loss 0.30779 Acc 0.86875 | Val acc 0.81907 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1068\n",
+      "Epoch   2 Batch   100 (  330) Loss 0.30074 Acc 0.88125 | Val acc 0.82152 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1069\n",
+      "Epoch   2 Batch   100 (  330) Loss 0.29439 Acc 0.88125 | Val acc 0.82641 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1070\n",
+      "Epoch   2 Batch   100 (  330) Loss 0.28360 Acc 0.89375 | Val acc 0.83374 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1071\n",
+      "INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn-1071\n",
+      "Round: 18\n",
+      "Epoch   2 Batch   100 (  330) Loss 0.28049 Acc 0.88750 | Val acc 0.83374 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1072\n",
+      "Epoch   2 Batch   100 (  330) Loss 0.27575 Acc 0.90000 | Val acc 0.83863 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1073\n",
+      "Epoch   2 Batch   100 (  330) Loss 0.27683 Acc 0.88750 | Val acc 0.84352 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1074\n",
+      "Epoch   2 Batch   100 (  330) Loss 0.27200 Acc 0.90000 | Val acc 0.84597 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1075\n",
+      "INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn-1075\n",
+      "Round: 19\n",
+      "Epoch   2 Batch   100 (  330) Loss 0.26647 Acc 0.93750 | Val acc 0.84108 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1076\n",
+      "Epoch   2 Batch   100 (  330) Loss 0.26391 Acc 0.93750 | Val acc 0.83863 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1077\n",
+      "Epoch   2 Batch   100 (  330) Loss 0.26801 Acc 0.92500 | Val acc 0.83619 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1078\n",
+      "Epoch   2 Batch   100 (  330) Loss 0.26092 Acc 0.92500 | Val acc 0.84108 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1079\n",
+      "INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn-1076\n",
+      "Round: 20\n",
+      "Epoch   2 Batch   100 (  330) Loss 0.26391 Acc 0.93750 | Val acc 0.83863 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1080\n",
+      "Epoch   2 Batch   100 (  330) Loss 0.26801 Acc 0.92500 | Val acc 0.83619 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1081\n",
+      "Epoch   2 Batch   100 (  330) Loss 0.26092 Acc 0.92500 | Val acc 0.84108 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1082\n",
+      "Epoch   2 Batch   100 (  330) Loss 0.26757 Acc 0.90625 | Val acc 0.85330 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1083\n",
+      "INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn-1083\n",
+      "Round: 21\n",
+      "Epoch   2 Batch   100 (  330) Loss 0.26281 Acc 0.91875 | Val acc 0.85575 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1084\n",
+      "Epoch   2 Batch   100 (  330) Loss 0.25737 Acc 0.93125 | Val acc 0.85575 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1085\n",
+      "Epoch   2 Batch   100 (  330) Loss 0.24958 Acc 0.93750 | Val acc 0.87042 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1086\n",
+      "Epoch   2 Batch   100 (  330) Loss 0.24881 Acc 0.93750 | Val acc 0.87042 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1087\n",
+      "INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn-1086\n",
+      "Round: 22\n",
+      "Epoch   2 Batch   100 (  330) Loss 0.24881 Acc 0.93750 | Val acc 0.87042 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1088\n",
+      "Epoch   2 Batch   100 (  330) Loss 0.24322 Acc 0.94375 | Val acc 0.85819 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1089\n",
+      "Epoch   2 Batch   100 (  330) Loss 0.24998 Acc 0.91875 | Val acc 0.86797 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1090\n",
+      "Epoch   2 Batch   100 (  330) Loss 0.24372 Acc 0.91875 | Val acc 0.86553 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1091\n",
+      "INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn-1088\n",
+      "Round: 23\n",
+      "Epoch   2 Batch   100 (  330) Loss 0.24322 Acc 0.94375 | Val acc 0.85819 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1092\n",
+      "Epoch   2 Batch   100 (  330) Loss 0.24998 Acc 0.91875 | Val acc 0.86797 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1093\n",
+      "Epoch   2 Batch   100 (  330) Loss 0.24372 Acc 0.91875 | Val acc 0.86553 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1094\n",
+      "Epoch   2 Batch   100 (  330) Loss 0.24040 Acc 0.93750 | Val acc 0.86308 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1095\n",
+      "INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn-1093\n",
+      "Round: 24\n",
+      "Epoch   2 Batch   100 (  330) Loss 0.24372 Acc 0.91875 | Val acc 0.86553 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1096\n",
+      "Epoch   2 Batch   100 (  330) Loss 0.24040 Acc 0.93750 | Val acc 0.86308 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1097\n",
+      "Epoch   2 Batch   100 (  330) Loss 0.22721 Acc 0.95000 | Val acc 0.86553 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1098\n",
+      "Epoch   2 Batch   100 (  330) Loss 0.22792 Acc 0.94375 | Val acc 0.86308 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1099\n",
+      "INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn-1096\n",
+      "Round: 25\n",
+      "Epoch   2 Batch   100 (  330) Loss 0.24040 Acc 0.93750 | Val acc 0.86308 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1100\n",
+      "Epoch   2 Batch   100 (  330) Loss 0.22721 Acc 0.95000 | Val acc 0.86553 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1101\n",
+      "Epoch   2 Batch   100 (  330) Loss 0.22792 Acc 0.94375 | Val acc 0.86308 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1102\n",
+      "Epoch   2 Batch   100 (  330) Loss 0.23063 Acc 0.94375 | Val acc 0.86553 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1103\n",
+      "INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn-1101\n",
+      "Round: 26\n",
+      "Epoch   2 Batch   100 (  330) Loss 0.22792 Acc 0.94375 | Val acc 0.86308 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1104\n",
+      "Epoch   2 Batch   100 (  330) Loss 0.23063 Acc 0.94375 | Val acc 0.86553 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1105\n",
+      "Epoch   2 Batch   100 (  330) Loss 0.24060 Acc 0.93125 | Val acc 0.87042 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1106\n",
+      "Epoch   2 Batch   100 (  330) Loss 0.25265 Acc 0.90625 | Val acc 0.87775 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1107\n",
+      "INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn-1107\n",
+      "Round: 27\n",
+      "Epoch   2 Batch   100 (  330) Loss 0.23588 Acc 0.91875 | Val acc 0.87531 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1108\n",
+      "Epoch   2 Batch   100 (  330) Loss 0.22337 Acc 0.93750 | Val acc 0.86797 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1109\n",
+      "Epoch   2 Batch   100 (  330) Loss 0.22149 Acc 0.96250 | Val acc 0.88020 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1110\n",
+      "Epoch   2 Batch   100 (  330) Loss 0.22539 Acc 0.92500 | Val acc 0.87775 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1111\n",
+      "INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn-1110\n",
+      "Round: 28\n",
+      "Epoch   2 Batch   100 (  330) Loss 0.22539 Acc 0.92500 | Val acc 0.87775 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1112\n",
+      "Epoch   2 Batch   100 (  330) Loss 0.21537 Acc 0.92500 | Val acc 0.87042 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1113\n",
+      "Epoch   2 Batch   100 (  330) Loss 0.20826 Acc 0.94375 | Val acc 0.88020 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1114\n",
+      "Epoch   2 Batch   100 (  330) Loss 0.24497 Acc 0.92500 | Val acc 0.87531 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1115\n",
+      "INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn-1114\n",
+      "Round: 29\n",
+      "Epoch   2 Batch   100 (  330) Loss 0.24497 Acc 0.92500 | Val acc 0.87531 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1116\n",
+      "Epoch   2 Batch   100 (  330) Loss 0.20842 Acc 0.95625 | Val acc 0.87775 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1117\n",
+      "Epoch   2 Batch   100 (  330) Loss 0.20437 Acc 0.97500 | Val acc 0.87531 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1118\n",
+      "Epoch   2 Batch   100 (  330) Loss 0.20019 Acc 0.96250 | Val acc 0.87042 | Model saved to /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn, global_step 1119\n",
+      "INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn-1117\n"
+     ]
+    }
+   ],
+   "source": [
+    "with g1.as_default():\n",
+    "    emiDriver = EMI_Driver(inputPipeline, emiFastGRNN, emiTrainer)\n",
+    "\n",
+    "emiDriver.initializeSession(g1)\n",
+    "y_updated, modelStats = emiDriver.run(numClasses=NUM_OUTPUT, x_train=x_train,\n",
+    "                                      y_train=y_train, bag_train=BAG_TRAIN,\n",
+    "                                      x_val=x_val, y_val=y_val, bag_val=BAG_VAL,\n",
+    "                                      numIter=NUM_ITER, keep_prob=KEEP_PROB,\n",
+    "                                      numRounds=NUM_ROUNDS, batchSize=BATCH_SIZE,\n",
+    "                                      numEpochs=NUM_EPOCHS, modelPrefix=MODEL_PREFIX,\n",
+    "                                      fracEMI=0.5, updatePolicy='top-k', k=1)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Evaluating the  trained model"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2018-08-19T12:51:45.832728Z",
+     "start_time": "2018-08-19T12:51:45.805984Z"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "# Early Prediction Policy: We make an early prediction based on the predicted classes\n",
+    "#     probability. If the predicted class probability > minProb at some step, we make\n",
+    "#     a prediction at that step.\n",
+    "def earlyPolicy_minProb(instanceOut, minProb, **kwargs):\n",
+    "    assert instanceOut.ndim == 2\n",
+    "    classes = np.argmax(instanceOut, axis=1)\n",
+    "    prob = np.max(instanceOut, axis=1)\n",
+    "    index = np.where(prob >= minProb)[0]\n",
+    "    if len(index) == 0:\n",
+    "        assert (len(instanceOut) - 1) == (len(classes) - 1)\n",
+    "        return classes[-1], len(instanceOut) - 1\n",
+    "    index = index[0]\n",
+    "    return classes[index], index\n",
+    "\n",
+    "def getEarlySaving(predictionStep, numTimeSteps, returnTotal=False):\n",
+    "    predictionStep = predictionStep + 1\n",
+    "    predictionStep = np.reshape(predictionStep, -1)\n",
+    "    totalSteps = np.sum(predictionStep)\n",
+    "    maxSteps = len(predictionStep) * numTimeSteps\n",
+    "    savings = 1.0 - (totalSteps / maxSteps)\n",
+    "    if returnTotal:\n",
+    "        return savings, totalSteps\n",
+    "    return savings"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2018-08-19T12:51:46.210240Z",
+     "start_time": "2018-08-19T12:51:45.834534Z"
+    }
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Accuracy at k = 2: 0.868885\n",
+      "Additional savings: 0.295475\n"
+     ]
+    }
+   ],
+   "source": [
+    "k = 2\n",
+    "predictions, predictionStep = emiDriver.getInstancePredictions(x_test, y_test, earlyPolicy_minProb, minProb=0.99)\n",
+    "bagPredictions = emiDriver.getBagPredictions(predictions, minSubsequenceLen=k, numClass=NUM_OUTPUT)\n",
+    "print('Accuracy at k = %d: %f' % (k,  np.mean((bagPredictions == BAG_TEST).astype(int))))\n",
+    "print('Additional savings: %f' % getEarlySaving(predictionStep, NUM_TIMESTEPS))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2018-08-19T12:51:46.677691Z",
+     "start_time": "2018-08-19T12:51:46.212285Z"
+    },
+    "scrolled": false
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "   len       acc  macro-fsc  macro-pre  macro-rec  micro-fsc  micro-pre  \\\n",
+      "0    1  0.860078   0.859642   0.868740   0.862055   0.860078   0.860078   \n",
+      "1    2  0.868885   0.868860   0.870591   0.869796   0.868885   0.868885   \n",
+      "2    3  0.875734   0.875619   0.875752   0.875529   0.875734   0.875734   \n",
+      "3    4  0.866928   0.866436   0.869102   0.865945   0.866928   0.866928   \n",
+      "4    5  0.863992   0.862807   0.871518   0.862185   0.863992   0.863992   \n",
+      "\n",
+      "   micro-rec  fscore_01  \n",
+      "0   0.860078   0.867470  \n",
+      "1   0.868885   0.870656  \n",
+      "2   0.875734   0.871847  \n",
+      "3   0.866928   0.858333  \n",
+      "4   0.863992   0.850054  \n",
+      "Max accuracy 0.875734 at subsequencelength 3\n",
+      "Max micro-f 0.875734 at subsequencelength 3\n",
+      "Micro-precision 0.875734 at subsequencelength 3\n",
+      "Micro-recall 0.875734 at subsequencelength 3\n",
+      "Max macro-f 0.875619 at subsequencelength 3\n",
+      "macro-precision 0.875752 at subsequencelength 3\n",
+      "macro-recall 0.875529 at subsequencelength 3\n"
+     ]
+    }
+   ],
+   "source": [
+    "# A slightly more detailed analysis method is provided. \n",
+    "df = emiDriver.analyseModel(predictions, BAG_TEST, NUM_SUBINSTANCE, NUM_OUTPUT)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Picking the best model"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 21,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2018-08-19T13:06:04.024660Z",
+     "start_time": "2018-08-19T13:04:47.045787Z"
+    },
+    "scrolled": true
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn-1032\n",
+      "INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn-1003\n",
+      "Round:  0, Validation accuracy: 0.6919, Test Accuracy (k = 3): 0.689824, Additional savings: 0.000024\n",
+      "INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn-1007\n",
+      "Round:  1, Validation accuracy: 0.7237, Test Accuracy (k = 3): 0.708415, Additional savings: 0.000171\n",
+      "INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn-1011\n",
+      "Round:  2, Validation accuracy: 0.7653, Test Accuracy (k = 3): 0.727006, Additional savings: 0.001492\n",
+      "INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn-1014\n",
+      "Round:  3, Validation accuracy: 0.7775, Test Accuracy (k = 3): 0.732877, Additional savings: 0.002275\n",
+      "INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn-1019\n",
+      "Round:  4, Validation accuracy: 0.7848, Test Accuracy (k = 3): 0.739726, Additional savings: 0.009247\n",
+      "INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn-1023\n",
+      "Round:  5, Validation accuracy: 0.7971, Test Accuracy (k = 3): 0.766145, Additional savings: 0.018493\n",
+      "INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn-1027\n",
+      "Round:  6, Validation accuracy: 0.8191, Test Accuracy (k = 3): 0.775930, Additional savings: 0.026272\n",
+      "INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn-1031\n",
+      "Round:  7, Validation accuracy: 0.8362, Test Accuracy (k = 3): 0.793542, Additional savings: 0.035152\n",
+      "INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn-1033\n",
+      "Round:  8, Validation accuracy: 0.8411, Test Accuracy (k = 3): 0.801370, Additional savings: 0.038454\n",
+      "INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn-1036\n",
+      "Round:  9, Validation accuracy: 0.8411, Test Accuracy (k = 3): 0.802348, Additional savings: 0.041120\n",
+      "INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn-1041\n",
+      "Round: 10, Validation accuracy: 0.8411, Test Accuracy (k = 3): 0.808219, Additional savings: 0.048483\n",
+      "INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn-1046\n",
+      "Round: 11, Validation accuracy: 0.8435, Test Accuracy (k = 3): 0.818982, Additional savings: 0.058953\n",
+      "INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn-1051\n",
+      "Round: 12, Validation accuracy: 0.8411, Test Accuracy (k = 3): 0.825832, Additional savings: 0.073679\n",
+      "INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn-1052\n",
+      "Round: 13, Validation accuracy: 0.8411, Test Accuracy (k = 3): 0.818004, Additional savings: 0.074046\n",
+      "INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn-1057\n",
+      "Round: 14, Validation accuracy: 0.8411, Test Accuracy (k = 3): 0.835616, Additional savings: 0.082021\n",
+      "INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn-1063\n",
+      "Round: 15, Validation accuracy: 0.8191, Test Accuracy (k = 3): 0.825832, Additional savings: 0.126981\n",
+      "INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn-1067\n",
+      "Round: 16, Validation accuracy: 0.8240, Test Accuracy (k = 3): 0.828767, Additional savings: 0.154892\n",
+      "INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn-1071\n",
+      "Round: 17, Validation accuracy: 0.8337, Test Accuracy (k = 3): 0.831703, Additional savings: 0.175636\n",
+      "INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn-1075\n",
+      "Round: 18, Validation accuracy: 0.8460, Test Accuracy (k = 3): 0.842466, Additional savings: 0.199902\n",
+      "INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn-1076\n",
+      "Round: 19, Validation accuracy: 0.8411, Test Accuracy (k = 3): 0.844423, Additional savings: 0.203082\n",
+      "INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn-1083\n",
+      "Round: 20, Validation accuracy: 0.8533, Test Accuracy (k = 3): 0.857143, Additional savings: 0.218567\n",
+      "INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn-1086\n",
+      "Round: 21, Validation accuracy: 0.8704, Test Accuracy (k = 3): 0.863014, Additional savings: 0.233439\n",
+      "INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn-1088\n",
+      "Round: 22, Validation accuracy: 0.8704, Test Accuracy (k = 3): 0.859100, Additional savings: 0.237696\n",
+      "INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn-1093\n",
+      "Round: 23, Validation accuracy: 0.8680, Test Accuracy (k = 3): 0.865949, Additional savings: 0.242343\n",
+      "INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn-1096\n",
+      "Round: 24, Validation accuracy: 0.8655, Test Accuracy (k = 3): 0.870841, Additional savings: 0.248679\n",
+      "INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn-1101\n",
+      "Round: 25, Validation accuracy: 0.8655, Test Accuracy (k = 3): 0.872798, Additional savings: 0.258708\n",
+      "INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn-1107\n",
+      "Round: 26, Validation accuracy: 0.8778, Test Accuracy (k = 3): 0.878669, Additional savings: 0.264873\n",
+      "INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn-1110\n",
+      "Round: 27, Validation accuracy: 0.8802, Test Accuracy (k = 3): 0.876712, Additional savings: 0.281238\n",
+      "INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn-1114\n",
+      "Round: 28, Validation accuracy: 0.8802, Test Accuracy (k = 3): 0.881605, Additional savings: 0.291389\n",
+      "INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn-1117\n",
+      "Round: 29, Validation accuracy: 0.8778, Test Accuracy (k = 3): 0.875734, Additional savings: 0.295475\n"
+     ]
+    }
+   ],
+   "source": [
+    "k=3\n",
+    "emiDriver.loadSavedGraphToNewSession(MODEL_PREFIX , 1032)\n",
+    "devnull = open(os.devnull, 'r')\n",
+    "for val in modelStats:\n",
+    "    round_, acc, modelPrefix, globalStep = val\n",
+    "    emiDriver.loadSavedGraphToNewSession(modelPrefix, globalStep, redirFile=devnull)\n",
+    "    predictions, predictionStep = emiDriver.getInstancePredictions(x_test, y_test, earlyPolicy_minProb,\n",
+    "                                                               minProb=0.99, keep_prob=1.0)\n",
+    " \n",
+    "    bagPredictions = emiDriver.getBagPredictions(predictions, minSubsequenceLen=k, numClass=NUM_OUTPUT)\n",
+    "    print(\"Round: %2d, Validation accuracy: %.4f\" % (round_, acc), end='')\n",
+    "    print(', Test Accuracy (k = %d): %f, ' % (k,  np.mean((bagPredictions == BAG_TEST).astype(int))), end='')\n",
+    "    print('Additional savings: %f' % getEarlySaving(predictionStep, NUM_TIMESTEPS)) "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 22,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "MODEL_PREFIX = '/home/sf/data/SWELL-KW/FG_8_13/model-fgrnn'"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 23,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/FG_8_13/model-fgrnn-1118\n",
+      "-1.243401288986206\n",
+      "Accuracy at k = 2: 0.870841\n"
+     ]
+    }
+   ],
+   "source": [
+    "import time\n",
+    "k=2\n",
+    "start = time.time()\n",
+    "emiDriver.loadSavedGraphToNewSession(MODEL_PREFIX , 1118)\n",
+    "predictions, predictionStep = emiDriver.getInstancePredictions(x_test, y_test, earlyPolicy_minProb,\n",
+    "                                                               minProb=0.99, keep_prob=1.0)###\n",
+    "bagPredictions = emiDriver.getBagPredictions(predictions, minSubsequenceLen=k, numClass=NUM_OUTPUT)###\n",
+    "end = time.time()\n",
+    "print(start-end)\n",
+    "print('Accuracy at k = %d: %f' % (k,  np.mean((bagPredictions == BAG_TEST).astype(int))))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 18,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "params = {\n",
+    "    \"NUM_HIDDEN\" : 128,\n",
+    "    \"NUM_TIMESTEPS\" : 8, #subinstance length.\n",
+    "    \"NUM_FEATS\" : 22,\n",
+    "    \"FORGET_BIAS\" : 1.0,\n",
+    "    \"UPDATE_NL\" : \"quantTanh\",\n",
+    "    \"GATE_NL\" : \"quantSigm\",\n",
+    "    \"NUM_OUTPUT\" : 3,\n",
+    "    \"WRANK\" : 5,\n",
+    "    \"URANK\" : 6,\n",
+    "    \"USE_DROPOUT\" : False,\n",
+    "    \"KEEP_PROB\" : 0.9,\n",
+    "    \"PREFETCH_NUM\" : 5,\n",
+    "    \"BATCH_SIZE\" : 32,\n",
+    "    \"NUM_EPOCHS\" : 2,\n",
+    "    \"NUM_ITER\" : 4,\n",
+    "    \"NUM_ROUNDS\" : 10,\n",
+    "    \"MODEL_PREFIX\" : '/home/sf/data/DREAMER/Dominance/48_16/models/Fast-GRNN/model-fgrnn'\n",
+    "}"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 24,
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "   len       acc  macro-fsc  macro-pre  macro-rec  micro-fsc  micro-pre  \\\n",
+      "0    1  0.853229   0.852651   0.863397   0.855376   0.853229   0.853229   \n",
+      "1    2  0.870841   0.870792   0.873219   0.871904   0.870841   0.870841   \n",
+      "2    3  0.878669   0.878602   0.878571   0.878641   0.878669   0.878669   \n",
+      "3    4  0.868885   0.868431   0.870852   0.867953   0.868885   0.868885   \n",
+      "4    5  0.869863   0.868957   0.875526   0.868309   0.869863   0.869863   \n",
+      "\n",
+      "   micro-rec  fscore_01  \n",
+      "0   0.853229   0.861878  \n",
+      "1   0.870841   0.873321  \n",
+      "2   0.878669   0.875752  \n",
+      "3   0.868885   0.860707  \n",
+      "4   0.869863   0.858058  \n",
+      "Max accuracy 0.878669 at subsequencelength 3\n",
+      "Max micro-f 0.878669 at subsequencelength 3\n",
+      "Micro-precision 0.878669 at subsequencelength 3\n",
+      "Micro-recall 0.878669 at subsequencelength 3\n",
+      "Max macro-f 0.878602 at subsequencelength 3\n",
+      "macro-precision 0.878571 at subsequencelength 3\n",
+      "macro-recall 0.878641 at subsequencelength 3\n",
+      "+----+-------+----------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+\n",
+      "|    |   len |      acc |   macro-fsc |   macro-pre |   macro-rec |   micro-fsc |   micro-pre |   micro-rec |   fscore_01 |\n",
+      "+====+=======+==========+=============+=============+=============+=============+=============+=============+=============+\n",
+      "|  0 |     1 | 0.853229 |    0.852651 |    0.863397 |    0.855376 |    0.853229 |    0.853229 |    0.853229 |    0.861878 |\n",
+      "+----+-------+----------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+\n",
+      "|  1 |     2 | 0.870841 |    0.870792 |    0.873219 |    0.871904 |    0.870841 |    0.870841 |    0.870841 |    0.873321 |\n",
+      "+----+-------+----------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+\n",
+      "|  2 |     3 | 0.878669 |    0.878602 |    0.878571 |    0.878641 |    0.878669 |    0.878669 |    0.878669 |    0.875752 |\n",
+      "+----+-------+----------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+\n",
+      "|  3 |     4 | 0.868885 |    0.868431 |    0.870852 |    0.867953 |    0.868885 |    0.868885 |    0.868885 |    0.860707 |\n",
+      "+----+-------+----------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+\n",
+      "|  4 |     5 | 0.869863 |    0.868957 |    0.875526 |    0.868309 |    0.869863 |    0.869863 |    0.869863 |    0.858058 |\n",
+      "+----+-------+----------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+\n"
+     ]
+    }
+   ],
+   "source": [
+    "fgrnn_dict = {**params}\n",
+    "fgrnn_dict[\"k\"] = k\n",
+    "fgrnn_dict[\"accuracy\"] = np.mean((bagPredictions == BAG_TEST).astype(int))\n",
+    "fgrnn_dict[\"total_savings\"] = getEarlySaving(predictionStep, NUM_TIMESTEPS)\n",
+    "fgrnn_dict[\"y_test\"] = BAG_TEST\n",
+    "fgrnn_dict[\"y_pred\"] = bagPredictions\n",
+    "\n",
+    "# A slightly more detailed analysis method is provided. \n",
+    "df = emiDriver.analyseModel(predictions, BAG_TEST, NUM_SUBINSTANCE, NUM_OUTPUT)\n",
+    "print (tabulate(df, headers=list(df.columns), tablefmt='grid'))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 25,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Results for this run have been saved at /home/sf/data/SWELL-KW/ .\n"
+     ]
+    }
+   ],
+   "source": [
+    "dirname = \"/home/sf/data/SWELL-KW/\"\n",
+    "pathlib.Path(dirname).mkdir(parents=True, exist_ok=True)\n",
+    "print (\"Results for this run have been saved at\" , dirname, \".\")\n",
+    "\n",
+    "now = datetime.datetime.now()\n",
+    "filename = list((str(now.year),\"-\",str(now.month),\"-\",str(now.day),\"|\",str(now.hour),\"-\",str(now.minute)))\n",
+    "filename = ''.join(filename)\n",
+    "\n",
+    "#Save the dictionary containing the params and the results.\n",
+    "pkl.dump(fgrnn_dict,open(dirname  + filename + \".pkl\",mode='wb'))"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.7.3"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}