Datasets
Models
Downloads: 1
872 lines (871 with data), 51.0 kB
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# SWELL-KW GRU"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Adapted from Microsoft's notebooks, available at https://github.com/microsoft/EdgeML authored by Dennis et al."
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"import pandas as pd\n",
"import numpy as np\n",
"from tabulate import tabulate\n",
"import os\n",
"import datetime as datetime\n",
"import pickle as pkl\n",
"import pathlib"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {
"ExecuteTime": {
"end_time": "2018-12-14T14:17:51.796585Z",
"start_time": "2018-12-14T14:17:49.648375Z"
}
},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"Using TensorFlow backend.\n"
]
}
],
"source": [
"from __future__ import print_function\n",
"import os\n",
"import sys\n",
"import tensorflow as tf\n",
"import numpy as np\n",
"# Making sure edgeml is part of python path\n",
"sys.path.insert(0, '../../')\n",
"#For processing on CPU.\n",
"os.environ['CUDA_VISIBLE_DEVICES'] ='0'\n",
"\n",
"np.random.seed(42)\n",
"tf.set_random_seed(42)\n",
"\n",
"# MI-RNN and EMI-RNN imports\n",
"from edgeml.graph.rnn import EMI_DataPipeline\n",
"from edgeml.graph.rnn import EMI_GRU\n",
"from edgeml.trainer.emirnnTrainer import EMI_Trainer, EMI_Driver\n",
"import edgeml.utils\n",
"\n",
"import keras.backend as K\n",
"cfg = K.tf.ConfigProto()\n",
"cfg.gpu_options.allow_growth = True\n",
"K.set_session(K.tf.Session(config=cfg))"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {
"ExecuteTime": {
"end_time": "2018-12-14T14:17:51.803381Z",
"start_time": "2018-12-14T14:17:51.798799Z"
}
},
"outputs": [],
"source": [
"# Network parameters for our LSTM + FC Layer\n",
"NUM_HIDDEN = 128\n",
"NUM_TIMESTEPS = 8\n",
"ORIGINAL_NUM_TIMESTEPS = 20\n",
"NUM_FEATS = 22\n",
"FORGET_BIAS = 1.0\n",
"NUM_OUTPUT = 2\n",
"USE_DROPOUT = True\n",
"KEEP_PROB = 0.75\n",
"\n",
"# For dataset API\n",
"PREFETCH_NUM = 5\n",
"BATCH_SIZE = 32\n",
"\n",
"# Number of epochs in *one iteration*\n",
"NUM_EPOCHS = 2\n",
"# Number of iterations in *one round*. After each iteration,\n",
"# the model is dumped to disk. At the end of the current\n",
"# round, the best model among all the dumped models in the\n",
"# current round is picked up..\n",
"NUM_ITER = 4\n",
"# A round consists of multiple training iterations and a belief\n",
"# update step using the best model from all of these iterations\n",
"NUM_ROUNDS = 30\n",
"LEARNING_RATE=0.001\n",
"\n",
"# A staging direcory to store models\n",
"MODEL_PREFIX = '/home/sf/data/SWELL-KW/models/GRU/model-gru'"
]
},
{
"cell_type": "markdown",
"metadata": {
"heading_collapsed": true
},
"source": [
"# Loading Data"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {
"ExecuteTime": {
"end_time": "2018-12-14T14:17:52.040352Z",
"start_time": "2018-12-14T14:17:51.805319Z"
},
"hidden": true
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"x_train shape is: (3679, 5, 8, 22)\n",
"y_train shape is: (3679, 5, 2)\n",
"x_test shape is: (409, 5, 8, 22)\n",
"y_test shape is: (409, 5, 2)\n"
]
}
],
"source": [
"# Loading the data\n",
"x_train, y_train = np.load('/home/sf/data/SWELL-KW/8_3/x_train.npy'), np.load('/home/sf/data/SWELL-KW/8_3/y_train.npy')\n",
"x_test, y_test = np.load('/home/sf/data/SWELL-KW/8_3/x_test.npy'), np.load('/home/sf/data/SWELL-KW/8_3/y_test.npy')\n",
"x_val, y_val = np.load('/home/sf/data/SWELL-KW/8_3/x_val.npy'), np.load('/home/sf/data/SWELL-KW/8_3/y_val.npy')\n",
"\n",
"# BAG_TEST, BAG_TRAIN, BAG_VAL represent bag_level labels. These are used for the label update\n",
"# step of EMI/MI RNN\n",
"BAG_TEST = np.argmax(y_test[:, 0, :], axis=1)\n",
"BAG_TRAIN = np.argmax(y_train[:, 0, :], axis=1)\n",
"BAG_VAL = np.argmax(y_val[:, 0, :], axis=1)\n",
"NUM_SUBINSTANCE = x_train.shape[1]\n",
"print(\"x_train shape is:\", x_train.shape)\n",
"print(\"y_train shape is:\", y_train.shape)\n",
"print(\"x_test shape is:\", x_val.shape)\n",
"print(\"y_test shape is:\", y_val.shape)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Computation Graph"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {
"ExecuteTime": {
"end_time": "2018-12-14T14:17:52.053161Z",
"start_time": "2018-12-14T14:17:52.042928Z"
}
},
"outputs": [],
"source": [
"# Define the linear secondary classifier\n",
"def createExtendedGraph(self, baseOutput, *args, **kwargs):\n",
" W1 = tf.Variable(np.random.normal(size=[NUM_HIDDEN, NUM_OUTPUT]).astype('float32'), name='W1')\n",
" B1 = tf.Variable(np.random.normal(size=[NUM_OUTPUT]).astype('float32'), name='B1')\n",
" y_cap = tf.add(tf.tensordot(baseOutput, W1, axes=1), B1, name='y_cap_tata')\n",
" self.output = y_cap\n",
" self.graphCreated = True\n",
"\n",
"def restoreExtendedGraph(self, graph, *args, **kwargs):\n",
" y_cap = graph.get_tensor_by_name('y_cap_tata:0')\n",
" self.output = y_cap\n",
" self.graphCreated = True\n",
" \n",
"def feedDictFunc(self, keep_prob=None, inference=False, **kwargs):\n",
" if inference is False:\n",
" feedDict = {self._emiGraph.keep_prob: keep_prob}\n",
" else:\n",
" feedDict = {self._emiGraph.keep_prob: 1.0}\n",
" return feedDict\n",
" \n",
"EMI_GRU._createExtendedGraph = createExtendedGraph\n",
"EMI_GRU._restoreExtendedGraph = restoreExtendedGraph\n",
"\n",
"if USE_DROPOUT is True:\n",
" EMI_Driver.feedDictFunc = feedDictFunc"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {
"ExecuteTime": {
"end_time": "2018-12-14T14:17:52.335299Z",
"start_time": "2018-12-14T14:17:52.055483Z"
}
},
"outputs": [],
"source": [
"inputPipeline = EMI_DataPipeline(NUM_SUBINSTANCE, NUM_TIMESTEPS, NUM_FEATS, NUM_OUTPUT)\n",
"emiGRU = EMI_GRU(NUM_SUBINSTANCE, NUM_HIDDEN, NUM_TIMESTEPS, NUM_FEATS,\n",
" useDropout=USE_DROPOUT)\n",
"emiTrainer = EMI_Trainer(NUM_TIMESTEPS, NUM_OUTPUT, lossType='xentropy',\n",
" stepSize=LEARNING_RATE)"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {
"ExecuteTime": {
"end_time": "2018-12-14T14:18:05.031382Z",
"start_time": "2018-12-14T14:17:52.338750Z"
}
},
"outputs": [],
"source": [
"tf.reset_default_graph()\n",
"g1 = tf.Graph() \n",
"with g1.as_default():\n",
" # Obtain the iterators to each batch of the data\n",
" x_batch, y_batch = inputPipeline()\n",
" # Create the forward computation graph based on the iterators\n",
" y_cap = emiGRU(x_batch)\n",
" # Create loss graphs and training routines\n",
" emiTrainer(y_cap, y_batch)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# EMI Driver"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {
"ExecuteTime": {
"end_time": "2018-12-14T14:35:15.209910Z",
"start_time": "2018-12-14T14:18:05.034359Z"
},
"scrolled": true
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Update policy: top-k\n",
"Training with MI-RNN loss for 15 rounds\n",
"Round: 0\n",
"Epoch 1 Batch 110 ( 225) Loss 0.08950 Acc 0.63125 | Val acc 0.58680 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1000\n",
"Epoch 1 Batch 110 ( 225) Loss 0.08044 Acc 0.61875 | Val acc 0.60636 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1001\n",
"Epoch 1 Batch 110 ( 225) Loss 0.08588 Acc 0.59375 | Val acc 0.61369 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1002\n",
"Epoch 1 Batch 110 ( 225) Loss 0.06992 Acc 0.70625 | Val acc 0.63570 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1003\n",
"INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/models/GRU/model-gru-1003\n",
"Round: 1\n",
"Epoch 1 Batch 110 ( 225) Loss 0.07058 Acc 0.65625 | Val acc 0.64792 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1004\n",
"Epoch 1 Batch 110 ( 225) Loss 0.06464 Acc 0.71250 | Val acc 0.66748 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1005\n",
"Epoch 1 Batch 110 ( 225) Loss 0.06516 Acc 0.70000 | Val acc 0.68704 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1006\n",
"Epoch 1 Batch 110 ( 225) Loss 0.06636 Acc 0.72500 | Val acc 0.69927 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1007\n",
"INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/models/GRU/model-gru-1007\n",
"Round: 2\n",
"Epoch 1 Batch 110 ( 225) Loss 0.06905 Acc 0.68750 | Val acc 0.69438 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1008\n",
"Epoch 1 Batch 110 ( 225) Loss 0.06327 Acc 0.75625 | Val acc 0.68949 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1009\n",
"Epoch 1 Batch 110 ( 225) Loss 0.06798 Acc 0.75625 | Val acc 0.70416 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1010\n",
"Epoch 1 Batch 110 ( 225) Loss 0.06369 Acc 0.71875 | Val acc 0.69193 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1011\n",
"INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/models/GRU/model-gru-1010\n",
"Round: 3\n",
"Epoch 1 Batch 110 ( 225) Loss 0.06522 Acc 0.73750 | Val acc 0.70171 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1012\n",
"Epoch 1 Batch 110 ( 225) Loss 0.06332 Acc 0.74375 | Val acc 0.70905 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1013\n",
"Epoch 1 Batch 110 ( 225) Loss 0.06348 Acc 0.75625 | Val acc 0.73105 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1014\n",
"Epoch 1 Batch 110 ( 225) Loss 0.06125 Acc 0.75625 | Val acc 0.72616 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1015\n",
"INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/models/GRU/model-gru-1014\n",
"Round: 4\n",
"Epoch 1 Batch 110 ( 225) Loss 0.06360 Acc 0.73750 | Val acc 0.72616 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1016\n",
"Epoch 1 Batch 110 ( 225) Loss 0.06455 Acc 0.76250 | Val acc 0.74328 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1017\n",
"Epoch 1 Batch 110 ( 225) Loss 0.06291 Acc 0.76250 | Val acc 0.74328 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1018\n",
"Epoch 1 Batch 110 ( 225) Loss 0.05980 Acc 0.78750 | Val acc 0.75061 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1019\n",
"INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/models/GRU/model-gru-1019\n",
"Round: 5\n",
"Epoch 1 Batch 110 ( 225) Loss 0.06897 Acc 0.76250 | Val acc 0.75061 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1020\n",
"Epoch 1 Batch 110 ( 225) Loss 0.06064 Acc 0.75625 | Val acc 0.76039 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1021\n",
"Epoch 1 Batch 110 ( 225) Loss 0.05520 Acc 0.78750 | Val acc 0.76773 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1022\n",
"Epoch 1 Batch 110 ( 225) Loss 0.05275 Acc 0.84375 | Val acc 0.75795 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1023\n",
"INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/models/GRU/model-gru-1022\n",
"Round: 6\n",
"Epoch 1 Batch 110 ( 225) Loss 0.05810 Acc 0.78750 | Val acc 0.77017 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1024\n",
"Epoch 1 Batch 110 ( 225) Loss 0.05748 Acc 0.76875 | Val acc 0.76528 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1025\n",
"Epoch 1 Batch 110 ( 225) Loss 0.06343 Acc 0.81250 | Val acc 0.77751 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1026\n",
"Epoch 1 Batch 110 ( 225) Loss 0.05767 Acc 0.78125 | Val acc 0.76039 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1027\n",
"INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/models/GRU/model-gru-1026\n",
"Round: 7\n",
"Epoch 1 Batch 110 ( 225) Loss 0.05564 Acc 0.80000 | Val acc 0.76528 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1028\n",
"Epoch 1 Batch 110 ( 225) Loss 0.06886 Acc 0.77500 | Val acc 0.76039 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1029\n",
"Epoch 1 Batch 110 ( 225) Loss 0.05440 Acc 0.78125 | Val acc 0.77751 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1030\n",
"Epoch 1 Batch 110 ( 225) Loss 0.05853 Acc 0.76875 | Val acc 0.78484 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1031\n",
"INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/models/GRU/model-gru-1031\n",
"Round: 8\n",
"Epoch 1 Batch 110 ( 225) Loss 0.05785 Acc 0.75000 | Val acc 0.77751 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1032\n",
"Epoch 1 Batch 110 ( 225) Loss 0.05716 Acc 0.78750 | Val acc 0.80196 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1033\n",
"Epoch 1 Batch 110 ( 225) Loss 0.05379 Acc 0.81875 | Val acc 0.79707 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1034\n",
"Epoch 1 Batch 110 ( 225) Loss 0.05206 Acc 0.81875 | Val acc 0.82152 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1035\n",
"INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/models/GRU/model-gru-1035\n",
"Round: 9\n",
"Epoch 1 Batch 110 ( 225) Loss 0.05389 Acc 0.80625 | Val acc 0.81418 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1036\n",
"Epoch 1 Batch 110 ( 225) Loss 0.05023 Acc 0.79375 | Val acc 0.82152 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1037\n",
"Epoch 1 Batch 110 ( 225) Loss 0.04696 Acc 0.80625 | Val acc 0.81907 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1038\n",
"Epoch 1 Batch 110 ( 225) Loss 0.04655 Acc 0.82500 | Val acc 0.82641 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1039\n",
"INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/models/GRU/model-gru-1039\n",
"Round: 10\n",
"Epoch 1 Batch 110 ( 225) Loss 0.04700 Acc 0.83125 | Val acc 0.81663 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1040\n",
"Epoch 1 Batch 110 ( 225) Loss 0.04426 Acc 0.83125 | Val acc 0.82641 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1041\n",
"Epoch 1 Batch 110 ( 225) Loss 0.05274 Acc 0.81875 | Val acc 0.82885 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1042\n",
"Epoch 1 Batch 110 ( 225) Loss 0.04178 Acc 0.82500 | Val acc 0.83619 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1043\n",
"INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/models/GRU/model-gru-1043\n",
"Round: 11\n",
"Epoch 1 Batch 110 ( 225) Loss 0.05123 Acc 0.78125 | Val acc 0.84597 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1044\n",
"Epoch 1 Batch 110 ( 225) Loss 0.05073 Acc 0.82500 | Val acc 0.84352 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1045\n",
"Epoch 1 Batch 110 ( 225) Loss 0.04487 Acc 0.82500 | Val acc 0.83374 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1046\n",
"Epoch 1 Batch 110 ( 225) Loss 0.04353 Acc 0.84375 | Val acc 0.84108 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1047\n",
"INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/models/GRU/model-gru-1044\n",
"Round: 12\n",
"Epoch 1 Batch 110 ( 225) Loss 0.04450 Acc 0.85625 | Val acc 0.85086 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1048\n",
"Epoch 1 Batch 110 ( 225) Loss 0.04483 Acc 0.84375 | Val acc 0.83619 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1049\n",
"Epoch 1 Batch 110 ( 225) Loss 0.03620 Acc 0.86250 | Val acc 0.83863 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1050\n",
"Epoch 1 Batch 110 ( 225) Loss 0.03725 Acc 0.87500 | Val acc 0.84108 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1051\n",
"INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/models/GRU/model-gru-1048\n",
"Round: 13\n",
"Epoch 1 Batch 110 ( 225) Loss 0.04427 Acc 0.81250 | Val acc 0.83619 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1052\n",
"Epoch 1 Batch 110 ( 225) Loss 0.03978 Acc 0.87500 | Val acc 0.84841 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1053\n",
"Epoch 1 Batch 110 ( 225) Loss 0.04293 Acc 0.83125 | Val acc 0.85086 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1054\n",
"Epoch 1 Batch 110 ( 225) Loss 0.03750 Acc 0.85625 | Val acc 0.84352 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1055\n",
"INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/models/GRU/model-gru-1054\n",
"Round: 14\n",
"Epoch 1 Batch 110 ( 225) Loss 0.04008 Acc 0.83125 | Val acc 0.84108 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1056\n",
"Epoch 1 Batch 110 ( 225) Loss 0.03921 Acc 0.85000 | Val acc 0.84597 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1057\n",
"Epoch 1 Batch 110 ( 225) Loss 0.03854 Acc 0.85625 | Val acc 0.84352 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1058\n",
"Epoch 1 Batch 110 ( 225) Loss 0.03663 Acc 0.88125 | Val acc 0.84352 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1059\n",
"INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/models/GRU/model-gru-1057\n",
"Round: 15\n",
"Switching to EMI-Loss function\n",
"Epoch 1 Batch 110 ( 225) Loss 0.45096 Acc 0.84375 | Val acc 0.81663 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1060\n",
"Epoch 1 Batch 110 ( 225) Loss 0.47173 Acc 0.79375 | Val acc 0.82396 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1061\n",
"Epoch 1 Batch 110 ( 225) Loss 0.43876 Acc 0.83750 | Val acc 0.82396 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1062\n",
"Epoch 1 Batch 110 ( 225) Loss 0.43396 Acc 0.80625 | Val acc 0.85330 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1063\n",
"INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/models/GRU/model-gru-1063\n",
"Round: 16\n",
"Epoch 1 Batch 110 ( 225) Loss 0.43497 Acc 0.84375 | Val acc 0.84352 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1064\n",
"Epoch 1 Batch 110 ( 225) Loss 0.42715 Acc 0.83750 | Val acc 0.83619 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1065\n",
"Epoch 1 Batch 110 ( 225) Loss 0.42176 Acc 0.82500 | Val acc 0.82885 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1066\n",
"Epoch 1 Batch 110 ( 225) Loss 0.39219 Acc 0.84375 | Val acc 0.84108 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1067\n",
"INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/models/GRU/model-gru-1064\n",
"Round: 17\n",
"Epoch 1 Batch 110 ( 225) Loss 0.39438 Acc 0.85625 | Val acc 0.83863 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1068\n",
"Epoch 1 Batch 110 ( 225) Loss 0.43058 Acc 0.82500 | Val acc 0.85086 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1069\n",
"Epoch 1 Batch 110 ( 225) Loss 0.40885 Acc 0.87500 | Val acc 0.84597 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1070\n",
"Epoch 1 Batch 110 ( 225) Loss 0.39980 Acc 0.83125 | Val acc 0.84352 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1071\n",
"INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/models/GRU/model-gru-1069\n",
"Round: 18\n",
"Epoch 1 Batch 110 ( 225) Loss 0.40417 Acc 0.84375 | Val acc 0.86064 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1072\n",
"Epoch 1 Batch 110 ( 225) Loss 0.43128 Acc 0.83125 | Val acc 0.84597 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1073\n",
"Epoch 1 Batch 110 ( 225) Loss 0.40664 Acc 0.86250 | Val acc 0.85330 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1074\n",
"Epoch 1 Batch 110 ( 225) Loss 0.41514 Acc 0.86250 | Val acc 0.85086 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1075\n",
"INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/models/GRU/model-gru-1072\n",
"Round: 19\n",
"Epoch 1 Batch 110 ( 225) Loss 0.38532 Acc 0.85000 | Val acc 0.84597 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1076\n",
"Epoch 1 Batch 110 ( 225) Loss 0.40468 Acc 0.85625 | Val acc 0.84841 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1077\n",
"Epoch 1 Batch 110 ( 225) Loss 0.44360 Acc 0.82500 | Val acc 0.85086 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1078\n",
"Epoch 1 Batch 110 ( 225) Loss 0.39450 Acc 0.85625 | Val acc 0.83863 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1079\n",
"INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/models/GRU/model-gru-1078\n",
"Round: 20\n",
"Epoch 1 Batch 110 ( 225) Loss 0.37850 Acc 0.83750 | Val acc 0.85330 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1080\n",
"Epoch 1 Batch 110 ( 225) Loss 0.37629 Acc 0.86250 | Val acc 0.85086 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1081\n",
"Epoch 1 Batch 110 ( 225) Loss 0.38814 Acc 0.85625 | Val acc 0.84108 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1082\n",
"Epoch 1 Batch 110 ( 225) Loss 0.37590 Acc 0.90000 | Val acc 0.84841 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1083\n",
"INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/models/GRU/model-gru-1080\n",
"Round: 21\n",
"Epoch 1 Batch 110 ( 225) Loss 0.39236 Acc 0.87500 | Val acc 0.85330 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1084\n",
"Epoch 1 Batch 110 ( 225) Loss 0.38403 Acc 0.86875 | Val acc 0.84597 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1085\n",
"Epoch 1 Batch 110 ( 225) Loss 0.39141 Acc 0.86875 | Val acc 0.84841 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1086\n",
"Epoch 1 Batch 110 ( 225) Loss 0.37983 Acc 0.85625 | Val acc 0.84841 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1087\n",
"INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/models/GRU/model-gru-1084\n",
"Round: 22\n",
"Epoch 1 Batch 110 ( 225) Loss 0.40008 Acc 0.87500 | Val acc 0.84597 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1088\n",
"Epoch 1 Batch 110 ( 225) Loss 0.34944 Acc 0.89375 | Val acc 0.85086 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1089\n",
"Epoch 1 Batch 110 ( 225) Loss 0.39081 Acc 0.89375 | Val acc 0.83130 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1090\n",
"Epoch 1 Batch 110 ( 225) Loss 0.38600 Acc 0.84375 | Val acc 0.83863 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1091\n",
"INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/models/GRU/model-gru-1089\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Round: 23\n",
"Epoch 1 Batch 110 ( 225) Loss 0.39712 Acc 0.86250 | Val acc 0.85330 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1092\n",
"Epoch 1 Batch 110 ( 225) Loss 0.35570 Acc 0.89375 | Val acc 0.84841 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1093\n",
"Epoch 1 Batch 110 ( 225) Loss 0.35381 Acc 0.88750 | Val acc 0.84841 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1094\n",
"Epoch 1 Batch 110 ( 225) Loss 0.37666 Acc 0.85625 | Val acc 0.84352 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1095\n",
"INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/models/GRU/model-gru-1092\n",
"Round: 24\n",
"Epoch 1 Batch 110 ( 225) Loss 0.38746 Acc 0.88750 | Val acc 0.85575 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1096\n",
"Epoch 1 Batch 110 ( 225) Loss 0.36476 Acc 0.91250 | Val acc 0.84597 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1097\n",
"Epoch 1 Batch 110 ( 225) Loss 0.37721 Acc 0.88750 | Val acc 0.83863 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1098\n",
"Epoch 1 Batch 110 ( 225) Loss 0.35381 Acc 0.90625 | Val acc 0.83130 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1099\n",
"INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/models/GRU/model-gru-1096\n",
"Round: 25\n",
"Epoch 1 Batch 110 ( 225) Loss 0.39612 Acc 0.85625 | Val acc 0.83863 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1100\n",
"Epoch 1 Batch 110 ( 225) Loss 0.38607 Acc 0.83750 | Val acc 0.85086 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1101\n",
"Epoch 1 Batch 110 ( 225) Loss 0.37138 Acc 0.90000 | Val acc 0.84841 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1102\n",
"Epoch 1 Batch 110 ( 225) Loss 0.36966 Acc 0.87500 | Val acc 0.85086 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1103\n",
"INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/models/GRU/model-gru-1101\n",
"Round: 26\n",
"Epoch 1 Batch 110 ( 225) Loss 0.33496 Acc 0.94375 | Val acc 0.84597 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1104\n",
"Epoch 1 Batch 110 ( 225) Loss 0.35933 Acc 0.87500 | Val acc 0.84841 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1105\n",
"Epoch 1 Batch 110 ( 225) Loss 0.35526 Acc 0.88750 | Val acc 0.84108 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1106\n",
"Epoch 1 Batch 110 ( 225) Loss 0.36529 Acc 0.87500 | Val acc 0.85330 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1107\n",
"INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/models/GRU/model-gru-1107\n",
"Round: 27\n",
"Epoch 1 Batch 110 ( 225) Loss 0.35815 Acc 0.90625 | Val acc 0.85330 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1108\n",
"Epoch 1 Batch 110 ( 225) Loss 0.35819 Acc 0.88125 | Val acc 0.85330 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1109\n",
"Epoch 1 Batch 110 ( 225) Loss 0.36880 Acc 0.90000 | Val acc 0.85086 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1110\n",
"Epoch 1 Batch 110 ( 225) Loss 0.35225 Acc 0.90625 | Val acc 0.84841 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1111\n",
"INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/models/GRU/model-gru-1108\n",
"Round: 28\n",
"Epoch 1 Batch 110 ( 225) Loss 0.36809 Acc 0.88750 | Val acc 0.85330 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1112\n",
"Epoch 1 Batch 110 ( 225) Loss 0.36660 Acc 0.88125 | Val acc 0.84841 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1113\n",
"Epoch 1 Batch 110 ( 225) Loss 0.33627 Acc 0.88125 | Val acc 0.85575 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1114\n",
"Epoch 1 Batch 110 ( 225) Loss 0.35610 Acc 0.88750 | Val acc 0.85330 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1115\n",
"INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/models/GRU/model-gru-1114\n",
"Round: 29\n",
"Epoch 1 Batch 110 ( 225) Loss 0.35923 Acc 0.87500 | Val acc 0.85086 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1116\n",
"Epoch 1 Batch 110 ( 225) Loss 0.34627 Acc 0.89375 | Val acc 0.84841 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1117\n",
"Epoch 1 Batch 110 ( 225) Loss 0.36224 Acc 0.88750 | Val acc 0.84352 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1118\n",
"Epoch 1 Batch 110 ( 225) Loss 0.36262 Acc 0.89375 | Val acc 0.84597 | Model saved to /home/sf/data/SWELL-KW/models/GRU/model-gru, global_step 1119\n",
"INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/models/GRU/model-gru-1116\n"
]
}
],
"source": [
"with g1.as_default():\n",
" emiDriver = EMI_Driver(inputPipeline, emiGRU, emiTrainer)\n",
"\n",
"emiDriver.initializeSession(g1)\n",
"y_updated, modelStats = emiDriver.run(numClasses=NUM_OUTPUT, x_train=x_train,\n",
" y_train=y_train, bag_train=BAG_TRAIN,\n",
" x_val=x_val, y_val=y_val, bag_val=BAG_VAL,\n",
" numIter=NUM_ITER, keep_prob=KEEP_PROB,\n",
" numRounds=NUM_ROUNDS, batchSize=BATCH_SIZE,\n",
" numEpochs=NUM_EPOCHS, modelPrefix=MODEL_PREFIX,\n",
" fracEMI=0.5, updatePolicy='top-k', k=1)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Evaluating the trained model"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {
"ExecuteTime": {
"end_time": "2018-12-14T14:35:15.218040Z",
"start_time": "2018-12-14T14:35:15.211771Z"
}
},
"outputs": [],
"source": [
"# Early Prediction Policy: We make an early prediction based on the predicted classes\n",
"# probability. If the predicted class probability > minProb at some step, we make\n",
"# a prediction at that step.\n",
"def earlyPolicy_minProb(instanceOut, minProb, **kwargs):\n",
" assert instanceOut.ndim == 2\n",
" classes = np.argmax(instanceOut, axis=1)\n",
" prob = np.max(instanceOut, axis=1)\n",
" index = np.where(prob >= minProb)[0]\n",
" if len(index) == 0:\n",
" assert (len(instanceOut) - 1) == (len(classes) - 1)\n",
" return classes[-1], len(instanceOut) - 1\n",
" index = index[0]\n",
" return classes[index], index\n",
"\n",
"def getEarlySaving(predictionStep, numTimeSteps, returnTotal=False):\n",
" predictionStep = predictionStep + 1\n",
" predictionStep = np.reshape(predictionStep, -1)\n",
" totalSteps = np.sum(predictionStep)\n",
" maxSteps = len(predictionStep) * numTimeSteps\n",
" savings = 1.0 - (totalSteps / maxSteps)\n",
" if returnTotal:\n",
" return savings, totalSteps\n",
" return savings"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {
"ExecuteTime": {
"end_time": "2018-12-14T14:35:16.257489Z",
"start_time": "2018-12-14T14:35:15.221029Z"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Accuracy at k = 2: 0.842466\n",
"Savings due to MI-RNN : 0.600000\n",
"Savings due to Early prediction: 0.162573\n",
"Total Savings: 0.665029\n"
]
}
],
"source": [
"k = 2\n",
"predictions, predictionStep = emiDriver.getInstancePredictions(x_test, y_test, earlyPolicy_minProb,\n",
" minProb=0.99, keep_prob=1.0)\n",
"bagPredictions = emiDriver.getBagPredictions(predictions, minSubsequenceLen=k, numClass=NUM_OUTPUT)\n",
"print('Accuracy at k = %d: %f' % (k, np.mean((bagPredictions == BAG_TEST).astype(int))))\n",
"mi_savings = (1 - NUM_TIMESTEPS / ORIGINAL_NUM_TIMESTEPS)\n",
"emi_savings = getEarlySaving(predictionStep, NUM_TIMESTEPS)\n",
"total_savings = mi_savings + (1 - mi_savings) * emi_savings\n",
"print('Savings due to MI-RNN : %f' % mi_savings)\n",
"print('Savings due to Early prediction: %f' % emi_savings)\n",
"print('Total Savings: %f' % (total_savings))"
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {
"ExecuteTime": {
"end_time": "2018-12-14T14:35:17.044115Z",
"start_time": "2018-12-14T14:35:16.259280Z"
},
"scrolled": true
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
" len acc macro-fsc macro-pre macro-rec micro-fsc micro-pre \\\n",
"0 1 0.831703 0.831557 0.835146 0.832988 0.831703 0.831703 \n",
"1 2 0.842466 0.842440 0.842466 0.842688 0.842466 0.842466 \n",
"2 3 0.846380 0.846156 0.846630 0.845956 0.846380 0.846380 \n",
"3 4 0.847358 0.846679 0.850044 0.846213 0.847358 0.847358 \n",
"4 5 0.839530 0.838276 0.845458 0.837832 0.839530 0.839530 \n",
"\n",
" micro-rec fscore_01 \n",
"0 0.831703 0.836502 \n",
"1 0.842466 0.840436 \n",
"2 0.846380 0.840285 \n",
"3 0.847358 0.836478 \n",
"4 0.839530 0.824034 \n",
"Max accuracy 0.847358 at subsequencelength 4\n",
"Max micro-f 0.847358 at subsequencelength 4\n",
"Micro-precision 0.847358 at subsequencelength 4\n",
"Micro-recall 0.847358 at subsequencelength 4\n",
"Max macro-f 0.846679 at subsequencelength 4\n",
"macro-precision 0.850044 at subsequencelength 4\n",
"macro-recall 0.846213 at subsequencelength 4\n"
]
}
],
"source": [
"# A slightly more detailed analysis method is provided. \n",
"df = emiDriver.analyseModel(predictions, BAG_TEST, NUM_SUBINSTANCE, NUM_OUTPUT)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Picking the best model"
]
},
{
"cell_type": "code",
"execution_count": 18,
"metadata": {
"ExecuteTime": {
"end_time": "2018-12-14T14:35:54.899340Z",
"start_time": "2018-12-14T14:35:17.047464Z"
},
"scrolled": true
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/models/GRU/model-gru-1003\n",
"Round: 0, Validation accuracy: 0.6357, Test Accuracy (k = 4): 0.667319, Total Savings: 0.602926\n",
"INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/models/GRU/model-gru-1007\n",
"Round: 1, Validation accuracy: 0.6993, Test Accuracy (k = 4): 0.693738, Total Savings: 0.603063\n",
"INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/models/GRU/model-gru-1010\n",
"Round: 2, Validation accuracy: 0.7042, Test Accuracy (k = 4): 0.685910, Total Savings: 0.603875\n",
"INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/models/GRU/model-gru-1014\n",
"Round: 3, Validation accuracy: 0.7311, Test Accuracy (k = 4): 0.693738, Total Savings: 0.603806\n",
"INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/models/GRU/model-gru-1019\n",
"Round: 4, Validation accuracy: 0.7506, Test Accuracy (k = 4): 0.709393, Total Savings: 0.604521\n",
"INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/models/GRU/model-gru-1022\n",
"Round: 5, Validation accuracy: 0.7677, Test Accuracy (k = 4): 0.726027, Total Savings: 0.605832\n",
"INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/models/GRU/model-gru-1026\n",
"Round: 6, Validation accuracy: 0.7775, Test Accuracy (k = 4): 0.745597, Total Savings: 0.607730\n",
"INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/models/GRU/model-gru-1031\n",
"Round: 7, Validation accuracy: 0.7848, Test Accuracy (k = 4): 0.772016, Total Savings: 0.609295\n",
"INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/models/GRU/model-gru-1035\n",
"Round: 8, Validation accuracy: 0.8215, Test Accuracy (k = 4): 0.777886, Total Savings: 0.611252\n",
"INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/models/GRU/model-gru-1039\n",
"Round: 9, Validation accuracy: 0.8264, Test Accuracy (k = 4): 0.798434, Total Savings: 0.614902\n",
"INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/models/GRU/model-gru-1043\n",
"Round: 10, Validation accuracy: 0.8362, Test Accuracy (k = 4): 0.799413, Total Savings: 0.617515\n",
"INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/models/GRU/model-gru-1044\n",
"Round: 11, Validation accuracy: 0.8460, Test Accuracy (k = 4): 0.814090, Total Savings: 0.618190\n",
"INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/models/GRU/model-gru-1048\n",
"Round: 12, Validation accuracy: 0.8509, Test Accuracy (k = 4): 0.810176, Total Savings: 0.619305\n",
"INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/models/GRU/model-gru-1054\n",
"Round: 13, Validation accuracy: 0.8509, Test Accuracy (k = 4): 0.825832, Total Savings: 0.620881\n",
"INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/models/GRU/model-gru-1057\n",
"Round: 14, Validation accuracy: 0.8460, Test Accuracy (k = 4): 0.828767, Total Savings: 0.623914\n",
"INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/models/GRU/model-gru-1063\n",
"Round: 15, Validation accuracy: 0.8533, Test Accuracy (k = 4): 0.823875, Total Savings: 0.634061\n",
"INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/models/GRU/model-gru-1064\n",
"Round: 16, Validation accuracy: 0.8435, Test Accuracy (k = 4): 0.836595, Total Savings: 0.637857\n",
"INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/models/GRU/model-gru-1069\n",
"Round: 17, Validation accuracy: 0.8509, Test Accuracy (k = 4): 0.828767, Total Savings: 0.641301\n",
"INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/models/GRU/model-gru-1072\n",
"Round: 18, Validation accuracy: 0.8606, Test Accuracy (k = 4): 0.831703, Total Savings: 0.640753\n",
"INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/models/GRU/model-gru-1078\n",
"Round: 19, Validation accuracy: 0.8509, Test Accuracy (k = 4): 0.832681, Total Savings: 0.646027\n",
"INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/models/GRU/model-gru-1080\n",
"Round: 20, Validation accuracy: 0.8533, Test Accuracy (k = 4): 0.836595, Total Savings: 0.648141\n",
"INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/models/GRU/model-gru-1084\n",
"Round: 21, Validation accuracy: 0.8533, Test Accuracy (k = 4): 0.846380, Total Savings: 0.648679\n",
"INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/models/GRU/model-gru-1089\n",
"Round: 22, Validation accuracy: 0.8509, Test Accuracy (k = 4): 0.839530, Total Savings: 0.651624\n",
"INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/models/GRU/model-gru-1092\n",
"Round: 23, Validation accuracy: 0.8533, Test Accuracy (k = 4): 0.835616, Total Savings: 0.653933\n",
"INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/models/GRU/model-gru-1096\n",
"Round: 24, Validation accuracy: 0.8557, Test Accuracy (k = 4): 0.839530, Total Savings: 0.655998\n",
"INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/models/GRU/model-gru-1101\n",
"Round: 25, Validation accuracy: 0.8509, Test Accuracy (k = 4): 0.833659, Total Savings: 0.657808\n",
"INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/models/GRU/model-gru-1107\n",
"Round: 26, Validation accuracy: 0.8533, Test Accuracy (k = 4): 0.840509, Total Savings: 0.661654\n",
"INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/models/GRU/model-gru-1108\n",
"Round: 27, Validation accuracy: 0.8533, Test Accuracy (k = 4): 0.841487, Total Savings: 0.663053\n",
"INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/models/GRU/model-gru-1114\n",
"Round: 28, Validation accuracy: 0.8557, Test Accuracy (k = 4): 0.839530, Total Savings: 0.663738\n",
"INFO:tensorflow:Restoring parameters from /home/sf/data/SWELL-KW/models/GRU/model-gru-1116\n",
"Round: 29, Validation accuracy: 0.8509, Test Accuracy (k = 4): 0.842466, Total Savings: 0.665029\n"
]
}
],
"source": [
"devnull = open(os.devnull, 'r')\n",
"for val in modelStats:\n",
" round_, acc, modelPrefix, globalStep = val\n",
" emiDriver.loadSavedGraphToNewSession(modelPrefix, globalStep, redirFile=devnull)\n",
" predictions, predictionStep = emiDriver.getInstancePredictions(x_test, y_test, earlyPolicy_minProb,\n",
" minProb=0.99, keep_prob=1.0)\n",
"\n",
" bagPredictions = emiDriver.getBagPredictions(predictions, minSubsequenceLen=k, numClass=NUM_OUTPUT)\n",
" print(\"Round: %2d, Validation accuracy: %.4f\" % (round_, acc), end='')\n",
" print(', Test Accuracy (k = %d): %f, ' % (4, np.mean((bagPredictions == BAG_TEST).astype(int))), end='')\n",
" mi_savings = (1 - NUM_TIMESTEPS / ORIGINAL_NUM_TIMESTEPS)\n",
" emi_savings = getEarlySaving(predictionStep, NUM_TIMESTEPS)\n",
" total_savings = mi_savings + (1 - mi_savings) * emi_savings\n",
" print(\"Total Savings: %f\" % total_savings)"
]
},
{
"cell_type": "code",
"execution_count": 14,
"metadata": {},
"outputs": [],
"source": [
"params = {\n",
" \"NUM_HIDDEN\" : 128,\n",
" \"NUM_TIMESTEPS\" : 64, #subinstance length.\n",
" \"ORIGINAL_NUM_TIMESTEPS\" : 128,\n",
" \"NUM_FEATS\" : 16,\n",
" \"FORGET_BIAS\" : 1.0,\n",
" \"NUM_OUTPUT\" : 5,\n",
" \"USE_DROPOUT\" : 1, # '1' -> True. '0' -> False\n",
" \"KEEP_PROB\" : 0.75,\n",
" \"PREFETCH_NUM\" : 5,\n",
" \"BATCH_SIZE\" : 32,\n",
" \"NUM_EPOCHS\" : 2,\n",
" \"NUM_ITER\" : 4,\n",
" \"NUM_ROUNDS\" : 10,\n",
" \"LEARNING_RATE\" : 0.001,\n",
" \"MODEL_PREFIX\" : '/home/sf/data/DREAMER/Dominance/model-gru'\n",
"}"
]
},
{
"cell_type": "code",
"execution_count": 15,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
" len acc macro-fsc macro-pre macro-rec micro-fsc micro-pre \\\n",
"0 1 0.831703 0.831557 0.835146 0.832988 0.831703 0.831703 \n",
"1 2 0.842466 0.842440 0.842466 0.842688 0.842466 0.842466 \n",
"2 3 0.846380 0.846156 0.846630 0.845956 0.846380 0.846380 \n",
"3 4 0.847358 0.846679 0.850044 0.846213 0.847358 0.847358 \n",
"4 5 0.839530 0.838276 0.845458 0.837832 0.839530 0.839530 \n",
"\n",
" micro-rec fscore_01 \n",
"0 0.831703 0.836502 \n",
"1 0.842466 0.840436 \n",
"2 0.846380 0.840285 \n",
"3 0.847358 0.836478 \n",
"4 0.839530 0.824034 \n",
"Max accuracy 0.847358 at subsequencelength 4\n",
"Max micro-f 0.847358 at subsequencelength 4\n",
"Micro-precision 0.847358 at subsequencelength 4\n",
"Micro-recall 0.847358 at subsequencelength 4\n",
"Max macro-f 0.846679 at subsequencelength 4\n",
"macro-precision 0.850044 at subsequencelength 4\n",
"macro-recall 0.846213 at subsequencelength 4\n",
"+----+-------+----------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+\n",
"| | len | acc | macro-fsc | macro-pre | macro-rec | micro-fsc | micro-pre | micro-rec | fscore_01 |\n",
"+====+=======+==========+=============+=============+=============+=============+=============+=============+=============+\n",
"| 0 | 1 | 0.831703 | 0.831557 | 0.835146 | 0.832988 | 0.831703 | 0.831703 | 0.831703 | 0.836502 |\n",
"+----+-------+----------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+\n",
"| 1 | 2 | 0.842466 | 0.84244 | 0.842466 | 0.842688 | 0.842466 | 0.842466 | 0.842466 | 0.840436 |\n",
"+----+-------+----------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+\n",
"| 2 | 3 | 0.84638 | 0.846156 | 0.84663 | 0.845956 | 0.84638 | 0.84638 | 0.84638 | 0.840285 |\n",
"+----+-------+----------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+\n",
"| 3 | 4 | 0.847358 | 0.846679 | 0.850044 | 0.846213 | 0.847358 | 0.847358 | 0.847358 | 0.836478 |\n",
"+----+-------+----------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+\n",
"| 4 | 5 | 0.83953 | 0.838276 | 0.845458 | 0.837832 | 0.83953 | 0.83953 | 0.83953 | 0.824034 |\n",
"+----+-------+----------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+\n"
]
}
],
"source": [
"gru_dict = {**params}\n",
"gru_dict[\"k\"] = k\n",
"gru_dict[\"accuracy\"] = np.mean((bagPredictions == BAG_TEST).astype(int))\n",
"gru_dict[\"total_savings\"] = total_savings\n",
"gru_dict[\"y_test\"] = BAG_TEST\n",
"gru_dict[\"y_pred\"] = bagPredictions\n",
"\n",
"# A slightly more detailed analysis method is provided. \n",
"df = emiDriver.analyseModel(predictions, BAG_TEST, NUM_SUBINSTANCE, NUM_OUTPUT)\n",
"print (tabulate(df, headers=list(df.columns), tablefmt='grid'))"
]
},
{
"cell_type": "code",
"execution_count": 16,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Results for this run have been saved at /home/sf/data/SWELL/GRU/ .\n"
]
}
],
"source": [
"dirname = \"/home/sf/data/SWELL/GRU/\"\n",
"pathlib.Path(dirname).mkdir(parents=True, exist_ok=True)\n",
"print (\"Results for this run have been saved at\" , dirname, \".\")\n",
"\n",
"now = datetime.datetime.now()\n",
"filename = list((str(now.year),\"-\",str(now.month),\"-\",str(now.day),\"|\",str(now.hour),\"-\",str(now.minute)))\n",
"filename = ''.join(filename)\n",
"\n",
"#Save the dictionary containing the params and the results.\n",
"pkl.dump(gru_dict,open(dirname + filename + \".pkl\",mode='wb'))"
]
},
{
"cell_type": "code",
"execution_count": 17,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"'/home/sf/data/SWELL/GRU/2019-9-3|15-1.pkl'"
]
},
"execution_count": 17,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"dirname+filename+'.pkl'"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.3"
},
"latex_envs": {
"LaTeX_envs_menu_present": true,
"autoclose": false,
"autocomplete": true,
"bibliofile": "biblio.bib",
"cite_by": "apalike",
"current_citInitial": 1,
"eqLabelWithNumbers": true,
"eqNumInitial": 1,
"hotkeys": {
"equation": "Ctrl-E",
"itemize": "Ctrl-I"
},
"labels_anchors": false,
"latex_user_defs": false,
"report_style_numbering": false,
"user_envs_cfg": false
}
},
"nbformat": 4,
"nbformat_minor": 2
}