--- a
+++ b/Preprocess_Deap.ipynb
@@ -0,0 +1,726 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Valence value regression based on Deap Dataset\n",
+    "\n",
+    "## 0. This notebook is based on DEAP database\n",
+    "\n",
+    "Anyone should refer to DEAP team first\n",
+    "\n",
+    "@article{koelstra2012deap,\n",
+    "  title={Deap: A database for emotion analysis; using physiological signals},\n",
+    "  author={Koelstra, Sander and Muhl, Christian and Soleymani, Mohammad and Lee, Jong-Seok and Yazdani, Ashkan and Ebrahimi, Touradj and Pun, Thierry and Nijholt, Anton and Patras, Ioannis},\n",
+    "  journal={IEEE Transactions on Affective Computing},\n",
+    "  volume={3},\n",
+    "  number={1},\n",
+    "  pages={18--31},\n",
+    "  year={2012},\n",
+    "  publisher={IEEE}\n",
+    "}\n",
+    "\n",
+    "## 1. Dependency\n",
+    "* numpy\n",
+    "* pyEEG\n",
+    "* sciki-learn"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import numpy as np\n",
+    "#import pyeeg as pe\n",
+    "import pickle as pickle\n",
+    "import pandas as pd\n",
+    "import math\n",
+    "\n",
+    "from sklearn import svm\n",
+    "from sklearn.preprocessing import normalize\n",
+    "from sklearn.ensemble import RandomForestRegressor\n",
+    "from sklearn.ensemble import AdaBoostRegressor\n",
+    "\n",
+    "import os\n",
+    "#import tensorflow as tf\n",
+    "import time"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## 2. Global Variables setup\n",
+    "File Name data\\SXX.dat, XX \\in [0,31]\n",
+    "* data: 40 x 40 x 8064: trial x channel x data\n",
+    "* label: 40 x 4: video/trial x label (valence, arousal, dominance, liking)\n",
+    "\n",
+    "Channel Indice: {\n",
+    "* 1 : AF3; 2: F3; 3: F7; 4: FC5; 7: T7; 11: P7; 13: O1\n",
+    "* 17: AF4; 19: F4; 20: F8; 21: FC6; 25: T8; 29: P8; 31: O2 }"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 48,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "channel = [1,2,3,4,6,11,13,17,19,20,21,25,29,31] #14 Channels chosen to fit Emotiv Epoch+\n",
+    "band = [4,8,12,16,25,45] #5 bands\n",
+    "window_size = 256 #Averaging band power of 2 sec\n",
+    "step_size = 16 #Each 0.125 sec update once\n",
+    "sample_rate = 128 #Sampling rate of 128 Hz\n",
+    "subjectList = ['01','02','03','04','05','06','07','08','09','10','11','12','13','14','15','16','17','18','19','20','21','22','23','24','25','26','27','28','29','30','31','32']\n",
+    "#List of subjects"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## 3. FFT with pyeeg\n",
+    "* [4-8]: theta band\n",
+    "* [8-12]: alpha band\n",
+    "* [12-16]: low beta band \n",
+    "* [16-25]: high beta band\n",
+    "* [25-45]: gamma band"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 60,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def FFT_Processing (sub, channel, band, window_size, step_size, sample_rate):\n",
+    "    '''\n",
+    "    arguments:  string subject\n",
+    "                list channel indice\n",
+    "                list band\n",
+    "                int window size for FFT\n",
+    "                int step size for FFT\n",
+    "                int sample rate for FFT\n",
+    "    return:     void\n",
+    "    '''\n",
+    "    meta = []\n",
+    "    with open('data\\s' + sub + '.dat', 'rb') as file:\n",
+    "\n",
+    "        subject = pickle.load(file, encoding='latin1') #resolve the python 2 data problem by encoding : latin1\n",
+    "\n",
+    "        for i in range (0,40):\n",
+    "            # loop over 0-39 trails\n",
+    "            data = subject[\"data\"][i]\n",
+    "            labels = subject[\"labels\"][i]\n",
+    "            start = 0;\n",
+    "\n",
+    "            while start + window_size < data.shape[1]:\n",
+    "                meta_array = []\n",
+    "                meta_data = [] #meta vector for analysis\n",
+    "                for j in channel:\n",
+    "                    X = data[j][start : start + window_size] #Slice raw data over 2 sec, at interval of 0.125 sec\n",
+    "                    Y = pe.bin_power(X, band, sample_rate) #FFT over 2 sec of channel j, in seq of theta, alpha, low beta, high beta, gamma\n",
+    "                    meta_data = meta_data + list(Y[0])\n",
+    "\n",
+    "                meta_array.append(np.array(meta_data))\n",
+    "                meta_array.append(labels)\n",
+    "\n",
+    "                meta.append(np.array(meta_array))    \n",
+    "                start = start + step_size\n",
+    "                \n",
+    "        meta = np.array(meta)\n",
+    "        np.save('out\\s' + sub, meta, allow_pickle=True, fix_imports=True)\n",
+    "\n",
+    "def testing (M, L, model):\n",
+    "    '''\n",
+    "    arguments:  M: testing dataset\n",
+    "                L: testing dataset label\n",
+    "                model: scikit-learn model\n",
+    "\n",
+    "    return:     void\n",
+    "    '''\n",
+    "    output = model.predict(M[0:78080:32])\n",
+    "    label = L[0:78080:32]\n",
+    "\n",
+    "    k = 0\n",
+    "    l = 0\n",
+    "\n",
+    "    for i in range(len(label)):\n",
+    "        k = k + (output[i] - label[i])*(output[i] - label[i]) #square difference \n",
+    "\n",
+    "        #a good guess\n",
+    "        if (output[i] > 5 and label[i] > 5):\n",
+    "            l = l + 1\n",
+    "        elif (output[i] < 5 and label[i] <5):\n",
+    "            l = l + 1\n",
+    "\n",
+    "    print (\"l2 error:\", k/len(label), \"classification accuracy:\", l / len(label),l, len(label))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "for subjects in subjectList:\n",
+    "    FFT_Processing (subjects, channel, band, window_size, step_size, sample_rate)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## 3.Segment of preprocessed data\n",
+    "* training dataset: 75 %\n",
+    "* validation dataset: 12.5%\n",
+    "* testing dataset: 12.5%\n",
+    "\n",
+    "Agrithom pool:\n",
+    "* Support Vector Machine (which kernal?)\n",
+    "* Ada-Boost\n",
+    "\n",
+    "Best practice could be refered to this paper: \n",
+    "\n",
+    "@article{alarcao2017emotions,\n",
+    "  title={Emotions recognition using EEG signals: A survey},\n",
+    "  author={Alarcao, Soraia M and Fonseca, Manuel J},\n",
+    "  journal={IEEE Transactions on Affective Computing},\n",
+    "  year={2017},\n",
+    "  publisher={IEEE}\n",
+    "}"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 54,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "training dataset: (468480, 70) (468480, 4)\n",
+      "testing dataset: (78080, 70) (78080, 4)\n",
+      "validation dataset: (78080, 70) (78080, 4)\n"
+     ]
+    }
+   ],
+   "source": [
+    "#for subjects in subjectList:\n",
+    "data_training = []\n",
+    "label_training = []\n",
+    "data_testing = []\n",
+    "label_testing = []\n",
+    "data_validation = []\n",
+    "label_validation = []\n",
+    "\n",
+    "for subjects in subjectList:\n",
+    "\n",
+    "    with open('out\\s' + subjects + '.npy', 'rb') as file:\n",
+    "        sub = np.load(file)\n",
+    "        for i in range (0,sub.shape[0]):\n",
+    "            if i % 8 == 0:\n",
+    "                data_testing.append(sub[i][0])\n",
+    "                label_testing.append(sub[i][1])\n",
+    "            elif i % 8 == 1:\n",
+    "                data_validation.append(sub[i][0])\n",
+    "                label_validation.append(sub[i][1])\n",
+    "            else:\n",
+    "                data_training.append(sub[i][0])\n",
+    "                label_training.append(sub[i][1])\n",
+    "\n",
+    "np.save('out\\data_training', np.array(data_training), allow_pickle=True, fix_imports=True)\n",
+    "np.save('out\\label_training', np.array(label_training), allow_pickle=True, fix_imports=True)\n",
+    "print(\"training dataset:\", np.array(data_training).shape, np.array(label_training).shape)\n",
+    "\n",
+    "np.save('out\\data_testing', np.array(data_testing), allow_pickle=True, fix_imports=True)\n",
+    "np.save('out\\label_testing', np.array(label_testing), allow_pickle=True, fix_imports=True)\n",
+    "print(\"testing dataset:\", np.array(data_testing).shape, np.array(label_testing).shape)\n",
+    "\n",
+    "np.save('out\\data_validation', np.array(data_validation), allow_pickle=True, fix_imports=True)\n",
+    "np.save('out\\label_validation', np.array(label_validation), allow_pickle=True, fix_imports=True)\n",
+    "print(\"validation dataset:\", np.array(data_validation).shape, np.array(label_validation).shape)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## 4.Regression\n",
+    "### 0. Loading Training and Testing dataset"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 59,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "with open('out\\data_training.npy', 'rb') as fileTrain:\n",
+    "    X  = np.load(fileTrain)\n",
+    "    \n",
+    "with open('out\\label_training.npy', 'rb') as fileTrainL:\n",
+    "    Y  = np.load(fileTrainL)\n",
+    "    \n",
+    "X = normalize(X)\n",
+    "Z = np.ravel(Y[:, [1]])\n",
+    "\n",
+    "Arousal_Train = np.ravel(Y[:, [0]])\n",
+    "Valence_Train = np.ravel(Y[:, [1]])\n",
+    "Domain_Train = np.ravel(Y[:, [2]])\n",
+    "Like_Train = np.ravel(Y[:, [3]])\n",
+    "\n",
+    "\n",
+    "\n",
+    "with open('out\\data_validation.npy', 'rb') as fileTrain:\n",
+    "    M  = np.load(fileTrain)\n",
+    "    \n",
+    "with open('out\\label_validation.npy', 'rb') as fileTrainL:\n",
+    "    N  = np.load(fileTrainL)\n",
+    "\n",
+    "M = normalize(M)\n",
+    "L = np.ravel(N[:, [1]])\n",
+    "\n",
+    "Arousal_Test = np.ravel(N[:, [0]])\n",
+    "Valence_Test = np.ravel(N[:, [1]])\n",
+    "Domain_Test = np.ravel(N[:, [2]])\n",
+    "Like_Test = np.ravel(N[:, [3]])"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "\n",
+    "### 1. Support Vector Regression\n",
+    "* default setting, l1 error: 1.621761042477756 classification error: 0.6057377049180328 1478 2440"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "SVR(C=1.0, cache_size=200, coef0=0.0, degree=3, epsilon=0.1, gamma='auto',\n",
+       "  kernel='rbf', max_iter=-1, shrinking=True, tol=0.001, verbose=False)"
+      ]
+     },
+     "execution_count": 15,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "clf = svm.SVR()\n",
+    "clf.fit(X[0:468480:32], Z[0:468480:32])  "
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### 2. Random Forest Regression\n",
+    "* n_estimators = 10, sample rate = 1/32, l1 error: 1.137919672131145 classification accuracy: 0.7774590163934426 1897 2440\n",
+    "* n_estimators = 100, sample rate = 1/32, l1 error: 1.1029040163934432 classification accuracy: 0.8147540983606557 1988 2440\n",
+    "* n_estimators = 100, min_samples_leaf=10, sample rate = 1/32, l1 error: 1.274458098574928 classification accuracy: 0.7622950819672131 1860 2440\n",
+    "* n_estimators = 100, min_samples_leaf=50, sample rate = 1/32, l1 error: 1.4575897309409926 classification accuracy: 0.6823770491803278 1665 2440\n",
+    "\n",
+    "* n_estimators = 250, sample rate = 1/32, l1 error: 1.0905590819672137 classification accuracy: 0.830327868852459 2026 2440\n",
+    "* n_estimators = 750, sample rate = 1/32, l1 error: 1.0953162021857932 classification accuracy: 0.8340163934426229 2035 2440\n",
+    "* n_estimators = 750, sample rate = 1/8, l1 error: l1 error: 1.066982950819674 classification accuracy: 0.8217213114754098 2005 2440\n",
+    "* __n_estimators = 512, sample rate = 1/32, l1 error: 1.092375304175206 classification accuracy: 0.8364754098360656 2041 2440\n",
+    "__\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 62,
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "l2 error: 1.876775658972537 classification accuracy: 0.8290983606557377 2023 2440\n"
+     ]
+    }
+   ],
+   "source": [
+    "Val_R = RandomForestRegressor(n_estimators=512, n_jobs=6)\n",
+    "Val_R.fit(X[0:468480:32], Valence_Train[0:468480:32])\n",
+    "testing (M, Valence_Test, Val_R)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 63,
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "l2 error: 2.0764509040715233 classification accuracy: 0.8266393442622951 2017 2440\n"
+     ]
+    }
+   ],
+   "source": [
+    "Aro_R = RandomForestRegressor(n_estimators=512, n_jobs=6)\n",
+    "Aro_R.fit(X[0:468480:32], Arousal_Train[0:468480:32])\n",
+    "testing (M, Arousal_Test, Aro_R)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 64,
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "l2 error: 1.813647083229937 classification accuracy: 0.8184426229508197 1997 2440\n"
+     ]
+    }
+   ],
+   "source": [
+    "Dom_R = RandomForestRegressor(n_estimators=512, n_jobs=6)\n",
+    "Dom_R.fit(X[0:468480:32], Domain_Train[0:468480:32])\n",
+    "testing (M, Domain_Test, Dom_R)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 65,
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "l2 error: 2.489005384276336 classification accuracy: 0.8512295081967213 2077 2440\n"
+     ]
+    }
+   ],
+   "source": [
+    "Lik_R = RandomForestRegressor(n_estimators=512, n_jobs=6)\n",
+    "Lik_R.fit(X[0:468480:32], Like_Train[0:468480:32])\n",
+    "testing (M, Like_Test, Lik_R)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### 3. AdaBoost Regression\n",
+    "* n = 50, lr = 1.0: l2 error: 3.8454054839726695 classification accuracy: 0.6147540983606558 1500 2440\n",
+    "* n = 50, lr = 1.0, square: l2 error: 4.015289218608164 classification accuracy: 0.5913934426229508 1443 2440\n",
+    "* n = 500, lr = 1.0: l2 error: 3.8861651269012594 classification accuracy: 0.6155737704918033 1502 2440\n",
+    "*\n",
+    "*"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 32,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "AdaBoostRegressor(base_estimator=None, learning_rate=0.01, loss='linear',\n",
+       "         n_estimators=5000, random_state=None)"
+      ]
+     },
+     "execution_count": 32,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "clf = AdaBoostRegressor(n_estimators=5000, learning_rate=0.01)\n",
+    "clf.fit(X[0:468480:32], Z[0:468480:32])"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Calculating accuracy and loss"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 58,
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "l2 error: 1.8832017200301692 classification accuracy: 0.8348360655737705 2037 2440\n"
+     ]
+    }
+   ],
+   "source": [
+    "output = Val_R.predict(M[0:78080:32])\n",
+    "label = L[0:78080:32]\n",
+    "\n",
+    "k = 0\n",
+    "l = 0\n",
+    "\n",
+    "for i in range(len(label)):\n",
+    "    k = k + (output[i] - label[i])*(output[i] - label[i]) #square difference \n",
+    "    \n",
+    "    #a good guess\n",
+    "    if (output[i] > 5 and label[i] > 5):\n",
+    "        l = l + 1\n",
+    "    elif (output[i] < 5 and label[i] <5):\n",
+    "        l = l + 1\n",
+    "\n",
+    "print (\"l2 error:\", k/len(label), \"classification accuracy:\", l / len(label),l, len(label))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### 4. ANN\n",
+    "* 500 epoch 0.005 128 - 256 - 256 - 128 loss = 3.1\n",
+    "* 3000 epoch 0.0001 256-512-512-256 Epoch: 3196 - Training Cost: 1.8372873067855835  Testing Cost: 2.231332540512085\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Pull out columns for X (data to train with) and Y (value to predict)\n",
+    "X_training = X[0:468480:32]\n",
+    "Y_training = Z[0:468480:32]\n",
+    "\n",
+    "# Pull out columns for X (data to train with) and Y (value to predict)\n",
+    "X_testing = M[0:78080:32]\n",
+    "Y_testing = L[0:78080:32]\n",
+    "\n",
+    "# DO Scale both the training inputs and outputs\n",
+    "X_scaled_training = pd.DataFrame (data = X_training).values\n",
+    "Y_scaled_training = pd.DataFrame (data = Y_training).values\n",
+    "\n",
+    "# It's very important that the training and test data are scaled with the same scaler.\n",
+    "X_scaled_testing = pd.DataFrame (data = X_testing).values\n",
+    "Y_scaled_testing = pd.DataFrame (data = Y_testing).values"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Turn off TensorFlow warning messages in program output\n",
+    "os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'\n",
+    "\n",
+    "# Define model parameters\n",
+    "t = time.time()\n",
+    "learning_rate = 0.0001\n",
+    "training_epochs = 5000\n",
+    "display_step = 1\n",
+    "\n",
+    "# Define how many inputs and outputs are in our neural network\n",
+    "number_of_inputs = 70\n",
+    "number_of_outputs = 1\n",
+    "\n",
+    "# Define how many neurons we want in each layer of our neural network\n",
+    "layer_1_nodes = 512\n",
+    "layer_2_nodes = 1024\n",
+    "layer_3_nodes = 1024\n",
+    "layer_4_nodes = 512\n",
+    "\n",
+    "# Section One: Define the layers of the neural network itself\n",
+    "RUN_NAME = str(int(round(t * 1000))) + '_' + str(layer_1_nodes) + '_' + str(layer_2_nodes) + '_' + str(layer_3_nodes) + '_' + str(layer_4_nodes) + '_' + str(learning_rate) + '_' + str(training_epochs) + '_' + 'Val'\n",
+    "\n",
+    "\n",
+    "# Input Layer\n",
+    "with tf.variable_scope('input'):\n",
+    "    X = tf.placeholder(tf.float32, shape=(None, number_of_inputs))\n",
+    "\n",
+    "# Layer 1\n",
+    "with tf.variable_scope('layer_1'):\n",
+    "    weights = tf.get_variable(\"weights1\", shape=[number_of_inputs, layer_1_nodes], initializer=tf.contrib.layers.xavier_initializer())\n",
+    "    biases = tf.get_variable(name=\"biases1\", shape=[layer_1_nodes], initializer=tf.zeros_initializer())\n",
+    "    layer_1_output = tf.nn.relu(tf.matmul(X, weights) + biases)\n",
+    "\n",
+    "# Layer 2\n",
+    "with tf.variable_scope('layer_2'):\n",
+    "    weights = tf.get_variable(\"weights2\", shape=[layer_1_nodes, layer_2_nodes], initializer=tf.contrib.layers.xavier_initializer())\n",
+    "    biases = tf.get_variable(name=\"biases2\", shape=[layer_2_nodes], initializer=tf.zeros_initializer())\n",
+    "    layer_2_output = tf.nn.relu(tf.matmul(layer_1_output, weights) + biases)\n",
+    "\n",
+    "# Layer 3\n",
+    "with tf.variable_scope('layer_3'):\n",
+    "    weights = tf.get_variable(\"weights3\", shape=[layer_2_nodes, layer_3_nodes], initializer=tf.contrib.layers.xavier_initializer())\n",
+    "    biases = tf.get_variable(name=\"biases3\", shape=[layer_3_nodes], initializer=tf.zeros_initializer())\n",
+    "    layer_3_output = tf.nn.relu(tf.matmul(layer_2_output, weights) + biases)\n",
+    "\n",
+    "# Layer 4\n",
+    "with tf.variable_scope('layer_4'):\n",
+    "    weights = tf.get_variable(\"weights4\", shape=[layer_3_nodes, layer_4_nodes], initializer=tf.contrib.layers.xavier_initializer())\n",
+    "    biases = tf.get_variable(name=\"biases4\", shape=[layer_4_nodes], initializer=tf.zeros_initializer())\n",
+    "    layer_4_output = tf.nn.relu(tf.matmul(layer_3_output, weights) + biases)\n",
+    "\n",
+    "# Output Layer\n",
+    "with tf.variable_scope('output'):\n",
+    "    weights = tf.get_variable(\"weights5\", shape=[layer_4_nodes, number_of_outputs], initializer=tf.contrib.layers.xavier_initializer())\n",
+    "    biases = tf.get_variable(name=\"biases5\", shape=[number_of_outputs], initializer=tf.zeros_initializer())\n",
+    "    prediction = tf.matmul(layer_4_output, weights) + biases\n",
+    "\n",
+    "# Section Two: Define the cost function of the neural network that will be optimized during training\n",
+    "\n",
+    "with tf.variable_scope('cost'):\n",
+    "    Y = tf.placeholder(tf.float32, shape=(None, 1))\n",
+    "    cost = tf.reduce_mean(tf.squared_difference(prediction, Y))\n",
+    "\n",
+    "# Section Three: Define the optimizer function that will be run to optimize the neural network\n",
+    "\n",
+    "with tf.variable_scope('train'):\n",
+    "    optimizer = tf.train.AdamOptimizer(learning_rate).minimize(cost)\n",
+    "\n",
+    "# Create a summary operation to log the progress of the network\n",
+    "with tf.variable_scope('logging'):\n",
+    "    tf.summary.scalar('current_cost', cost)\n",
+    "    summary = tf.summary.merge_all()\n",
+    "\n",
+    "saver = tf.train.Saver()\n",
+    "\n",
+    "# Initialize a session so that we can run TensorFlow operations\n",
+    "with tf.Session() as session:\n",
+    "\n",
+    "    # Run the global variable initializer to initialize all variables and layers of the neural network\n",
+    "    session.run(tf.global_variables_initializer())\n",
+    "\n",
+    "    # Create log file writers to record training progress.\n",
+    "    # We'll store training and testing log data separately.\n",
+    "    training_writer = tf.summary.FileWriter(\"./{}/logs/training\".format(RUN_NAME), session.graph)\n",
+    "    testing_writer = tf.summary.FileWriter(\"./{}/logs/testing\".format(RUN_NAME), session.graph)\n",
+    "\n",
+    "    # Run the optimizer over and over to train the network.\n",
+    "    # One epoch is one full run through the training data set.\n",
+    "    for epoch in range(training_epochs):\n",
+    "\n",
+    "        # Feed in the training data and do one step of neural network training\n",
+    "        session.run(optimizer, feed_dict={X: X_scaled_training, Y: Y_scaled_training})\n",
+    "\n",
+    "        # Every few training steps, log our progress\n",
+    "        if epoch % display_step == 0:\n",
+    "            # Get the current accuracy scores by running the \"cost\" operation on the training and test data sets\n",
+    "            training_cost, training_summary = session.run([cost, summary], feed_dict={X: X_scaled_training, Y:Y_scaled_training})\n",
+    "            testing_cost, testing_summary = session.run([cost, summary], feed_dict={X: X_scaled_testing, Y:Y_scaled_testing})\n",
+    "\n",
+    "            # Write the current training status to the log files (Which we can view with TensorBoard)\n",
+    "            training_writer.add_summary(training_summary, epoch)\n",
+    "            testing_writer.add_summary(testing_summary, epoch)\n",
+    "\n",
+    "            # Print the current training status to the screen\n",
+    "            print(\"Epoch: {} - Training Cost: {}  Testing Cost: {}\".format(epoch, training_cost, testing_cost))\n",
+    "\n",
+    "    # Training is now complete!\n",
+    "\n",
+    "    # Get the final accuracy scores by running the \"cost\" operation on the training and test data sets\n",
+    "    final_training_cost = session.run(cost, feed_dict={X: X_scaled_training, Y: Y_scaled_training})\n",
+    "    final_testing_cost = session.run(cost, feed_dict={X: X_scaled_testing, Y: Y_scaled_testing})\n",
+    "\n",
+    "    print(\"Final Training cost: {}\".format(final_training_cost))\n",
+    "    print(\"Final Testing cost: {}\".format(final_testing_cost))\n",
+    "\n",
+    "    save_path = saver.save(session, \"./{}/logs/trained_model.ckpt\".format(RUN_NAME))\n",
+    "    print(\"Model saved: {}\".format(save_path))\n",
+    "\n",
+    "    '''\n",
+    "    # Now that the neural network is trained, let's use it to make predictions for our test data.\n",
+    "    # Pass in the X testing data and run the \"prediciton\" operation\n",
+    "    Y_predicted_scaled = session.run(prediction, feed_dict={X: X_scaled_testing})\n",
+    "    # Unscale the data back to it's original units (dollars)\n",
+    "    Y_predicted = Y_scaler.inverse_transform(Y_predicted_scaled)\n",
+    "    real_earnings = test_data_df['total_earnings'].values[0]\n",
+    "    predicted_earnings = Y_predicted[0][0]\n",
+    "    print(\"The actual earnings of Game #1 were ${}\".format(real_earnings))\n",
+    "    print(\"Our neural network predicted earnings of ${}\".format(predicted_earnings))\n",
+    "    \n",
+    "'''\n",
+    "    model_builder = tf.saved_model.builder.SavedModelBuilder(\"./{}/exported_model\".format(RUN_NAME))\n",
+    "\n",
+    "    inputs = {\n",
+    "        'input': tf.saved_model.utils.build_tensor_info(X)\n",
+    "        }\n",
+    "    outputs = {\n",
+    "        'earnings': tf.saved_model.utils.build_tensor_info(prediction)\n",
+    "        }\n",
+    "\n",
+    "    signature_def = tf.saved_model.signature_def_utils.build_signature_def(\n",
+    "        inputs=inputs,\n",
+    "        outputs=outputs,\n",
+    "        method_name=tf.saved_model.signature_constants.PREDICT_METHOD_NAME\n",
+    "    )\n",
+    "\n",
+    "    model_builder.add_meta_graph_and_variables(\n",
+    "        session,\n",
+    "        tags=[tf.saved_model.tag_constants.SERVING],\n",
+    "        signature_def_map={\n",
+    "            tf.saved_model.signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY: signature_def\n",
+    "        }\n",
+    "    )\n",
+    "\n",
+    "    model_builder.save()\n",
+    "    print('model saved')\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.6.5"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}