{

 "cells": [

  {

   "cell_type": "code",

   "execution_count": 1,

   "metadata": {},

   "outputs": [],

   "source": [

    "import numpy as np \n",

    "import scipy.io as sio\n",

    "import matplotlib.pyplot as plt \n",

    "import seaborn as sn\n",

    "import pandas as pd\n",

    "\n",

    "import torch\n",

    "import os \n",

    "\n",

    "import torch.optim as optim\n",

    "import torch.nn as nn\n",

    "import torch.nn.functional as F\n",

    "\n",

    "from torch.autograd import Variable\n",

    "from torch.utils.data.dataset import Dataset\n",

    "from torch.utils.data import DataLoader,random_split\n",

    "\n",

    "from sklearn.model_selection import train_test_split\n",

    "\n",

    "torch.manual_seed(1234)\n",

    "np.random.seed(1234)"

   ]

  },

  {

   "cell_type": "markdown",

   "metadata": {},

   "source": [

    "## Instancing on the GPU"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": 2,

   "metadata": {},

   "outputs": [],

   "source": [

    "device = torch.device(\"cuda\")"

   ]

  },

  {

   "cell_type": "markdown",

   "metadata": {},

   "source": [

    "## Loading the images"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": 5,

   "metadata": {},

   "outputs": [

    {

     "name": "stdout",

     "output_type": "stream",

     "text": [

      "(2670, 3, 32, 32)\n",

      "(2670, 7, 3, 32, 32)\n",

      "(2670,)\n",

      "(2670,)\n"

     ]

    }

   ],

   "source": [

    "Mean_Images = sio.loadmat(\"images.mat\")[\"img\"] #corresponding to the images mean for all the seven windows\n",

    "print(np.shape(Mean_Images)) \n",

    "\n",

    "Images = sio.loadmat(\"images_time.mat\")[\"img\"] #corresponding to the images mean for all the seven windows\n",

    "print(np.shape(Images)) \n",

    "\n",

    "\n",

    "Label = (sio.loadmat(\"FeatureMat_timeWin\")[\"features\"][:,-1]-1).astype(int)\n",

    "print(np.shape(Label)) \n",

    "\n",

    "Patient_id = sio.loadmat(\"trials_subNums.mat\")['subjectNum'][0]\n",

    "print(np.shape(Patient_id))"

   ]

  },

  {

   "cell_type": "markdown",

   "metadata": {},

   "source": [

    "### Dataloader"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": 6,

   "metadata": {},

   "outputs": [],

   "source": [

    "class EEGImagesDataset(Dataset):\n",

    "    \"\"\"EEGLearn Images Dataset from EEG.\"\"\"\n",

    "    \n",

    "    def __init__(self, label, image):\n",

    "        self.label = Label\n",

    "        self.Images = image\n",

    "        \n",

    "    def __len__(self):\n",

    "        return len(self.label)\n",

    "    \n",

    "    def __getitem__(self, idx):\n",

    "        if torch.is_tensor(idx):\n",

    "            idx = idx.tolist()\n",

    "        image = self.Images[idx]\n",

    "        label = self.label[idx]\n",

    "        sample = (image, label)\n",

    "        \n",

    "        return sample"

   ]

  },

  {

   "cell_type": "markdown",

   "metadata": {},

   "source": [

    "### K-Fold Validation"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": 7,

   "metadata": {},

   "outputs": [],

   "source": [

    "def kfold(length, n_fold):\n",

    "    tot_id = np.arange(length)\n",

    "    np.random.shuffle(tot_id)\n",

    "    len_fold = int(length/n_fold)\n",

    "    train_id = []\n",

    "    test_id = []\n",

    "    for i in range(n_fold):\n",

    "        test_id.append(tot_id[i*len_fold:(i+1)*len_fold])\n",

    "        train_id.append(np.hstack([tot_id[0:i*len_fold],tot_id[(i+1)*len_fold:-1]]))\n",

    "    return train_id, test_id"

   ]

  },

  {

   "cell_type": "markdown",

   "metadata": {},

   "source": [

    "# Mean Image"

   ]

  },

  {

   "cell_type": "markdown",

   "metadata": {},

   "source": [

    "## Basic Model"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": 8,

   "metadata": {},

   "outputs": [],

   "source": [

    "class BasicCNN(nn.Module):\n",

    "    '''\n",

    "    Build the  Mean Basic model performing a classification with CNN \n",

    "\n",

    "    param input_image: list of EEG image [batch_size, n_window, n_channel, h, w]\n",

    "    param kernel: kernel size used for the convolutional layers\n",

    "    param stride: stride apply during the convolutions\n",

    "    param padding: padding used during the convolutions\n",

    "    param max_kernel: kernel used for the maxpooling steps\n",

    "    param n_classes: number of classes\n",

    "    return x: output of the last layers after the log softmax\n",

    "    '''\n",

    "    def __init__(self, input_image=torch.zeros(1, 3, 32, 32), kernel=(3,3), stride=1, padding=1,max_kernel=(2,2), n_classes=4):\n",

    "        super(BasicCNN, self).__init__()\n",

    "\n",

    "        n_window = input_image.shape[1]\n",

    "        n_channel = input_image.shape[2]\n",

    "\n",

    "        self.conv1 = nn.Conv2d(3,32,kernel,stride=stride, padding=padding)\n",

    "        self.conv2 = nn.Conv2d(32,32,kernel,stride=stride, padding=padding)\n",

    "        self.conv3 = nn.Conv2d(32,32,kernel,stride=stride, padding=padding)\n",

    "        self.conv4 = nn.Conv2d(32,32,kernel,stride=stride, padding=padding)\n",

    "        self.pool1 = nn.MaxPool2d(max_kernel)\n",

    "        self.conv5 = nn.Conv2d(32,64,kernel,stride=stride,padding=padding)\n",

    "        self.conv6 = nn.Conv2d(64,64,kernel,stride=stride,padding=padding)\n",

    "        self.conv7 = nn.Conv2d(64,128,kernel,stride=stride,padding=padding)\n",

    "\n",

    "        self.pool = nn.MaxPool2d((1,1))\n",

    "        self.drop = nn.Dropout(p=0.5)\n",

    "\n",

    "        self.fc1 = nn.Linear(2048,512)\n",

    "        self.fc2 = nn.Linear(512,n_classes)\n",

    "        self.max = nn.LogSoftmax()\n",

    "    \n",

    "    def forward(self, x):\n",

    "        batch_size = x.shape[0]\n",

    "        x = F.relu(self.conv1(x))\n",

    "        x = F.relu(self.conv2(x))\n",

    "        x = F.relu(self.conv3(x))\n",

    "        x = F.relu(self.conv4(x))\n",

    "        x = self.pool1(x)\n",

    "        x = F.relu(self.conv5(x))\n",

    "        x = F.relu(self.conv6(x))\n",

    "        x = self.pool1(x)\n",

    "        x = F.relu(self.conv7(x))\n",

    "        x = self.pool1(x)\n",

    "        x = x.reshape(x.shape[0],x.shape[1], -1)\n",

    "        x = self.pool(x)\n",

    "        x = x.reshape(x.shape[0],-1)\n",

    "        x = self.fc1(x)\n",

    "        x = self.fc2(x)\n",

    "        x = self.max(x)\n",

    "        return x"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": 10,

   "metadata": {},

   "outputs": [],

   "source": [

    "EEG = EEGImagesDataset(label=Label, image=Mean_Images)\n",

    "\n",

    "lengths = [int(2670*0.8), int(2670*0.2)]\n",

    "Train, Test = random_split(EEG, lengths)\n",

    "\n",

    "Trainloader = DataLoader(Train,batch_size=32)\n",

    "Testloader = DataLoader(Test, batch_size=32)"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": 140,

   "metadata": {},

   "outputs": [

    {

     "name": "stderr",

     "output_type": "stream",

     "text": [

      "/home/vdelv/anaconda3/envs/Pytorch_EEG/lib/python3.7/site-packages/ipykernel_launcher.py:52: UserWarning: Implicit dimension choice for log_softmax has been deprecated. Change the call to include dim=X as an argument.\n"

     ]

    },

    {

     "name": "stdout",

     "output_type": "stream",

     "text": [

      "Finished Training\n"

     ]

    }

   ],

   "source": [

    "net = BasicCNN().cuda()\n",

    "criterion = nn.CrossEntropyLoss()\n",

    "optimizer = optim.SGD(net.parameters(), lr=0.001, momentum=0.9)\n",

    "\n",

    "for epoch in range(30):  # loop over the dataset multiple times\n",

    "    running_loss = 0.0\n",

    "    evaluation = []\n",

    "    for i, data in enumerate(Trainloader, 0):\n",

    "        # get the inputs; data is a list of [inputs, labels]\n",

    "        inputs, labels = data\n",

    "        # zero the parameter gradients\n",

    "        optimizer.zero_grad()\n",

    "\n",

    "        # forward + backward + optimize\n",

    "        outputs = net(inputs.to(torch.float32).cuda())\n",

    "        _, predicted = torch.max(outputs.cpu().data, 1)\n",

    "        evaluation.append((predicted==labels).tolist())\n",

    "        loss = criterion(outputs, labels.cuda())\n",

    "        loss.backward()\n",

    "        optimizer.step()\n",

    "        \n",

    "        running_loss += loss.item()\n",

    "    running_loss = running_loss/(i+1)\n",

    "    evaluation = [item for sublist in evaluation for item in sublist]\n",

    "    running_acc = sum(evaluation)/len(evaluation)\n",

    "    validation_loss, validation_acc = Test_Model(net, Testloader,True)\n",

    "\n",

    "print('Finished Training')"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": 139,

   "metadata": {},

   "outputs": [

    {

     "name": "stderr",

     "output_type": "stream",

     "text": [

      "/home/vdelv/anaconda3/envs/Pytorch_EEG/lib/python3.7/site-packages/ipykernel_launcher.py:52: UserWarning: Implicit dimension choice for log_softmax has been deprecated. Change the call to include dim=X as an argument.\n"

     ]

    },

    {

     "name": "stdout",

     "output_type": "stream",

     "text": [

      "[5,  10]\tloss: 1.384\tAccuracy : 0.277\t\tval-loss: 1.383\tval-Accuracy : 0.311\n",

      "[10,  10]\tloss: 1.383\tAccuracy : 0.277\t\tval-loss: 1.382\tval-Accuracy : 0.311\n",

      "Finished Training \n",

      " loss: 1.383\tAccuracy : 0.277\t\tval-loss: 1.382\tval-Accuracy : 0.311\n"

     ]

    }

   ],

   "source": [

    "res = TrainTest_Model(BasicCNN, Trainloader, Testloader, n_epoch=10)"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": 111,

   "metadata": {},

   "outputs": [],

   "source": [

    "def Test_Model(net, Testloader, is_cuda=True):\n",

    "    running_loss = 0.0 \n",

    "    evaluation = []\n",

    "    for i, data in enumerate(Testloader, 0):\n",

    "        input_img, labels = data\n",

    "        optimizer.zero_grad()\n",

    "        input_img = input_img.to(torch.float32)\n",

    "        if is_cuda:\n",

    "            input_img = input_img.cuda()\n",

    "        outputs = net(input_img)\n",

    "        _, predicted = torch.max(outputs.cpu().data, 1)\n",

    "        evaluation.append((predicted==labels).tolist())\n",

    "        loss = criterion(outputs, labels.cuda())\n",

    "        running_loss += loss.item()\n",

    "    running_loss = running_loss/(i+1)\n",

    "    evaluation = [item for sublist in evaluation for item in sublist]\n",

    "    running_acc = sum(evaluation)/len(evaluation)\n",

    "    return running_loss, running_acc"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": 138,

   "metadata": {},

   "outputs": [],

   "source": [

    "def TrainTest_Model(model, trainloader, testloader, n_epoch=30, opti='SGD', learning_rate=0.0001, is_cuda=True, print_epoch =5):\n",

    "    if is_cuda:\n",

    "        net = model().cuda()\n",

    "    else :\n",

    "        net = model()\n",

    "        \n",

    "    criterion = nn.CrossEntropyLoss()\n",

    "    \n",

    "    if opti=='SGD':\n",

    "        optimizer = optim.SGD(net.parameters(), lr=learning_rate, momentum=0.9)\n",

    "    elif opti =='Adam':\n",

    "        optimizer = optim.Adam(CNN.parameters(), lr=learning_rate)\n",

    "    else: \n",

    "        print(\"Optimizer: \"+optim+\" not implemented.\")\n",

    "    \n",

    "    for epoch in range(n_epoch):\n",

    "        running_loss = 0.0\n",

    "        evaluation = []\n",

    "        for i, data in enumerate(Trainloader, 0):\n",

    "            # get the inputs; data is a list of [inputs, labels]\n",

    "            inputs, labels = data\n",

    "            # zero the parameter gradients\n",

    "            optimizer.zero_grad()\n",

    "\n",

    "            # forward + backward + optimize\n",

    "            outputs = net(inputs.to(torch.float32).cuda())\n",

    "            _, predicted = torch.max(outputs.cpu().data, 1)\n",

    "            evaluation.append((predicted==labels).tolist())\n",

    "            loss = criterion(outputs, labels.cuda())\n",

    "            loss.backward()\n",

    "            optimizer.step()\n",

    "\n",

    "            running_loss += loss.item()\n",

    "\n",

    "        running_loss = running_loss/(i+1)\n",

    "        evaluation = [item for sublist in evaluation for item in sublist]\n",

    "        running_acc = sum(evaluation)/len(evaluation)\n",

    "        validation_loss, validation_acc = Test_Model(net, Testloader,True)\n",

    "        \n",

    "        if epoch%print_epoch==(print_epoch-1):\n",

    "            print('[%d, %3d]\\tloss: %.3f\\tAccuracy : %.3f\\t\\tval-loss: %.3f\\tval-Accuracy : %.3f' %\n",

    "             (epoch+1, n_epoch, running_loss, running_acc, validation_loss, validation_acc))\n",

    "    \n",

    "    print('Finished Training \\n loss: %.3f\\tAccuracy : %.3f\\t\\tval-loss: %.3f\\tval-Accuracy : %.3f' %\n",

    "                 (running_loss, running_acc, validation_loss,validation_acc))\n",

    "    \n",

    "    return (running_loss, running_acc, validation_loss,validation_acc)"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": 477,

   "metadata": {

    "scrolled": true

   },

   "outputs": [

    {

     "name": "stdout",

     "output_type": "stream",

     "text": [

      "Begin Training Fold 1/5\t of Patient 1\n"

     ]

    },

    {

     "name": "stderr",

     "output_type": "stream",

     "text": [

      "c:\\users\\victo\\.conda\\envs\\pytorch_eeg\\lib\\site-packages\\ipykernel_launcher.py:19: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n"

     ]

    },

    {

     "name": "stdout",

     "output_type": "stream",

     "text": [

      "Finish Training Fold 1/5\t of Patient 1\n",

      "Begin Training Fold 2/5\t of Patient 1\n",

      "Finish Training Fold 2/5\t of Patient 1\n",

      "Begin Training Fold 3/5\t of Patient 1\n",

      "Finish Training Fold 3/5\t of Patient 1\n",

      "Begin Training Fold 4/5\t of Patient 1\n",

      "Finish Training Fold 4/5\t of Patient 1\n",

      "Begin Training Fold 5/5\t of Patient 1\n",

      "Finish Training Fold 5/5\t of Patient 1\n",

      "loss: 0.854\tAccuracy : 0.855\t\tval-loss: 0.956\tval-Accuracy : 0.805\n",

      "Begin Training Fold 1/5\t of Patient 2\n",

      "Finish Training Fold 1/5\t of Patient 2\n",

      "Begin Training Fold 2/5\t of Patient 2\n",

      "Finish Training Fold 2/5\t of Patient 2\n",

      "Begin Training Fold 3/5\t of Patient 2\n",

      "Finish Training Fold 3/5\t of Patient 2\n",

      "Begin Training Fold 4/5\t of Patient 2\n",

      "Finish Training Fold 4/5\t of Patient 2\n",

      "Begin Training Fold 5/5\t of Patient 2\n",

      "Finish Training Fold 5/5\t of Patient 2\n",

      "loss: 0.824\tAccuracy : 0.872\t\tval-loss: 0.897\tval-Accuracy : 0.852\n",

      "Begin Training Fold 1/5\t of Patient 3\n",

      "Finish Training Fold 1/5\t of Patient 3\n",

      "Begin Training Fold 2/5\t of Patient 3\n",

      "Finish Training Fold 2/5\t of Patient 3\n",

      "Begin Training Fold 3/5\t of Patient 3\n",

      "Finish Training Fold 3/5\t of Patient 3\n",

      "Begin Training Fold 4/5\t of Patient 3\n",

      "Finish Training Fold 4/5\t of Patient 3\n",

      "Begin Training Fold 5/5\t of Patient 3\n",

      "Finish Training Fold 5/5\t of Patient 3\n",

      "loss: 0.778\tAccuracy : 0.883\t\tval-loss: 0.900\tval-Accuracy : 0.841\n",

      "Begin Training Fold 1/5\t of Patient 4\n",

      "Finish Training Fold 1/5\t of Patient 4\n",

      "Begin Training Fold 2/5\t of Patient 4\n",

      "Finish Training Fold 2/5\t of Patient 4\n",

      "Begin Training Fold 3/5\t of Patient 4\n",

      "Finish Training Fold 3/5\t of Patient 4\n",

      "Begin Training Fold 4/5\t of Patient 4\n",

      "Finish Training Fold 4/5\t of Patient 4\n",

      "Begin Training Fold 5/5\t of Patient 4\n",

      "Finish Training Fold 5/5\t of Patient 4\n",

      "loss: 0.770\tAccuracy : 0.960\t\tval-loss: 0.797\tval-Accuracy : 0.950\n",

      "Begin Training Fold 1/5\t of Patient 6\n",

      "Finish Training Fold 1/5\t of Patient 6\n",

      "Begin Training Fold 2/5\t of Patient 6\n",

      "Finish Training Fold 2/5\t of Patient 6\n",

      "Begin Training Fold 3/5\t of Patient 6\n",

      "Finish Training Fold 3/5\t of Patient 6\n",

      "Begin Training Fold 4/5\t of Patient 6\n",

      "Finish Training Fold 4/5\t of Patient 6\n",

      "Begin Training Fold 5/5\t of Patient 6\n",

      "Finish Training Fold 5/5\t of Patient 6\n",

      "loss: 0.814\tAccuracy : 0.888\t\tval-loss: 0.872\tval-Accuracy : 0.872\n",

      "Begin Training Fold 1/5\t of Patient 7\n",

      "Finish Training Fold 1/5\t of Patient 7\n",

      "Begin Training Fold 2/5\t of Patient 7\n",

      "Finish Training Fold 2/5\t of Patient 7\n",

      "Begin Training Fold 3/5\t of Patient 7\n",

      "Finish Training Fold 3/5\t of Patient 7\n",

      "Begin Training Fold 4/5\t of Patient 7\n",

      "Finish Training Fold 4/5\t of Patient 7\n",

      "Begin Training Fold 5/5\t of Patient 7\n",

      "Finish Training Fold 5/5\t of Patient 7\n",

      "loss: 0.785\tAccuracy : 0.893\t\tval-loss: 0.862\tval-Accuracy : 0.885\n",

      "Begin Training Fold 1/5\t of Patient 8\n",

      "Finish Training Fold 1/5\t of Patient 8\n",

      "Begin Training Fold 2/5\t of Patient 8\n",

      "Finish Training Fold 2/5\t of Patient 8\n",

      "Begin Training Fold 3/5\t of Patient 8\n",

      "Finish Training Fold 3/5\t of Patient 8\n",

      "Begin Training Fold 4/5\t of Patient 8\n",

      "Finish Training Fold 4/5\t of Patient 8\n",

      "Begin Training Fold 5/5\t of Patient 8\n",

      "Finish Training Fold 5/5\t of Patient 8\n",

      "loss: 0.780\tAccuracy : 0.951\t\tval-loss: 0.795\tval-Accuracy : 0.953\n",

      "Begin Training Fold 1/5\t of Patient 9\n",

      "Finish Training Fold 1/5\t of Patient 9\n",

      "Begin Training Fold 2/5\t of Patient 9\n",

      "Finish Training Fold 2/5\t of Patient 9\n",

      "Begin Training Fold 3/5\t of Patient 9\n",

      "Finish Training Fold 3/5\t of Patient 9\n",

      "Begin Training Fold 4/5\t of Patient 9\n",

      "Finish Training Fold 4/5\t of Patient 9\n",

      "Begin Training Fold 5/5\t of Patient 9\n",

      "Finish Training Fold 5/5\t of Patient 9\n",

      "loss: 0.769\tAccuracy : 0.965\t\tval-loss: 0.808\tval-Accuracy : 0.930\n",

      "Begin Training Fold 1/5\t of Patient 10\n",

      "Finish Training Fold 1/5\t of Patient 10\n",

      "Begin Training Fold 2/5\t of Patient 10\n",

      "Finish Training Fold 2/5\t of Patient 10\n",

      "Begin Training Fold 3/5\t of Patient 10\n",

      "Finish Training Fold 3/5\t of Patient 10\n",

      "Begin Training Fold 4/5\t of Patient 10\n",

      "Finish Training Fold 4/5\t of Patient 10\n",

      "Begin Training Fold 5/5\t of Patient 10\n",

      "Finish Training Fold 5/5\t of Patient 10\n",

      "loss: 0.774\tAccuracy : 0.939\t\tval-loss: 0.803\tval-Accuracy : 0.943\n",

      "Begin Training Fold 1/5\t of Patient 11\n",

      "Finish Training Fold 1/5\t of Patient 11\n",

      "Begin Training Fold 2/5\t of Patient 11\n",

      "Finish Training Fold 2/5\t of Patient 11\n",

      "Begin Training Fold 3/5\t of Patient 11\n",

      "Finish Training Fold 3/5\t of Patient 11\n",

      "Begin Training Fold 4/5\t of Patient 11\n",

      "Finish Training Fold 4/5\t of Patient 11\n",

      "Begin Training Fold 5/5\t of Patient 11\n",

      "Finish Training Fold 5/5\t of Patient 11\n",

      "loss: 0.768\tAccuracy : 0.944\t\tval-loss: 0.832\tval-Accuracy : 0.920\n",

      "Begin Training Fold 1/5\t of Patient 12\n",

      "Finish Training Fold 1/5\t of Patient 12\n",

      "Begin Training Fold 2/5\t of Patient 12\n",

      "Finish Training Fold 2/5\t of Patient 12\n",

      "Begin Training Fold 3/5\t of Patient 12\n",

      "Finish Training Fold 3/5\t of Patient 12\n",

      "Begin Training Fold 4/5\t of Patient 12\n",

      "Finish Training Fold 4/5\t of Patient 12\n",

      "Begin Training Fold 5/5\t of Patient 12\n",

      "Finish Training Fold 5/5\t of Patient 12\n",

      "loss: 0.767\tAccuracy : 0.924\t\tval-loss: 0.810\tval-Accuracy : 0.944\n",

      "Begin Training Fold 1/5\t of Patient 14\n",

      "Finish Training Fold 1/5\t of Patient 14\n",

      "Begin Training Fold 2/5\t of Patient 14\n",

      "Finish Training Fold 2/5\t of Patient 14\n",

      "Begin Training Fold 3/5\t of Patient 14\n",

      "Finish Training Fold 3/5\t of Patient 14\n",

      "Begin Training Fold 4/5\t of Patient 14\n",

      "Finish Training Fold 4/5\t of Patient 14\n",

      "Begin Training Fold 5/5\t of Patient 14\n",

      "Finish Training Fold 5/5\t of Patient 14\n",

      "loss: 0.795\tAccuracy : 0.904\t\tval-loss: 0.895\tval-Accuracy : 0.854\n",

      "Begin Training Fold 1/5\t of Patient 15\n",

      "Finish Training Fold 1/5\t of Patient 15\n",

      "Begin Training Fold 2/5\t of Patient 15\n",

      "Finish Training Fold 2/5\t of Patient 15\n",

      "Begin Training Fold 3/5\t of Patient 15\n",

      "Finish Training Fold 3/5\t of Patient 15\n",

      "Begin Training Fold 4/5\t of Patient 15\n",

      "Finish Training Fold 4/5\t of Patient 15\n",

      "Begin Training Fold 5/5\t of Patient 15\n",

      "Finish Training Fold 5/5\t of Patient 15\n",

      "loss: 0.772\tAccuracy : 0.967\t\tval-loss: 0.768\tval-Accuracy : 0.986\n"

     ]

    }

   ],

   "source": [

    "p = 0\n",

    "fold_vloss = np.zeros((n_fold,n_patient))\n",

    "fold_loss = np.zeros((n_fold,n_patient))\n",

    "fold_vacc = np.zeros((n_fold,n_patient))\n",

    "fold_acc = np.zeros((n_fold,n_patient))\n",

    "for patient in np.unique(Patient):\n",

    "    id_patient = np.arange(len(Mean_Images))[Patient==patient]\n",

    "    n_fold = 5\n",

    "    length = len(id_patient)\n",

    "    \n",

    "    n_patient = len(np.unique(Patient))\n",

    "    \n",

    "    train_id, test_id = kfold(length,n_fold)\n",

    "    \n",

    "    for fold in range(n_fold):\n",

    "        X_train = Mean_Images[id_patient[train_id[fold]]]\n",

    "        X_test = Mean_Images[id_patient[test_id[fold]]]\n",

    "        y_train = Label[id_patient[train_id[fold]]]\n",

    "        y_test = Label[id_patient[test_id[fold]]] \n",

    "\n",

    "        print(\"Begin Training Fold %d/%d\\t of Patient %d\" % \n",

    "             (fold+1,n_fold, patient))\n",

    "\n",

    "        CNN = BasicCNN().cuda(0)\n",

    "        criterion = nn.CrossEntropyLoss()\n",

    "        optimizer = optim.SGD(CNN.parameters(), lr=0.001, momentum=0.9)\n",

    "\n",

    "        n_epochs = 50\n",

    "        for epoch in range(n_epochs):\n",

    "            running_loss = 0.0\n",

    "            batchsize = 5\n",

    "            for i in range(int(len(y_train)/batchsize)):\n",

    "                optimizer.zero_grad()\n",

    "\n",

    "                # forward + backward + optimize\n",

    "                outputs = CNN(torch.from_numpy(X_train[i:i+batchsize]).to(torch.float32).cuda())\n",

    "                loss = criterion(outputs, torch.from_numpy(y_train[i:i+batchsize]).to(torch.long).cuda())\n",

    "                loss.backward()\n",

    "                optimizer.step()\n",

    "                running_loss += loss.item()\n",

    "\n",

    "            #acc\n",

    "            _, idx = torch.max(CNN(torch.from_numpy(X_train[:]).to(torch.float32).cuda()).data,1)\n",

    "            acc = (idx == torch.from_numpy(y_train).cuda()).sum().item()/len(y_train)\n",

    "\n",

    "            #val Loss\n",

    "            val_outputs = CNN(torch.from_numpy(X_test[:]).to(torch.float32).cuda())\n",

    "            val_loss = criterion(val_outputs, torch.from_numpy(y_test[:]).to(torch.long).cuda())\n",

    "            _, idx = torch.max(val_outputs.data,1)\n",

    "            val_acc = (idx == torch.from_numpy(y_test).cuda()).sum().item()/len(y_test)\n",

    "\n",

    "            #if epoch%10==0:\n",

    "            #    print('[%d, %3d] loss: %.3f\\tAccuracy : %.3f\\t\\tval-loss: %.3f\\tval-Accuracy : %.3f' %\n",

    "            #     (epoch+1, n_epochs, running_loss/i, float(acc), val_loss, val_acc))\n",

    "        fold_vloss[fold, p ] = val_loss.item()\n",

    "        fold_loss[fold, p] = running_loss/i\n",

    "        fold_vacc[fold, p] = val_acc\n",

    "        fold_acc[fold, p] = acc\n",

    "        print('Finish Training Fold %d/%d\\t of Patient %d' % \n",

    "             (fold+1,n_fold, patient))\n",

    "    print('loss: %.3f\\tAccuracy : %.3f\\t\\tval-loss: %.3f\\tval-Accuracy : %.3f' %\n",

    "                 (np.mean(fold_loss[:,p]), np.mean(fold_acc[:,p]), np.mean(fold_vloss[:,p]),np.mean(fold_vacc[:,p])))\n",

    "    \n",

    "    p = p + 1"

   ]

  },

  {

   "cell_type": "markdown",

   "metadata": {},

   "source": [

    "### Peresented Results"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": 493,

   "metadata": {},

   "outputs": [

    {

     "data": {

      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAtcAAAKUCAYAAADPQhSfAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nOzdfZild10f/vfHXR4qIWGX4CokkICRDhkrmhTaH2u744pEfq2pIpWxRqCj+dmf2bZAqeBACNFRfKjaKrVFJi0IzooppalNDUjOqFv1MknFBzJFQ3haowLZ5WERhF2+/eOcCbPDzO7M5D7zcPb1uq5z7bmfP585M5P33Pne912ttQAAAA/el2x1AQAAMCqEawAA6IhwDQAAHRGuAQCgI8I1AAB0RLgGAICOCNcAm6yqXlBVR5ZMn6iqJ65l3Q0c639W1fM3uj0A6yNcA9teVX1nVd05CKF/PgiM+7eolodX1ceq6htWWPbTVXXzevfZWjuvtXZvB7XdUFVvWrbvb26tveHB7vssx2xV9bRhHQNgJxGugW2tql6c5GeS/EiSfUken+TfJ7l6lfV3D7Oe1tpnkvxyku9edtxdSSaTDC3IbjdVVUmuSXIsyaaeHa8+/w0Dth2/mIBtq6ouSHJjku9vrb21tfap1trnWmv/vbX20sE6N1TVzVX1pqr6RJIXVNXDqupnquq+wetnquphg/UvrKpfHZx9PlZVv7UY0qrqB6rqz6rqk1X1nqo6uEppb0jynKr60iXznpX+79T/OdjXy6rqvYN93V1V33qGPltVfeXg/aOr6paq+kRV/V6SJy1b999W1YcGy++qqq8fzL8qyQ8m+Y7BGf4/GMyfr6rvGbz/kqp6RVV9oKo+XFVvHHyNU1WXDOp4flV9sKo+WlXTZ/mIvj7JY5P8iyTPq6qHLqv1e6tqYcnX4OsG8y+uqrdW1Ueq6v6q+rnB/NPOvC+pafeSXmaq6n8l+askT6yqFy45xr1V9f8tq+HqqnrX4Ov13qq6qqqeW1V3LVvvJVX1trP0C3BWwjWwnf3dJA9P8l/Pst7VSW5O8qgkb04yneTvJHlqkq9J8rQkrxis+5IkR5M8Jv0z4T+YpFXVk5Ncl+Rvt9YemX5Yfv9KB2ut/XaSP0/ybUtmX5Pkl1prJwfT700/fF6Q5NVJ3lRVX7GGnl+b5DNJviLJPx28lrpj0NfeJL+U5Feq6uGttV9L/+z+Lw+GmXzNCvt+weA1keSJSc5L8nPL1tmf5MlJDia5vqrGzlDr85P89/TP5CfJP1hcUFXPTXJD+mf4z0/yLUnuH5zh/9UkH0hySZLHJTl8hmMsd02Sa5M8crCPDw+Oe36SFyb56SUh/mlJ3pjkpel/b/y99D/TW5Jcuqy370ryi+uoA2BFwjWwnT06yUeXBNbV/E5r7W2ttc+31j6d5J8kubG19uHW2kfSD7fXDNb9XPrB9QmDs+C/1VprSU4leViSp1TVQ1pr72+tvfcMx3xjBkNDqur89AP+A0NCWmu/0lq7b1DTLyf50/RD/qoGwfM5Sa4fnKX/4ywbZtJae1Nr7f7W2snW2r8Z1Pzks3x9Fv2TJD/VWru3tXYiycvTP+O8dCjNq1trn26t/UGSP0j/j5OVav3SJM9N/w+Kz6X/x83SoSHfk+THW2t3tL57WmsfGHwNHpvkpYMeP9NaW88Fm/+5tfbuQf+fa639j9baewfH+I0kb0//j5okmUpyU2vtHYPP4c9aa/+ntfbX6f9B8F2DXi5PP+j/6jrqAFiRcA1sZ/cnuXAN46g/tGz6semf1Vz0gcG8JPmJJPckeftgGMHLkqS1dk+Sf5n+2dYPV9Xhqnps8sDdPBZfjx/s541JJqrqcUm+Pck9rbXfXzxgVX33YDjCx6rqY0nGk1x4lj4ek2T3sn6W9rE4fGGhqj4+2O8Fa9jvopW+LrvTP4O/6C+WvP+r9M9ur+Rbk5xMcutg+s1JvrmqHjOYvjj9s/fLXZzkA2v4g2k1p33WVfXNVfW7gyE+H0vy7Hzh67FaDUn/j5bvrHpg3PhbBqEb4EERroHt7HfSHyLxj86yXls2fV+SJyyZfvxgXlprn2ytvaS19sQk/zDJixfHVrfWfqm1tn+wbUvyY4P55y15fXAw74NJfiv9s8HXpB+2kyRV9YQkv5D+MJNHt9YeleSPk9RZ+vhI+oH14mW1L+7365P8QJJ/nGTPYL8fX7Lf5V+H5Vb6upxM8pdn2W4lz08/eH+wqv4iya8keUj6F3Um/RD8pBW2+1CSx6/yB9Onkiwdx/7lK6zzQI/VH0f/X5L8ZJJ9g6/HrfnC12O1GtJa+90kn03/LPd3xpAQoCPCNbBttdY+nuT6JK+tqn9UVV9aVQ8ZnK388TNsOpfkFVX1mKq6cLCPNyVJVf2DqvrKwRnLT6Q/HORUVT25qr5hENg+k+TTg2Vn8ob0A/Qz0j9zu+gR6YfAjwyO+cL0z1yfrd9TSd6a5IZBr0/J6UMtHpl+GP5Ikt1VdX36Y40X/WWSS2r1u2jMJXlRVV1aVeflC2O013UWeXC2/mD6Y52fmi+Mbf+xJfW+Psm/qqorqu8rB390/F7649VfU1WPqP6tDZ8x2OZdSf5eVT1+cKHly89SykPTHxbzkSQnq+qbk3zTkuWzSV5YVQerfzHn46rqby5Z/sb0x5yfXOfQFIBVCdfAttZa+6kkL07/gsSPpH828rokZ7qzww8nuTPJHyb5oyT/ezAvSS5L8utJTqR/Zvzft9bm0w9pr0ny0fSHRnxZ+hc7nsnNSfYkeWdr7c+X1Hx3kn8z2P9fJvnqJP9rLf0OejtvUMN/TvKfliy7Lf27kfxJ+kM6PpPTh0n8yuDf+6vqf6+w75vSP0P7m0neN9j+0BrrWuqaJO9qrb29tfYXi68k/y7J36qq8dbarySZSf+iy0+m/3ntHfwB8Q+TfGWSD6Z/cel3JElr7R3pj4X+wyR35SxjoFtrn0zyz5O8Jcnx9M9A37Jk+e9lcJFj+mf4fyOnn7n/xfT/6HHWGuhM9a/jAYBzS1X9jfTvNvJ1rbU/3ep6gNHgzDUA56p/luQOwRro0lCfZAYA21FVvT/9Cx/PdrEswLoYFgIAAB0xLAQAADoiXANsgqpqVfWVHe/zB6vq9V3uE4AHR7gG2KFaaz/SWvuejWxbVc+qqt+sqk9W1Ueq6jeq6lsGy14w+GPgpcu2OVpVBwbvbxis89wly3cP5l2y4aYAdjjhGuAcU1Xfnv49sd+Y5KL0H39+ffr3n150LMkPVNX5X7yH09a5sap2DatWgJ1GuAbYPM+uqnur6qNV9ROLT1KsqidV1e1Vdf9g2Zur6lGLG1XVD1TVnw3OMr9n8XHtg7PHb1qy3v6q+u2q+lhVfaiqXrC8gMGTKX8qyQ+11l7fWvt4a+3zrbXfaK1975JVF9J/CM6LztDPr6X/CPHvehBfE4CRIlwDbJ5vTXJlkq9LcnWSfzqYX0l+NMljk4wluTjJDUlSVU9O/6mNf7u19sgkz0ry/uU7rqrHp//0xp9N8pj0H0n+rhVqePJg/zevod5Xpv+49L2rLG+DdV5VVQ9Zw/4ARp5wDbB5fqy1dqy19sEkP5NkMklaa/e01t7RWvvr1tpH0j+z/PcH25xK/9HsT6mqh7TW3t9ae+8K+/4nSX69tTbXWvtca+3+1tpK4frRg3//fIVlpxls//YkP3CGdW5J/7H0Gxr7DTBqhGuAzfOhJe8/kP6Z6lTVl1XV4cHQj08keVOSC5N+8E7yL9M/k/3hwXqPXWHfFydZKXQvd//g369YY83XJ/lnVfXlZ1jnFUmmkzx8jfsEGFnCNcDmuXjJ+8cnuW/w/kfTH2Lxt1pr56c/hrkWV2yt/VJrbX+SJwzW+7EV9v2hJE9aQw3vGaz7nLUU3Fr7P0nemuQHz7DOO5Lck+T/X8s+AUaZcA2weV5aVXuq6uIk/yLJLw/mPzLJiSQfq6rHJXngFnhV9eSq+oaqeliSzyT5dPpDRZZ7c5JvrKp/PLgl3qOr6qnLV2r9x/K+OMkrq+qFVXV+VX3J4GLI161S96uTvDDJo1ZZnvTPXP/rMzUPcC4QrgE2z39Lclf6Fxr+jySzg/mvTv8ix48P5r91yTYPS/KaJB9N8hdJviwrnEUejON+dpKXpH+LvHcl+ZqVimit3ZzkO9K/oPK+JH+Z5IcH9a20/vuS/GKSR6zWWGvtfyX5vdWWA5wrqn8SAwAAeLCcuQYAgI4I1wAA0BHhGgAAOiJcAwBAR4RrAADoiHANAAAdEa4BAKAjwjUAAHREuAYAgI4I1wAA0BHhGgAAOiJcAwBAR4RrAADoiHANAAAdEa4BAKAjwjUAAHREuAYAgI4I1wAA0BHhGgAAOiJcAwBAR4RrAADoiHANAAAdEa4BAKAjwjUAAHREuAYAgI4I1wAA0BHhGgAAOiJcAwBAR4RrAADoiHANAAAdEa4BAKAjwjUAAHREuAYAgI4I1wAA0BHhGgAAOiJcAwBAR4RrAADoiHANAAAdEa4BAKAjwjUAAHREuAYAgI4I1wAA0BHhGgAAOiJcAwBAR4RrAADoyO6tLqArF154YbvkkkuGfpxPfepTecQjHjH042yWUepnlHpJRqufUeolGa1+RqmXZLT6GaVektHqZ5R6SUarn83q5a677vpoa+0xKy0bmXB9ySWX5M477xz6cebn53PgwIGhH2ezjFI/o9RLMlr9jFIvyWj1M0q9JKPVzyj1koxWP6PUSzJa/WxWL1X1gdWWGRYCAAAdEa4BAKAjwjUAAHREuAYAgI4I1wAA0BHhGgAAOiJcAwBAR4RrAADoiHANAAAdEa4BAKAjwjUAAHREuAYAgI4I1wAA0BHhGgAAOiJcAwBAR4RrAADoiHANAAAdEa4BAKAjwjUAAHREuAYAgI4MLVxX1U1V9eGq+uNVlldV/buquqeq/rCqvm7JsudX1Z8OXs8fVo0A56q5ubmMj4/n4MGDGR8fz9zc3FaXBNuenxvWYvcQ9/2fk/xckjeusvybk1w2eD09yc8neXpV7U3yqiRXJmlJ7qqqW1prx4dYK8A5Y25uLtPT05mdnc2pU6eya9euTE1NJUkmJye3uDrYnvzcsFZDO3PdWvvNJMfOsMrVSd7Y+n43yaOq6iuSPCvJO1prxwaB+h1JrhpWnQDnmpmZmczOzmZiYiK7d+/OxMREZmdnMzMzs9Wlwbbl54a1qtba8HZedUmSX22tja+w7FeTvKa1dmQw/c4kP5DkQJKHt9Z+eDD/lUk+3Vr7yRX2cW2Sa5Nk3759Vxw+fHg4jSxx4sSJnHfeeUM/zmYZpX5GqZdkZ/YzMTGxoe16vV7HlTx4o9TLcgcPHsxtt92W3bt3P/B9dvLkyTzrWc/KO9/5zq0u70HZiT83qxmlXpKd34+fm813YP7qTTvW/IH/tq71JyYm7mqtXbnSsmEOCzmbWmFeO8P8L57Z2uuSvC5JrrzyynbgwIHOilvN/Px8NuM4m2WU+hmlXpKd2c9qf6xX1arLtqsz1bsT+1lqbGwsu3btyoEDBx74Puv1ehkbG9tx33PL7cSfm9WMUi/Jzu/Hz80WOPDxdW+y0V7Wv8XqtvJuIUeTXLxk+qIk951hPgAdmJ6eztTUVHq9Xk6ePJler5epqalMT09vdWmwbfm5Ya228sz1LUmuq6rD6V/Q+PHW2p9X1W1JfqSq9gzW+6YkL9+qIgFGzeLFV4cOHcrCwkLGxsYyMzPjoiw4Az83rNXQwnVVzaV/lv3Cqjqa/h1AHpIkrbX/kOTWJM9Ock+Sv0rywsGyY1X1Q0nuGOzqxtbamS6MBGCdJicnMzk5uX3/dzBsQ35uWIuhhevW2hn/lGv9AYvfv8qym5LcNIy6AABgWDyhEQAAOiJcAwBAR4RrAADoiHANAAAdEa4BAKAjwjUAAHREuAYAgI4I1wAA0BHhGgAAOiJcAwBAR4RrAADoiHANAAAdEa4BAKAjwjUAAHREuAYAgI4I1wAA0BHhGgAAOiJcAwBAR4RrAADoiHANAAzN3NxcxsfHc/DgwYyPj2dubm6rS4Kh2r3VBQAAo2lubi7T09OZnZ3NqVOnsmvXrkxNTSVJJicnt7g6GA5nrgGAoZiZmcns7GwmJiaye/fuTExMZHZ2NjMzM1tdGgyNcA0ADMXCwkL2799/2rz9+/dnYWFhiyqC4ROuAYChGBsby5EjR06bd+TIkYyNjW1RRTB8xlwDwJDt3bs3x48fH/px9uzZk2PHjg39OGs1PT2dqampB8Zc93q9TE1NGRbCSBOuAWDIjh8/ntbauraZn5/PgQMH1rVNVa1r/WFbvGjx0KFDWVhYyNjYWGZmZlzMyEgTrmETnKtnrQAmJyczOTm5oT8WYCcy5ho2weJZq/W8er3eurfZjAAPAKxOuAYAgI4I1wAA0BHhGgAAOiJcAwBAR4RrAADoiHANAAAdEa4BAKAjwjUAAHREuAYAgI4I1wAA0BHhGgAAOiJcAwBAR4RrAADoiHANAAAdEa4BAKAjwjUAAHREuAYAgI4I1wAA0BHhGgAAOiJcAwBAR4RrAADoiHANAAAdEa4BAKAjwjUAAHREuAYAgI7s3uoCAAC2zA0XrHuTA0kyv5FjfXwDG7HTDDVcV9VVSf5tkl1JXt9ae82y5U9IclOSxyQ5luS7WmtHB8tOJfmjwaofbK19yzBrBWDnmpuby8zMTBYWFjI2Npbp6elMTk5udVnsAPXqT6S1tq5t5ufnc+DAgfUdpyrthnVtwg41tHBdVbuSvDbJM5McTXJHVd3SWrt7yWo/meSNrbU3VNU3JPnRJNcMln26tfbUYdUHwGiYm5vL9PR0Zmdnc+rUqezatStTU1NJImADm26YY66fluSe1tq9rbXPJjmc5Opl6zwlyTsH73srLAeAM5qZmcns7GwmJiaye/fuTExMZHZ2NjMzM1tdGmwbVbXqa2JiYtVlrN8wh4U8LsmHlkwfTfL0Zev8QZLnpD905FuTPLKqHt1auz/Jw6vqziQnk7ymtfa25QeoqmuTXJsk+/bty/z8fOdNLHfixIlNOc5mGaV+tnMv7VXnr3tc34Fk3WP62qvO37Zfg+1a10aNSj/b+edmrRYWFnLq1KnMz88/0M+pU6eysLCwrXpbby0b/Wy2U89LbefvtXPhs+n1eqsum5iYWHX5dv3MVrMtvs9aa0N5JXlu+uOsF6evSfKzy9Z5bJK3Jvn99AP20SQXLC4b/PvEJO9P8qQzHe+KK65om6HX623KcTbLKPUzSr20Nlr99H/VjI5R6mcUvs8uv/zydvvtt7fWvtDP7bff3i6//PItrOp0G/me2chns52/N7fr95rPZnvXtl6b9X2W5M62SiYd5pnro0kuXjJ9UZL7lq7QWrsvybclSVWdl+Q5rbWPL1mW1tq9VTWf5GuTvHeI9bJDbPR/U7V1XrDCyvbu3Zvjx4+ve7v1fm579uzJsWPH1n0czj3T09OZmpp6YMx1r9fL1NSUYSHAlhhmuL4jyWVVdWmSP0vyvCTfuXSFqrowybHW2ueTvDz9O4ekqvYk+avW2l8P1nlGkh8fYq3sIKuF5KoSoDfB8ePHN+3KeliLxYsWDx069MDdQmZmZlzMCGyJoYXr1trJqrouyW3p34rvptbau6vqxvRPpd+S/rDSH62qluQ3k3z/YPOxJP+xqj6f/kWXr2mn32UEgOXO4fv1Tk5OZnJyckN/yAF0aaj3uW6t3Zrk1mXzrl/y/uYkN6+w3W8n+eph1gYwcjYQeIVRgG55/DkAAHREuAYAgI4I1wAA0BHhGgAAOiJcAwBAR4RrAADoiHANAAAdEa4BAKAjwjUAAHREuAYAgI4I1wAA0BHhGgAAOiJcAwBAR4RrAADoiHANAMCONjc3l/Hx8Rw8eDDj4+OZm5vbslp2b9mRAYAdae/evTl+/PjQj7Nnz54cO3Zs6MdhZ5ubm8v09HRmZ2dz6tSp7Nq1K1NTU0mSycnJTa/HmWsAYF2OHz+e1tq6Xr1eb93bbEaAZ+ebmZnJ7OxsJiYmsnv37kxMTGR2djYzMzNbUo9wDQDAjrWwsJD9+/efNm///v1ZWFjYknqEawAAdqyxsbEcOXLktHlHjhzJ2NjYltRjzDUAcE6rqqEfY8+ePUM/xrlqeno6U1NTD4y57vV6mZqa2rJhIcI1AHDOaq2te5v5+fkcOHCg+2LYkMWLFg8dOpSFhYWMjY1lZmZmSy5mTIRrABi69qrzkxsuWNc2B5JkfgPHgXPQ5ORkJicnt8UfPsI1AAzbDR9f9yZVtaGzqsDWckEjAAB0RLgGABgRe/fuTVWt65Vk3dvs3bt3izvdvoRrAIAR4QE/W8+Ya+CcttHHOK/31l0e47wB67wAMNnYRYD9Y61/TDTASoRr4Jy2eJZnPTZyNfpm3Ed35Gwg8G6HOwUA5zbDQgDOQXNzcxkfH8/BgwczPj6eubm5rS4JYCQ4cw1wjpmbm8v09PQDTzPbtWtXpqamkmTLHroAMCqcuQY4x8zMzGR2djYTExPZvXt3JiYmMjs7u2WPCgYYJcI1wDlmYWEh+/fvP23e/v37s7CwsEUVAYwO4RrgHDM2NpYjR46cNu/IkSMZGxvboooARodwDXCOmZ6eztTUVHq9Xk6ePJler5epqalMT09vdWkAO54LGgHOMYsXLR46dCgLCwsZGxvLzMyMixkBOiBcA5yDJicnMzk56b7QAB0zLAQAADoiXAMAQEeEawAA6IhwDQAAHRGuAQCgI8I1AAB0RLgGAICOuM81ALAu7VXnJzdcsK5tDiTJ/AaOAzuMcA0ArM8NH1/3Jh5YxLnCsBAAAOiIcA0AAB0RrgEAoCPCNQAAdES4BgCAjgjXAADQEeEaAAA64j7Xo2SdN/RPNnZT//6x1n+PUwC+WFVtaFlrbRjlAA+ScD1C6tWfWPcv243c1L+q0m5Y1yYArGK139seugI7k3ANADAiPJp+6w01XFfVVUn+bZJdSV7fWnvNsuVPSHJTksckOZbku1prRwfLnp/kFYNVf7i19oZh1grbxdzcXGZmZrKwsJCxsbFMT09ncnJyq8sCYAfwf7G33tDCdVXtSvLaJM9McjTJHVV1S2vt7iWr/WSSN7bW3lBV35DkR5NcU1V7k7wqyZVJWpK7BtseH1a9sB3Mzc1leno6s7OzOXXqVHbt2pWpqakkEbABYAcY5t1Cnpbkntbava21zyY5nOTqZes8Jck7B+97S5Y/K8k7WmvHBoH6HUmuGmKtsC3MzMxkdnY2ExMT2b17dyYmJjI7O5uZmZmtLg0AWINhDgt5XJIPLZk+muTpy9b5gyTPSX/oyLcmeWRVPXqVbR+3/ABVdW2Sa5Nk3759mZ+f76r2VZ04cWJTjrNR661to/1s16/Bdq1rrRYWFnLq1KnMz88/8NmcOnUqCwsL26q3Ufs+G7V+1mO7/05br1HqZ5R6SUarn+3ei99p81tbRGttKK8kz01/nPXi9DVJfnbZOo9N8tYkv59+wD6a5IIkL03yiiXrvTLJS850vCuuuKJthl6vtynH2Yj+x7k+G+lnI8fZDNu1rvW4/PLL2+23395a+8Jnc/vtt7fLL798C6s63ah9n41aP+u1nX+nbcQo9TNKvbQ2Wv1s5178TuttynGS3NlWyaTDHBZyNMnFS6YvSnLf0hVaa/e11r6ttfa1SaYH8z6+lm1hFE1PT2dqaiq9Xi8nT55Mr9fL1NRUpqent7o0AGANhjks5I4kl1XVpUn+LMnzknzn0hWq6sIkx1prn0/y8vTvHJIktyX5karaM5j+psFyGGmLFy0eOnTogbuFzMzMbKuLGd3mCQBWN7Rw3Vo7WVXXpR+UdyW5qbX27qq6Mf1T6bek/9/cH62qluQ3k3z/YNtjVfVD6Qf0JLmxtXZsWLXCdjI5OZnJyclt+wAJt3kCgNUN9T7XrbVbk9y6bN71S97fnOTmVba9KV84kw0AANveMMdcAwDAOUW4BgCAjgjXAADQEeEaAAA6MtQLGneyqtrQduu9iwIAAKNDuF7FaiG5qgRoAABWZFgIAAB0RLgGAICOCNcAANAR4RoAADoiXAMAQEeEawCANZibm8v4+HgOHjyY8fHxzM3NbXVJbENuxQcAcBZzc3OZnp7O7OxsTp06lV27dmVqaipJMjk5ucXVsZ04cw0AcBYzMzOZnZ3NxMREdu/enYmJiczOzmZmZmarS2ObEa4BAM5iYWEh+/fvP23e/v37s7CwsEUVsV0J1wAAZzE2NpYjR46cNu/IkSMZGxvboorYroRrAICzmJ6eztTUVHq9Xk6ePJler5epqalMT09vdWlsMy5oBAA4i8WLFg8dOpSFhYWMjY1lZmbGxYx8EeEaAGANJicnMzk5mfn5+Rw4cGCry2GbMiwEAAA6IlwDAEBHDAsBABghVTX0Y+zZs2fox9ipnLkGABgRrbV1v3q93rq3OXbs2Fa3um0J1wAA0BHhGgAAOmLMNXBOa686P7nhgnVtcyBJ5jdwHABGnnANnNPq1Z9Ia21d22zkHrdVlXbDujYBYAcyLAQAADoiXAMAQEeEa7alvXv3pqrW9Uqy7m327t27xZ0CAKNEuGZbOn78+Kbcp/P48eNb3SoAMEKEawAA6IhwDQAAHRGuAYChmZuby/j4eA4ePJjx8fHMzc1tdUkwVO5zDQAMxdzcXKanpzM7O5tTp05l165dmZqaSpJMTk5ucXUwHM5cAwBDMTMzk9nZ2UxMTGT37t2ZmJjI7OxsZmZmtro0GBrhGgAYioWFhezfv/+0efv378/CwsIWVQTDJ1wDAEMxNjaWI0eOnDbvyJEjGRsb26KKYPjO6XDtQSUAMDzT09OZmppKr9fLyZMn0+v1MjU1lenp6a0uDYbmnL6gcfFBJesxPz+fAwcOrGubxVAOAOeSxYsWDx06lIWFhYyNjWVmZsbFjIy0czpcAwDDNTk5mcnJyQ2dnIKd6JweFgIAAF0SrgEAoCPCNQAAdES4BgCAjgjXAADQEeEaAAA6IlwDAEBHhGsAAOiIcA0AAB0RrgEAoCPCNQAAdES4BgCAjgjXAADQkaGG66q6qqreU1X3VNXLVlj++KrqVdXvV9UfVq/LnxQAACAASURBVNWzB/MvqapPV9W7Bq//MMw6AQCgC7uHteOq2pXktUmemeRokjuq6pbW2t1LVntFkre01n6+qp6S5NYklwyWvbe19tRh1QcAAF0b5pnrpyW5p7V2b2vts0kOJ7l62TotyfmD9xckuW+I9QAAwFAN7cx1kscl+dCS6aNJnr5snRuSvL2qDiV5RJJvXLLs0qr6/SSfSPKK1tpvLT9AVV2b5Nok2bdvX+bn59dd5Hq3OXHixKYcZ6NGqZ9R6mUjNtrPZhi1z2bU+lmP7fx9thGj1M8o9ZKMVj+j1EsyWv1si15aa0N5JXluktcvmb4myc8uW+fFSV4yeP93k9yd/tn0hyV59GD+FemH9PPPdLwrrriirVe//fXp9XqbcpyNGKV+RqmXjdpIP5th1D6bUetnvbbr99lGjVI/o9RLa6PVzyj10tpo9bNZvSS5s62SSYc5LORokouXTF+ULx72MZXkLen/V+d3kjw8yYWttb9urd0/mH9Xkvcm+aoh1goAAA/aMMP1HUkuq6pLq+qhSZ6X5JZl63wwycEkqaqx9MP1R6rqMYMLIlNVT0xyWZJ7h1grAAA8aEMbc91aO1lV1yW5LcmuJDe11t5dVTemfyr9liQvSfILVfWi9C9ufEFrrVXV30tyY1WdTHIqyfe11o4Nq1YAAOjCMC9oTGvt1vRvr7d03vVL3t+d5BkrbPdfkvyXYdbG9tZedX5ywwXr2uZAksxv4DisW1UN/Rh79uwZ+jEAoGtDDdewUfXqTyxe9Lpm8/PzOXDgwPqOU5V2w7o2Oeet93NJNvbZAMBO5PHnAADQEeEaAAA6IlwDAEBHhGsAAOiIcA0AAB0RrgEAoCPCNQAAdES4BgCAjpzTD5HxFEAAALp0TodrTwEEAKBLhoUAAEBHhGsAAOiIcA0AAB0RrgEAoCPCNQAAdES4BgCAjgjXAADQEeEaAAA6IlwDAEBHhGsAAOiIcA0AAB0RrgEAoCPCNQAAdES4BgCAjgjXAADQkd1bXQDAVquqoR9jz549Qz8GAFtPuAbOaa21dW9TVRvaDoDRZ1gIAAB0RLgGAICOCNcAANCRs4brqrquqlyJAwAAZ7GWM9dfnuSOqnpLVV1Vm3FZPQAA7EBnDdettVckuSzJbJIXJPnTqvqRqnrSkGsDAIAdZU1jrlv/nlN/MXidTLInyc1V9eNDrA0AAHaUs97nuqr+eZLnJ/loktcneWlr7XNV9SVJ/jTJvx5uiQAAsDOs5SEyFyb5ttbaB5bObK19vqr+wXDKAgCAnWctw0JuTXJscaKqHllVT0+S1trCsAoDAICdZi3h+ueTnFgy/anBPAAAYIm1hOsaXNCYpD8cJGsbTgIAAOeUtYTre6vqn1fVQwavf5Hk3mEXBgAAO81awvX3Jfl/kvxZkqNJnp7k2mEWBQAAO9FZh3e01j6c5HmbUAsAAOxoa7nP9cOTTCW5PMnDF+e31v7pEOsCAIAdZy3DQn4xyZcneVaS30hyUZJPDrMoAADYidYSrr+ytfbKJJ9qrb0hyf+b5KuHWxYAAOw8awnXnxv8+7GqGk9yQZJLhlYRAADsUGu5X/XrqmpPklckuSXJeUleOdSqAABgBzpjuK6qL0nyidba8SS/meSJm1IVAADsQGccFjJ4GuN1m1QLAADsaGsZc/2OqvpXVXVxVe1dfA29MgAA2GHWMuZ68X7W379kXoshIgAAcJq1PKHx0s0oBAAAdrq1PKHxu1ea31p7Y/flAADAzrWWMdd/e8nr65PckORb1rLzqrqqqt5TVfdU1ctWWP74qupV1e9X1R9W1bOXLHv5YLv3VNWz1tQNAABsobUMCzm0dLqqLkj/kehnVFW7krw2yTOTHE1yR1Xd0lq7e8lqr0jyltbaz1fVU5LcmuSSwfvnJbk8yWOT/HpVfVVr7dQa+wIAgE23ljPXy/1VksvWsN7TktzTWru3tfbZJIeTXL1snZbk/MH7C5LcN3h/dZLDrbW/bq29L8k9g/0BAMC2tZYx1/89/RCc9MP4U5K8ZQ37flySDy2ZPprk6cvWuSHJ26vqUJJHJPnGJdv+7rJtH7dCbdcmuTZJ9u3bl/n5+TWUdbr1bnPixIlNOc5GjVI/o9TLRmy0n+1oJ/YyMTFxxuVVteL8Xq83jHKGZid+NmcySv2MUi/JaPUzSr0ko9XPtuiltXbGV5K/v+T1jCQXnW2bwXbPTfL6JdPXJPnZZeu8OMlLBu//bpK70w/wr03yXUvWm03ynDMd74orrmjr1W9/fXq93qYcZyNGqZ9R6mWjNtLPdjVKvbQ2Wv2MUi+tjVY/o9RLa6PVzyj10tpo9bNZvSS5s62SSddyn+sPJvnz1tpnkqSq/kZVXdJae/9Ztjua5OIl0xflC8M+Fk0luWoQ8n+nqh6e5MI1bgsAANvKWsZc/0qSzy+ZPjWYdzZ3JLmsqi6tqoemf4HiLcvW+WCSg0lSVWNJHp7kI4P1nldVD6uqS9Mf4/17azgmAABsmbWcud7d+hckJklaa58dhOUzaq2drKrrktyWZFeSm1pr766qG9M/lX5Lkpck+YWqelH647pfMDjV/u6qekv6w0ROJvn+5k4hAABsc2sJ1x+pqm8ZhOFU1dVJPrqWnbfWbk3/9npL512/5P3d6Y/jXmnbmSQzazkOAABsB2sJ19+X5M1V9XOD6aNJVnxqIwAAnMvW8hCZ9yb5O1V1XpJqrX1y+GXB6rc669KePXuGfgwA4Nxx1gsaq+pHqupRrbUTrbVPVtWeqvrhzSiOc9dqt7c502sj2x07dmyLOwUARsla7hbyza21jy1OtNaOJ3n28EoCAICdaS3heldVPWxxoqr+RpKHnWF9AAA4J63lgsY3JXlnVf2nwfQLk7xheCUBAMDOtJYLGn+8qv4wyTcmqSS/luQJwy4MAAB2mrUMC0mSv0j/KY3PSf+JigtDqwgAAHaoVc9cV9VXpf/I8skk9yf55fRvxTexSbUBAMCOcqZhIf8nyW8l+YettXuSZPCYcgAAYAVnGhbynPSHg/Sq6heq6mD6Y64BAIAVrBquW2v/tbX2HUn+ZpL5JC9Ksq+qfr6qvmmT6gMAgB1jLXcL+VSSNyd5c1XtTfLcJC9L8vYh18YGeGQ4AMDWWct9rh/QWjuW5D8OXmwzi48AX4+q2tB2AAB8sbXeig8AADiLdZ25HkWGUQAA0JVz+sx1a23dr16vt+5tjh07ttWtAgCwCc7pcA0AAF0SrgEAoCPCNQAAdES4BgCAjgjXAADQEeEaAAA6IlwDAEBHhGsAAOiIcA0AAB0RrgEAoCPCNQAAdES4BgCAjgjXAADQEeEaAAA6IlwDAEBHhGsAAOiIcA0AAB0RrgEAoCPCNQAAdES4BgCAjgjXAADQEeEaAAA6IlwDAEBHhGsAAOiIcA0AAB3ZvdUFMHxVtaHlrbVhlAMAMLKcuT4HtNZWffV6vVWXAQCwPsI1AAB0RLgGAICOCNcAANAR4RoAADoiXAMAQEeEawAA6IhwDQAAHRGuAQCgI8I1AAB0RLgGAICODDVcV9VVVfWeqrqnql62wvKfrqp3DV5/UlUfW7Ls1JJltwyzTgAA6MLuYe24qnYleW2SZyY5muSOqrqltXb34jqttRctWf9Qkq9dsotPt9aeOqz6AACga8M8c/20JPe01u5trX02yeEkV59h/ckkc0OsBwAAhqpaa8PZcdW3J7mqtfY9g+lrkjy9tXbdCus+IcnvJrmotXZqMO9kknclOZnkNa21t62w3bVJrk2Sffv2XXH48OGh9LLUiRMnct555w39OJtllPqZmJhIr9fb6jI6M0qfzSj1koxWP6PUSzJa/YxSL8lo9TNKvSSj1c9m9TIxMXFXa+3KlZYNbVhIklph3mpJ/nlJbl4M1gOPb63dV1VPTHJ7Vf1Ra+29p+2stdcleV2SXHnlle3AgQMdlH1m8/Pz2YzjbJZR62eUehmlz2aUeklGq59R6iUZrX5GqZdktPoZpV6S0epnO/QyzGEhR5NcvGT6oiT3rbLu87JsSEhr7b7Bv/cmmc/p47EBAGDbGWa4viPJZVV1aVU9NP0A/UV3/aiqJyfZk+R3lszbU1UPG7y/MMkzkty9fFsAANhOhjYspLV2sqquS3Jbkl1JbmqtvbuqbkxyZ2ttMWhPJjncTh/8PZbkP1bV59P/A+A1S+8yAgAA29Ewx1yntXZrkluXzbt+2fQNK2z320m+epi1AQBA1zyhEQAAOiJcAwBAR4RrAADoyFDHXMMwVK10C/WzLxvWA5MAABY5c82O01pb8dXr9VZdJlgDAJtBuAYAgI4I1wAA0BHhGgAAOiJcAwBAR4RrAADoiHANAAAdEa4BAKAjwjUAAHREuAYAgI4I1wAA0BHhGgAAOiJcAwBAR4RrAADoiHANAAAdEa4BAKAjwjUAAHREuAYAgI4I1wAA0BHhGgAAOiJcAwBAR4RrAADoiHANAAAdEa4BAKAjwjUAAHREuAYAgI4I1wAA0BHhGgAAOiJcAwBAR4RrAADoiHANAAAdEa4BAKAjwjUAAHREuAYAgI4I1wAA0BHhGgAAOiJcAwBAR4RrAADoiHANAAAdEa4BAKAjwjUAAHREuAYAgI4I1wAA0BHhGgAAOiJcAwBAR4RrAADoiHANAAAdEa4BAKAjwjUAAHREuAYAgI4I1wAA0JGhhuuquqqq3lNV91TVy1ZY/tNV9a7B60+q6mNLlj2/qv508Hr+MOsEAIAu7B7WjqtqV5LXJnlmkqNJ7qiqW1prdy+u01p70ZL1DyX52sH7vUleleTKJC3JXYNtjw+rXgAAeLCGeeb6aUnuaa3d21r7bJLDSa4+w/qTSeYG75+V5B2ttWODQP2OJFcNsVYAAHjQqrU2nB1XfXuSq1pr3zOYvibJ01tr162w7hOS/G6Si1prp6rqXyV5eGvthwfLX5nk0621n1y23bVJrk2Sffv2XXH48OGh9LLUiRMnct555w39OJtllPoZpV6S0epnlHpJRqufUeolGa1+RqmXZLT6GaVektHqZ7N6mZiYuKu1duVKy4Y2LCRJrTBvtST/vCQ3t9ZOrWfb1trrkrwuSa688sp24MCBDZS5PvPz89mM42yWUepnlHpJRqufUeolGa1+RqmXZLT6GaVektHqZ5R6SUarn+3QyzCHhRxNcvGS6YuS3LfKus/LF4aErHdbAADYFoYZru9IcllVXVpVD00/QN+yfKWqenKSPUl+Z8ns25J8U1Xtqao9Sb5pMA8AALatoQ0Laa2drKrr0g/Fu5Lc1Fp7d1XdmOTO1tpi0J5McrgtGfzdWjtWVT+UfkBPkhtba8eGVSsAAHRhmGOu01q7Ncmty+Zdv2z6hlW2vSnJTUMrDgAAOuYJjQAA0BHhGgAAOiJcAwBAR4RrAADoiHANAAAdEa4BAKAjwjUAAHREuAYAgI4I1wAA0BHhGgAAOiJcAwBAR4RrAADoiHANAAAdEa4BAKAjwjUAAHREuAYAgI4I1wAA0BHhGgAAOiJcAwBAR4RrAADoiHANAAAdEa4BAKAjwjUAAHREuAYAgI4I1wAA0BHhGgAAOiJcAwBAR4RrAADoiHANAAAdEa4BAKAjwjUAAHREuAYAgI4I1wAA0BHhGhiaubm5jI+P5+DBgxkfH8/c3NxWlwQAQ7V7qwsARtPc3Fymp6czOzubU6dOZdeuXZmamkqSTE5ObnF1ADAczlwDQzEzM5PZ2dlMTExk9+7dmZiYyOzsbGZmZra6NAAYGuEaGIqFhYXs37//tHn79+/PwsLCFlUEAMMnXANDMTY2liNHjpw278iRIxkbG9uiigBg+IRrYCimp6czNTWVXq+XkydPptfrZWpqKtPT01tdGgAMjQsagaFYvGjx0KFDWVhYyNjYWGZmZlzMCMBIE66BoZmcnMzk5GTm5+dz4MCBrS4HAIbOsBAAAOiIcA0AAB0RrgEAoCPCNQAAdES4BgCAjgjXAADQEeEaAAA6IlwDAEBHhGsAAOiIcA0AAB0RrgEAoCPCNQAAdES4BgCAjgw1XFfVVVX1nqq6p6petso6/7iq7q6qd1fVLy2Zf6qq3jV43TLMOgEAoAu7h7XjqtqV5LVJnpnkaJI7quqW1trdS9a5LMnLkzyjtXa8qr5syS4+3Vp76rDqAwCArg3zzPXTktzTWru3tfbZJIeTXL1sne9N8trW2vEkaa19eIj1PChzc3MZHx/PwYMHMz4+nrm5ua0uCQCAbaZaa8PZcdW3J7mqtfY9g+lrkjy9tXbdknXeluRPkjwjya4kN7TWfm2w7GSSdyU5meQ1rbW3rXCMa5NcmyT79u274vDhw0Pp5Z3vfGdmZ2fz0pe+NJdeemne97735Sd+4icyNTWVgwcPDuWYm+XEiRM577zztrqMToxSL8lo9TNKvSSj1c8o9ZKMVj+j1EsyWv2MUi/JaPWzWb1MTEzc1Vq7csWFrbWhvJI8N8nrl0xfk+Rnl63zq0n+a5KHJLk0/eEjjxose+zg3ycmeX+SJ53peFdccUUblssvv7zdfvvtrbXWer1ea62122+/vV1++eVDO+ZmWexnFIxSL62NVj+j1Etro9XPKPXS2mj1M0q9tDZa/YxSL62NVj+b1UuSO9sqmXSYw0KOJrl4yfRFSe5bYZ3/1lr7XGvtfUnek+SyJGmt3Tf4994k80m+doi1ntHCwkL2799/2rz9+/dnYWFhiyoCAGA7Gma4viPJZVV1aVU9NMnzkiy/68fbkkwkSVVdmOSrktxbVXuq6mFL5j8jyd3ZImNjYzly5Mhp844cOZKxsbEtqggAgO1oaOG6tXYyyXVJbkuykOQtrbV3V9WNVfUtg9VuS3J/Vd2dpJfkpa21+5OMJbmzqv5gMP81bcldRjbb9PR0pqam0uv1cvLkyfR6vUxNTWV6enqrSgIAYBsa2q34kqS1dmuSW5fNu37J+5bkxYPX0nV+O8lXD7O29ZicnEySHDp0KAsLCxkbG8vMzMwD8wEAIBlyuB4lk5OTmZyczPz8fA4cOLDV5QAAsA15/DkAAHREuAYAgI4I1wAA0BHhGgAAOiJcAwBAR4RrAADoiHANAAAdEa4BAKAjwjUAAHREuAYAgI4I1wAA0BHhGgAAOiJcAwBAR4RrAADoiHANAAAdEa4BAKAjwjUAAHREuAYAgI4I1wAA0BHhGgAAOiJcAwBAR4RrAADoiHANAAAdEa4BAKAjwjUAAHREuAYAgI4I1wAA0BHhGgAAOiJcAwBAR4RrAADoiHANAAAdEa4BAKAjwjUAAHREuAYAgI4I1wAA0BHhGgAAOiJcAwBAR4RrgP/b3t0H2VXQZxz/PpPFJhBbkZdUSNrQNpVtdzRCJsUmzWQNxmAp1NdhhzqxbE07Iy/2xRbcKVboTu3oOO10mCq6NMwIm9EoyCjGpDapZlosBAIkLIgFwRRqaMFqChUSnv5xzg7LZtds9p69J/fs85nZ2XvPPffe55mb7P723HPuiYiIqEiG64iIiIiIimS4joiIiIioSIbriIiIiIiKZLiOiIiImIWGh4fp6elhzZo19PT0MDw8XHekRuiqO0BEREREtNfw8DADAwMMDQ1x6NAh5syZQ39/PwB9fX01p+ts2XIdERERMcsMDg4yNDREb28vXV1d9Pb2MjQ0xODgYN3ROl6G64iIiIhZZmRkhJUrV75s2cqVKxkZGakpUXNkuI6IiIiYZbq7u9m5c+fLlu3cuZPu7u6aEjVHhuuIiIiIWWZgYID+/n62b9/OwYMH2b59O/39/QwMDNQdrePlgMaIiIiIWWb0oMXLLruMkZERuru7GRwczMGMFchwHRERETEL9fX10dfXx44dO1i9enXdcRoju4VERERERFQkw3VEREREREUyXEdEREREVCTDdURERERERWZ0uJa0TtJDkr4j6cpJ1nm3pAck7ZV085jl6yU9XH6tn8mcERERERFVmLFPC5E0B7gOeDOwD7hT0m22HxizzhLgKmCF7WcknVoufzXwYWAZYGBXed9nZipvRERERESrZnLL9XLgO7Yfsf08sAm4cNw67wOuGx2abe8vl78F2Gb76fK2bcC6GcwaEREREdEy2Z6ZB5beCayz/Xvl9fcAv2b70jHr3Ap8G1gBzAH+wvYWSX8CzLX9l+V6fw48Z/vj455jA7ABYMGCBWdv2rRpRrqMdeDAAebPnz/jz9MuTerTpC7QrD5N6gLN6tOkLtCsPk3qAs3q06Qu0Kw+7erS29u7y/ayiW6byZPIaIJl4yf5LmAJsBpYCHxTUs8U74vt64HrAZYtW+Z2fAB60z5ovUl9mtQFmtWnSV2gWX2a1AWa1adJXaBZfZrUBZrV51joMpO7hewDFo25vhB4YoJ1vmT7BduPAg9RDNtTuW9ERERExDFlJofrO4Elks6Q9ArgIuC2cevcCvQCSDoZ+GXgEeBrwFpJJ0o6EVhbLouIiIiIOGbN2G4htg9KupRiKJ4D3GB7r6RrgLts38ZLQ/QDwCHgg7b/G0DStRQDOsA1tp+eqawREREREVWYyX2usX07cPu4ZVePuWzgj8qv8fe9AbhhJvNFRERERFQpZ2iMiIiIiKhIhuuIiIiIiIpkuI6IiIiIqEiG64iIiIiIimS4joiIiIioSIbriIiIiIiKZLiOiIiIiKiIio+a7nySngIea8NTnQz8Vxuep12a1KdJXaBZfZrUBZrVp0ldoFl9mtQFmtWnSV2gWX3a1eXnbZ8y0Q2NGa7bRdJdtpfVnaMqTerTpC7QrD5N6gLN6tOkLtCsPk3qAs3q06Qu0Kw+x0KX7BYSEREREVGRDNcRERERERXJcH30rq87QMWa1KdJXaBZfZrUBZrVp0ldoFl9mtQFmtWnSV2gWX1q75J9riMiIiIiKpIt1xERERERFclwHRERERFRkQzXUyTpBkn7Je2pO0urJC2StF3SiKS9kq6oO1MrJM2V9G+S7i37fKTuTK2SNEfSPZK+XHeWVkn6rqT7Je2WdFfdeVoh6VWSNkt6sPz/88a6M02XpNeWr8no1w8lfaDuXNMl6Q/L//97JA1Lmlt3plZIuqLssrcTX5eJfmdKerWkbZIeLr+fWGfGqZqky7vK1+ZFSR31EXaT9PlY+XPtPkm3SHpVnRmnapIu15Y9dkvaKum0dufKcD11G4F1dYeoyEHgj213A+cA75f0KzVnasWPgTfZfj2wFFgn6ZyaM7XqCmCk7hAV6rW9tO7PHq3A3wJbbJ8JvJ4Ofo1sP1S+JkuBs4FngVtqjjUtkk4HLgeW2e4B5gAX1Ztq+iT1AO8DllP8Oztf0pJ6Ux21jRz+O/NK4Ou2lwBfL693go0c3mUP8HbgG21P07qNHN5nG9Bj+3XAt4Gr2h1qmjZyeJeP2X5d+bPty8DV7Q6V4XqKbH8DeLruHFWw/aTtu8vLP6IYEE6vN9X0uXCgvHpc+dWxR+pKWgj8JvCZurPESyT9NLAKGAKw/bztH9SbqjJrgH+33Y6z3M6ULmCepC7geOCJmvO0ohu4w/aztg8C/wy8reZMR2WS35kXAjeWl28EfrutoaZpoi62R2w/VFOklkzSZ2v5bw3gDmBh24NNwyRdfjjm6gnUMA9kuJ7lJC0G3gB8q94krSl3o9gN7Ae22e7kPn8D/CnwYt1BKmJgq6RdkjbUHaYFvwA8BfxDucvOZySdUHeoilwEDNcdYrps/wfwceBx4Engf2xvrTdVS/YAqySdJOl44K3AopozVWGB7Seh2MgDnFpznpjYJcBX6w7RCkmDkr4HXEy2XEc7SZoPfAH4wLi/9DqO7UPlW0ALgeXl26odR9L5wH7bu+rOUqEVts8CzqPYBWlV3YGmqQs4C/h7228A/pfOeVt7UpJeAVwAfL7uLNNV7rt7IXAGcBpwgqTfqTfV9NkeAf6a4q36LcC9FLvzRcwoSQMU/9ZuqjtLK2wP2F5E0ePSdj9/hutZStJxFIP1Tba/WHeeqpRv0++gc/ePXwFcIOm7wCbgTZI+W2+k1th+ovy+n2Kf3uX1Jpq2fcC+Me+KbKYYtjvdecDdtr9fd5AWnAs8avsp2y8AXwR+veZMLbE9ZPss26so3vZ+uO5MFfi+pNcAlN/315wnxpC0HjgfuNjNOQnKzcA72v2kGa5nIUmi2G90xPYn6s7TKkmnjB7ZLGkexS/aB+tNNT22r7K90PZiirfq/8l2x26Bk3SCpFeOXgbWUrzl3XFs/yfwPUmvLRetAR6oMVJV+ujgXUJKjwPnSDq+/Pm2hg4+2BRA0qnl95+jOHCu018jgNuA9eXl9cCXaswSY0haB/wZcIHtZ+vO04pxB/9eQA3zQFe7n7BTSRoGVgMnS9oHfNj2UL2ppm0F8B7g/nI/ZYAP2b69xkyteA1wo6Q5FH8wfs52x3+EXUMsAG4p5h26gJttb6k3UksuA24qd6V4BPjdmvO0pNyf983A79edpRW2vyVpM3A3xVva93AMnAK5RV+QdBLwAvB+28/UHehoTPQ7E/go8DlJ/RR/EL2rvoRTN0mXp4G/A04BviJpt+231Jdy6ibpcxXwU8C28uf1Hbb/oLaQUzRJl7eWG0FeBB4D2t4jpz+PiIiIiKhIdguJiIiIiKhIhuuIiIiIiIpkuI6IiIiIqEiG64iIiIiIimS4joiIiIioSIbriIhjnKRDknZL2iPp8+VH6P2k9T807vq/tPDc75V02iS3XSPp3AmWr5aUj8OMiFkpw3VExLHvOdtLbfcAz3Pkz2192XBtu5WzFb6X4pTih7F9te1/bOGxIyIaJ8N1RERn+SbwSwCSbpW0S9JeSRvKZR8F5pVbum8qlx0YvbOkD0q6U9J9kj5SLlssaUTSp8vH2ippnqR3AssoTpyzuzwDKmMea2O5DpLWSXpQ0k6KMwpGRMxKGa4jIjqEpC7gPOD+ctElts+mGIAvl3SS7St5aUv3xePuvxZYoYLxRwAAAXNJREFUAiwHlgJnS1pV3rwEuM72rwI/AN5hezNwF3Bx+XjPTZJrLvBp4LeA3wB+trrWERGdJcN1RMSxb56k3RSD7uPAULn8ckn3AncAiygG5J9kbfl1D8Wpws8cc59Hbe8uL+8CFh9FvjPL+z/s4rS/nz2K+0ZENEpX3QEiIuKInrO9dOwCSauBc4E32n5W0g5g7hEeR8Bf2f7UuMdaDPx4zKJDwMt2AZkCH+X6ERGNlC3XERGd6WeAZ8rB+kzgnDG3vSDpuAnu8zXgEknzASSdLunUIzzPj4BXHmGdB4EzJP1ieb3vyPEjIpopw3VERGfaAnRJug+4lmLXkFHXA/eNHtA4yvZW4GbgXyXdD2zmyIPzRuCTEx3QOOZx/w/YAHylPKDxsWn0iYhoBBW7x0VERERERKuy5ToiIiIioiIZriMiIiIiKpLhOiIiIiKiIhmuIyIiIiIqkuE6IiIiIqIiGa4jIiIiIiqS4ToiIiIioiL/D0+/g1YpUHEKAAAAAElFTkSuQmCC\n",

      "text/plain": [

       "<Figure size 864x720 with 1 Axes>"

      ]

     },

     "metadata": {

      "needs_background": "light"

     },

     "output_type": "display_data"

    }

   ],

   "source": [

    "fig = plt.figure(figsize=(12,10))\n",

    "plt.grid()\n",

    "plt.boxplot(fold_vacc)\n",

    "plt.suptitle('Cross-Validation Accuracy\\n basic CNN')\n",

    "ax = plt.gca()\n",

    "plt.xlabel('Patient id')\n",

    "plt.ylabel('Accuracy')\n",

    "plt.savefig('fig.png')\n",

    "plt.show()"

   ]

  },

  {

   "cell_type": "markdown",

   "metadata": {},

   "source": [

    "# Image Time Window"

   ]

  },

  {

   "cell_type": "markdown",

   "metadata": {},

   "source": [

    "## CNN Parallel Model (A)"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": 8,

   "metadata": {},

   "outputs": [],

   "source": [

    "class ParallelCNN(nn.Module):\n",

    "    def __init__(self):\n",

    "        super(ParallelCNN, self).__init__()\n",

    "        self.conv1 = nn.Conv2d(3,3,3, padding=1)\n",

    "        self.conv2 = nn.Conv2d(3,3,5, padding=2)\n",

    "        self.conv3 = nn.Conv2d(3,3,3, padding=1)\n",

    "        self.conv4 = nn.Conv2d(3,3,5, padding=2)\n",

    "        self.conv5 = nn.Conv2d(3,3,3, padding=1)\n",

    "        self.conv6 = nn.Conv2d(3,3,5, padding=2)\n",

    "        self.conv7 = nn.Conv2d(3,3,3, padding=1)\n",

    "        self.conv8 = nn.Conv2d(3,3,5, padding=2)\n",

    "        self.conv9 = nn.Conv2d(3,3,3, padding=1)\n",

    "        self.conv10 = nn.Conv2d(3,3,5, padding=2)\n",

    "        self.conv11 = nn.Conv2d(3,3,3, padding=1)\n",

    "        self.conv12 = nn.Conv2d(3,3,5, padding=2)\n",

    "        self.conv13 = nn.Conv2d(3,3,3, padding=1)\n",

    "        self.conv14 = nn.Conv2d(3,3,5, padding=2)\n",

    "        self.pool = nn.MaxPool2d(2,2)\n",

    "        self.fc1 = nn.Linear(3549,512)\n",

    "        self.fc2 = nn.Linear(512,4)\n",

    "        self.max = nn.Softmax()\n",

    "    \n",

    "    def forward(self, x):\n",

    "        batch_size = x.shape[0]\n",

    "        x[:,0] = F.relu(self.conv1(x[:,0]))\n",

    "        x[:,1] = F.relu(self.conv3(x[:,1]))\n",

    "        x[:,2] = F.relu(self.conv5(x[:,2]))\n",

    "        x[:,3] = F.relu(self.conv7(x[:,3]))\n",

    "        x[:,4] = F.relu(self.conv9(x[:,4]))\n",

    "        x[:,5] = F.relu(self.conv11(x[:,5]))\n",

    "        x[:,6] = F.relu(self.conv13(x[:,6]))\n",

    "        x[:,0] = F.relu(self.conv2(x[:,0]))\n",

    "        x[:,1] = F.relu(self.conv4(x[:,1]))\n",

    "        x[:,2] = F.relu(self.conv6(x[:,2]))\n",

    "        x[:,3] = F.relu(self.conv8(x[:,3]))\n",

    "        x[:,4] = F.relu(self.conv10(x[:,4]))\n",

    "        x[:,5] = F.relu(self.conv12(x[:,5]))\n",

    "        x[:,6] = F.relu(self.conv14(x[:,6]))\n",

    "        x = x[:,:,:,3:29,3:29]\n",

    "        x = x.reshape(batch_size, x.shape[2], x.shape[1]*x.shape[3],-1) # img reshape\n",

    "        x = self.pool(x)\n",

    "        x = x.view(batch_size,-1)\n",

    "        x = self.fc1(x)\n",

    "        x = self.fc2(x)\n",

    "        x = self.max(x)\n",

    "        return x"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": 10,

   "metadata": {

    "collapsed": true,

    "jupyter": {

     "outputs_hidden": true

    }

   },

   "outputs": [

    {

     "name": "stderr",

     "output_type": "stream",

     "text": [

      "c:\\users\\victo\\.conda\\envs\\pytorch_eeg\\lib\\site-packages\\ipykernel_launcher.py:45: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n"

     ]

    },

    {

     "name": "stdout",

     "output_type": "stream",

     "text": [

      "Begin Training Fold 1/5\t of Patient 1\n",

      "[1,  30] loss: 1.414\tAccuracy : 0.517\t\tval-loss: 1.359\tval-Accuracy : 0.459\n",

      "[6,  30] loss: 1.117\tAccuracy : 0.748\t\tval-loss: 1.088\tval-Accuracy : 0.757\n",

      "[11,  30] loss: 0.913\tAccuracy : 0.830\t\tval-loss: 0.968\tval-Accuracy : 0.784\n",

      "[16,  30] loss: 0.862\tAccuracy : 0.830\t\tval-loss: 0.925\tval-Accuracy : 0.811\n"

     ]

    },

    {

     "ename": "KeyboardInterrupt",

     "evalue": "",

     "output_type": "error",

     "traceback": [

      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",

      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",

      "\u001b[1;32m<ipython-input-10-a0847679b8ae>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m     36\u001b[0m                 \u001b[0moutputs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mCNN\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtorch\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfrom_numpy\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX_train\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m:\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m+\u001b[0m\u001b[0mbatchsize\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mto\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtorch\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfloat32\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcuda\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     37\u001b[0m                 \u001b[0mloss\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcriterion\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0moutputs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtorch\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfrom_numpy\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0my_train\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m:\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m+\u001b[0m\u001b[0mbatchsize\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mto\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtorch\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlong\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcuda\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 38\u001b[1;33m                 \u001b[0mloss\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     39\u001b[0m                 \u001b[0moptimizer\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     40\u001b[0m                 \u001b[0mrunning_loss\u001b[0m \u001b[1;33m+=\u001b[0m \u001b[0mloss\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mitem\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",

      "\u001b[1;32mc:\\users\\victo\\.conda\\envs\\pytorch_eeg\\lib\\site-packages\\torch\\tensor.py\u001b[0m in \u001b[0;36mbackward\u001b[1;34m(self, gradient, retain_graph, create_graph)\u001b[0m\n\u001b[0;32m    164\u001b[0m                 \u001b[0mproducts\u001b[0m\u001b[1;33m.\u001b[0m \u001b[0mDefaults\u001b[0m \u001b[0mto\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;31m`\u001b[0m\u001b[1;32mFalse\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;31m`\u001b[0m\u001b[1;33m.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    165\u001b[0m         \"\"\"\n\u001b[1;32m--> 166\u001b[1;33m         \u001b[0mtorch\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mautograd\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mgradient\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    167\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    168\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mregister_hook\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mhook\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",

      "\u001b[1;32mc:\\users\\victo\\.conda\\envs\\pytorch_eeg\\lib\\site-packages\\torch\\autograd\\__init__.py\u001b[0m in \u001b[0;36mbackward\u001b[1;34m(tensors, grad_tensors, retain_graph, create_graph, grad_variables)\u001b[0m\n\u001b[0;32m     97\u001b[0m     Variable._execution_engine.run_backward(\n\u001b[0;32m     98\u001b[0m         \u001b[0mtensors\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mgrad_tensors\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 99\u001b[1;33m         allow_unreachable=True)  # allow_unreachable flag\n\u001b[0m\u001b[0;32m    100\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    101\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",

      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "

     ]

    }

   ],

   "source": [

    "p = 0\n",

    "n_fold = 5    \n",

    "n_patient = len(np.unique(Patient))\n",

    "fold_vloss = np.zeros((n_fold,n_patient))\n",

    "fold_loss = np.zeros((n_fold,n_patient))\n",

    "fold_vacc = np.zeros((n_fold,n_patient))\n",

    "fold_acc = np.zeros((n_fold,n_patient))\n",

    "for patient in np.unique(Patient):\n",

    "    id_patient = np.arange(len(tmp))[Patient==patient]\n",

    "\n",

    "    length = len(id_patient)\n",

    "    \n",

    "    train_id, test_id = kfold(length,n_fold)\n",

    "    \n",

    "    for fold in range(n_fold):\n",

    "        X_train = tmp[id_patient[train_id[fold]]]\n",

    "        X_test = tmp[id_patient[test_id[fold]]]\n",

    "        y_train = Label[id_patient[train_id[fold]]]\n",

    "        y_test = Label[id_patient[test_id[fold]]] \n",

    "\n",

    "        print(\"Begin Training Fold %d/%d\\t of Patient %d\" % \n",

    "             (fold+1,n_fold, patient))\n",

    "\n",

    "        CNN = ParallelCNN().cuda(0)\n",

    "        criterion = nn.CrossEntropyLoss()\n",

    "        optimizer = optim.SGD(CNN.parameters(), lr=0.001, momentum=0.9)\n",

    "\n",

    "        n_epochs = 30\n",

    "        for epoch in range(n_epochs):\n",

    "            running_loss = 0.0\n",

    "            batchsize = 4\n",

    "            for i in range(int(len(y_train)/batchsize)):\n",

    "                optimizer.zero_grad()\n",

    "\n",

    "                # forward + backward + optimize\n",

    "                outputs = CNN(torch.from_numpy(X_train[i:i+batchsize]).to(torch.float32).cuda())\n",

    "                loss = criterion(outputs, torch.from_numpy(y_train[i:i+batchsize]).to(torch.long).cuda())\n",

    "                loss.backward()\n",

    "                optimizer.step()\n",

    "                running_loss += loss.item()\n",

    "\n",

    "            #acc\n",

    "            _, idx = torch.max(CNN(torch.from_numpy(X_train[:]).to(torch.float32).cuda()).data,1)\n",

    "            acc = (idx == torch.from_numpy(y_train).cuda()).sum().item()/len(y_train)\n",

    "\n",

    "            #val Loss\n",

    "            val_outputs = CNN(torch.from_numpy(X_test[:]).to(torch.float32).cuda())\n",

    "            val_loss = criterion(val_outputs, torch.from_numpy(y_test[:]).to(torch.long).cuda())\n",

    "            _, idx = torch.max(val_outputs.data,1)\n",

    "            val_acc = (idx == torch.from_numpy(y_test).cuda()).sum().item()/len(y_test)\n",

    "\n",

    "            if epoch%5==0:\n",

    "                print('[%d, %3d] loss: %.3f\\tAccuracy : %.3f\\t\\tval-loss: %.3f\\tval-Accuracy : %.3f' %\n",

    "             (epoch+1, n_epochs, running_loss/i, float(acc), val_loss, val_acc))\n",

    "        fold_vloss[fold, p ] = val_loss.item()\n",

    "        fold_loss[fold, p] = running_loss/i\n",

    "        fold_vacc[fold, p] = val_acc\n",

    "        fold_acc[fold, p] = acc\n",

    "        print('Finish Training Fold %d/%d\\t of Patient %d' % \n",

    "             (fold+1,n_fold, patient))\n",

    "    print('loss: %.3f\\tAccuracy : %.3f\\t\\tval-loss: %.3f\\tval-Accuracy : %.3f' %\n",

    "                 (np.mean(fold_loss[:,p]), np.mean(fold_acc[:,p]), np.mean(fold_vloss[:,p]),np.mean(fold_vacc[:,p])))\n",

    "    \n",

    "    p = p + 1"

   ]

  },

  {

   "cell_type": "markdown",

   "metadata": {},

   "source": [

    "### Peresented Results"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": 91,

   "metadata": {},

   "outputs": [

    {

     "data": {

      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAtAAAAKUCAYAAAAtng/mAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nOzdfZyld10f/M/XXR6UhLBLMAIBEgXpyNYHSBFl9Z5xfQjcCFVKyyAIdJT2fkm0YqngeMNCO61PVVul9kZGAQMTAdFGTAVKZtRVqTwINGREQ8pDBCWQJSGghV1+9x/nLEwm+zC/2TlzZs6+36/Xee2c61zXub7fOTNnP/M7v+u6qrUWAABgY75k3AUAAMBuIkADAEAHARoAADoI0AAA0EGABgCADgI0AAB0EKABRqCqnlFVR9bcv72qvnIj625iX/+9qp6+2e0B6CNAAztCVT2lqt4+DJofHYbCg2Oq5e5V9cmq+raTPPYLVfW63udsrZ3XWrtxC2o7XFVXrnvux7TWXnG2z32GfbaqeuSo9gGwmwjQwNhV1XOS/GKSf5/koiQPTPJfkjzhFOvvHWU9rbW/T/KbSb5/3X73JJlNMrKwutNUVSV5WpJbkmzrKHcN+H8K2HG8MQFjVVUXJHlxkh9qrb2+tfbp1trnWmu/21p77nCdw1X1uqq6sqpuS/KMqrpbVf1iVX1kePvFqrrbcP0Lq+oNw1HkW6rqj04Esar68ar666r6VFW9r6oOnaK0VyR5YlV92Zpl35XB++Z/Hz7X86rq/cPnur6qvuc0fbaqevDw63tX1dVVdVtV/VmSr1q37n+qqg8PH39HVX3LcPnlSX4iyT8bjtS/e7h8pap+YPj1l1TVT1bVB6vqY1X1yuH3OFV1ybCOp1fVh6rq41U1f4aX6FuS3C/JjyR5clXddV2tP1hVq2u+Bw8fLn9AVb2+qm6uqk9U1S8Pl99hBH1NTXvX9LJQVX+c5DNJvrKqnrlmHzdW1b9YV8MTqupdw+/X+6vq8qp6UlW9Y916P1ZVv3OGfgHOSIAGxu2bktw9yW+fYb0nJHldknsleVWS+SSPSvL1Sb4uySOT/ORw3R9LclOS+2Qwov0TSVpVPTTJs5P8o9ba+RkE4g+cbGettT9J8tEk37tm8dOSvLq1dmx4//0ZBMwLkrwoyZVVdd8N9PySJH+f5L5J/vnwttbbhn3tT/LqJK+tqru31n4/g1H63xxOCfm6kzz3M4a3mSRfmeS8JL+8bp2DSR6a5FCSF1TV1GlqfXqS381gRD5JHnfigap6UpLDGYzU3zPJ45N8YjhS/4YkH0xySZL7J7nqNPtY72lJnpXk/OFzfGy433smeWaSX1gT1B+Z5JVJnpvBz8a3ZvCaXp3k0nW9PTXJb3TUAXBSAjQwbvdO8vE1ofRU/rS19juttc+31v4uyfcleXFr7WOttZszCLBPG677uQzC6YOGo9l/1FprSY4nuVuSr6mqu7TWPtBae/9p9vnKDKdxVNU9MwjxX5i+0Vp7bWvtI8OafjPJX2UQ5E9pGC6fmOQFw9H267JuSkhr7crW2idaa8daa/9xWPNDz/D9OeH7kvx8a+3G1trtSZ6fwcjx2mkvL2qt/V1r7d1J3p3BHyAnq/XLkjwpgz8aPpfBHzBrp3H8QJKfaa29rQ3c0Fr74PB7cL8kzx32+PettZ6DJF/eWnvvsP/PtdZ+r7X2/uE+/iDJmzL4wyVJ5pL8WmvtzcPX4a9ba3/RWvs/GYT+pw57eVgGYf4NHXUAnJQADYzbJ5JcuIF5zR9ed/9+GYxOnvDB4bIk+dkkNyR50/Aj/+clSWvthiT/KoNR049V1VVVdb/kC2fJOHF74PB5Xplkpqrun+SfJLmhtfbnJ3ZYVd8/nDrwyar6ZJIDSS48Qx/3SbJ3XT9r+zgx1WC1qm4dPu8FG3jeE072fdmbwUj8CX+z5uvPZDBKfTLfk+RYkmuG91+V5DFVdZ/h/QdkMAq/3gOSfHADfxSdyh1e66p6TFW9dTgd55NJHpsvfj9OVUMy+MPkKVVfmMf9mmGwBjgrAjQwbn+awXSGf3yG9dq6+x9J8qA19x84XJbW2qdaaz/WWvvKJN+d5Dkn5jq31l7dWjs43LYl+enh8vPW3D40XPahJH+Uwaju0zII1EmSqnpQkl/NYErIvVtr90pyXZI6Qx83ZxBKH7Cu9hPP+y1JfjzJP02yb/i8t6553vXfh/VO9n05luRvz7DdyTw9g3D9oar6mySvTXKXDA6kTAZB96tOst2HkzzwFH8UfTrJ2nnlX3GSdb7QYw3mtf9Wkp9LctHw+3FNvvj9OFUNaa29NclnMxitfkpM3wC2iAANjFVr7dYkL0jykqr6x1X1ZVV1l+Go48+cZtOlJD9ZVfepqguHz3FlklTV46rqwcORx9symLpxvKoeWlXfNgxlf5/k74aPnc4rMgjJj85gBPaEe2QQ9G4e7vOZGYxAn6nf40len+TwsNevyR2nRZyfQeC9OcneqnpBBnN/T/jbJJfUqc9OsZTkR6vq0qo6L1+cM901GjwcdT+Uwdzjr88X55r/9Jp6X5bkX1fVI2rgwcM/LP4sg/njP1VV96jBaQEfPdzmXUm+taoeODy48flnKOWuGUxhuTnJsap6TJLvXPP4YpJnVtWhGhxAef+q+gdrHn9lBnPAj3VOIwE4JQEaGLvW2s8neU4GBwHenMGo4rOTnO6MCf8uyduTvCfJ/0ryzuGyJHlIkv+R5PYMRrj/S2ttJYMg9lNJPp7BNIYvz+AAw9N5XZJ9Sd7SWvvompqvT/Ifh8//t0n+YZI/3ki/w97OG9bw8iS/vuaxN2Zwlo+/zGD6xd/njlMaXjv89xNV9c6TPPevZTDS+odJ/vdw+ys2WNdaT0vyrtbam1prf3PiluQ/J/naqjrQWnttkoUMDnT8VAav1/7hHwnfneTBST6UwQGd/yxJWmtvzmBu8nuSvCNnmJPcWvtUkh9O8pokRzMYSb56zeN/luGBhRmM1P9B7jgC/xsZ/GFj9BnYMjU4rgYAJk9VfWkGZ/F4eGvtr8ZdDzAZjEADMMn+nyRvE56BrTTSq3kBwLhU1QcyONjwTAeoAnQxhQMAADqYwgEAAB0EaIAdrKo+UFXfPvz6cFVducHtVqrqB0ZbHcC5SYAG6FRVrao+Pbxq4V9X1c8PL9G9a1TVV1fVa6vq48MrHr6nqp5TVXuq6pJhj7+3bpsrq+rw8Ovp4TovWbfOkap6xvZ1ArD9BGiAzfm61tp5GVxs5ClJfrD3CTZw+fKRqKqvSvI/Mzi/9D9srV2Q5ElJLsvgQi4nPGrNBVBO5tNJvr+qLhlRqQA7kgANcBZaa3+RweW+DyRJVT2vqt5fVZ+qquur6ntOrFtVz6iqP66qX6iqWzK4GuFXVdW1VfWJ4Wjwq6rqXhvZd1U9qqr+pKo+WVXvrqrpDZb9oiR/0lp7zomLw7TW3tdae0pr7ZNr1vuZfPHiNCfzyQwuBPPCDe4XYCII0ABnYXgp7m9J8ufDRe8f3r8gg6B6ZVXdd80m35jkxgyugriQwWnW/kOS+yWZSvKAJIc3sN/7J/m9DALu/iT/OslvVdV9NlD2t2dwhcUzeUmSrz4xB/sUFpI8saoeuoHnA5gIAjTA5ryzqo4m+d0kL8vwctyttde21j7SWvt8a+03k/xVkkeu2e4jrbVfaq0da639XWvthtbam1tr/6e1dnOSn0/yf21g/09Nck1r7Zrhvt6cwaXNH7uBbe+d5KNnXGtwGfCFnGYUenh57/+a5MUbeD6AieBCKgCb8/DW2g3rF1bV9yd5TpJLhovOS3LhmlU+vG79L0/ynzMYtT4/g4GNoxvY/4OSPKmqvnvNsrskWd7Atp9Ict8zrjXwq0meu24/6/10kvdX1ddt8DkBdjUj0ABbpKoelEHgfHaSe7fW7pXkugymaZyw/upV/2G47Gtba/fMYGS5cmYfTvIbrbV7rbndo7X2UxvY9n8keeIG1ktr7XMZTEX5t6eqq7X2iSS/OFwHYOIJ0ABb5x4ZhOGbk6SqnpnhwYWncX6S25N8cjiv+bkb3NeVSb67qr5reOq5uw9PLXfxBrZ9YZJvrqqfraqvGNb64OFp6k52AONvJLlbkstP85w/n+SbM5jHDTDRBGiALdJauz7Jf0zyp0n+Nsk/TPLHZ9jsRUkenuTWDA4KfP0G9/XhJE9I8hMZBPYPZxC+z/i+3lp7f5JvymCayXur6tYkv5XBHOpPnWT94xmE7v2nec7bMjhrxynXAZgU1dr6TxMBAIBTMQINAAAdBGgAAOggQAMAQAcBGgAAOgjQAADQQYAGAIAOAjQAAHQQoAEAoIMADQAAHQRoAADoIEADAEAHARoAADoI0AAA0EGABgCADgI0AAB0EKABAKCDAA0AAB0EaAAA6CBAAwBABwEaAAA6CNAAANBBgAYAgA4CNAAAdBCgAQCggwANAAAdBGgAAOggQAMAQAcBGgAAOgjQAADQQYAGAIAOAjQAAHQQoAEAoIMADQAAHQRoAADoIEADAEAHARoAADoI0AAA0EGABgCADgI0AAB0EKABAKCDAA0AAB0EaAAA6CBAAwBABwEaAAA6CNAAANBh77gL6HXhhRe2Sy65ZOT7+fSnP5173OMeI9/PdpikXpLJ6meSekkmq59J6iWZrH4mqZdksvqZpF6SyepnknpJtq+fd7zjHR9vrd1n/fJdF6AvueSSvP3tbx/5flZWVjI9PT3y/WyHSeolmax+JqmXZLL6maReksnqZ5J6SSarn0nqJZmsfiapl2T7+qmqD55suSkcAADQQYAGAIAOAjQAAHQQoAEAoIMADQAAHQRoAADoIEADAEAHARoAADoI0AAA0EGABgCADgI0AAB0EKABAKCDAA0AAB0EaAAA6CBAAwBABwEaAAA6CNAAANBBgAYAgA4CNAAAdBCgAQCgw8gCdFX9WlV9rKquO8XjVVX/uapuqKr3VNXDR1ULAABslVGOQL88yeWnefwxSR4yvD0rya+MsBYAANgSIwvQrbU/THLLaVZ5QpJXtoG3JrlXVd13VPUAAMBWqNba6J686pIkb2itHTjJY29I8lOttSPD+29J8uOttbefZN1nZTBKnYsuuugRV1111chqPuH222/PeeedN/L9bIdJ6iXZnf3MzMxsarvl5eUtruTsTVIvp7Mbf85OZ5L6maReksnqZ5J6SSarn0nqJdm+fmZmZt7RWrts/fK9I9/zqdVJlp00zbfWXprkpUly2WWXtenp6RGWNbCyspLt2M92mKRekt3Zz6n+UK2qUz62U52u3t3Yz6nsxp+z05mkfiapl2Sy+pmkXpLJ6meSeknG3884z8JxU5IHrLl/cZKPjKkWAADYkHEG6KuTfP/wbByPSnJra+2jY6wHAADOaGRTOKpqKcl0kgur6qYkL0xylyRprf3XJNckeWySG5J8JskzR1ULAABslZEF6Nba7Bkeb0l+aFT7BwCAUXAlQgAA6CBAAwBABwEaAAA6CNAAANBBgAYAgA4CNAAAdBCgAQCggwANAAAdBGgAAOggQAMAQAcBGgAAOgjQAADQQYAGAIAOAjQAAHQQoAEAoIMADQAAHQRoAADoIEADAEAHARoAADoI0AAA0EGABgCADnvHXQAAAKxXVZvarrW2xZXcmQANAMCOc7ogXFXbEpRPxRQOAADoIEADAEAHARo4qf3796equm5JurfZv3//mDsFgD4CNHBSR48eTWut67a8vNy9zdGjR8fdKgB0EaABAKCDAA0AAB0EaAAA6CBAAwBABwEaAAA6CNAAANBBgAYAgA4CNAAAdBCgAQCggwANAAAdBGgAAOggQAMAQAcBGgAAOgjQAADQQYAGAIAOAjQAAHQQoAEAoIMADQAAHQRoAADoIEADAEAHARoAADoI0AAA0EGABgBgbPbv35+q6rol6d5m//79W1azAA0AwNgcPXo0rbWu2/Lycvc2R48e3bKa927ZMzF2J/4i69Va2+JKAFjL+zNMFgF6gpzqjbaqvAkDjNHp3oO9R8PuYwoHAAB0MAINW2T//v2bml/V+9Huvn37csstt3TvB3Yb0x6AncoINGyR3XgQBOxkp/odON1jwjOwHYxAAyfVXnjP5PAFXdtMJ8nKJvYDALuIAA2c3OFbuzdZWVnJ9PT01tcCADuIKRwAANBBgAYAgA4CNAAAdBCgAQCggwANAAAdBGgAAOggQAMAQAcBGgAAOgjQAADQQYAGAIAOAjQAAHQQoAEAoIMADQAAHQRoAADoIEADAEAHARoAADoI0AAA0EGABgCADnvHXQAAAOeu9sJ7Jocv6NpmOklWNrGfLSJAAwAwPodv7d5kZWUl09PTW1/LBpnCAQAAHQRoAADoIEADwBbZv39/qqrrlqR7m/3794+5Uzi3CdAAsEWOHj2a1lrXbXl5uXubo0ePjrtVOKc5iBAAYJfZv3//yP+Q2rdvX2655ZaR7mO3MgINALDL9H7a4ZOOrWUEGrbIbjyPJfnCHNRerbUtruTctZmRtN7XzUgasJUEaNgi9aLbukPVZs5jWVVph7s24TRO9ZpVlZC8TU6MpG3UZn9vALaKKRwAANBBgAYAgA4CNAAAdBCgAQCggwANAAAdBGgAAOggQAMAQAcBGgAAOgjQAADQQYAGAIAOAjQAAHQQoAEAoIMADQAAHQRoAADoIEADAEAHARoAADoI0AAA0EGABgCADgI0AAB0EKABAKCDAA0AAB0EaAAA6CBAAwBAh73jLoA++/fvz9GjR7u3q6qu9fft25dbbrmlez8A57L2wnsmhy/o2mY6SVY2sR9gbAToXebo0aNprXVts7Kykunp6a5tegM3AEm96LZte49uh7s2AbbQSAN0VV2e5D8l2ZPkZa21n1r3+IOS/FqS+yS5JclTW2s3jbImAIDdrvfTjunEJx1baGQBuqr2JHlJku9IclOSt1XV1a2169es9nNJXtlae0VVfVuS/5DkaaOqCQBgIhy+tWv1qur+dIRTG+VBhI9MckNr7cbW2meTXJXkCevW+Zokbxl+vXySxwEAYEcZ5RSO+yf58Jr7NyX5xnXrvDvJEzOY5vE9Sc6vqnu31j6xdqWqelaSZyXJRRddlJWVlVHV/AW33377tuxnM3rr2mwvO7X/nfzabMfc8fPPP3/H9f+Wt7wlV155ZT70oQ/lgQ98YJ761Kfm0KFD4y7rCx7/+MfnU5/6VPd2va/n+eefn6uvvrp7P9thJ//eJH3vNzv9Pe1cfo/e6T9nvSatn0nqZeyvTWttJLckT8pg3vOJ+09L8kvr1rlfktcn+fMMQvRNSS443fM+4hGPaNtheXl5W/bTa/CS9dlML5vZz3bZqa/NZkxCL69+9avbpZde2q699tr25je/uV177bXt0ksvba9+9avHXdoX+L3Z2T9rvd+3nfzanOs/azv552wzJqmfnfozs1nb9dokeXs7SR4d5RSOm5I8YM39i5N8ZO0KrbWPtNa+t7X2DUnmh8v6JvUAY7WwsJDFxcXMzMxk7969mZmZyeLiYhYWFsZdGgCMxCgD9NuSPKSqLq2quyZ5cpI7fLZZVRdW1Ykanp/BGTmAXWR1dTUHDx68w7KDBw9mdXV1TBUBwGiNLEC31o4leXaSNyZZTfKa1tp7q+rFVfX44WrTSd5XVX+Z5KIkhqxgl5mamsqRI0fusOzIkSOZmpoaU0UAMFojPQ90a+2aJNesW/aCNV+/LsnrRlkDMFrz8/OZm5vL4uJijh8/nuXl5czNzZnCAcDEciVC4KzMzs4mSa644oqsrq5mamoqCwsLX1gOAJNGgAbO2uzsbGZnZzd1SWK2zmZPo9hcXAGgiwANMCFOF4RdhQxg64zyLBwAADBxzukRaB93AgDQ65wegT7ZlWVO3E73OADwRUtLSzlw4EAOHTqUAwcOZGlpadwlwUid0yPQAMDZWVpayvz8/BdOZblnz57Mzc0libPxMLHO6RFoAODsLCwsZHFxMTMzM9m7d29mZmayuLjoXPBMNAEaANi01dXVHDx48A7LDh48mNXV1TFVBKMnQAMAmzY1NZUjR47cYdmRI0cyNTU1popg9ARoAGDT5ufnMzc3l+Xl5Rw7dizLy8uZm5vL/Pz8uEuDkXEQIQCwaScOFLziiiuyurqaqampLCwsOICQiSZAAwBnZXZ2NrOzs1lZWcn09PS4y4GRM4UDAAA6CNAAANBBgAYAgA4CNAAAdHAQ4S7TXnjP5PAFXdtMJ8nKJvYDAMCdCNC7TL3otrTWurbZzFHRVZV2uGsTAIBzgikcAADQQYAGAIAOpnAAAEyIqtrUY73TQ891RqABACZEa+2kt+Xl5VM+Jjz3E6ABAKCDKRwAjFXv6TmnE6fmBMZKgGbXWFpaysLCQlZXVzM1NZX5+fnMzs6OuyzgbB2+tWv1qvKRMzBWAjS7wtLSUubn57O4uJjjx49nz549mZubSxIhGgDYVuZAsyssLCxkcXExMzMz2bt3b2ZmZrK4uJiFhYVxlwYAnGMEaHaF1dXVHDx48A7LDh48mNXV1TFVBACcqwRodoWpqakcOXLkDsuOHDmSqampMVUEAJyrBGh2hfn5+czNzWV5eTnHjh3L8vJy5ubmMj8/P+7SAIBzjIMI2RVOHCh4xRVXfOEsHAsLCw4gBAC2nQDNrjE7O5vZ2dmsrKxkenp63OXA+HScM/mE3nMtf3FffaeYAzgXCNAAu0y96Lbu8yBv5g/Pqko73LUJwDnBHGgAAOggQAMAQAcBGgAAOpwTAXr//v2pqq5bku5t9u/fP+ZOAQAYtXMiQB89ejStta7b8vJy9zZHjx4dd6sAAIzYORGgAQBgqwjQAADQQYAGAIAOLqQCAJyzTpw4oFfvxYyYLAI0AHDOOl0QripBmZMyhQMAADoI0AAA0MEUjl1os/O1euzbt2/k+wCYRN6jYfIJ0LvMZuZimcMFsD028167srKS6enprS8GGBlTOAAAoIMADQAAHQRoAADoIEADAEAHARoAADoI0AAA0EGABgCADs4DDUy89sJ7Jocv6NpmOklWNrEfACaeAA1MvHrRbd0XuNjMxS2qKu1w1yYA7EKmcAAAQAcBGgAAOgjQAADQQYAGAIAOAjQAAHQQoAEAoIMADQAAHQRoAADoIEADAEAHARoAADoI0AAA0EGABgCADgI0AAB0EKABAKCDAA0AAB0EaAAA6CBAAwBABwEaAAA67B13AWydqtrUY621UZQDADCRjEBPkNbaSW/Ly8unfEx4BgDoI0ADAEAHARoAADoI0AAA0EGABgCADgI0AAB0EKBhDJaWlnLgwIEcOnQoBw4cyNLS0rhLAgA2yHmgYZstLS1lfn4+i4uLOX78ePbs2ZO5ubkkyezs7JirAwDOxAg0bLOFhYUsLi5mZmYme/fuzczMTBYXF7OwsDDu0gCADTACDdtsdXU1Bw8evMOygwcPZnV1dUwVwc7k6qrATmUEGrbZ1NRUjhw5codlR44cydTU1Jgqgp3J1VWBnUqAhm02Pz+fubm5LC8v59ixY1leXs7c3Fzm5+fHXRoAsAGmcMA2O3Gg4BVXXJHV1dVMTU1lYWHBAYTAznP4gq7Vp5NkZTP7uXUTG8H4CNAwBrOzs5mdnc3Kykqmp6fHXQ7AyXUGW+9pnCtM4QAAgA4CNAAAdBCgAQCggwANAAAdBGgAAOggQAMAQAcBGgAAOgjQAADQQYAGACbe/v37U1VdtyTd2+zfv3/MnbIdBGgAYOIdPXo0rbWu2/Lycvc2R48eHXerbAMBGgAAOpwxQFfVs6tq33YUAwAAO93eDazzFUneVlXvTPJrSd7YWmujLWtrtRfeMzl8Qdc200myson9AAAw0c4YoFtrP1lV/2+S70zyzCS/XFWvSbLYWnv/qAvcEodv7d6kqrLL/k4AAGAbbGgO9HDE+W+Gt2NJ9iV5XVX9zAhrAwCAHeeMI9BV9cNJnp7k40leluS5rbXPVdWXJPmrJP9mtCUCAMDOsZE50Bcm+d7W2gfXLmytfb6qHjeasgAAYGfayBSOa5LccuJOVZ1fVd+YJK211VEVBgAAO9FGAvSvJLl9zf1PD5cBAMA5ZyMButaetq619vlsbOoHAABMnI0E6Bur6oer6i7D248kuXHUhQEAwE60kQD9L5N8c5K/TnJTkm9M8qxRFgUAADvVGQN0a+1jrbUnt9a+vLV2UWvtKa21j23kyavq8qp6X1XdUFXPO8njD6yq5ar686p6T1U9djNNAADjs7S0lAMHDuTQoUM5cOBAlpaWxl0SjNRGzgN99yRzSR6W5O4nlrfW/vkZttuT5CVJviODkeu3VdXVrbXr16z2k0le01r7lar6mgzO+HFJbxMAwHgsLS1lfn4+i4uLOX78ePbs2ZO5ubkkyezs7Jirg9HYyBSO30jyFUm+K8kfJLk4yac2sN0jk9zQWruxtfbZJFclecK6dVqSew6/viDJRzZSNACwMywsLGRxcTEzMzPZu3dvZmZmsri4mIWFhXGXBiOzkbNpPLi19qSqekJr7RVV9eokb9zAdvdP8uE190/Mn17rcJI3VdUVSe6R5NtP9kRV9awM511fdNFFWVlZ2cDuz9527WfUbr/99onpJZmsfiapl2Rn99Nb12Z72anvTzu9nx47+edsM3Z7P6urqzl+/HhWVla+0Mvx48ezurq64/rye7My7jK2zNj7aa2d9pbkz4b//mGSAxlcmfDGDWz3pCQvW3P/aUl+ad06z0nyY8OvvynJ9Um+5HTP+4hHPKJth8G3ZjIsLy+Pu4QtNUn9TFIvre3cfjbz+7yZXrbrfWPS+um1U3/ONmu39/Owhz2sXXvtta21L/Zy7bXXtoc97GFjrOrO/N4sj7uELbVd/SR5eztJHt3IFI6XVtW+DOYrXz0MuT+9ge1uSvKANfcvzp2naMwlec0wyP9pBnOsL9zAcwMAO8D8/Hzm5uayvLycY8eOZXl5OXNzc5mfnx93aTAyp53CUVVfkuS21trRDEagv7Ljud+W5CFVdWkGp8B7cpKnrFvnQ0kOJXl5VU1lEKBv7tgHADBGJw4UvOKKK7K6upqpqaksLCw4gJCJdtoR6C7o5GAAABZxSURBVDa46uCzN/PErbVjw23fmGQ1g7NtvLeqXlxVjx+u9mNJfrCq3p1kKckzhsPlAMAuMTs7m+uuuy5vectbct111wnPTLyNHET45qr610l+M8mnTyxsrd1ypg1ba9dkcGq6tctesObr65M8esPVAgDAmG0kQJ843/MPrVnW0jedAwAAJsIZA3Rr7dLtKAQAAHaDjVyJ8PtPtry19sqtLwcAAHa2jUzh+Edrvr57BmfNeGcSARoAgHPORqZwXLH2flVdkMHlvQEA4JyzkRHo9T6T5CFbXQgAwKi0F94zOXxB1zbTSbKyif0w8TYyB/p3MzjrRjI4b/TXZHj1QACA3aBedFt6LzWxsrKS6enpvv1UpR3u2oRdaCMj0D+35utjST7YWrtpRPUAAMCOtpEA/aEkH22t/X2SVNWXVtUlrbUPjLQyAADYgU57Ke+h1yb5/Jr7x4fLAADgnLORAL23tfbZE3eGX991dCUBAMDOtZEAfXNVPf7Enap6QpKPj64kAADYuTYyB/pfJnlVVf3y8P5NSU56dUIAAJh0G7mQyvuTPKqqzktSrbVPjb6s7VFVm3q89zQ4AABMjjNO4aiqf19V92qt3d5a+1RV7auqf7cdxY1aa+2Ut+Xl5VM+BgDAuWsjc6Af01r75Ik7rbWjSR47upIAAGDn2kiA3lNVdztxp6q+NMndTrM+AABMrI0cRHhlkrdU1a8P7z8zyStGVxIAAOxcGzmI8Geq6j1Jvj1JJfn9JA8adWEAALATbWQKR5L8TQZXI3xikkNJVkdWEQAA7GCnHIGuqq9O8uQks0k+keQ3MziN3cw21QYAADvO6aZw/EWSP0ry3a21G5Kkqn50W6oCAIAd6nRTOJ6YwdSN5ar61ao6lMEcaAAAOGedMkC31n67tfbPkvyDJCtJfjTJRVX1K1X1ndtUHwAA7ChnPIiwtfbp1tqrWmuPS3Jxkncled7IKwMAgDWWlpZy4MCBHDp0KAcOHMjS0tJY6tjIeaC/oLV2S5L/b3gDAIBtsbS0lPn5+SwuLub48ePZs2dP5ubmkiSzs7PbWstGT2MHAABjs7CwkMXFxczMzGTv3r2ZmZnJ4uJiFhYWtr0WARoAgB1vdXU1Bw8evMOygwcPZnV1+y9PIkADALDjTU1N5ciRI3dYduTIkUxNTW17LQI0AAA73vz8fObm5rK8vJxjx45leXk5c3NzmZ+f3/Zaug4iBACAcThxoOAVV1yR1dXVTE1NZWFhYdsPIEwEaAAAdonZ2dnMzs5mZWUl09PTY6vDFA4AAOggQAMAQAcBGgAAOgjQAADQQYAGAIAOAjQAAHRwGjuAXaiqRr6Pffv2jXwfALuRAA2wy7TWurepqk1tB8CdmcIBAAAdBGgAAOggQAMAQAcBGgAAOgjQAADQQYAGAIAOAjQAAHQQoAEAoIMADQAAHQRoAADoIEADAEAHARoAADoI0AAA0EGABgCADgI0AAB0EKABAKCDAA0AAB0EaAAA6CBAAwBABwEaAAA6CNAAANBh77gLAADYDlU18n3s27dv5Ptg/ARoxuvwBd2bTCfJymb2desmNgJgErTWurepqk1tx+QToBmretFt3W9OKysrmZ6e7ttPVdrhrk0AAE7KHGgAAOggQAMAQAcBGgAAOgjQAADQQYAGAIAOAjQAAHRwGjuACXGmi0Sc6nHnuQXoYwQaYEK01k55W15ePuVjAPQRoAEAoIMADQAAHQRoAADoIEADAEAHARoAADoI0AAA0EGABgCADgI0AAB0EKABAKCDAA0AAB0EaAAA6CBAAwBABwEaAAA6CNAAANBBgAYAgA4CNAAAdBCgAQCgw95xFwAAMC5VtanHW2ujKIddwgg0AHDOaq2d8ra8vHzKxzi3CdAAANBBgAYAgA7mQDN2Z5p/thX27ds38n0AAOcGI9CM1enmnm1mTtqpbrfccsu4WwUAJoQADQAAHQRoAADoIEADAEAHARoAADoI0AAA0EGABgCADgI0AAB0EKABAKCDAA0AAB0EaAAA6CBAAwBABwEaAAA6CNAAANBBgAYAgA4CNAAAdBCgAQCgw0gDdFVdXlXvq6obqup5J3n8F6rqXcPbX1bVJ0dZDwAAnK29o3riqtqT5CVJviPJTUneVlVXt9auP7FOa+1H16x/RZJvGFU9AACwFUY5Av3IJDe01m5srX02yVVJnnCa9WeTLI2wHgAAOGvVWhvNE1f9kySXt9Z+YHj/aUm+sbX27JOs+6Akb01ycWvt+Ekef1aSZyXJRRdd9IirrrpqJDWvdfvtt+e8884b+X62wyT1kkxWP5PUS7Jz+5mZmdmW/Zx//vm5+uqrt2VfvXbqa7MZk9RLMln9TFIvyWT1M0m9JNvXz8zMzDtaa5etXz6yKRxJ6iTLTpXWn5zkdScLz0nSWntpkpcmyWWXXdamp6e3pMDTWVlZyXbsZztMUi/JZPUzSb0kO7efzQwU7NReNmuS+pmkXpLJ6meSekkmq59J6iUZfz+jnMJxU5IHrLl/cZKPnGLdJ8f0DQAAdoFRBui3JXlIVV1aVXfNICTf6bPNqnpokn1J/nSEtQAAwJYYWYBurR1L8uwkb0yymuQ1rbX3VtWLq+rxa1adTXJVG9VkbAAA2EKjnAOd1to1Sa5Zt+wF6+4fHmUNAACwlVyJEAAAOgjQAADQQYAGAIAOAjQAAHQQoAEAoIMADQAAHQRoAADoIEADAEAHARoAADoI0AAA0EGABgCADgI0AAB0EKABAKCDAA0AAB0EaAAA6CBAAwBABwEaAAA6CNAAANBBgAYAgA4CNAAAdBCgAQCggwANAAAdBGgAAOggQAMAQAcBGgAAOgjQAADQQYAGAIAOAjQAAHQQoAEAoIMADQAAHQRoAADoIEADAEAHARoAADoI0AAA0EGABgCADgI0AAB0EKABAKCDAA0AAB0EaAAA6CBAAwBABwEaAAA6CNAAANBBgAYAgA4CNAAAdBCgAQCggwANAAAdBGgAAOggQAMAQAcBGgAAOgjQAADQQYAGAIAOAjQAAHQQoAEAoIMADQAAHQRoAADoIEADAEAHARoAADoI0AAA0EGABgCADgI0AAB0EKABAKCDAA0AAB0EaAAA6CBAAwBABwEaAAA6CNAAANBBgAYAgA4CNAAAdBCgAQCggwANAAAdBGgAAOggQAMAQAcBGgAAOgjQAADQQYAGAIAOAjQAAHQQoAEAoIMADQAAHQRoAADoIEADAEAHARoAADoI0AAA0EGABgCADgI0AAB0EKABAKCDAA0AAB0EaAAA6CBAAwBABwEaAAA6CNAAANBBgAYAgA4CNAAAdBCgAQCggwANAAAdBGgAAOggQAMAQAcBGgAAOgjQAADQQYAGAIAOAjQAAHQQoAEAoIMADQAAHQRoAADoIEADAEAHARoAADoI0AAA0EGABgCADgI0AAB0EKABAKCDAA0AAB0EaAAA6CBAAwBAh5EG6Kq6vKreV1U3VNXzTrHOP62q66vqvVX16lHWAwAAZ2vvqJ64qvYkeUmS70hyU5K3VdXVrbXr16zzkCTPT/Lo1trRqvryUdUDAABbYZQj0I9MckNr7cbW2meTXJXkCevW+cEkL2mtHU2S1trHRlgPAACctWqtjeaJq/5Jkstbaz8wvP+0JN/YWnv2mnV+J8lfJnl0kj1JDrfWfv8kz/WsJM9KkosuuugRV1111UhqXuv222/PeeedN/L9bIdJ6iWZrH4mqZdksvqZpF6SyepnknpJJqufSeolmax+JqmXZPv6mZmZeUdr7bL1y0c2hSNJnWTZ+rS+N8lDkkwnuTjJH1XVgdbaJ++wUWsvTfLSJLnsssva9PT0lhe73srKSrZjP9thknpJJqufSeolmax+JqmXZLL6maReksnqZ5J6SSarn0nqJRl/P6OcwnFTkgesuX9xko+cZJ3/1lr7XGvtfyd5XwaBGgAAdqRRBui3JXlIVV1aVXdN8uQkV69b53eSzCRJVV2Y5KuT3DjCmgAA4KyMLEC31o4leXaSNyZZTfKa1tp7q+rFVfX44WpvTPKJqro+yXKS57bWPjGqmgAA4GyNcg50WmvXJLlm3bIXrPm6JXnO8AYAADueKxECAEAHARoAADoI0AAA0EGAZtdYWlrKgQMHcujQoRw4cCBLS0vjLgkAOAeN9CBC2CpLS0uZn5/P4uJijh8/nj179mRubi5JMjs7O+bqAIBziRFodoWFhYUsLi5mZmYme/fuzczMTBYXF7OwsDDu0gCAc4wAza6wurqagwcP3mHZwYMHs7q6OqaKAIBzlQDNrjA1NZUjR47cYdmRI0cyNTU1pooAgHOVAM2uMD8/n7m5uSwvL+fYsWNZXl7O3Nxc5ufnx10aAHCOcRAhu8KJAwWvuOKKrK6uZmpqKgsLCw4gBAC2nQDNrjE7O5vZ2dmsrKxkenp63OUAAOcoUzgAAKCDAA0AAB0EaAAA6CBAAwBABwEaAAA6CNAAANBBgAYAgA4CNAAAdBCgAQCggwANAAAdBGgAAOggQAMAQAcBGgAAOgjQAADQQYAGAIAOAjQAAHQQoAEAoIMADQAAHQRoAADoIEADAECHaq2Nu4YuVXVzkg9uw64uTPLxbdjPdpikXpLJ6meSekkmq59J6iWZrH4mqZdksvqZpF6SyepnknpJtq+fB7XW7rN+4a4L0Nulqt7eWrts3HVshUnqJZmsfiapl2Sy+pmkXpLJ6meSekkmq59J6iWZrH4mqZdk/P2YwgEAAB0EaAAA6CBAn9pLx13AFpqkXpLJ6meSekkmq59J6iWZrH4mqZdksvqZpF6SyepnknpJxtyPOdAAANDBCDQAAHQQoAEAoIMAvU5V/VpVfayqrht3LWerqh5QVctVtVpV762qHxl3TZtVVXevqj+rqncPe3nRuGvaClW1p6r+vKreMO5azkZVfaCq/ldVvauq3j7ues5WVd2rql5XVX8x/P35pnHXtBlV9dDha3LidltV/atx13U2qupHh+8B11XVUlXdfdw1bVZV/ciwj/fuxtflZP9fVtX+qnpzVf3V8N9946yxxyn6edLw9fl8Ve2aU8CdopefHb6nvaeqfruq7jXOGnucop9/O+zlXVX1pqq633bWJEDf2cuTXD7uIrbIsSQ/1lqbSvKoJD9UVV8z5po26/8k+bbW2tcl+fokl1fVo8Zc01b4kSSr4y5ii8y01r5+Qs4z+p+S/H5r7R8k+brs0teotfa+4Wvy9UkekeQzSX57zGVtWlXdP8kPJ7mstXYgyZ4kTx5vVZtTVQeS/GCSR2bwM/a4qnrIeKvq9vLc+f/L5yV5S2vtIUneMry/W7w8d+7nuiTfm+QPt72as/Py3LmXNyc50Fr72iR/meT5213UWXh57tzPz7bWvnb4/vaGJC/YzoIE6HVaa3+Y5JZx17EVWmsfba29c/j1pzIIAfcfb1Wb0wZuH969y/C2q4+AraqLk/zfSV427lr4oqq6Z5JvTbKYJK21z7bWPjneqrbEoSTvb61tx5VcR2lvki+tqr1JvizJR8Zcz2ZNJXlra+0zrbVjSf4gyfeMuaYup/j/8glJXjH8+hVJ/vG2FnUWTtZPa221tfa+MZW0aafo5U3Dn7UkeWuSi7e9sE06RT+3rbl7j2xzJhCgzxFVdUmSb0jyP8dbyeYNpzu8K8nHkry5tbZrexn6xST/Jsnnx13IFmhJ3lRV76iqZ427mLP0lUluTvLrw+k1L6uqe4y7qC3w5CRL4y7ibLTW/jrJzyX5UJKPJrm1tfam8Va1adcl+daqundVfVmSxyZ5wJhr2goXtdY+mgwGcZJ8+Zjr4eT+eZL/Pu4izlZVLVTVh5N8X4xAs9Wq6rwkv5XkX637i21Xaa0dH35Uc3GSRw4/At2VqupxST7WWnvHuGvZIo9urT08yWMymCr0reMu6CzsTfLwJL/SWvuGJJ/O7voY+k6q6q5JHp/kteOu5WwM59M+IcmlSe6X5B5V9dTxVrU5rbXVJD+dwcfqv5/k3RlMu4ORqqr5DH7WXjXuWs5Wa22+tfaADHp59nbuW4CecFV1lwzC86taa68fdz1bYfhx+kp291z1Ryd5fFV9IMlVSb6tqq4cb0mb11r7yPDfj2Uwx/aR463orNyU5KY1n3C8LoNAvZs9Jsk7W2t/O+5CztK3J/nfrbWbW2ufS/L6JN885po2rbW22Fp7eGvtWzP4ePqvxl3TFvjbqrpvkgz//diY62GNqnp6kscl+b42WRcCeXWSJ27nDgXoCVZVlcE8ztXW2s+Pu56zUVX3OXHEcFV9aQb/kf7FeKvavNba81trF7fWLsngo/VrW2u7ciStqu5RVeef+DrJd2bw8fSu1Fr7myQfrqqHDhcdSnL9GEvaCrPZ5dM3hj6U5FFV9WXD97dD2aUHeCZJVX358N8HZnCg2iS8Rlcnefrw66cn+W9jrIU1quryJD+e5PGttc+Mu56zte6g28dnmzPB3u3c2W5QVUtJppNcWFU3JXlha21xvFVt2qOTPC3J/xrOHU6Sn2itXTPGmjbrvkleUVV7MvjD7zWttV196rcJclGS3x7kmexN8urW2u+Pt6SzdkWSVw2nPtyY5JljrmfThvNrvyPJvxh3LWertfY/q+p1Sd6ZwUfQf57dfXni36qqeyf5XJIfaq0dHXdBPU72/2WSn0rymqqay+APnieNr8I+p+jnliS/lOQ+SX6vqt7VWvuu8VW5Mafo5flJ7pbkzcP367e21v7l2IrscIp+Hjsc6Ph8kg8m2dZeXMobAAA6mMIBAAAdBGgAAOggQAMAQAcBGgAAOgjQAADQQYAG2CGq6nhVvauqrquq1w5PQXe69X9i3f0/OYt9P6Oq7neKx15cVd9+kuXTVeV0ksA5R4AG2Dn+rrX29a21A0k+mzOf1/QOAbq1djZX5XtGBpfHvpPW2gtaa//jLJ4bYKII0AA70x8leXCSVNXvVNU7quq9/397d8waVRBGYfg9ECEBxUIQQYSIBAI2gYhooVhI0MJKG0mXImU6wVYs9A8IahBSqFVaQVNZCAoqxqQwYCH6BxQUo4iMxZ3VNUQ3VwST+D6wsPvdmdm53WGYYZJM1tplYKCuWN+qtQ+dzknOJXmcZCHJhVobTPIiyXQday7JQJIzwAGay2Pm622fdI01U9uQ5ESSpSQPaG7Pk6T/jgFaktaZJH3ASWCxliZKKaM0IXcqyY5Synl+rFiPr+g/BgwBB4ERYDTJ0fp4CLhSStkPvANOl1JmgSfAeB1v+Rfz6gemgVPAEWDX33trSdo4DNCStH4MJJmnCbNvgBu1PpXkOfAI2EMTgn9nrH6e0Vx7PdzV51UpZb5+fwoMtpjfcO3/sjTX2N5s0VeSNo2+fz0BSdJ3y6WUke5CkmPAceBwKeVjkvtAf49xAlwqpVxbMdYg8Lmr9BX4abvGGpSW7SVp03EFWpLWt+3A2xqeh4FDXc++JNmySp97wESSrQBJdifZ2eN/3gPberRZAvYm2Vd/n+09fUnafAzQkrS+3QX6kiwAF2m2cXRcBxY6hwg7SilzwG3gYZJFYJbe4XgGuLraIcKucT8Bk8Cdeojw9R+8jyRteGm2sUmSJElaC1egJUmSpBYM0JIkSVILBmhJkiSpBQO0JEmS1IIBWpIkSWrBAC1JkiS1YICWJEmSWvgGED8z1eXyZtQAAAAASUVORK5CYII=\n",

      "text/plain": [

       "<Figure size 864x720 with 1 Axes>"

      ]

     },

     "metadata": {

      "needs_background": "light"

     },

     "output_type": "display_data"

    }

   ],

   "source": [

    "sio.savemat('Result/Res_ParallelCNN.mat',{\"loss\":fold_loss,\"acc\":fold_acc,\"val loss\":fold_vloss,\"val acc\":fold_vacc})\n",

    "\n",

    "fig = plt.figure(figsize=(12,10))\n",

    "plt.grid()\n",

    "plt.boxplot(fold_vacc)\n",

    "plt.suptitle('Cross-Validation Accuracy\\n Parallel CNN')\n",

    "ax = plt.gca()\n",

    "plt.xlabel('Patient id')\n",

    "plt.ylabel('Accuracy')\n",

    "plt.savefig('Result/ParallelCNN.png')\n",

    "plt.show()"

   ]

  },

  {

   "cell_type": "markdown",

   "metadata": {},

   "source": [

    "## CNN Temporal Model (B)"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": 276,

   "metadata": {},

   "outputs": [],

   "source": [

    "class TemporalCNN(nn.Module):\n",

    "    def __init__(self):\n",

    "        super(TemporalCNN, self).__init__()\n",

    "        self.conv1 = nn.Conv2d(3,3,3)\n",

    "        self.conv2 = nn.Conv2d(3,3,5)\n",

    "        self.conv3 = nn.Conv2d(3,3,3)\n",

    "        self.conv4 = nn.Conv2d(3,3,5)\n",

    "        self.conv5 = nn.Conv2d(3,3,3)\n",

    "        self.conv6 = nn.Conv2d(3,3,5)\n",

    "        self.conv7 = nn.Conv2d(3,3,3)\n",

    "        self.conv8 = nn.Conv2d(3,3,5)\n",

    "        self.conv9 = nn.Conv2d(3,3,3)\n",

    "        self.conv10 = nn.Conv2d(3,3,5)\n",

    "        self.conv11 = nn.Conv2d(3,3,3)\n",

    "        self.conv12 = nn.Conv2d(3,3,5)\n",

    "        self.conv13 = nn.Conv2d(3,3,3)\n",

    "        self.conv14 = nn.Conv2d(3,3,5)     \n",

    "        self.pool1 = nn.MaxPool2d(2)\n",

    "        self.pool2 = nn.MaxPool2d(2)   \n",

    "        self.conv15 = nn.Conv2d(3,3,5)\n",

    "        self.conv16 = nn.Conv2d(3,3,7)\n",

    "        self.fc1 = nn.Linear(120,512)\n",

    "        self.fc2 = nn.Linear(512,4)\n",

    "        self.max = nn.Softmax()\n",

    "    \n",

    "    def forward(self, x):\n",

    "        batch_size = x.shape[0]\n",

    "        tmp = torch.zeros(batch_size, x.shape[1], x.shape[2],26,26).cuda()\n",

    "        tmp[:,0] = F.relu(self.conv2(F.relu(self.conv1(x[:,0]))))\n",

    "        tmp[:,1] = F.relu(self.conv4(F.relu(self.conv3(x[:,1]))))\n",

    "        tmp[:,2] = F.relu(self.conv6(F.relu(self.conv5(x[:,2]))))\n",

    "        tmp[:,3] = F.relu(self.conv8(F.relu(self.conv7(x[:,3]))))\n",

    "        tmp[:,4] = F.relu(self.conv10(F.relu(self.conv9(x[:,4]))))\n",

    "        tmp[:,5] = F.relu(self.conv12(F.relu(self.conv11(x[:,5]))))\n",

    "        tmp[:,6] = F.relu(self.conv14(F.relu(self.conv13(x[:,6]))))\n",

    "        x = torch.zeros(batch_size, x.shape[1], x.shape[2],26,26).cuda()\n",

    "        for i in range(7):\n",

    "            x[:,i] = tmp[:,i]\n",

    "        #x[:,0] = F.relu(self.conv1(x[:,0]))\n",

    "        #x[:,1] = F.relu(self.conv3(x[:,1]))\n",

    "        #x[:,2] = F.relu(self.conv5(x[:,2]))\n",

    "        #x[:,3] = F.relu(self.conv7(x[:,3]))\n",

    "        #x[:,4] = F.relu(self.conv9(x[:,4]))\n",

    "        #x[:,5] = F.relu(self.conv11(x[:,5]))\n",

    "        #x[:,6] = F.relu(self.conv13(x[:,6]))\n",

    "        #x[:,0] = F.relu(self.conv2(x[:,0]))\n",

    "        #x[:,1] = F.relu(self.conv4(x[:,1]))\n",

    "        #x[:,2] = F.relu(self.conv6(x[:,2]))\n",

    "        #x[:,3] = F.relu(self.conv8(x[:,3]))\n",

    "        #x[:,4] = F.relu(self.conv10(x[:,4]))\n",

    "        #x[:,5] = F.relu(self.conv12(x[:,5]))\n",

    "        #x[:,6] = F.relu(self.conv14(x[:,6]))\n",

    "        #x = x[:,:,:,3:29,3:29]\n",

    "        #tmp = torch.zeros(batch_size, x.shape[2], x.shape[1]*x.shape[3],x.shape[4]).cuda()\n",

    "        #for i in range(x.shape[1]):\n",

    "        #    tmp[:,:,i*x.shape[3]:(i+1)*x.shape[3], :] = x[:,i]\n",

    "        x = x.reshape(batch_size, x.shape[2], x.shape[1]*x.shape[3],-1) # img reshape\n",

    "        x = self.pool1(x)\n",

    "        x = F.relu(self.conv15(x))\n",

    "        x = F.relu(self.conv16(x))\n",

    "        x = self.pool2(x)\n",

    "        x = x.view(batch_size,-1)\n",

    "        x = self.fc1(x)\n",

    "        x = self.fc2(x)\n",

    "        x = self.max(x)\n",

    "        return x"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": 277,

   "metadata": {},

   "outputs": [

    {

     "name": "stderr",

     "output_type": "stream",

     "text": [

      "c:\\users\\victo\\.conda\\envs\\pytorch_eeg\\lib\\site-packages\\ipykernel_launcher.py:65: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n"

     ]

    },

    {

     "data": {

      "text/plain": [

       "tensor([[0.2572, 0.2522, 0.2505, 0.2401],\n",

       "        [0.2591, 0.2499, 0.2525, 0.2385]], device='cuda:0',\n",

       "       grad_fn=<SoftmaxBackward>)"

      ]

     },

     "execution_count": 277,

     "metadata": {},

     "output_type": "execute_result"

    }

   ],

   "source": [

    "net = TemporalCNN().cuda()\n",

    "net(torch.from_numpy(tmp[0:2]).to(torch.float32).cuda())"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": 280,

   "metadata": {

    "scrolled": true

   },

   "outputs": [

    {

     "name": "stdout",

     "output_type": "stream",

     "text": [

      "Begin Training Fold 1/5\t of Patient 1\n"

     ]

    },

    {

     "name": "stderr",

     "output_type": "stream",

     "text": [

      "c:\\users\\victo\\.conda\\envs\\pytorch_eeg\\lib\\site-packages\\ipykernel_launcher.py:65: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n"

     ]

    },

    {

     "name": "stdout",

     "output_type": "stream",

     "text": [

      "[1, 100] loss: 1.424\tAccuracy : 0.327\t\tval-loss: 1.384\tval-Accuracy : 0.243\n",

      "[11, 100] loss: 1.422\tAccuracy : 0.320\t\tval-loss: 1.384\tval-Accuracy : 0.270\n",

      "[21, 100] loss: 1.420\tAccuracy : 0.320\t\tval-loss: 1.384\tval-Accuracy : 0.270\n",

      "[31, 100] loss: 1.418\tAccuracy : 0.320\t\tval-loss: 1.383\tval-Accuracy : 0.270\n",

      "[41, 100] loss: 1.417\tAccuracy : 0.320\t\tval-loss: 1.383\tval-Accuracy : 0.270\n",

      "[51, 100] loss: 1.415\tAccuracy : 0.320\t\tval-loss: 1.383\tval-Accuracy : 0.270\n",

      "[61, 100] loss: 1.413\tAccuracy : 0.320\t\tval-loss: 1.383\tval-Accuracy : 0.270\n",

      "[71, 100] loss: 1.411\tAccuracy : 0.320\t\tval-loss: 1.383\tval-Accuracy : 0.270\n",

      "[81, 100] loss: 1.409\tAccuracy : 0.320\t\tval-loss: 1.383\tval-Accuracy : 0.270\n",

      "[91, 100] loss: 1.408\tAccuracy : 0.320\t\tval-loss: 1.383\tval-Accuracy : 0.270\n",

      "Finish Training Fold 1/5\t of Patient 1\n",

      "Begin Training Fold 2/5\t of Patient 1\n",

      "[1, 100] loss: 1.427\tAccuracy : 0.265\t\tval-loss: 1.390\tval-Accuracy : 0.216\n"

     ]

    },

    {

     "ename": "KeyboardInterrupt",

     "evalue": "",

     "output_type": "error",

     "traceback": [

      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",

      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",

      "\u001b[1;32m<ipython-input-280-7232a97298b5>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m     37\u001b[0m                 \u001b[0moutputs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mCNN\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtorch\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfrom_numpy\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX_train\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m:\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m+\u001b[0m\u001b[0mbatchsize\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mto\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtorch\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfloat32\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcuda\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     38\u001b[0m                 \u001b[0mloss\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcriterion\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0moutputs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtorch\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfrom_numpy\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0my_train\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m:\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m+\u001b[0m\u001b[0mbatchsize\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mto\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtorch\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlong\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcuda\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 39\u001b[1;33m                 \u001b[0mloss\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     40\u001b[0m                 \u001b[0moptimizer\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     41\u001b[0m                 \u001b[0mrunning_loss\u001b[0m \u001b[1;33m+=\u001b[0m \u001b[0mloss\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mitem\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",

      "\u001b[1;32mc:\\users\\victo\\.conda\\envs\\pytorch_eeg\\lib\\site-packages\\torch\\tensor.py\u001b[0m in \u001b[0;36mbackward\u001b[1;34m(self, gradient, retain_graph, create_graph)\u001b[0m\n\u001b[0;32m    164\u001b[0m                 \u001b[0mproducts\u001b[0m\u001b[1;33m.\u001b[0m \u001b[0mDefaults\u001b[0m \u001b[0mto\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;31m`\u001b[0m\u001b[1;32mFalse\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;31m`\u001b[0m\u001b[1;33m.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    165\u001b[0m         \"\"\"\n\u001b[1;32m--> 166\u001b[1;33m         \u001b[0mtorch\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mautograd\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mgradient\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    167\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    168\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mregister_hook\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mhook\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",

      "\u001b[1;32mc:\\users\\victo\\.conda\\envs\\pytorch_eeg\\lib\\site-packages\\torch\\autograd\\__init__.py\u001b[0m in \u001b[0;36mbackward\u001b[1;34m(tensors, grad_tensors, retain_graph, create_graph, grad_variables)\u001b[0m\n\u001b[0;32m     97\u001b[0m     Variable._execution_engine.run_backward(\n\u001b[0;32m     98\u001b[0m         \u001b[0mtensors\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mgrad_tensors\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 99\u001b[1;33m         allow_unreachable=True)  # allow_unreachable flag\n\u001b[0m\u001b[0;32m    100\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    101\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",

      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "

     ]

    }

   ],

   "source": [

    "    if opti=='SGD':\n",

    "        optimizer = optim.SGD(net.parameters(), lr=learning_rate, momentum=0.9)\n",

    "    elif opti =='Adam':\n",

    "        optimizer = optip = 0\n",

    "n_fold = 5    \n",

    "n_patient = len(np.unique(Patient))\n",

    "fold_vloss = np.zeros((n_fold,n_patient))\n",

    "fold_loss = np.zeros((n_fold,n_patient))\n",

    "fold_vacc = np.zeros((n_fold,n_patient))\n",

    "fold_acc = np.zeros((n_fold,n_patient))\n",

    "for patient in np.unique(Patient):\n",

    "    id_patient = np.arange(len(tmp))[Patient==patient]\n",

    "\n",

    "    length = len(id_patient)\n",

    "    \n",

    "    train_id, test_id = kfold(length,n_fold)\n",

    "    \n",

    "    for fold in range(n_fold):\n",

    "        X_train = tmp[id_patient[train_id[fold]]]\n",

    "        X_test = tmp[id_patient[test_id[fold]]]\n",

    "        y_train = Label[id_patient[train_id[fold]]]\n",

    "        y_test = Label[id_patient[test_id[fold]]] \n",

    "\n",

    "        print(\"Begin Training Fold %d/%d\\t of Patient %d\" % \n",

    "             (fold+1,n_fold, patient))\n",

    "\n",

    "        CNN = TemporalCNN().cuda(0)\n",

    "        criterion = nn.CrossEntropyLoss()\n",

    "        optimizer = optim.SGD(CNN.parameters(), lr=0.001)\n",

    "#        optimizer = optim.SGD(CNN.parameters(), lr=0.001, momentum=0.9)\n",

    "\n",

    "        n_epochs = 100\n",

    "        for epoch in range(n_epochs):\n",

    "            running_loss = 0.0\n",

    "            batchsize = 4\n",

    "            for i in range(int(len(y_train)/batchsize)):\n",

    "                optimizer.zero_grad()\n",

    "\n",

    "                # forward + backward + optimize\n",

    "                outputs = CNN(torch.from_numpy(X_train[i:i+batchsize]).to(torch.float32).cuda())\n",

    "                loss = criterion(outputs, torch.from_numpy(y_train[i:i+batchsize]).to(torch.long).cuda())\n",

    "                loss.backward()\n",

    "                optimizer.step()\n",

    "                running_loss += loss.item()\n",

    "\n",

    "            #acc\n",

    "            _, idx = torch.max(CNN(torch.from_numpy(X_train[:]).to(torch.float32).cuda()).data,1)\n",

    "            acc = (idx == torch.from_numpy(y_train).cuda()).sum().item()/len(y_train)\n",

    "\n",

    "            #val Loss\n",

    "            val_outputs = CNN(torch.from_numpy(X_test[:]).to(torch.float32).cuda())\n",

    "            val_loss = criterion(val_outputs, torch.from_numpy(y_test[:]).to(torch.long).cuda())\n",

    "            _, idx = torch.max(val_outputs.data,1)\n",

    "            val_acc = (idx == torch.from_numpy(y_test).cuda()).sum().item()/len(y_test)\n",

    "\n",

    "            if epoch%10==0:\n",

    "                print('[%d, %3d] loss: %.3f\\tAccuracy : %.3f\\t\\tval-loss: %.3f\\tval-Accuracy : %.3f' %\n",

    "             (epoch+1, n_epochs, running_loss/i, float(acc), val_loss, val_acc))\n",

    "        fold_vloss[fold, p ] = val_loss.item()\n",

    "        fold_loss[fold, p] = running_loss/i\n",

    "        fold_vacc[fold, p] = val_acc\n",

    "        fold_acc[fold, p] = acc\n",

    "        print('Finish Training Fold %d/%d\\t of Patient %d' % \n",

    "             (fold+1,n_fold, patient))\n",

    "    print('loss: %.3f\\tAccuracy : %.3f\\t\\tval-loss: %.3f\\tval-Accuracy : %.3f' %\n",

    "                 (np.mean(fold_loss[:,p]), np.mean(fold_acc[:,p]), np.mean(fold_vloss[:,p]),np.mean(fold_vacc[:,p])))\n",

    "    \n",

    "    p = p + 1"

   ]

  },

  {

   "cell_type": "markdown",

   "metadata": {},

   "source": [

    "### Peresented Results"

   ]