2851 lines (2850 with data), 176.3 kB
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"import torch\n",
"import torch.nn as nn\n",
"from torch.utils.data import TensorDataset\n",
"import torch.optim as optim\n",
"from torch.optim import lr_scheduler\n",
"import numpy as np\n",
"import torchvision\n",
"import torch.nn.functional as F\n",
"from torch.utils.data.sampler import SubsetRandomSampler\n",
"from torch.utils.data import DataLoader\n",
"from torchvision import datasets, models, transforms\n",
"from torchvision.transforms import Resize, ToTensor, Normalize\n",
"import matplotlib.pyplot as plt\n",
"#from imblearn.under_sampling import RandomUnderSampler\n",
"\n",
"from sklearn.metrics import accuracy_score, precision_recall_fscore_support, confusion_matrix, roc_auc_score, \\\n",
" average_precision_score\n",
"from sklearn.model_selection import train_test_split\n",
"import time\n",
"import os\n",
"from pathlib import Path\n",
"from skimage import io\n",
"import copy\n",
"from torch import optim, cuda\n",
"import pandas as pd\n",
"import glob\n",
"from collections import Counter\n",
"# Useful for examining network\n",
"from functools import reduce\n",
"from operator import __add__\n",
"# from torchsummary import summary\n",
"import seaborn as sns\n",
"import warnings\n",
"warnings.filterwarnings('ignore', category=FutureWarning)\n",
"from PIL import Image\n",
"from timeit import default_timer as timer\n",
"import matplotlib.pyplot as plt\n",
"import gc\n",
"import cv2\n",
"from scipy import stats"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"from torch.optim.lr_scheduler import StepLR"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
"def add_gaussian_noise(signal):\n",
" noise = np.random.normal(0, 0.05, signal.shape)\n",
" return (signal + noise)"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"make train_df_available 3.213082475\n",
"make train_df_available 3.222219173\n",
"Successfully loaded pre-built trainData and evalData\n",
"time to generate arrays: 0.0005355969997253851\n"
]
}
],
"source": [
"def split_train_eval_test(metadata_df_path):\n",
" df = pd.read_csv(metadata_df_path)\n",
"\n",
" pat_ids = list(set(df['PatientID']))\n",
"\n",
" permuted_pat_ids = np.random.permutation(pat_ids)\n",
"\n",
" last_train_idx = int(len(permuted_pat_ids) * .80)\n",
"\n",
" last_eval_idx = int(last_train_idx +\n",
" ((len(permuted_pat_ids) - last_train_idx) * .70))\n",
"\n",
" train_pat_ids = permuted_pat_ids[:last_train_idx]\n",
" eval_pat_ids = permuted_pat_ids[last_train_idx:last_eval_idx]\n",
" test_pat_ids = permuted_pat_ids[last_eval_idx:]\n",
"\n",
" df_train = df[df['PatientID'].isin(train_pat_ids)]\n",
" df_eval = df[df['PatientID'].isin(eval_pat_ids)]\n",
" df_test = df[df['PatientID'].isin(test_pat_ids)]\n",
"\n",
" print(\n",
" \"Train pts/pct: {}, {} \\n Eval pts/pct: {}, {} \\n Test pts/pct: {}, {}\"\n",
" .format(len(df_train),\n",
" len(df_train) / float(len(df)), len(df_eval),\n",
" len(df_eval) / float(len(df)), len(df_test),\n",
" len(df_test) / float(len(df))))\n",
"\n",
" df_train.to_csv('/data/ECGnet/data/train_metadata_full_no_balance_DEIDENTIFIED.csv', index=False)\n",
" df_eval.to_csv('/data/ECGnet/data/eval_metadata_full_no_balance_DEIDENTIFIED.csv', index=False)\n",
" df_test.to_csv('/data/ECGnet/data/test_metadata_full_no_balance_DEIDENTIFIED.csv', index=False)\n",
"\n",
"\n",
"# train_subsample_path = \"/data/ECGnet/data/train_metadata_full_no_balance_DEIDENTIFIED.csv\"\n",
"# if not Path(train_subsample_path).exists():\n",
"# split_train_eval_test('/data/proj/cardiac-amyloid/amyloid_simple_run_DEIDENTIFIED.csv')\n",
"\n",
"train_df = pd.read_csv('/data/ECGnet/data/train_df_pace_removed_any_patient_negative_DEIDENTIFIED.csv')\n",
"eval_df = pd.read_csv('/data/ECGnet/data/eval_df_pace_removed_any_patient_negative_DEIDENTIFIED.csv')\n",
"\n",
"\n",
"def balance_binary(df, label, sampling_strategy=1):\n",
" y = df[label]\n",
" X = df\n",
" binary_mask = np.bitwise_or(y == 0, y == 1)\n",
" binary_y = y[binary_mask]\n",
" binary_X = X[binary_mask]\n",
" rus = RandomUnderSampler(sampling_strategy=sampling_strategy)\n",
" X_res, y_res = rus.fit_resample(binary_X, binary_y)\n",
" return X_res\n",
"\n",
"train_available_path = '/data/ECGnet/data/train_df_pace_removed_any_patient_negative_available_DEIDENTIFIED.csv'\n",
"print(\"make train_df_available\", time.process_time())\n",
"if not Path(train_available_path).exists():\n",
" print(\"No train_df_available found, creating now...\")\n",
" npyfilespath = \"/data/proj/cardiac-amyloid/ekg_waveforms_output/\"\n",
" os.chdir(npyfilespath)\n",
" npfiles = glob.glob(\"*.npy\")\n",
" npfiles.sort()\n",
" npdf = pd.DataFrame({'filename': npfiles})\n",
" train_df_available = train_df.merge(npdf)\n",
" eval_df_available = eval_df.merge(npdf)\n",
"\n",
" # print(\"balance datasets\", time.process_time())\n",
" label = \"ANY_AMYLOID\"\n",
" # train_df_available = balance_binary(train_df_available, label)\n",
" # eval_df_available = balance_binary(eval_df_available,label)\n",
" train_df_available = train_df_available\n",
" eval_df_available = eval_df_available\n",
" train_df_available.to_csv(\"/data/ECGnet/data/train_df_pace_removed_any_patient_negative_available_DEIDENTIFIED.csv\")\n",
" eval_df_available.to_csv(\"/data/ECGnet/data/eval_df_pace_removed_any_patient_negative_available_DEIDENTIFIED.csv\")\n",
" # print(\"balance datasets\", time.process_time())\n",
"else:\n",
" train_df_available = pd.read_csv('/data/ECGnet/data/train_df_pace_removed_any_patient_negative_available_DEIDENTIFIED.csv')\n",
" eval_df_available = pd.read_csv('/data/ECGnet/data/eval_df_pace_removed_any_patient_negative_available_DEIDENTIFIED.csv')\n",
"print(\"make train_df_available\", time.process_time())\n",
"\n",
"# filelist_train = train_df_available['filename']\n",
"# filelist_eval = eval_df_available['filename']\n",
"#\n",
"# print('train DF length:', len(filelist_train), '\\n eval DF length:', len(filelist_eval))\n",
"# print('train ECG count:', len(train_df_available), '\\n eval ECG count:', len(eval_df_available))\n",
"\n",
"npyfilespath = \"/data/proj/cardiac-amyloid/ekg_waveforms_output/\"\n",
"fpath_train = \"/data/ECGnet/data/amyloid_concat_train_pace_removed_any_patient_negative.npy\"\n",
"fpath_eval = \"/data/ECGnet/data/amyloid_concat_eval_pace_removed_any_patient_negative.npy\"\n",
"\n",
"try:\n",
" trainData = np.load(fpath_train) # Load input data. Input data should be compiled as numpy arrays with (sample numbers, time, lead, 1)\n",
" evalData = np.load(fpath_eval)\n",
" print('Successfully loaded pre-built trainData and evalData')\n",
"except:\n",
" trainData = []\n",
" evalData = []\n",
"\n",
"time1 = timer()\n",
"if not Path(fpath_train).exists(): # or (len(trainData)!=len(train_df_available)) or (len(evalData)!=len(eval_df_available)):\n",
" print(\"no train array found, building now...\")\n",
" for npfile in filelist_train:\n",
" i = 0\n",
" try:\n",
" path = os.path.join(npyfilespath + npfile)\n",
" file = np.load(path)\n",
" trainData.append(file)\n",
" i += 1\n",
" if i % 1 == 100:\n",
" print(\"{i} EKGs have been written to array\")\n",
" except:\n",
" continue\n",
" trainData = np.array(trainData)\n",
" np.save(fpath_train, trainData)\n",
"\n",
"if not Path(fpath_eval).exists(): # or (len(trainData)!=len(train_df_available)) or (len(evalData)!=len(eval_df_available)):\n",
" print(\"no eval array found, building now...\")\n",
" for npfile in filelist_eval:\n",
" i = 0\n",
" try:\n",
" path = os.path.join(npyfilespath + npfile)\n",
" file = np.load(path)\n",
" evalData.append(file)\n",
" i += 1\n",
" if i % 1 == 100:\n",
" print(\"{i} EKGs have been written to array\")\n",
" except:\n",
" continue\n",
" evalData = np.array(evalData)\n",
" np.save(fpath_eval, evalData)\n",
"print(\"time to generate arrays: \", timer() - time1)"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"trainData initial shape: (4465, 2500, 12, 1)\n",
"trainData norm_prep shape: (4465, 12, 2500, 1)\n",
"(4465, 12, 2500, 1) (12,) (12,)\n",
"Mean and STD after normalization [ 1.04420378e-13 -4.35152501e-15 3.41003085e-14 -1.21242727e-14\n",
" 1.14753556e-14 -2.43490832e-17 -1.67866714e-14 -1.79234147e-13\n",
" -2.84146375e-13 7.58005498e-14 1.49721463e-14 -3.36270833e-14] [1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1.]\n",
"Median and MAD after normalization [ 1.04420378e-13 -4.35152501e-15 3.41003085e-14 -1.21242727e-14\n",
" 1.14753556e-14 -2.43490832e-17 -1.67866714e-14 -1.79234147e-13\n",
" -2.84146375e-13 7.58005498e-14 1.49721463e-14 -3.36270833e-14] [1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1.]\n",
"trainData shape for model: (4465, 1, 2500, 12)\n",
"evalData shape for model: (934, 1, 2500, 12)\n",
"X_train: (4465, 1, 2500, 12)\n",
"X_test: (934, 1, 2500, 12)\n",
"train label count: Counter({0: 3479, 1: 986}) \n",
" eval label count: Counter({0: 706, 1: 228})\n"
]
}
],
"source": [
"trainData = np.array(trainData)\n",
"evalData = np.array(evalData)\n",
"print('trainData initial shape:', trainData.shape)\n",
"\n",
"trainData = np.transpose(trainData, axes=[0, 2, 1, 3])\n",
"evalData = np.transpose(evalData, axes=[0, 2, 1, 3])\n",
"print('trainData norm_prep shape:', trainData.shape)\n",
"assert trainData.shape[1:] == (12, 2500, 1), \"train is not X,12,2500\"\n",
"assert evalData.shape[1:] == (12, 2500, 1), \"eval is not X,12,2500\"\n",
"#mean_tr, std_tr = np.mean(trainData, axis=(0,2,3)), np.std(trainData, axis=(0,2,3))\n",
"\n",
"# Normalization\n",
"\n",
"# print(\"eval sum/mean before norm: \", evalData.sum(),evalData.mean())\n",
"# evalData = normalize_array(trainData, evalData)\n",
"# print(\"eval sum/mean after norm: \", evalData.sum(),evalData.mean())\n",
"#\n",
"# print(\"train sum/mean before norm: \", trainData.sum(), trainData.mean())\n",
"# trainData = normalize_array(trainData, trainData)\n",
"# print(\"train sum/mean after norm: \", trainData.sum(), trainData.mean())\n",
"\n",
"def normalize_array(x, y):\n",
" xmax, xmean, xmin = x.max(), x.mean(), x.min()\n",
" return (y - xmean) / (xmax - xmin)\n",
"\n",
"def normalization_with_limits(source_array, target_array):\n",
" mean = np.mean(source_array)\n",
" sd = np.std(source_array)\n",
" datasdlimit = target_array\n",
" datasdlimit = np.where(datasdlimit > mean + (3*sd), mean + (3*sd), datasdlimit)\n",
" datasdlimit = np.where(datasdlimit < mean - (3*sd), mean - (3*sd), datasdlimit)\n",
" datanorm2 = cv2.normalize(datasdlimit, datasdlimit, -1, 1, cv2.NORM_MINMAX)\n",
" return datanorm2\n",
"\n",
"def truncate_data(source_array, target_array):\n",
" mean = np.mean(source_array)\n",
" sd = np.std(source_array)\n",
" datasdlimit = target_array\n",
" datasdlimit = np.where(datasdlimit > mean + (3*sd), mean + (3*sd), datasdlimit)\n",
" datasdlimit = np.where(datasdlimit < mean - (3*sd), mean - (3*sd), datasdlimit)\n",
" #datanorm2 = cv2.normalize(datasdlimit, datasdlimit, -1, 1, cv2.NORM_MINMAX)\n",
" return datasdlimit\n",
"\n",
"def normalization_with_limits_median(source_array, target_array):\n",
" median = np.median(source_array)\n",
" mad = stats.median_absolute_deviation(source_array)\n",
" datamadlimit = target_array\n",
" #Truncate data to 3*mad to limit outlier effect and use robust statistical approach over mean\n",
" datamadlimit = np.where(datamadlimit > median + (3*mad), median + (3*mad), datamadlimit)\n",
" datamadlimit = np.where(datamadlimit < median - (3*mad), median - (3*mad), datamadlimit)\n",
" datanorm2 = cv2.normalize(datamadlimit, datamadlimit, -1, 1, cv2.NORM_MINMAX)\n",
" return datanorm2\n",
"\n",
"#Per-lead normalization using mean and standard deviation\n",
"# mean_tr, std_tr = np.mean(trainData, axis=(0,2,3)), np.std(trainData, axis=(0,2,3))\n",
"# print(trainData.shape, mean_tr.shape, std_tr.shape)\n",
"# for i in range(trainData.shape[1]):\n",
"# trainData[:,i,:,:] = (trainData[:,i,:,:] - mean_tr[i]) / std_tr[i]\n",
"# evalData[:,i,:,:] = (evalData[:,i,:,:] - mean_tr[i]) / std_tr[i]\n",
"\n",
"def truncate_data_median(source_array, target_array):\n",
" median = np.median(source_array)\n",
" mad = stats.median_absolute_deviation(source_array)\n",
" datamadlimit = target_array\n",
" #Truncate data to 3*mad to limit outlier effect and use robust statistical approach over mean\n",
" datamadlimit = np.where(datamadlimit > median + (3*mad), median + (3*mad), datamadlimit)\n",
" datamadlimit = np.where(datamadlimit < median - (3*mad), median - (3*mad), datamadlimit)\n",
" # datanorm2 = cv2.normalize(datamadlimit, datamadlimit, -1, 1, cv2.NORM_MINMAX)\n",
" return datamadlimit\n",
"\n",
"evalData = truncate_data(trainData, evalData)\n",
"trainData = truncate_data(trainData, trainData)\n",
"\n",
"mean_tr, std_tr = np.mean(trainData, axis=(0,2,3)), np.std(trainData, axis=(0,2,3))\n",
"#med_va, mad_va = np.median(evalData, axis=(0,2,3)), stats.std(evalData, axis=(0,2,3))\n",
"print(trainData.shape, mean_tr.shape, std_tr.shape)\n",
"for i in range(trainData.shape[1]):\n",
" trainData[:,i,:,:] = (trainData[:,i,:,:] - mean_tr[i]) / std_tr[i]\n",
" evalData[:,i,:,:] = (evalData[:,i,:,:] - mean_tr[i]) / std_tr[i]\n",
"\n",
"\n",
"mean_norm, std_norm = np.mean(trainData, axis=(0,2,3)), np.std(trainData, axis=(0,2,3))\n",
"print(\"Mean and STD after normalization\", mean_norm, std_norm)\n",
"med_norm, mad_norm = np.median(trainData, axis=(0,2,3)), stats.median_absolute_deviation(trainData, axis=(0,2,3))\n",
"print(\"Median and MAD after normalization\", mean_norm, std_norm)\n",
"\n",
"trainData = np.transpose(trainData, axes=[0, 3, 2, 1])\n",
"print('trainData shape for model:', trainData.shape)\n",
"evalData = np.transpose(evalData, axes=[0, 3, 2, 1])\n",
"print('evalData shape for model:', evalData.shape)\n",
"\n",
"assert trainData.shape[1:] == (1, 2500, 12), \"train is not X,12,2500\"\n",
"assert evalData.shape[1:] == (1, 2500, 12), \"eval is not X,12,2500\"\n",
"\n",
"\n",
"\n",
"#MODIFYING DATA TO SIMPLIFY\n",
"def ecg_modifier(data, labels, list):\n",
" for i,ecg in enumerate(data):\n",
" if labels[i]==1:\n",
" list.append(ecg)\n",
" else:\n",
" list.append(ecg)\n",
"\n",
"trainlist = []\n",
"evallist = []\n",
"\n",
"# ecg_modifier(trainData,y_train,trainlist)\n",
"# trainData = np.array(trainlist)\n",
"# ecg_modifier(evalData,y_test,evallist)\n",
"# evalData = np.array(evallist)\n",
"\n",
"y_train = [round(x) for x in train_df_available['ANY_AMYLOID']]\n",
"y_train = np.array(y_train)\n",
"\n",
"y_test = [round(x) for x in eval_df_available['ANY_AMYLOID']]\n",
"y_test = np.array(y_test)\n",
"\n",
"# Change the place holder names\n",
"X_train = np.array(trainData)\n",
"X_test = np.array(evalData)\n",
"\n",
"# Return unique values and count\n",
"# u_train, train_counts = np.unique(label_train, return_counts=True)\n",
"# u_eval, eval_counts = np.unique(label_eval, return_counts=True)\n",
"# print('Training class counts:', train_counts)\n",
"# print('Evaluation class counts:', eval_counts)\n",
"print('X_train:', X_train.shape)\n",
"print('X_test:', X_test.shape)\n",
"print('train label count: ', Counter(y_train), '\\n eval label count: ', Counter(y_test))"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [],
"source": [
"SHAPE = (1, 12, 2500) # Shape of the input. ECG model takes 2500 points sampled at 4 micro-seconds/point for 12 lead.\n",
"batch_size = 10\n",
"dlen = X_train.shape[0]\n",
"\n",
"# print(X_test.shape, y_test.shape)\n",
"# print(\"X_test.size(0): \", X_test.size(0), \"y_test.size(0): \", y_test.size(0))\n",
"y_test = torch.FloatTensor(y_test)\n",
"X_test = TensorDataset(torch.from_numpy(X_test), y_test)\n",
"test_loader = DataLoader(X_test, batch_size=batch_size, pin_memory=True, shuffle=True)\n",
"# a,b=next(iter(test_loader))\n",
"\n",
"y_train = torch.FloatTensor(y_train)\n",
"X_train = TensorDataset(torch.from_numpy(X_train), y_train)\n",
"train_loader = DataLoader(X_train, batch_size=batch_size, pin_memory=True, shuffle=True)"
]
},
{
"cell_type": "code",
"execution_count": 25,
"metadata": {},
"outputs": [],
"source": [
"try:\n",
" del model\n",
" gc.collect()\n",
" torch.cuda.empty_cache()\n",
"except:\n",
" pass"
]
},
{
"cell_type": "code",
"execution_count": 26,
"metadata": {},
"outputs": [],
"source": [
"class Net(nn.Module):\n",
"\n",
" # Convolution as a whole should span at least 1 beat, preferably more\n",
" # Input shape = (1,2500,12)\n",
" # summary(model, input_size=(1, 2500, 12))\n",
" # CLEAR EXPERIMENTS TO TRY\n",
" # Alter kernel size, right now drops to 10 channels at beginning then stays there\n",
" # Try increasing output channel size as you go deeper\n",
" # Alter stride to have larger image at FC layer\n",
"\n",
" def __init__(self):\n",
" super(Net, self).__init__()\n",
"\n",
" base_conv = 64\n",
" self.conv1 = nn.Sequential(\n",
" nn.Conv2d(in_channels=1, out_channels=base_conv, kernel_size=(12,3), stride=1),\n",
" nn.ReLU(),\n",
" nn.BatchNorm2d(base_conv),\n",
" nn.Conv2d(in_channels=base_conv, out_channels=base_conv, kernel_size=(1, 1), stride=1),\n",
" nn.ReLU(),\n",
" nn.BatchNorm2d(base_conv),\n",
" nn.MaxPool2d(kernel_size=(3, 1), stride=(3, 1))\n",
" )\n",
"\n",
" self.conv2 = nn.Sequential(\n",
" nn.Conv2d(in_channels=base_conv, out_channels=base_conv, kernel_size=(1, 1), stride=1),\n",
" nn.ReLU(),\n",
" nn.BatchNorm2d(base_conv),\n",
" nn.Conv2d(in_channels=base_conv, out_channels=base_conv, kernel_size=(1, 1), stride=1),\n",
" nn.ReLU(),\n",
" nn.BatchNorm2d(base_conv),\n",
" nn.MaxPool2d(kernel_size=(1, 1), stride=(1, 1))\n",
" )\n",
"\n",
" self.conv3 = nn.Sequential(\n",
" nn.Conv2d(in_channels=base_conv, out_channels=base_conv, kernel_size=(1, 1), stride=1),\n",
" nn.ReLU(),\n",
" nn.BatchNorm2d(base_conv),\n",
" nn.Conv2d(in_channels=base_conv, out_channels=base_conv, kernel_size=(1, 1), stride=1),\n",
" nn.ReLU(),\n",
" nn.BatchNorm2d(base_conv),\n",
" nn.MaxPool2d(kernel_size=(3, 1), stride=(3, 1))\n",
" )\n",
"\n",
" self.conv4 = nn.Sequential(\n",
" nn.Conv2d(in_channels=base_conv, out_channels=base_conv, kernel_size=(1, 1), stride=1),\n",
" nn.ReLU(),\n",
" nn.BatchNorm2d(base_conv),\n",
" nn.Conv2d(in_channels=base_conv, out_channels=base_conv, kernel_size=(1, 1), stride=1),\n",
" nn.ReLU(),\n",
" nn.BatchNorm2d(base_conv),\n",
" nn.MaxPool2d(kernel_size=(1, 1), stride=(1, 1))\n",
" )\n",
"\n",
" self.conv5 = nn.Sequential(\n",
" nn.Conv2d(in_channels=base_conv, out_channels=base_conv, kernel_size=(1, 1), stride=1),\n",
" nn.ReLU(),\n",
" nn.BatchNorm2d(base_conv),\n",
" nn.Conv2d(in_channels=base_conv, out_channels=base_conv, kernel_size=(1, 1), stride=1),\n",
" nn.ReLU(),\n",
" nn.BatchNorm2d(base_conv),\n",
" nn.MaxPool2d(kernel_size=(3, 1), stride=(3, 1))\n",
" )\n",
" self.conv6 = nn.Sequential(\n",
" nn.Conv2d(in_channels=base_conv, out_channels=base_conv, kernel_size=(1, 1), stride=1),\n",
" nn.ReLU(),\n",
" nn.BatchNorm2d(base_conv),\n",
" nn.Conv2d(in_channels=base_conv, out_channels=base_conv, kernel_size=(1, 1), stride=1),\n",
" nn.ReLU(),\n",
" nn.BatchNorm2d(base_conv),\n",
" nn.MaxPool2d(kernel_size=(1, 1), stride=(1, 1))\n",
" )\n",
"\n",
" self.conv7 = nn.Sequential(\n",
" nn.Conv2d(in_channels=base_conv, out_channels=base_conv, kernel_size=(1,1), stride=1),\n",
" nn.ReLU(),\n",
" nn.BatchNorm2d(base_conv),\n",
" nn.Conv2d(in_channels=base_conv, out_channels=base_conv, kernel_size=(1,1), stride=1),\n",
" nn.ReLU(),\n",
" nn.BatchNorm2d(base_conv),\n",
" nn.MaxPool2d(kernel_size=(3, 1), stride=(3, 1))\n",
" )\n",
" \n",
" self.conv8 = nn.Sequential(\n",
" nn.Conv2d(in_channels=base_conv, out_channels=base_conv, kernel_size=(1, 1), stride=1),\n",
" nn.ReLU(),\n",
" nn.BatchNorm2d(base_conv),\n",
" nn.Conv2d(in_channels=base_conv, out_channels=base_conv, kernel_size=(1, 1), stride=1),\n",
" nn.ReLU(),\n",
" nn.BatchNorm2d(base_conv),\n",
" nn.MaxPool2d(kernel_size=(1, 1), stride=(1, 1))\n",
" )\n",
"\n",
" self.fc1 = nn.Sequential(\n",
" # nn.AdaptiveAvgPool2d((1, 1)),\n",
" nn.Flatten(),\n",
" nn.Dropout(p=0.5),\n",
" nn.Linear(19200, 4096), # 64 kernel size, 2500 pooled to 29,\n",
" #nn.Dropout(p=0.5),\n",
" nn.ReLU(),\n",
" nn.Dropout(p=0.5),\n",
" nn.Linear(4096, 512),\n",
" #nn.Dropout(p=0.5),\n",
" nn.ReLU(),\n",
" nn.Dropout(p=0.5),\n",
" nn.Linear(512, 32),\n",
" #nn.Dropout(p=0.5),\n",
" nn.ReLU(),\n",
" nn.Dropout(p=0.2),\n",
" nn.Linear(32, 8),\n",
" #nn.Dropout(p=0.2),\n",
" nn.ReLU(),\n",
" nn.Linear(8, 1))\n",
"\n",
" def forward(self, x):\n",
" out = self.conv1(x)\n",
" # print(out.shape)\n",
" out = self.conv2(out)\n",
" # print(out.shape)\n",
" out = self.conv3(out)\n",
" # print(out.shape)\n",
" out = self.conv4(out)\n",
" # print(out.shape)\n",
" out = self.conv5(out)\n",
" # print(out.shape)\n",
" out = self.conv6(out)\n",
" # print(out.shape)\n",
" out = self.conv7(out)\n",
" # print(out.shape)\n",
" out = self.conv8(out)\n",
" #out = self.conv9(out)\n",
" out = self.fc1(out)\n",
" return out"
]
},
{
"cell_type": "code",
"execution_count": 27,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Net(\n",
" (conv1): Sequential(\n",
" (0): Conv2d(1, 64, kernel_size=(12, 3), stride=(1, 1))\n",
" (1): ReLU()\n",
" (2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
" (3): Conv2d(64, 64, kernel_size=(1, 1), stride=(1, 1))\n",
" (4): ReLU()\n",
" (5): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
" (6): MaxPool2d(kernel_size=(3, 1), stride=(3, 1), padding=0, dilation=1, ceil_mode=False)\n",
" )\n",
" (conv2): Sequential(\n",
" (0): Conv2d(64, 64, kernel_size=(1, 1), stride=(1, 1))\n",
" (1): ReLU()\n",
" (2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
" (3): Conv2d(64, 64, kernel_size=(1, 1), stride=(1, 1))\n",
" (4): ReLU()\n",
" (5): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
" (6): MaxPool2d(kernel_size=(1, 1), stride=(1, 1), padding=0, dilation=1, ceil_mode=False)\n",
" )\n",
" (conv3): Sequential(\n",
" (0): Conv2d(64, 64, kernel_size=(1, 1), stride=(1, 1))\n",
" (1): ReLU()\n",
" (2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
" (3): Conv2d(64, 64, kernel_size=(1, 1), stride=(1, 1))\n",
" (4): ReLU()\n",
" (5): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
" (6): MaxPool2d(kernel_size=(3, 1), stride=(3, 1), padding=0, dilation=1, ceil_mode=False)\n",
" )\n",
" (conv4): Sequential(\n",
" (0): Conv2d(64, 64, kernel_size=(1, 1), stride=(1, 1))\n",
" (1): ReLU()\n",
" (2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
" (3): Conv2d(64, 64, kernel_size=(1, 1), stride=(1, 1))\n",
" (4): ReLU()\n",
" (5): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
" (6): MaxPool2d(kernel_size=(1, 1), stride=(1, 1), padding=0, dilation=1, ceil_mode=False)\n",
" )\n",
" (conv5): Sequential(\n",
" (0): Conv2d(64, 64, kernel_size=(1, 1), stride=(1, 1))\n",
" (1): ReLU()\n",
" (2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
" (3): Conv2d(64, 64, kernel_size=(1, 1), stride=(1, 1))\n",
" (4): ReLU()\n",
" (5): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
" (6): MaxPool2d(kernel_size=(3, 1), stride=(3, 1), padding=0, dilation=1, ceil_mode=False)\n",
" )\n",
" (conv6): Sequential(\n",
" (0): Conv2d(64, 64, kernel_size=(1, 1), stride=(1, 1))\n",
" (1): ReLU()\n",
" (2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
" (3): Conv2d(64, 64, kernel_size=(1, 1), stride=(1, 1))\n",
" (4): ReLU()\n",
" (5): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
" (6): MaxPool2d(kernel_size=(1, 1), stride=(1, 1), padding=0, dilation=1, ceil_mode=False)\n",
" )\n",
" (conv7): Sequential(\n",
" (0): Conv2d(64, 64, kernel_size=(1, 1), stride=(1, 1))\n",
" (1): ReLU()\n",
" (2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
" (3): Conv2d(64, 64, kernel_size=(1, 1), stride=(1, 1))\n",
" (4): ReLU()\n",
" (5): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
" (6): MaxPool2d(kernel_size=(3, 1), stride=(3, 1), padding=0, dilation=1, ceil_mode=False)\n",
" )\n",
" (conv8): Sequential(\n",
" (0): Conv2d(64, 64, kernel_size=(1, 1), stride=(1, 1))\n",
" (1): ReLU()\n",
" (2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
" (3): Conv2d(64, 64, kernel_size=(1, 1), stride=(1, 1))\n",
" (4): ReLU()\n",
" (5): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
" (6): MaxPool2d(kernel_size=(1, 1), stride=(1, 1), padding=0, dilation=1, ceil_mode=False)\n",
" )\n",
" (fc1): Sequential(\n",
" (0): Flatten()\n",
" (1): Dropout(p=0.5, inplace=False)\n",
" (2): Linear(in_features=19200, out_features=4096, bias=True)\n",
" (3): ReLU()\n",
" (4): Dropout(p=0.5, inplace=False)\n",
" (5): Linear(in_features=4096, out_features=512, bias=True)\n",
" (6): ReLU()\n",
" (7): Dropout(p=0.5, inplace=False)\n",
" (8): Linear(in_features=512, out_features=32, bias=True)\n",
" (9): ReLU()\n",
" (10): Dropout(p=0.2, inplace=False)\n",
" (11): Linear(in_features=32, out_features=8, bias=True)\n",
" (12): ReLU()\n",
" (13): Linear(in_features=8, out_features=1, bias=True)\n",
" )\n",
")\n"
]
}
],
"source": [
"model = Net()\n",
"print(model)"
]
},
{
"cell_type": "code",
"execution_count": 28,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"(6, 0)"
]
},
"execution_count": 28,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"#model.summary\n",
"torch.cuda.get_device_capability()"
]
},
{
"cell_type": "code",
"execution_count": 29,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Train on: cuda:0\n"
]
},
{
"data": {
"text/plain": [
"Net(\n",
" (conv1): Sequential(\n",
" (0): Conv2d(1, 64, kernel_size=(12, 3), stride=(1, 1))\n",
" (1): ReLU()\n",
" (2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
" (3): Conv2d(64, 64, kernel_size=(1, 1), stride=(1, 1))\n",
" (4): ReLU()\n",
" (5): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
" (6): MaxPool2d(kernel_size=(3, 1), stride=(3, 1), padding=0, dilation=1, ceil_mode=False)\n",
" )\n",
" (conv2): Sequential(\n",
" (0): Conv2d(64, 64, kernel_size=(1, 1), stride=(1, 1))\n",
" (1): ReLU()\n",
" (2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
" (3): Conv2d(64, 64, kernel_size=(1, 1), stride=(1, 1))\n",
" (4): ReLU()\n",
" (5): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
" (6): MaxPool2d(kernel_size=(1, 1), stride=(1, 1), padding=0, dilation=1, ceil_mode=False)\n",
" )\n",
" (conv3): Sequential(\n",
" (0): Conv2d(64, 64, kernel_size=(1, 1), stride=(1, 1))\n",
" (1): ReLU()\n",
" (2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
" (3): Conv2d(64, 64, kernel_size=(1, 1), stride=(1, 1))\n",
" (4): ReLU()\n",
" (5): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
" (6): MaxPool2d(kernel_size=(3, 1), stride=(3, 1), padding=0, dilation=1, ceil_mode=False)\n",
" )\n",
" (conv4): Sequential(\n",
" (0): Conv2d(64, 64, kernel_size=(1, 1), stride=(1, 1))\n",
" (1): ReLU()\n",
" (2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
" (3): Conv2d(64, 64, kernel_size=(1, 1), stride=(1, 1))\n",
" (4): ReLU()\n",
" (5): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
" (6): MaxPool2d(kernel_size=(1, 1), stride=(1, 1), padding=0, dilation=1, ceil_mode=False)\n",
" )\n",
" (conv5): Sequential(\n",
" (0): Conv2d(64, 64, kernel_size=(1, 1), stride=(1, 1))\n",
" (1): ReLU()\n",
" (2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
" (3): Conv2d(64, 64, kernel_size=(1, 1), stride=(1, 1))\n",
" (4): ReLU()\n",
" (5): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
" (6): MaxPool2d(kernel_size=(3, 1), stride=(3, 1), padding=0, dilation=1, ceil_mode=False)\n",
" )\n",
" (conv6): Sequential(\n",
" (0): Conv2d(64, 64, kernel_size=(1, 1), stride=(1, 1))\n",
" (1): ReLU()\n",
" (2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
" (3): Conv2d(64, 64, kernel_size=(1, 1), stride=(1, 1))\n",
" (4): ReLU()\n",
" (5): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
" (6): MaxPool2d(kernel_size=(1, 1), stride=(1, 1), padding=0, dilation=1, ceil_mode=False)\n",
" )\n",
" (conv7): Sequential(\n",
" (0): Conv2d(64, 64, kernel_size=(1, 1), stride=(1, 1))\n",
" (1): ReLU()\n",
" (2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
" (3): Conv2d(64, 64, kernel_size=(1, 1), stride=(1, 1))\n",
" (4): ReLU()\n",
" (5): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
" (6): MaxPool2d(kernel_size=(3, 1), stride=(3, 1), padding=0, dilation=1, ceil_mode=False)\n",
" )\n",
" (conv8): Sequential(\n",
" (0): Conv2d(64, 64, kernel_size=(1, 1), stride=(1, 1))\n",
" (1): ReLU()\n",
" (2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
" (3): Conv2d(64, 64, kernel_size=(1, 1), stride=(1, 1))\n",
" (4): ReLU()\n",
" (5): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
" (6): MaxPool2d(kernel_size=(1, 1), stride=(1, 1), padding=0, dilation=1, ceil_mode=False)\n",
" )\n",
" (fc1): Sequential(\n",
" (0): Flatten()\n",
" (1): Dropout(p=0.5, inplace=False)\n",
" (2): Linear(in_features=19200, out_features=4096, bias=True)\n",
" (3): ReLU()\n",
" (4): Dropout(p=0.5, inplace=False)\n",
" (5): Linear(in_features=4096, out_features=512, bias=True)\n",
" (6): ReLU()\n",
" (7): Dropout(p=0.5, inplace=False)\n",
" (8): Linear(in_features=512, out_features=32, bias=True)\n",
" (9): ReLU()\n",
" (10): Dropout(p=0.2, inplace=False)\n",
" (11): Linear(in_features=32, out_features=8, bias=True)\n",
" (12): ReLU()\n",
" (13): Linear(in_features=8, out_features=1, bias=True)\n",
" )\n",
")"
]
},
"execution_count": 29,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
"print(f'Train on: {device}')\n",
"# send model to device\n",
"model.to(device)"
]
},
{
"cell_type": "code",
"execution_count": 30,
"metadata": {},
"outputs": [],
"source": [
"def update_outputs(output_dict,\n",
" new_outputs,\n",
" outputs_to_update=['scores', 'predictions', 'labels']):\n",
" for x in outputs_to_update:\n",
" if (type(new_outputs[x]) is list):\n",
" output_dict[x] += list(new_outputs[x])\n",
" else:\n",
" output_dict[x].append(new_outputs[x])\n",
"\n",
" return output_dict\n",
"\n",
"def accuracyFromLogits(output, target):\n",
" scores = torch.sigmoid(output)\n",
" pred = torch.round(scores)\n",
" correct_tensor = pred.eq(target.data.view_as(pred))\n",
" # Need to convert correct tensor from int to float to average\n",
" accuracy = torch.mean(correct_tensor.type(torch.FloatTensor))\n",
" return accuracy, scores, pred"
]
},
{
"cell_type": "code",
"execution_count": 31,
"metadata": {},
"outputs": [],
"source": [
"def train(model,\n",
" device,\n",
" criterion,\n",
" optimizer,\n",
" scheduler,\n",
" train_loader,\n",
" valid_loader,\n",
" save_file_name,\n",
" max_epochs_stop=5,\n",
" n_epochs=20,\n",
" print_every=1):\n",
" \"\"\"Train a PyTorch Model\n",
"\n",
" Params\n",
" --------\n",
" model (PyTorch model): cnn to train\n",
" criterion (PyTorch loss): objective to minimize\n",
" optimizer (PyTorch optimizier): optimizer to compute gradients of model parameters\n",
" train_loader (PyTorch dataloader): training dataloader to iterate through\n",
" valid_loader (PyTorch dataloader): validation dataloader used for early stopping\n",
" save_file_name (str ending in '.pt'): file path to save the model state dict\n",
" max_epochs_stop (int): maximum number of epochs with no improvement in validation loss for early stopping\n",
" n_epochs (int): maximum number of training epochs\n",
" print_every (int): frequency of epochs to print training stats\n",
"\n",
" Returns\n",
" --------\n",
" model (PyTorch model): trained cnn with best weights\n",
" history (DataFrame): history of train and validation loss and accuracy\n",
" \"\"\"\n",
"\n",
" # Early stopping intialization\n",
" epochs_no_improve = 0\n",
" valid_loss_min = np.Inf\n",
"\n",
" # Create empty history\n",
" history = []\n",
"\n",
" # Number of epochs already trained (if using loaded in model weights)\n",
" try:\n",
" print(f'Model has been trained for: {model.epochs} epochs.\\n')\n",
" except:\n",
" model.epochs = 0\n",
" print(f'Starting Training from Scratch on \\n')\n",
"\n",
" overall_start = timer()\n",
"\n",
" # Main loop\n",
" for epoch in range(n_epochs):\n",
" #scheduler.step()\n",
" # keep track of training and validation loss each epoch\n",
" train_step_counter = 0\n",
" valid_step_counter = 0\n",
"\n",
" train_loss = 0.0\n",
" valid_loss = 0.0\n",
"\n",
" train_acc = 0\n",
" valid_acc = 0\n",
" running_corrects = 0\n",
"\n",
" # Set to training\n",
" model.train()\n",
" start = timer()\n",
"\n",
" # Training loop\n",
" for ii, (input, target) in enumerate(train_loader):\n",
"\n",
" # Increment counter\n",
" train_step_counter += 1\n",
"\n",
" # Set inputs dtype. Check input.dtype and target.dtype\n",
" # NOTE: This match the type of input and model weights\n",
" input, target = input.to(torch.float32), target.to(torch.float32)\n",
"\n",
" # Send inputs to device\n",
" input, target = input.to(device), target.to(device)\n",
"\n",
" # Clear gradients\n",
" optimizer.zero_grad()\n",
"\n",
" # Calculate output\n",
" output = model(input)\n",
"\n",
" # Loss and backpropagation of gradients\n",
" # WARNING: Be careful about .squeeze(). In this case, since the output is\n",
" # [BATCHSIZE, 1] the scenario is easy!\n",
" matched_output = output.squeeze()\n",
" assert matched_output.shape == target.shape\n",
" loss = criterion(matched_output, target) # save a copy, plot the \n",
"\n",
" # Backward computation\n",
" loss.backward()\n",
"\n",
" # Update the parameters\n",
" optimizer.step()\n",
" #scheduler.step()\n",
"\n",
" # Track train loss by multiplying average loss by number of examples in batch\n",
" train_loss += loss.item()\n",
"\n",
" # Calculate accuracy by finding max log probability\n",
" accuracy, scores, pred = accuracyFromLogits(output, target)\n",
" train_acc += accuracy.item()\n",
"\n",
" # Track training progress\n",
" print(\n",
" f'Epoch: {epoch}\\t{100 * (ii + 1) / len(train_loader):.2f}% complete. {timer() - start:.2f} seconds elapsed in epoch.',\n",
" end='\\r')\n",
"\n",
" # Calculate accuracy and loss over the batch\n",
" train_acc = train_acc / train_step_counter\n",
" train_loss = train_loss / train_step_counter\n",
"\n",
"\n",
" # After training loops ends, start validation\n",
" model.epochs += 1\n",
"\n",
" # Don't need to keep track of gradients\n",
" output_dict = {\n",
" \"scores\": [],\n",
" \"predictions\": [],\n",
" \"labels\": []\n",
" }\n",
" with torch.no_grad():\n",
" # Set to evaluation mode\n",
" model.eval()\n",
"\n",
" # Validation loop\n",
" for input, target in valid_loader:\n",
"\n",
" # Increment counter\n",
" valid_step_counter += 1\n",
" #print()\n",
"\n",
" # Set inputs dtype. Check input.dtype and target.dtype\n",
" # NOTE: This match the type of input and model weights\n",
" input, target = input.to(torch.float32), target.to(torch.float32)\n",
"\n",
" # Send inputs to device\n",
" input, target = input.to(device), target.to(device)\n",
"\n",
" # Calculate output\n",
" output = model(input)\n",
"\n",
" # Loss and backpropagation of gradients\n",
" loss = criterion(output.squeeze(), target) # save copy\n",
"\n",
" # Track train loss by multiplying average loss by number of examples in batch\n",
" valid_loss += loss.item()\n",
"\n",
" # Calculate accuracy by finding max log probability\n",
" accuracy, scores, pred = accuracyFromLogits(output, target)\n",
" #print(\"accuracy, scores, pred: \", accuracy, scores, pred)\n",
" valid_acc += accuracy.item()\n",
"\n",
" output_dict = update_outputs(output_dict, new_outputs={\n",
" \"predictions\": pred.detach().cpu().numpy().squeeze().tolist(),\n",
" \"scores\": scores.detach().cpu().numpy().squeeze().tolist(),\n",
" \"labels\": target.detach().cpu().numpy().squeeze().tolist()\n",
" })\n",
"\n",
" # Calculate accuracy and loss over the batch\n",
" valid_acc = valid_acc / valid_step_counter\n",
" valid_loss = valid_loss / valid_step_counter\n",
"\n",
" # Calculate average accuracy\n",
" roc_auc_curve = roc_auc_score(output_dict[\"labels\"], output_dict[\"scores\"])\n",
" history.append([train_loss, valid_loss, train_acc, valid_acc, roc_auc_curve])\n",
"\n",
" # Print training and validation results\n",
" if (epoch + 1) % print_every == 0:\n",
" # print(\"epoch_acc\",epoch_acc)\n",
" print(\"\\n---------------------------------------------------------------------------------------------\"\n",
" f'Epoch {epoch} Results'\n",
" \"---------------------------------------------------------------------------------------------\\n\")\n",
" print(\n",
" f'Training Loss: {train_loss:.4f} \\t Validation Loss: {valid_loss:.4f}'\n",
" )\n",
" print(\n",
" f'Training Accuracy: {100 * train_acc:.2f}%\\t Validation Accuracy: {100 * valid_acc:.2f}%'\n",
" )\n",
"\n",
" precision, recall, f1_score, _ = precision_recall_fscore_support(output_dict[\"labels\"],\n",
" output_dict[\"predictions\"])\n",
" print(f'roc_auc_score :{roc_auc_curve}')\n",
"\n",
" for j in range(len(precision)):\n",
" print(f'\\nClass {j} Precision :{precision[j] * 100:.2f} ')\n",
" print(f'Class {j} Recall :{recall[j] * 100:.2f} ')\n",
" print(f'Class {j} f1_score :{f1_score[j] * 100:.2f} ')\n",
"\n",
" # Save the model if validation loss decreases\n",
" if valid_loss < valid_loss_min:\n",
" # Save model\n",
" torch.save(model.state_dict(), save_file_name)\n",
" # Track improvement\n",
" epochs_no_improve = 0\n",
" valid_loss_min = valid_loss\n",
" valid_best_acc = valid_acc\n",
" best_epoch = epoch\n",
"\n",
" # Otherwise increment count of epochs with no improvement\n",
" else:\n",
" epochs_no_improve += 1\n",
"\n",
" scheduler.step()\n",
" # # Attach the optimizer\n",
" # model.optimizer = optimizer\n",
" # Record overall time and print out stats\n",
" total_time = timer() - overall_start\n",
" print(\n",
" f'\\nBest epoch: {best_epoch} with loss: {valid_loss_min:.2f} and acc: {100 * valid_acc:.2f}%'\n",
" )\n",
" print(\n",
" f'{total_time:.2f} total seconds elapsed. {total_time / (epoch):.2f} seconds per epoch.'\n",
" )\n",
" # eval_metrics_dict, b = (get_metrics(eval_outputs_dict))\n",
" # for idx, class_idx in enumerate(eval_metrics_dict):\n",
" # for metric_name in eval_metrics_dict[class_idx]:\n",
" # print(\"Class \", class_idx, \"Evaluation\", metric_name, \":\", eval_metrics_dict[class_idx][metric_name])\n",
"\n",
" print(\"\\n\\n----------------\\n\\n\")\n",
" # Format history\n",
" history = pd.DataFrame(\n",
" history,\n",
" columns=['train_loss', 'valid_loss', 'train_acc', 'valid_acc', 'roc'])\n",
" print(\"history df shape: \", history.shape)\n",
" return model, history"
]
},
{
"cell_type": "code",
"execution_count": 32,
"metadata": {},
"outputs": [],
"source": [
"criterion = nn.BCEWithLogitsLoss()\n",
"\n",
"\n",
"# WARNING: It is necessary as arguments like pos_weight may change the device if not defined on the correct device\n",
"criterion.to(device)\n",
"\n",
"# WARNING: Check optimization parameters\n",
"optimizer_ft = optim.Adam(model.parameters(), lr=1e-5, betas=(0.9, 0.999), eps=1e-08, weight_decay=0.01,\n",
" amsgrad=False)\n",
"scheduler = StepLR(optimizer_ft, step_size = 2, gamma = 0.95)\n",
"# add L2 regularization penalty through weight_decay\n",
"NUM_EPOCHS = 100\n",
"save_file_name = 'ECGnet_torch_2DCNN_train_balanced_synthetic.pt'\n",
"checkpoint_path = 'ECGnet_torch_2DCNN_train_balanced_synthetic.pth'"
]
},
{
"cell_type": "code",
"execution_count": 33,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Starting Training from Scratch on \n",
"\n",
"Epoch: 0\t100.00% complete. 21.94 seconds elapsed in epoch.\n",
"---------------------------------------------------------------------------------------------Epoch 0 Results---------------------------------------------------------------------------------------------\n",
"\n",
"Training Loss: 0.5451 \t Validation Loss: 0.4977\n",
"Training Accuracy: 75.03%\t Validation Accuracy: 75.74%\n",
"roc_auc_score :0.7377615426668654\n",
"\n",
"Class 0 Precision :75.59 \n",
"Class 0 Recall :100.00 \n",
"Class 0 f1_score :86.10 \n",
"\n",
"Class 1 Precision :0.00 \n",
"Class 1 Recall :0.00 \n",
"Class 1 f1_score :0.00 \n",
"Epoch: 1\t100.00% complete. 21.96 seconds elapsed in epoch.\n",
"---------------------------------------------------------------------------------------------Epoch 1 Results---------------------------------------------------------------------------------------------\n",
"\n",
"Training Loss: 0.4235 \t Validation Loss: 0.4449\n",
"Training Accuracy: 78.01%\t Validation Accuracy: 76.17%\n",
"roc_auc_score :0.8048618358928483\n",
"\n",
"Class 0 Precision :76.48 \n",
"Class 0 Recall :98.58 \n",
"Class 0 f1_score :86.14 \n",
"\n",
"Class 1 Precision :58.33 \n",
"Class 1 Recall :6.14 \n",
"Class 1 f1_score :11.11 \n",
"Epoch: 2\t100.00% complete. 21.92 seconds elapsed in epoch.\n",
"---------------------------------------------------------------------------------------------Epoch 2 Results---------------------------------------------------------------------------------------------\n",
"\n",
"Training Loss: 0.3742 \t Validation Loss: 0.4385\n",
"Training Accuracy: 81.48%\t Validation Accuracy: 77.23%\n",
"roc_auc_score :0.8137269519407584\n",
"\n",
"Class 0 Precision :79.01 \n",
"Class 0 Recall :94.90 \n",
"Class 0 f1_score :86.23 \n",
"\n",
"Class 1 Precision :58.14 \n",
"Class 1 Recall :21.93 \n",
"Class 1 f1_score :31.85 \n",
"Epoch: 3\t100.00% complete. 21.80 seconds elapsed in epoch.\n",
"---------------------------------------------------------------------------------------------Epoch 3 Results---------------------------------------------------------------------------------------------\n",
"\n",
"Training Loss: 0.3352 \t Validation Loss: 0.4366\n",
"Training Accuracy: 85.57%\t Validation Accuracy: 78.14%\n",
"roc_auc_score :0.8222690224143929\n",
"\n",
"Class 0 Precision :80.61 \n",
"Class 0 Recall :93.63 \n",
"Class 0 f1_score :86.63 \n",
"\n",
"Class 1 Precision :60.53 \n",
"Class 1 Recall :30.26 \n",
"Class 1 f1_score :40.35 \n",
"Epoch: 4\t100.00% complete. 21.98 seconds elapsed in epoch.\n",
"---------------------------------------------------------------------------------------------Epoch 4 Results---------------------------------------------------------------------------------------------\n",
"\n",
"Training Loss: 0.3027 \t Validation Loss: 0.4304\n",
"Training Accuracy: 87.79%\t Validation Accuracy: 78.35%\n",
"roc_auc_score :0.8319541772277721\n",
"\n",
"Class 0 Precision :81.90 \n",
"Class 0 Recall :91.64 \n",
"Class 0 f1_score :86.50 \n",
"\n",
"Class 1 Precision :59.03 \n",
"Class 1 Recall :37.28 \n",
"Class 1 f1_score :45.70 \n",
"Epoch: 5\t100.00% complete. 21.85 seconds elapsed in epoch.\n",
"---------------------------------------------------------------------------------------------Epoch 5 Results---------------------------------------------------------------------------------------------\n",
"\n",
"Training Loss: 0.2652 \t Validation Loss: 0.5016\n",
"Training Accuracy: 90.60%\t Validation Accuracy: 76.91%\n",
"roc_auc_score :0.8172866656726803\n",
"\n",
"Class 0 Precision :80.00 \n",
"Class 0 Recall :92.92 \n",
"Class 0 f1_score :85.98 \n",
"\n",
"Class 1 Precision :56.14 \n",
"Class 1 Recall :28.07 \n",
"Class 1 f1_score :37.43 \n",
"Epoch: 6\t100.00% complete. 22.06 seconds elapsed in epoch.\n",
"---------------------------------------------------------------------------------------------Epoch 6 Results---------------------------------------------------------------------------------------------\n",
"\n",
"Training Loss: 0.2390 \t Validation Loss: 0.4960\n",
"Training Accuracy: 92.01%\t Validation Accuracy: 79.15%\n",
"roc_auc_score :0.82544977883803\n",
"\n",
"Class 0 Precision :81.71 \n",
"Class 0 Recall :93.63 \n",
"Class 0 f1_score :87.26 \n",
"\n",
"Class 1 Precision :64.00 \n",
"Class 1 Recall :35.09 \n",
"Class 1 f1_score :45.33 \n",
"Epoch: 7\t100.00% complete. 22.09 seconds elapsed in epoch.\n",
"---------------------------------------------------------------------------------------------Epoch 7 Results---------------------------------------------------------------------------------------------\n",
"\n",
"Training Loss: 0.2130 \t Validation Loss: 0.4742\n",
"Training Accuracy: 93.67%\t Validation Accuracy: 80.16%\n",
"roc_auc_score :0.8212563987873367\n",
"\n",
"Class 0 Precision :84.14 \n",
"Class 0 Recall :90.93 \n",
"Class 0 f1_score :87.41 \n",
"\n",
"Class 1 Precision :62.57 \n",
"Class 1 Recall :46.93 \n",
"Class 1 f1_score :53.63 \n",
"Epoch: 8\t100.00% complete. 21.81 seconds elapsed in epoch.\n",
"---------------------------------------------------------------------------------------------Epoch 8 Results---------------------------------------------------------------------------------------------\n",
"\n",
"Training Loss: 0.1845 \t Validation Loss: 0.5647\n",
"Training Accuracy: 94.56%\t Validation Accuracy: 80.16%\n",
"roc_auc_score :0.8285870483574376\n",
"\n",
"Class 0 Precision :82.36 \n",
"Class 0 Recall :93.91 \n",
"Class 0 f1_score :87.76 \n",
"\n",
"Class 1 Precision :66.67 \n",
"Class 1 Recall :37.72 \n",
"Class 1 f1_score :48.18 \n",
"Epoch: 9\t100.00% complete. 21.97 seconds elapsed in epoch.\n",
"---------------------------------------------------------------------------------------------Epoch 9 Results---------------------------------------------------------------------------------------------\n",
"\n",
"Training Loss: 0.1670 \t Validation Loss: 0.6113\n",
"Training Accuracy: 94.97%\t Validation Accuracy: 79.15%\n",
"roc_auc_score :0.8203804482878585\n",
"\n",
"Class 0 Precision :81.33 \n",
"Class 0 Recall :93.77 \n",
"Class 0 f1_score :87.11 \n",
"\n",
"Class 1 Precision :63.33 \n",
"Class 1 Recall :33.33 \n",
"Class 1 f1_score :43.68 \n",
"Epoch: 10\t100.00% complete. 21.96 seconds elapsed in epoch.\n",
"---------------------------------------------------------------------------------------------Epoch 10 Results---------------------------------------------------------------------------------------------\n",
"\n",
"Training Loss: 0.1549 \t Validation Loss: 0.6409\n",
"Training Accuracy: 95.64%\t Validation Accuracy: 79.10%\n",
"roc_auc_score :0.8103722479002037\n",
"\n",
"Class 0 Precision :82.14 \n",
"Class 0 Recall :92.49 \n",
"Class 0 f1_score :87.01 \n",
"\n",
"Class 1 Precision :61.87 \n",
"Class 1 Recall :37.72 \n",
"Class 1 f1_score :46.87 \n",
"Epoch: 11\t100.00% complete. 21.88 seconds elapsed in epoch.\n",
"---------------------------------------------------------------------------------------------Epoch 11 Results---------------------------------------------------------------------------------------------\n",
"\n",
"Training Loss: 0.1354 \t Validation Loss: 0.6322\n",
"Training Accuracy: 95.79%\t Validation Accuracy: 80.27%\n",
"roc_auc_score :0.8126397793350232\n",
"\n",
"Class 0 Precision :83.04 \n",
"Class 0 Recall :92.92 \n",
"Class 0 f1_score :87.70 \n",
"\n",
"Class 1 Precision :65.28 \n",
"Class 1 Recall :41.23 \n",
"Class 1 f1_score :50.54 \n",
"Epoch: 12\t100.00% complete. 21.84 seconds elapsed in epoch.\n",
"---------------------------------------------------------------------------------------------Epoch 12 Results---------------------------------------------------------------------------------------------\n",
"\n",
"Training Loss: 0.1244 \t Validation Loss: 0.6262\n",
"Training Accuracy: 96.44%\t Validation Accuracy: 80.05%\n",
"roc_auc_score :0.820125739277372\n",
"\n",
"Class 0 Precision :83.33 \n",
"Class 0 Recall :92.07 \n",
"Class 0 f1_score :87.48 \n",
"\n",
"Class 1 Precision :63.64 \n",
"Class 1 Recall :42.98 \n",
"Class 1 f1_score :51.31 \n",
"Epoch: 13\t100.00% complete. 21.76 seconds elapsed in epoch.\n",
"---------------------------------------------------------------------------------------------Epoch 13 Results---------------------------------------------------------------------------------------------\n",
"\n",
"Training Loss: 0.1148 \t Validation Loss: 0.6662\n",
"Training Accuracy: 96.94%\t Validation Accuracy: 79.57%\n",
"roc_auc_score :0.8241203220515878\n",
"\n",
"Class 0 Precision :81.34 \n",
"Class 0 Recall :94.48 \n",
"Class 0 f1_score :87.42 \n",
"\n",
"Class 1 Precision :65.79 \n",
"Class 1 Recall :32.89 \n",
"Class 1 f1_score :43.86 \n",
"Epoch: 14\t100.00% complete. 21.92 seconds elapsed in epoch.\n",
"---------------------------------------------------------------------------------------------Epoch 14 Results---------------------------------------------------------------------------------------------\n",
"\n",
"Training Loss: 0.1127 \t Validation Loss: 0.6754\n",
"Training Accuracy: 96.42%\t Validation Accuracy: 80.32%\n",
"roc_auc_score :0.8201754385964912\n",
"\n",
"Class 0 Precision :82.60 \n",
"Class 0 Recall :93.48 \n",
"Class 0 f1_score :87.71 \n",
"\n",
"Class 1 Precision :65.93 \n",
"Class 1 Recall :39.04 \n",
"Class 1 f1_score :49.04 \n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Epoch: 15\t100.00% complete. 21.89 seconds elapsed in epoch.\n",
"---------------------------------------------------------------------------------------------Epoch 15 Results---------------------------------------------------------------------------------------------\n",
"\n",
"Training Loss: 0.1043 \t Validation Loss: 0.7724\n",
"Training Accuracy: 96.58%\t Validation Accuracy: 79.10%\n",
"roc_auc_score :0.8133355698026936\n",
"\n",
"Class 0 Precision :80.75 \n",
"Class 0 Recall :95.04 \n",
"Class 0 f1_score :87.31 \n",
"\n",
"Class 1 Precision :66.02 \n",
"Class 1 Recall :29.82 \n",
"Class 1 f1_score :41.09 \n",
"Epoch: 16\t100.00% complete. 22.03 seconds elapsed in epoch.\n",
"---------------------------------------------------------------------------------------------Epoch 16 Results---------------------------------------------------------------------------------------------\n",
"\n",
"Training Loss: 0.0997 \t Validation Loss: 0.7270\n",
"Training Accuracy: 96.87%\t Validation Accuracy: 79.04%\n",
"roc_auc_score :0.804588489637692\n",
"\n",
"Class 0 Precision :80.92 \n",
"Class 0 Recall :94.33 \n",
"Class 0 f1_score :87.12 \n",
"\n",
"Class 1 Precision :63.96 \n",
"Class 1 Recall :31.14 \n",
"Class 1 f1_score :41.89 \n",
"Epoch: 17\t100.00% complete. 21.87 seconds elapsed in epoch.\n",
"---------------------------------------------------------------------------------------------Epoch 17 Results---------------------------------------------------------------------------------------------\n",
"\n",
"Training Loss: 0.0962 \t Validation Loss: 0.6965\n",
"Training Accuracy: 96.91%\t Validation Accuracy: 80.05%\n",
"roc_auc_score :0.823070423935192\n",
"\n",
"Class 0 Precision :82.50 \n",
"Class 0 Recall :93.48 \n",
"Class 0 f1_score :87.65 \n",
"\n",
"Class 1 Precision :65.67 \n",
"Class 1 Recall :38.60 \n",
"Class 1 f1_score :48.62 \n",
"Epoch: 18\t100.00% complete. 21.96 seconds elapsed in epoch.\n",
"---------------------------------------------------------------------------------------------Epoch 18 Results---------------------------------------------------------------------------------------------\n",
"\n",
"Training Loss: 0.0845 \t Validation Loss: 0.6860\n",
"Training Accuracy: 97.63%\t Validation Accuracy: 79.57%\n",
"roc_auc_score :0.8057315739774364\n",
"\n",
"Class 0 Precision :83.55 \n",
"Class 0 Recall :90.65 \n",
"Class 0 f1_score :86.96 \n",
"\n",
"Class 1 Precision :60.71 \n",
"Class 1 Recall :44.74 \n",
"Class 1 f1_score :51.52 \n",
"Epoch: 19\t100.00% complete. 21.56 seconds elapsed in epoch.\n",
"---------------------------------------------------------------------------------------------Epoch 19 Results---------------------------------------------------------------------------------------------\n",
"\n",
"Training Loss: 0.0836 \t Validation Loss: 0.6240\n",
"Training Accuracy: 97.32%\t Validation Accuracy: 79.89%\n",
"roc_auc_score :0.8257169126782964\n",
"\n",
"Class 0 Precision :83.79 \n",
"Class 0 Recall :90.79 \n",
"Class 0 f1_score :87.15 \n",
"\n",
"Class 1 Precision :61.54 \n",
"Class 1 Recall :45.61 \n",
"Class 1 f1_score :52.39 \n",
"Epoch: 20\t100.00% complete. 20.17 seconds elapsed in epoch.\n",
"---------------------------------------------------------------------------------------------Epoch 20 Results---------------------------------------------------------------------------------------------\n",
"\n",
"Training Loss: 0.0781 \t Validation Loss: 0.7997\n",
"Training Accuracy: 97.81%\t Validation Accuracy: 79.57%\n",
"roc_auc_score :0.8019792753839272\n",
"\n",
"Class 0 Precision :82.03 \n",
"Class 0 Recall :93.77 \n",
"Class 0 f1_score :87.51 \n",
"\n",
"Class 1 Precision :65.35 \n",
"Class 1 Recall :36.40 \n",
"Class 1 f1_score :46.76 \n",
"Epoch: 21\t100.00% complete. 20.69 seconds elapsed in epoch.\n",
"---------------------------------------------------------------------------------------------Epoch 21 Results---------------------------------------------------------------------------------------------\n",
"\n",
"Training Loss: 0.0789 \t Validation Loss: 0.7253\n",
"Training Accuracy: 97.54%\t Validation Accuracy: 79.57%\n",
"roc_auc_score :0.812062024750261\n",
"\n",
"Class 0 Precision :82.43 \n",
"Class 0 Recall :93.06 \n",
"Class 0 f1_score :87.43 \n",
"\n",
"Class 1 Precision :64.23 \n",
"Class 1 Recall :38.60 \n",
"Class 1 f1_score :48.22 \n",
"Epoch: 22\t100.00% complete. 20.25 seconds elapsed in epoch.\n",
"---------------------------------------------------------------------------------------------Epoch 22 Results---------------------------------------------------------------------------------------------\n",
"\n",
"Training Loss: 0.0751 \t Validation Loss: 0.7650\n",
"Training Accuracy: 97.83%\t Validation Accuracy: 79.95%\n",
"roc_auc_score :0.8063155409770886\n",
"\n",
"Class 0 Precision :81.68 \n",
"Class 0 Recall :94.76 \n",
"Class 0 f1_score :87.74 \n",
"\n",
"Class 1 Precision :67.83 \n",
"Class 1 Recall :34.21 \n",
"Class 1 f1_score :45.48 \n",
"Epoch: 23\t100.00% complete. 20.00 seconds elapsed in epoch.\n",
"---------------------------------------------------------------------------------------------Epoch 23 Results---------------------------------------------------------------------------------------------\n",
"\n",
"Training Loss: 0.0683 \t Validation Loss: 0.7674\n",
"Training Accuracy: 98.17%\t Validation Accuracy: 80.74%\n",
"roc_auc_score :0.815913721982009\n",
"\n",
"Class 0 Precision :82.61 \n",
"Class 0 Recall :94.19 \n",
"Class 0 f1_score :88.02 \n",
"\n",
"Class 1 Precision :68.22 \n",
"Class 1 Recall :38.60 \n",
"Class 1 f1_score :49.30 \n",
"Epoch: 24\t100.00% complete. 20.55 seconds elapsed in epoch.\n",
"---------------------------------------------------------------------------------------------Epoch 24 Results---------------------------------------------------------------------------------------------\n",
"\n",
"Training Loss: 0.0656 \t Validation Loss: 0.7198\n",
"Training Accuracy: 98.10%\t Validation Accuracy: 79.84%\n",
"roc_auc_score :0.8144600168977685\n",
"\n",
"Class 0 Precision :82.85 \n",
"Class 0 Recall :93.06 \n",
"Class 0 f1_score :87.66 \n",
"\n",
"Class 1 Precision :65.25 \n",
"Class 1 Recall :40.35 \n",
"Class 1 f1_score :49.86 \n",
"Epoch: 25\t100.00% complete. 20.62 seconds elapsed in epoch.\n",
"---------------------------------------------------------------------------------------------Epoch 25 Results---------------------------------------------------------------------------------------------\n",
"\n",
"Training Loss: 0.0651 \t Validation Loss: 0.7393\n",
"Training Accuracy: 98.14%\t Validation Accuracy: 78.83%\n",
"roc_auc_score :0.8113041101336912\n",
"\n",
"Class 0 Precision :82.21 \n",
"Class 0 Recall :91.64 \n",
"Class 0 f1_score :86.67 \n",
"\n",
"Class 1 Precision :59.86 \n",
"Class 1 Recall :38.60 \n",
"Class 1 f1_score :46.93 \n",
"Epoch: 26\t100.00% complete. 20.54 seconds elapsed in epoch.\n",
"---------------------------------------------------------------------------------------------Epoch 26 Results---------------------------------------------------------------------------------------------\n",
"\n",
"Training Loss: 0.0659 \t Validation Loss: 0.7362\n",
"Training Accuracy: 98.05%\t Validation Accuracy: 79.57%\n",
"roc_auc_score :0.8175724367576165\n",
"\n",
"Class 0 Precision :82.21 \n",
"Class 0 Recall :92.92 \n",
"Class 0 f1_score :87.23 \n",
"\n",
"Class 1 Precision :63.24 \n",
"Class 1 Recall :37.72 \n",
"Class 1 f1_score :47.25 \n",
"Epoch: 27\t100.00% complete. 20.08 seconds elapsed in epoch.\n",
"---------------------------------------------------------------------------------------------Epoch 27 Results---------------------------------------------------------------------------------------------\n",
"\n",
"Training Loss: 0.0627 \t Validation Loss: 0.7662\n",
"Training Accuracy: 98.26%\t Validation Accuracy: 78.56%\n",
"roc_auc_score :0.8137145271109786\n",
"\n",
"Class 0 Precision :81.23 \n",
"Class 0 Recall :93.20 \n",
"Class 0 f1_score :86.81 \n",
"\n",
"Class 1 Precision :61.29 \n",
"Class 1 Recall :33.33 \n",
"Class 1 f1_score :43.18 \n",
"Epoch: 28\t100.00% complete. 19.99 seconds elapsed in epoch.\n",
"---------------------------------------------------------------------------------------------Epoch 28 Results---------------------------------------------------------------------------------------------\n",
"\n",
"Training Loss: 0.0688 \t Validation Loss: 0.8259\n",
"Training Accuracy: 97.70%\t Validation Accuracy: 78.40%\n",
"roc_auc_score :0.8132920828984643\n",
"\n",
"Class 0 Precision :80.26 \n",
"Class 0 Recall :94.48 \n",
"Class 0 f1_score :86.79 \n",
"\n",
"Class 1 Precision :62.14 \n",
"Class 1 Recall :28.07 \n",
"Class 1 f1_score :38.67 \n",
"Epoch: 29\t100.00% complete. 20.61 seconds elapsed in epoch.\n",
"---------------------------------------------------------------------------------------------Epoch 29 Results---------------------------------------------------------------------------------------------\n",
"\n",
"Training Loss: 0.0611 \t Validation Loss: 0.7604\n",
"Training Accuracy: 98.32%\t Validation Accuracy: 78.67%\n",
"roc_auc_score :0.8007554296506137\n",
"\n",
"Class 0 Precision :81.89 \n",
"Class 0 Recall :92.21 \n",
"Class 0 f1_score :86.74 \n",
"\n",
"Class 1 Precision :60.43 \n",
"Class 1 Recall :36.84 \n",
"Class 1 f1_score :45.78 \n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Epoch: 30\t100.00% complete. 20.51 seconds elapsed in epoch.\n",
"---------------------------------------------------------------------------------------------Epoch 30 Results---------------------------------------------------------------------------------------------\n",
"\n",
"Training Loss: 0.0603 \t Validation Loss: 0.7687\n",
"Training Accuracy: 98.30%\t Validation Accuracy: 79.57%\n",
"roc_auc_score :0.810950002484966\n",
"\n",
"Class 0 Precision :81.27 \n",
"Class 0 Recall :94.62 \n",
"Class 0 f1_score :87.43 \n",
"\n",
"Class 1 Precision :66.07 \n",
"Class 1 Recall :32.46 \n",
"Class 1 f1_score :43.53 \n",
"Epoch: 31\t100.00% complete. 20.15 seconds elapsed in epoch.\n",
"---------------------------------------------------------------------------------------------Epoch 31 Results---------------------------------------------------------------------------------------------\n",
"\n",
"Training Loss: 0.0593 \t Validation Loss: 0.7236\n",
"Training Accuracy: 98.37%\t Validation Accuracy: 80.37%\n",
"roc_auc_score :0.8175165250236072\n",
"\n",
"Class 0 Precision :82.89 \n",
"Class 0 Recall :93.34 \n",
"Class 0 f1_score :87.81 \n",
"\n",
"Class 1 Precision :66.19 \n",
"Class 1 Recall :40.35 \n",
"Class 1 f1_score :50.14 \n",
"Epoch: 32\t100.00% complete. 20.03 seconds elapsed in epoch.\n",
"---------------------------------------------------------------------------------------------Epoch 32 Results---------------------------------------------------------------------------------------------\n",
"\n",
"Training Loss: 0.0565 \t Validation Loss: 0.7014\n",
"Training Accuracy: 98.32%\t Validation Accuracy: 79.79%\n",
"roc_auc_score :0.828183241389593\n",
"\n",
"Class 0 Precision :82.33 \n",
"Class 0 Recall :93.06 \n",
"Class 0 f1_score :87.37 \n",
"\n",
"Class 1 Precision :63.97 \n",
"Class 1 Recall :38.16 \n",
"Class 1 f1_score :47.80 \n",
"Epoch: 33\t100.00% complete. 20.55 seconds elapsed in epoch.\n",
"---------------------------------------------------------------------------------------------Epoch 33 Results---------------------------------------------------------------------------------------------\n",
"\n",
"Training Loss: 0.0571 \t Validation Loss: 0.8409\n",
"Training Accuracy: 98.34%\t Validation Accuracy: 78.30%\n",
"roc_auc_score :0.8063093285621987\n",
"\n",
"Class 0 Precision :80.10 \n",
"Class 0 Recall :94.62 \n",
"Class 0 f1_score :86.75 \n",
"\n",
"Class 1 Precision :62.00 \n",
"Class 1 Recall :27.19 \n",
"Class 1 f1_score :37.80 \n",
"Epoch: 34\t100.00% complete. 20.55 seconds elapsed in epoch.\n",
"---------------------------------------------------------------------------------------------Epoch 34 Results---------------------------------------------------------------------------------------------\n",
"\n",
"Training Loss: 0.0585 \t Validation Loss: 0.8204\n",
"Training Accuracy: 98.17%\t Validation Accuracy: 79.20%\n",
"roc_auc_score :0.813174047015556\n",
"\n",
"Class 0 Precision :80.77 \n",
"Class 0 Recall :95.18 \n",
"Class 0 f1_score :87.39 \n",
"\n",
"Class 1 Precision :66.67 \n",
"Class 1 Recall :29.82 \n",
"Class 1 f1_score :41.21 \n",
"Epoch: 35\t100.00% complete. 20.60 seconds elapsed in epoch.\n",
"---------------------------------------------------------------------------------------------Epoch 35 Results---------------------------------------------------------------------------------------------\n",
"\n",
"Training Loss: 0.0587 \t Validation Loss: 0.7496\n",
"Training Accuracy: 98.26%\t Validation Accuracy: 79.89%\n",
"roc_auc_score :0.8202499875751703\n",
"\n",
"Class 0 Precision :82.03 \n",
"Class 0 Recall :93.77 \n",
"Class 0 f1_score :87.51 \n",
"\n",
"Class 1 Precision :65.35 \n",
"Class 1 Recall :36.40 \n",
"Class 1 f1_score :46.76 \n",
"Epoch: 36\t100.00% complete. 20.14 seconds elapsed in epoch.\n",
"---------------------------------------------------------------------------------------------Epoch 36 Results---------------------------------------------------------------------------------------------\n",
"\n",
"Training Loss: 0.0480 \t Validation Loss: 0.8689\n",
"Training Accuracy: 98.93%\t Validation Accuracy: 79.04%\n",
"roc_auc_score :0.8109934893891952\n",
"\n",
"Class 0 Precision :80.62 \n",
"Class 0 Recall :95.47 \n",
"Class 0 f1_score :87.42 \n",
"\n",
"Class 1 Precision :67.35 \n",
"Class 1 Recall :28.95 \n",
"Class 1 f1_score :40.49 \n",
"Epoch: 37\t100.00% complete. 20.03 seconds elapsed in epoch.\n",
"---------------------------------------------------------------------------------------------Epoch 37 Results---------------------------------------------------------------------------------------------\n",
"\n",
"Training Loss: 0.0488 \t Validation Loss: 0.7923\n",
"Training Accuracy: 98.70%\t Validation Accuracy: 78.83%\n",
"roc_auc_score :0.8161808558222752\n",
"\n",
"Class 0 Precision :81.73 \n",
"Class 0 Recall :92.49 \n",
"Class 0 f1_score :86.78 \n",
"\n",
"Class 1 Precision :60.74 \n",
"Class 1 Recall :35.96 \n",
"Class 1 f1_score :45.18 \n",
"Epoch: 38\t100.00% complete. 20.06 seconds elapsed in epoch.\n",
"---------------------------------------------------------------------------------------------Epoch 38 Results---------------------------------------------------------------------------------------------\n",
"\n",
"Training Loss: 0.0453 \t Validation Loss: 0.7235\n",
"Training Accuracy: 98.64%\t Validation Accuracy: 80.32%\n",
"roc_auc_score :0.8210886635853087\n",
"\n",
"Class 0 Precision :82.93 \n",
"Class 0 Recall :92.92 \n",
"Class 0 f1_score :87.64 \n",
"\n",
"Class 1 Precision :65.03 \n",
"Class 1 Recall :40.79 \n",
"Class 1 f1_score :50.13 \n",
"Epoch: 39\t100.00% complete. 19.98 seconds elapsed in epoch.\n",
"---------------------------------------------------------------------------------------------Epoch 39 Results---------------------------------------------------------------------------------------------\n",
"\n",
"Training Loss: 0.0490 \t Validation Loss: 0.7612\n",
"Training Accuracy: 98.61%\t Validation Accuracy: 79.57%\n",
"roc_auc_score :0.822119924457035\n",
"\n",
"Class 0 Precision :81.87 \n",
"Class 0 Recall :94.05 \n",
"Class 0 f1_score :87.54 \n",
"\n",
"Class 1 Precision :65.85 \n",
"Class 1 Recall :35.53 \n",
"Class 1 f1_score :46.15 \n",
"Epoch: 40\t100.00% complete. 20.04 seconds elapsed in epoch.\n",
"---------------------------------------------------------------------------------------------Epoch 40 Results---------------------------------------------------------------------------------------------\n",
"\n",
"Training Loss: 0.0516 \t Validation Loss: 0.7799\n",
"Training Accuracy: 98.41%\t Validation Accuracy: 80.37%\n",
"roc_auc_score :0.815168232195219\n",
"\n",
"Class 0 Precision :82.48 \n",
"Class 0 Recall :94.05 \n",
"Class 0 f1_score :87.89 \n",
"\n",
"Class 1 Precision :67.44 \n",
"Class 1 Recall :38.16 \n",
"Class 1 f1_score :48.74 \n",
"Epoch: 41\t100.00% complete. 20.00 seconds elapsed in epoch.\n",
"---------------------------------------------------------------------------------------------Epoch 41 Results---------------------------------------------------------------------------------------------\n",
"\n",
"Training Loss: 0.0451 \t Validation Loss: 0.7786\n",
"Training Accuracy: 98.68%\t Validation Accuracy: 80.53%\n",
"roc_auc_score :0.8219024899358878\n",
"\n",
"Class 0 Precision :81.93 \n",
"Class 0 Recall :95.04 \n",
"Class 0 f1_score :88.00 \n",
"\n",
"Class 1 Precision :69.57 \n",
"Class 1 Recall :35.09 \n",
"Class 1 f1_score :46.65 \n",
"Epoch: 42\t100.00% complete. 20.00 seconds elapsed in epoch.\n",
"---------------------------------------------------------------------------------------------Epoch 42 Results---------------------------------------------------------------------------------------------\n",
"\n",
"Training Loss: 0.0450 \t Validation Loss: 0.7583\n",
"Training Accuracy: 98.79%\t Validation Accuracy: 80.96%\n",
"roc_auc_score :0.8170692311515332\n",
"\n",
"Class 0 Precision :82.81 \n",
"Class 0 Recall :94.19 \n",
"Class 0 f1_score :88.14 \n",
"\n",
"Class 1 Precision :68.70 \n",
"Class 1 Recall :39.47 \n",
"Class 1 f1_score :50.14 \n",
"Epoch: 43\t100.00% complete. 20.06 seconds elapsed in epoch.\n",
"---------------------------------------------------------------------------------------------Epoch 43 Results---------------------------------------------------------------------------------------------\n",
"\n",
"Training Loss: 0.0418 \t Validation Loss: 0.7837\n",
"Training Accuracy: 99.04%\t Validation Accuracy: 80.32%\n",
"roc_auc_score :0.8116085184632971\n",
"\n",
"Class 0 Precision :82.20 \n",
"Class 0 Recall :94.19 \n",
"Class 0 f1_score :87.79 \n",
"\n",
"Class 1 Precision :67.20 \n",
"Class 1 Recall :36.84 \n",
"Class 1 f1_score :47.59 \n",
"Epoch: 44\t100.00% complete. 20.00 seconds elapsed in epoch.\n",
"---------------------------------------------------------------------------------------------Epoch 44 Results---------------------------------------------------------------------------------------------\n",
"\n",
"Training Loss: 0.0419 \t Validation Loss: 0.8265\n",
"Training Accuracy: 98.90%\t Validation Accuracy: 78.56%\n",
"roc_auc_score :0.7998919039809154\n",
"\n",
"Class 0 Precision :80.93 \n",
"Class 0 Recall :93.77 \n",
"Class 0 f1_score :86.88 \n",
"\n",
"Class 1 Precision :62.07 \n",
"Class 1 Recall :31.58 \n",
"Class 1 f1_score :41.86 \n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Epoch: 45\t100.00% complete. 19.99 seconds elapsed in epoch.\n",
"---------------------------------------------------------------------------------------------Epoch 45 Results---------------------------------------------------------------------------------------------\n",
"\n",
"Training Loss: 0.0469 \t Validation Loss: 0.8442\n",
"Training Accuracy: 98.61%\t Validation Accuracy: 78.40%\n",
"roc_auc_score :0.8025694547984692\n",
"\n",
"Class 0 Precision :80.63 \n",
"Class 0 Recall :94.33 \n",
"Class 0 f1_score :86.95 \n",
"\n",
"Class 1 Precision :62.96 \n",
"Class 1 Recall :29.82 \n",
"Class 1 f1_score :40.48 \n",
"Epoch: 46\t100.00% complete. 20.01 seconds elapsed in epoch.\n",
"---------------------------------------------------------------------------------------------Epoch 46 Results---------------------------------------------------------------------------------------------\n",
"\n",
"Training Loss: 0.0406 \t Validation Loss: 0.7878\n",
"Training Accuracy: 98.95%\t Validation Accuracy: 80.43%\n",
"roc_auc_score :0.8059552209134735\n",
"\n",
"Class 0 Precision :82.38 \n",
"Class 0 Recall :94.05 \n",
"Class 0 f1_score :87.83 \n",
"\n",
"Class 1 Precision :67.19 \n",
"Class 1 Recall :37.72 \n",
"Class 1 f1_score :48.31 \n",
"Epoch: 47\t100.00% complete. 19.99 seconds elapsed in epoch.\n",
"---------------------------------------------------------------------------------------------Epoch 47 Results---------------------------------------------------------------------------------------------\n",
"\n",
"Training Loss: 0.0409 \t Validation Loss: 0.7808\n",
"Training Accuracy: 98.88%\t Validation Accuracy: 78.99%\n",
"roc_auc_score :0.8038678495104616\n",
"\n",
"Class 0 Precision :81.80 \n",
"Class 0 Recall :92.92 \n",
"Class 0 f1_score :87.00 \n",
"\n",
"Class 1 Precision :62.12 \n",
"Class 1 Recall :35.96 \n",
"Class 1 f1_score :45.56 \n",
"Epoch: 48\t100.00% complete. 19.98 seconds elapsed in epoch.\n",
"---------------------------------------------------------------------------------------------Epoch 48 Results---------------------------------------------------------------------------------------------\n",
"\n",
"Training Loss: 0.0409 \t Validation Loss: 0.8661\n",
"Training Accuracy: 98.90%\t Validation Accuracy: 78.62%\n",
"roc_auc_score :0.8087197455394861\n",
"\n",
"Class 0 Precision :80.46 \n",
"Class 0 Recall :94.48 \n",
"Class 0 f1_score :86.91 \n",
"\n",
"Class 1 Precision :62.86 \n",
"Class 1 Recall :28.95 \n",
"Class 1 f1_score :39.64 \n",
"Epoch: 49\t100.00% complete. 19.93 seconds elapsed in epoch.\n",
"---------------------------------------------------------------------------------------------Epoch 49 Results---------------------------------------------------------------------------------------------\n",
"\n",
"Training Loss: 0.0393 \t Validation Loss: 0.8274\n",
"Training Accuracy: 98.99%\t Validation Accuracy: 78.88%\n",
"roc_auc_score :0.803029173500323\n",
"\n",
"Class 0 Precision :81.15 \n",
"Class 0 Recall :93.91 \n",
"Class 0 f1_score :87.07 \n",
"\n",
"Class 1 Precision :63.25 \n",
"Class 1 Recall :32.46 \n",
"Class 1 f1_score :42.90 \n",
"Epoch: 50\t100.00% complete. 19.94 seconds elapsed in epoch.\n",
"---------------------------------------------------------------------------------------------Epoch 50 Results---------------------------------------------------------------------------------------------\n",
"\n",
"Training Loss: 0.0366 \t Validation Loss: 0.7627\n",
"Training Accuracy: 98.88%\t Validation Accuracy: 80.11%\n",
"roc_auc_score :0.8142363699617317\n",
"\n",
"Class 0 Precision :82.72 \n",
"Class 0 Recall :92.92 \n",
"Class 0 f1_score :87.53 \n",
"\n",
"Class 1 Precision :64.54 \n",
"Class 1 Recall :39.91 \n",
"Class 1 f1_score :49.32 \n",
"Epoch: 51\t100.00% complete. 19.97 seconds elapsed in epoch.\n",
"---------------------------------------------------------------------------------------------Epoch 51 Results---------------------------------------------------------------------------------------------\n",
"\n",
"Training Loss: 0.0368 \t Validation Loss: 0.7529\n",
"Training Accuracy: 99.08%\t Validation Accuracy: 81.44%\n",
"roc_auc_score :0.819454798469261\n",
"\n",
"Class 0 Precision :83.27 \n",
"Class 0 Recall :94.48 \n",
"Class 0 f1_score :88.52 \n",
"\n",
"Class 1 Precision :70.68 \n",
"Class 1 Recall :41.23 \n",
"Class 1 f1_score :52.08 \n",
"Epoch: 52\t100.00% complete. 20.02 seconds elapsed in epoch.\n",
"---------------------------------------------------------------------------------------------Epoch 52 Results---------------------------------------------------------------------------------------------\n",
"\n",
"Training Loss: 0.0359 \t Validation Loss: 0.7974\n",
"Training Accuracy: 98.97%\t Validation Accuracy: 80.64%\n",
"roc_auc_score :0.8203307489687393\n",
"\n",
"Class 0 Precision :82.03 \n",
"Class 0 Recall :95.04 \n",
"Class 0 f1_score :88.06 \n",
"\n",
"Class 1 Precision :69.83 \n",
"Class 1 Recall :35.53 \n",
"Class 1 f1_score :47.09 \n",
"Epoch: 53\t100.00% complete. 19.99 seconds elapsed in epoch.\n",
"---------------------------------------------------------------------------------------------Epoch 53 Results---------------------------------------------------------------------------------------------\n",
"\n",
"Training Loss: 0.0400 \t Validation Loss: 0.7680\n",
"Training Accuracy: 98.90%\t Validation Accuracy: 80.64%\n",
"roc_auc_score :0.8256299388698374\n",
"\n",
"Class 0 Precision :82.35 \n",
"Class 0 Recall :94.48 \n",
"Class 0 f1_score :87.99 \n",
"\n",
"Class 1 Precision :68.55 \n",
"Class 1 Recall :37.28 \n",
"Class 1 f1_score :48.30 \n",
"Epoch: 54\t100.00% complete. 20.01 seconds elapsed in epoch.\n",
"---------------------------------------------------------------------------------------------Epoch 54 Results---------------------------------------------------------------------------------------------\n",
"\n",
"Training Loss: 0.0429 \t Validation Loss: 0.7824\n",
"Training Accuracy: 98.70%\t Validation Accuracy: 79.52%\n",
"roc_auc_score :0.8167026986730281\n",
"\n",
"Class 0 Precision :81.75 \n",
"Class 0 Recall :93.91 \n",
"Class 0 f1_score :87.41 \n",
"\n",
"Class 1 Precision :65.04 \n",
"Class 1 Recall :35.09 \n",
"Class 1 f1_score :45.58 \n",
"Epoch: 55\t100.00% complete. 19.99 seconds elapsed in epoch.\n",
"---------------------------------------------------------------------------------------------Epoch 55 Results---------------------------------------------------------------------------------------------\n",
"\n",
"Training Loss: 0.0354 \t Validation Loss: 0.8072\n",
"Training Accuracy: 99.02%\t Validation Accuracy: 77.98%\n",
"roc_auc_score :0.8009852890015408\n",
"\n",
"Class 0 Precision :80.99 \n",
"Class 0 Recall :92.92 \n",
"Class 0 f1_score :86.54 \n",
"\n",
"Class 1 Precision :59.68 \n",
"Class 1 Recall :32.46 \n",
"Class 1 f1_score :42.05 \n",
"Epoch: 56\t100.00% complete. 20.00 seconds elapsed in epoch.\n",
"---------------------------------------------------------------------------------------------Epoch 56 Results---------------------------------------------------------------------------------------------\n",
"\n",
"Training Loss: 0.0367 \t Validation Loss: 0.8518\n",
"Training Accuracy: 99.04%\t Validation Accuracy: 78.56%\n",
"roc_auc_score :0.8076263605188609\n",
"\n",
"Class 0 Precision :80.78 \n",
"Class 0 Recall :94.05 \n",
"Class 0 f1_score :86.91 \n",
"\n",
"Class 1 Precision :62.50 \n",
"Class 1 Recall :30.70 \n",
"Class 1 f1_score :41.18 \n",
"Epoch: 57\t100.00% complete. 19.99 seconds elapsed in epoch.\n",
"---------------------------------------------------------------------------------------------Epoch 57 Results---------------------------------------------------------------------------------------------\n",
"\n",
"Training Loss: 0.0397 \t Validation Loss: 0.7943\n",
"Training Accuracy: 98.97%\t Validation Accuracy: 80.00%\n",
"roc_auc_score :0.8108878783360668\n",
"\n",
"Class 0 Precision :81.66 \n",
"Class 0 Recall :94.62 \n",
"Class 0 f1_score :87.66 \n",
"\n",
"Class 1 Precision :67.24 \n",
"Class 1 Recall :34.21 \n",
"Class 1 f1_score :45.35 \n",
"Epoch: 58\t100.00% complete. 19.96 seconds elapsed in epoch.\n",
"---------------------------------------------------------------------------------------------Epoch 58 Results---------------------------------------------------------------------------------------------\n",
"\n",
"Training Loss: 0.0359 \t Validation Loss: 0.8203\n",
"Training Accuracy: 98.93%\t Validation Accuracy: 79.10%\n",
"roc_auc_score :0.8122980965160777\n",
"\n",
"Class 0 Precision :81.04 \n",
"Class 0 Recall :94.48 \n",
"Class 0 f1_score :87.25 \n",
"\n",
"Class 1 Precision :64.86 \n",
"Class 1 Recall :31.58 \n",
"Class 1 f1_score :42.48 \n",
"Epoch: 59\t100.00% complete. 20.08 seconds elapsed in epoch.\n",
"---------------------------------------------------------------------------------------------Epoch 59 Results---------------------------------------------------------------------------------------------\n",
"\n",
"Training Loss: 0.0354 \t Validation Loss: 0.7775\n",
"Training Accuracy: 99.04%\t Validation Accuracy: 79.79%\n",
"roc_auc_score :0.8134349684409323\n",
"\n",
"Class 0 Precision :81.70 \n",
"Class 0 Recall :94.19 \n",
"Class 0 f1_score :87.50 \n",
"\n",
"Class 1 Precision :65.83 \n",
"Class 1 Recall :34.65 \n",
"Class 1 f1_score :45.40 \n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Epoch: 60\t100.00% complete. 20.01 seconds elapsed in epoch.\n",
"---------------------------------------------------------------------------------------------Epoch 60 Results---------------------------------------------------------------------------------------------\n",
"\n",
"Training Loss: 0.0295 \t Validation Loss: 0.7803\n",
"Training Accuracy: 99.37%\t Validation Accuracy: 80.00%\n",
"roc_auc_score :0.8167772476517071\n",
"\n",
"Class 0 Precision :81.43 \n",
"Class 0 Recall :95.04 \n",
"Class 0 f1_score :87.71 \n",
"\n",
"Class 1 Precision :68.18 \n",
"Class 1 Recall :32.89 \n",
"Class 1 f1_score :44.38 \n",
"Epoch: 61\t100.00% complete. 20.06 seconds elapsed in epoch.\n",
"---------------------------------------------------------------------------------------------Epoch 61 Results---------------------------------------------------------------------------------------------\n",
"\n",
"Training Loss: 0.0399 \t Validation Loss: 0.8105\n",
"Training Accuracy: 98.79%\t Validation Accuracy: 79.63%\n",
"roc_auc_score :0.8074337756572734\n",
"\n",
"Class 0 Precision :81.54 \n",
"Class 0 Recall :94.48 \n",
"Class 0 f1_score :87.53 \n",
"\n",
"Class 1 Precision :66.38 \n",
"Class 1 Recall :33.77 \n",
"Class 1 f1_score :44.77 \n",
"Epoch: 62\t100.00% complete. 19.98 seconds elapsed in epoch.\n",
"---------------------------------------------------------------------------------------------Epoch 62 Results---------------------------------------------------------------------------------------------\n",
"\n",
"Training Loss: 0.0335 \t Validation Loss: 0.7650\n",
"Training Accuracy: 99.19%\t Validation Accuracy: 80.43%\n",
"roc_auc_score :0.8112295611550122\n",
"\n",
"Class 0 Precision :82.14 \n",
"Class 0 Recall :94.48 \n",
"Class 0 f1_score :87.88 \n",
"\n",
"Class 1 Precision :68.03 \n",
"Class 1 Recall :36.40 \n",
"Class 1 f1_score :47.43 \n",
"Epoch: 63\t100.00% complete. 19.99 seconds elapsed in epoch.\n",
"---------------------------------------------------------------------------------------------Epoch 63 Results---------------------------------------------------------------------------------------------\n",
"\n",
"Training Loss: 0.0353 \t Validation Loss: 0.7998\n",
"Training Accuracy: 99.06%\t Validation Accuracy: 79.89%\n",
"roc_auc_score :0.8134846677600517\n",
"\n",
"Class 0 Precision :81.55 \n",
"Class 0 Recall :95.18 \n",
"Class 0 f1_score :87.84 \n",
"\n",
"Class 1 Precision :69.09 \n",
"Class 1 Recall :33.33 \n",
"Class 1 f1_score :44.97 \n",
"Epoch: 64\t100.00% complete. 20.05 seconds elapsed in epoch.\n",
"---------------------------------------------------------------------------------------------Epoch 64 Results---------------------------------------------------------------------------------------------\n",
"\n",
"Training Loss: 0.0307 \t Validation Loss: 0.8475\n",
"Training Accuracy: 99.26%\t Validation Accuracy: 79.41%\n",
"roc_auc_score :0.8069367824660802\n",
"\n",
"Class 0 Precision :81.27 \n",
"Class 0 Recall :94.62 \n",
"Class 0 f1_score :87.43 \n",
"\n",
"Class 1 Precision :66.07 \n",
"Class 1 Recall :32.46 \n",
"Class 1 f1_score :43.53 \n",
"Epoch: 65\t100.00% complete. 20.06 seconds elapsed in epoch.\n",
"---------------------------------------------------------------------------------------------Epoch 65 Results---------------------------------------------------------------------------------------------\n",
"\n",
"Training Loss: 0.0342 \t Validation Loss: 0.8037\n",
"Training Accuracy: 99.15%\t Validation Accuracy: 79.95%\n",
"roc_auc_score :0.8169139207792854\n",
"\n",
"Class 0 Precision :81.92 \n",
"Class 0 Recall :94.33 \n",
"Class 0 f1_score :87.69 \n",
"\n",
"Class 1 Precision :66.94 \n",
"Class 1 Recall :35.53 \n",
"Class 1 f1_score :46.42 \n",
"Epoch: 66\t100.00% complete. 20.22 seconds elapsed in epoch.\n",
"---------------------------------------------------------------------------------------------Epoch 66 Results---------------------------------------------------------------------------------------------\n",
"\n",
"Training Loss: 0.0325 \t Validation Loss: 0.8694\n",
"Training Accuracy: 99.22%\t Validation Accuracy: 79.36%\n",
"roc_auc_score :0.8109251528254062\n",
"\n",
"Class 0 Precision :80.77 \n",
"Class 0 Recall :95.18 \n",
"Class 0 f1_score :87.39 \n",
"\n",
"Class 1 Precision :66.67 \n",
"Class 1 Recall :29.82 \n",
"Class 1 f1_score :41.21 \n",
"Epoch: 67\t100.00% complete. 20.48 seconds elapsed in epoch.\n",
"---------------------------------------------------------------------------------------------Epoch 67 Results---------------------------------------------------------------------------------------------\n",
"\n",
"Training Loss: 0.0333 \t Validation Loss: 0.8188\n",
"Training Accuracy: 99.19%\t Validation Accuracy: 79.79%\n",
"roc_auc_score :0.8118197405695542\n",
"\n",
"Class 0 Precision :81.77 \n",
"Class 0 Recall :94.05 \n",
"Class 0 f1_score :87.48 \n",
"\n",
"Class 1 Precision :65.57 \n",
"Class 1 Recall :35.09 \n",
"Class 1 f1_score :45.71 \n",
"Epoch: 68\t100.00% complete. 19.95 seconds elapsed in epoch.\n",
"---------------------------------------------------------------------------------------------Epoch 68 Results---------------------------------------------------------------------------------------------\n",
"\n",
"Training Loss: 0.0315 \t Validation Loss: 0.7890\n",
"Training Accuracy: 99.22%\t Validation Accuracy: 79.15%\n",
"roc_auc_score :0.8120930868247105\n",
"\n",
"Class 0 Precision :81.94 \n",
"Class 0 Recall :93.20 \n",
"Class 0 f1_score :87.21 \n",
"\n",
"Class 1 Precision :63.36 \n",
"Class 1 Recall :36.40 \n",
"Class 1 f1_score :46.24 \n",
"Epoch: 69\t100.00% complete. 19.98 seconds elapsed in epoch.\n",
"---------------------------------------------------------------------------------------------Epoch 69 Results---------------------------------------------------------------------------------------------\n",
"\n",
"Training Loss: 0.0298 \t Validation Loss: 0.7909\n",
"Training Accuracy: 99.17%\t Validation Accuracy: 78.99%\n",
"roc_auc_score :0.8159385716415686\n",
"\n",
"Class 0 Precision :81.48 \n",
"Class 0 Recall :93.48 \n",
"Class 0 f1_score :87.07 \n",
"\n",
"Class 1 Precision :62.90 \n",
"Class 1 Recall :34.21 \n",
"Class 1 f1_score :44.32 \n",
"Epoch: 70\t100.00% complete. 20.00 seconds elapsed in epoch.\n",
"---------------------------------------------------------------------------------------------Epoch 70 Results---------------------------------------------------------------------------------------------\n",
"\n",
"Training Loss: 0.0297 \t Validation Loss: 0.7797\n",
"Training Accuracy: 99.26%\t Validation Accuracy: 80.64%\n",
"roc_auc_score :0.82691590875205\n",
"\n",
"Class 0 Precision :82.41 \n",
"Class 0 Recall :94.90 \n",
"Class 0 f1_score :88.22 \n",
"\n",
"Class 1 Precision :70.25 \n",
"Class 1 Recall :37.28 \n",
"Class 1 f1_score :48.71 \n",
"Epoch: 71\t100.00% complete. 20.00 seconds elapsed in epoch.\n",
"---------------------------------------------------------------------------------------------Epoch 71 Results---------------------------------------------------------------------------------------------\n",
"\n",
"Training Loss: 0.0380 \t Validation Loss: 0.7924\n",
"Training Accuracy: 98.93%\t Validation Accuracy: 79.79%\n",
"roc_auc_score :0.8210451766810795\n",
"\n",
"Class 0 Precision :81.70 \n",
"Class 0 Recall :94.19 \n",
"Class 0 f1_score :87.50 \n",
"\n",
"Class 1 Precision :65.83 \n",
"Class 1 Recall :34.65 \n",
"Class 1 f1_score :45.40 \n",
"Epoch: 72\t100.00% complete. 19.99 seconds elapsed in epoch.\n",
"---------------------------------------------------------------------------------------------Epoch 72 Results---------------------------------------------------------------------------------------------\n",
"\n",
"Training Loss: 0.0319 \t Validation Loss: 0.8082\n",
"Training Accuracy: 99.19%\t Validation Accuracy: 79.47%\n",
"roc_auc_score :0.814273644451071\n",
"\n",
"Class 0 Precision :81.63 \n",
"Class 0 Recall :93.77 \n",
"Class 0 f1_score :87.28 \n",
"\n",
"Class 1 Precision :64.23 \n",
"Class 1 Recall :34.65 \n",
"Class 1 f1_score :45.01 \n",
"Epoch: 73\t100.00% complete. 19.97 seconds elapsed in epoch.\n",
"---------------------------------------------------------------------------------------------Epoch 73 Results---------------------------------------------------------------------------------------------\n",
"\n",
"Training Loss: 0.0336 \t Validation Loss: 0.7939\n",
"Training Accuracy: 99.17%\t Validation Accuracy: 79.36%\n",
"roc_auc_score :0.8117265543462056\n",
"\n",
"Class 0 Precision :81.45 \n",
"Class 0 Recall :93.91 \n",
"Class 0 f1_score :87.24 \n",
"\n",
"Class 1 Precision :64.17 \n",
"Class 1 Recall :33.77 \n",
"Class 1 f1_score :44.25 \n",
"Epoch: 74\t100.00% complete. 20.06 seconds elapsed in epoch.\n",
"---------------------------------------------------------------------------------------------Epoch 74 Results---------------------------------------------------------------------------------------------\n",
"\n",
"Training Loss: 0.0331 \t Validation Loss: 0.8664\n",
"Training Accuracy: 99.19%\t Validation Accuracy: 77.98%\n",
"roc_auc_score :0.8004199592465583\n",
"\n",
"Class 0 Precision :80.17 \n",
"Class 0 Recall :94.48 \n",
"Class 0 f1_score :86.74 \n",
"\n",
"Class 1 Precision :61.76 \n",
"Class 1 Recall :27.63 \n",
"Class 1 f1_score :38.18 \n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Epoch: 75\t100.00% complete. 20.07 seconds elapsed in epoch.\n",
"---------------------------------------------------------------------------------------------Epoch 75 Results---------------------------------------------------------------------------------------------\n",
"\n",
"Training Loss: 0.0295 \t Validation Loss: 0.8167\n",
"Training Accuracy: 99.24%\t Validation Accuracy: 79.79%\n",
"roc_auc_score :0.8174792505342677\n",
"\n",
"Class 0 Precision :81.23 \n",
"Class 0 Recall :95.04 \n",
"Class 0 f1_score :87.60 \n",
"\n",
"Class 1 Precision :67.59 \n",
"Class 1 Recall :32.02 \n",
"Class 1 f1_score :43.45 \n",
"Epoch: 76\t100.00% complete. 20.01 seconds elapsed in epoch.\n",
"---------------------------------------------------------------------------------------------Epoch 76 Results---------------------------------------------------------------------------------------------\n",
"\n",
"Training Loss: 0.0306 \t Validation Loss: 0.7587\n",
"Training Accuracy: 99.24%\t Validation Accuracy: 80.11%\n",
"roc_auc_score :0.824474429700313\n",
"\n",
"Class 0 Precision :82.24 \n",
"Class 0 Recall :93.77 \n",
"Class 0 f1_score :87.62 \n",
"\n",
"Class 1 Precision :65.89 \n",
"Class 1 Recall :37.28 \n",
"Class 1 f1_score :47.62 \n",
"Epoch: 77\t100.00% complete. 20.04 seconds elapsed in epoch.\n",
"---------------------------------------------------------------------------------------------Epoch 77 Results---------------------------------------------------------------------------------------------\n",
"\n",
"Training Loss: 0.0264 \t Validation Loss: 0.8102\n",
"Training Accuracy: 99.44%\t Validation Accuracy: 78.88%\n",
"roc_auc_score :0.8116520053675265\n",
"\n",
"Class 0 Precision :81.46 \n",
"Class 0 Recall :93.34 \n",
"Class 0 f1_score :87.00 \n",
"\n",
"Class 1 Precision :62.40 \n",
"Class 1 Recall :34.21 \n",
"Class 1 f1_score :44.19 \n",
"Epoch: 78\t100.00% complete. 19.99 seconds elapsed in epoch.\n",
"---------------------------------------------------------------------------------------------Epoch 78 Results---------------------------------------------------------------------------------------------\n",
"\n",
"Training Loss: 0.0299 \t Validation Loss: 0.8146\n",
"Training Accuracy: 99.33%\t Validation Accuracy: 80.43%\n",
"roc_auc_score :0.8146588141742458\n",
"\n",
"Class 0 Precision :81.37 \n",
"Class 0 Recall :95.89 \n",
"Class 0 f1_score :88.04 \n",
"\n",
"Class 1 Precision :71.57 \n",
"Class 1 Recall :32.02 \n",
"Class 1 f1_score :44.24 \n",
"Epoch: 79\t100.00% complete. 20.14 seconds elapsed in epoch.\n",
"---------------------------------------------------------------------------------------------Epoch 79 Results---------------------------------------------------------------------------------------------\n",
"\n",
"Training Loss: 0.0306 \t Validation Loss: 0.8154\n",
"Training Accuracy: 99.40%\t Validation Accuracy: 79.47%\n",
"roc_auc_score :0.8084898861885592\n",
"\n",
"Class 0 Precision :81.78 \n",
"Class 0 Recall :93.48 \n",
"Class 0 f1_score :87.24 \n",
"\n",
"Class 1 Precision :63.78 \n",
"Class 1 Recall :35.53 \n",
"Class 1 f1_score :45.63 \n",
"Epoch: 80\t100.00% complete. 20.66 seconds elapsed in epoch.\n",
"---------------------------------------------------------------------------------------------Epoch 80 Results---------------------------------------------------------------------------------------------\n",
"\n",
"Training Loss: 0.0299 \t Validation Loss: 0.8219\n",
"Training Accuracy: 99.33%\t Validation Accuracy: 80.27%\n",
"roc_auc_score :0.8150377714825306\n",
"\n",
"Class 0 Precision :81.67 \n",
"Class 0 Recall :95.33 \n",
"Class 0 f1_score :87.97 \n",
"\n",
"Class 1 Precision :70.00 \n",
"Class 1 Recall :33.77 \n",
"Class 1 f1_score :45.56 \n",
"Epoch: 81\t100.00% complete. 20.32 seconds elapsed in epoch.\n",
"---------------------------------------------------------------------------------------------Epoch 81 Results---------------------------------------------------------------------------------------------\n",
"\n",
"Training Loss: 0.0299 \t Validation Loss: 0.7977\n",
"Training Accuracy: 99.06%\t Validation Accuracy: 79.95%\n",
"roc_auc_score :0.8174978877789374\n",
"\n",
"Class 0 Precision :82.24 \n",
"Class 0 Recall :93.77 \n",
"Class 0 f1_score :87.62 \n",
"\n",
"Class 1 Precision :65.89 \n",
"Class 1 Recall :37.28 \n",
"Class 1 f1_score :47.62 \n",
"Epoch: 82\t100.00% complete. 20.06 seconds elapsed in epoch.\n",
"---------------------------------------------------------------------------------------------Epoch 82 Results---------------------------------------------------------------------------------------------\n",
"\n",
"Training Loss: 0.0261 \t Validation Loss: 0.8321\n",
"Training Accuracy: 99.46%\t Validation Accuracy: 79.73%\n",
"roc_auc_score :0.8106766562298096\n",
"\n",
"Class 0 Precision :81.56 \n",
"Class 0 Recall :94.62 \n",
"Class 0 f1_score :87.61 \n",
"\n",
"Class 1 Precision :66.96 \n",
"Class 1 Recall :33.77 \n",
"Class 1 f1_score :44.90 \n",
"Epoch: 83\t100.00% complete. 20.24 seconds elapsed in epoch.\n",
"---------------------------------------------------------------------------------------------Epoch 83 Results---------------------------------------------------------------------------------------------\n",
"\n",
"Training Loss: 0.0287 \t Validation Loss: 0.7902\n",
"Training Accuracy: 99.33%\t Validation Accuracy: 79.79%\n",
"roc_auc_score :0.8152862680781273\n",
"\n",
"Class 0 Precision :82.40 \n",
"Class 0 Recall :93.48 \n",
"Class 0 f1_score :87.59 \n",
"\n",
"Class 1 Precision :65.41 \n",
"Class 1 Recall :38.16 \n",
"Class 1 f1_score :48.20 \n",
"Epoch: 84\t100.00% complete. 20.70 seconds elapsed in epoch.\n",
"---------------------------------------------------------------------------------------------Epoch 84 Results---------------------------------------------------------------------------------------------\n",
"\n",
"Training Loss: 0.0294 \t Validation Loss: 0.8150\n",
"Training Accuracy: 99.31%\t Validation Accuracy: 79.63%\n",
"roc_auc_score :0.8167710352368173\n",
"\n",
"Class 0 Precision :81.70 \n",
"Class 0 Recall :94.19 \n",
"Class 0 f1_score :87.50 \n",
"\n",
"Class 1 Precision :65.83 \n",
"Class 1 Recall :34.65 \n",
"Class 1 f1_score :45.40 \n",
"Epoch: 85\t100.00% complete. 20.35 seconds elapsed in epoch.\n",
"---------------------------------------------------------------------------------------------Epoch 85 Results---------------------------------------------------------------------------------------------\n",
"\n",
"Training Loss: 0.0272 \t Validation Loss: 0.7500\n",
"Training Accuracy: 99.40%\t Validation Accuracy: 80.59%\n",
"roc_auc_score :0.822629342478008\n",
"\n",
"Class 0 Precision :83.02 \n",
"Class 0 Recall :93.48 \n",
"Class 0 f1_score :87.94 \n",
"\n",
"Class 1 Precision :66.91 \n",
"Class 1 Recall :40.79 \n",
"Class 1 f1_score :50.68 \n",
"Epoch: 86\t100.00% complete. 19.98 seconds elapsed in epoch.\n",
"---------------------------------------------------------------------------------------------Epoch 86 Results---------------------------------------------------------------------------------------------\n",
"\n",
"Training Loss: 0.0294 \t Validation Loss: 0.8381\n",
"Training Accuracy: 99.28%\t Validation Accuracy: 78.19%\n",
"roc_auc_score :0.8079494060931365\n",
"\n",
"Class 0 Precision :80.58 \n",
"Class 0 Recall :94.05 \n",
"Class 0 f1_score :86.80 \n",
"\n",
"Class 1 Precision :61.82 \n",
"Class 1 Recall :29.82 \n",
"Class 1 f1_score :40.24 \n",
"Epoch: 87\t100.00% complete. 19.99 seconds elapsed in epoch.\n",
"---------------------------------------------------------------------------------------------Epoch 87 Results---------------------------------------------------------------------------------------------\n",
"\n",
"Training Loss: 0.0242 \t Validation Loss: 0.8519\n",
"Training Accuracy: 99.57%\t Validation Accuracy: 79.68%\n",
"roc_auc_score :0.8093223497838079\n",
"\n",
"Class 0 Precision :81.74 \n",
"Class 0 Recall :94.48 \n",
"Class 0 f1_score :87.65 \n",
"\n",
"Class 1 Precision :66.95 \n",
"Class 1 Recall :34.65 \n",
"Class 1 f1_score :45.66 \n",
"Epoch: 88\t100.00% complete. 19.97 seconds elapsed in epoch.\n",
"---------------------------------------------------------------------------------------------Epoch 88 Results---------------------------------------------------------------------------------------------\n",
"\n",
"Training Loss: 0.0285 \t Validation Loss: 0.7911\n",
"Training Accuracy: 99.37%\t Validation Accuracy: 80.05%\n",
"roc_auc_score :0.8161932806520551\n",
"\n",
"Class 0 Precision :82.26 \n",
"Class 0 Recall :93.91 \n",
"Class 0 f1_score :87.70 \n",
"\n",
"Class 1 Precision :66.41 \n",
"Class 1 Recall :37.28 \n",
"Class 1 f1_score :47.75 \n",
"Epoch: 89\t100.00% complete. 19.97 seconds elapsed in epoch.\n",
"---------------------------------------------------------------------------------------------Epoch 89 Results---------------------------------------------------------------------------------------------\n",
"\n",
"Training Loss: 0.0267 \t Validation Loss: 0.7789\n",
"Training Accuracy: 99.37%\t Validation Accuracy: 79.95%\n",
"roc_auc_score :0.8178644202574424\n",
"\n",
"Class 0 Precision :82.40 \n",
"Class 0 Recall :93.48 \n",
"Class 0 f1_score :87.59 \n",
"\n",
"Class 1 Precision :65.41 \n",
"Class 1 Recall :38.16 \n",
"Class 1 f1_score :48.20 \n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Epoch: 90\t100.00% complete. 19.98 seconds elapsed in epoch.\n",
"---------------------------------------------------------------------------------------------Epoch 90 Results---------------------------------------------------------------------------------------------\n",
"\n",
"Training Loss: 0.0251 \t Validation Loss: 0.7689\n",
"Training Accuracy: 99.46%\t Validation Accuracy: 80.32%\n",
"roc_auc_score :0.8211383629044282\n",
"\n",
"Class 0 Precision :82.12 \n",
"Class 0 Recall :94.33 \n",
"Class 0 f1_score :87.80 \n",
"\n",
"Class 1 Precision :67.48 \n",
"Class 1 Recall :36.40 \n",
"Class 1 f1_score :47.29 \n",
"Epoch: 91\t100.00% complete. 19.98 seconds elapsed in epoch.\n",
"---------------------------------------------------------------------------------------------Epoch 91 Results---------------------------------------------------------------------------------------------\n",
"\n",
"Training Loss: 0.0273 \t Validation Loss: 0.7538\n",
"Training Accuracy: 99.37%\t Validation Accuracy: 80.69%\n",
"roc_auc_score :0.830177426569256\n",
"\n",
"Class 0 Precision :82.88 \n",
"Class 0 Recall :93.91 \n",
"Class 0 f1_score :88.05 \n",
"\n",
"Class 1 Precision :67.91 \n",
"Class 1 Recall :39.91 \n",
"Class 1 f1_score :50.28 \n",
"Epoch: 92\t100.00% complete. 20.02 seconds elapsed in epoch.\n",
"---------------------------------------------------------------------------------------------Epoch 92 Results---------------------------------------------------------------------------------------------\n",
"\n",
"Training Loss: 0.0308 \t Validation Loss: 0.7863\n",
"Training Accuracy: 99.15%\t Validation Accuracy: 79.89%\n",
"roc_auc_score :0.8186968838526912\n",
"\n",
"Class 0 Precision :81.56 \n",
"Class 0 Recall :94.62 \n",
"Class 0 f1_score :87.61 \n",
"\n",
"Class 1 Precision :66.96 \n",
"Class 1 Recall :33.77 \n",
"Class 1 f1_score :44.90 \n",
"Epoch: 93\t100.00% complete. 20.00 seconds elapsed in epoch.\n",
"---------------------------------------------------------------------------------------------Epoch 93 Results---------------------------------------------------------------------------------------------\n",
"\n",
"Training Loss: 0.0298 \t Validation Loss: 0.8290\n",
"Training Accuracy: 99.24%\t Validation Accuracy: 79.36%\n",
"roc_auc_score :0.8142363699617314\n",
"\n",
"Class 0 Precision :81.53 \n",
"Class 0 Recall :93.77 \n",
"Class 0 f1_score :87.22 \n",
"\n",
"Class 1 Precision :63.93 \n",
"Class 1 Recall :34.21 \n",
"Class 1 f1_score :44.57 \n",
"Epoch: 94\t100.00% complete. 19.95 seconds elapsed in epoch.\n",
"---------------------------------------------------------------------------------------------Epoch 94 Results---------------------------------------------------------------------------------------------\n",
"\n",
"Training Loss: 0.0280 \t Validation Loss: 0.8295\n",
"Training Accuracy: 99.24%\t Validation Accuracy: 80.32%\n",
"roc_auc_score :0.8182620148103971\n",
"\n",
"Class 0 Precision :81.96 \n",
"Class 0 Recall :94.62 \n",
"Class 0 f1_score :87.84 \n",
"\n",
"Class 1 Precision :68.07 \n",
"Class 1 Recall :35.53 \n",
"Class 1 f1_score :46.69 \n",
"Epoch: 95\t100.00% complete. 20.47 seconds elapsed in epoch.\n",
"---------------------------------------------------------------------------------------------Epoch 95 Results---------------------------------------------------------------------------------------------\n",
"\n",
"Training Loss: 0.0286 \t Validation Loss: 0.8599\n",
"Training Accuracy: 99.26%\t Validation Accuracy: 77.93%\n",
"roc_auc_score :0.8132920828984643\n",
"\n",
"Class 0 Precision :80.49 \n",
"Class 0 Recall :93.48 \n",
"Class 0 f1_score :86.50 \n",
"\n",
"Class 1 Precision :59.65 \n",
"Class 1 Recall :29.82 \n",
"Class 1 f1_score :39.77 \n",
"Epoch: 96\t100.00% complete. 20.73 seconds elapsed in epoch.\n",
"---------------------------------------------------------------------------------------------Epoch 96 Results---------------------------------------------------------------------------------------------\n",
"\n",
"Training Loss: 0.0263 \t Validation Loss: 0.8357\n",
"Training Accuracy: 99.44%\t Validation Accuracy: 78.78%\n",
"roc_auc_score :0.8083035137418617\n",
"\n",
"Class 0 Precision :81.43 \n",
"Class 0 Recall :93.77 \n",
"Class 0 f1_score :87.16 \n",
"\n",
"Class 1 Precision :63.64 \n",
"Class 1 Recall :33.77 \n",
"Class 1 f1_score :44.13 \n",
"Epoch: 97\t100.00% complete. 20.54 seconds elapsed in epoch.\n",
"---------------------------------------------------------------------------------------------Epoch 97 Results---------------------------------------------------------------------------------------------\n",
"\n",
"Training Loss: 0.0272 \t Validation Loss: 0.8160\n",
"Training Accuracy: 99.31%\t Validation Accuracy: 78.94%\n",
"roc_auc_score :0.8099249540281299\n",
"\n",
"Class 0 Precision :81.13 \n",
"Class 0 Recall :93.77 \n",
"Class 0 f1_score :86.99 \n",
"\n",
"Class 1 Precision :62.71 \n",
"Class 1 Recall :32.46 \n",
"Class 1 f1_score :42.77 \n",
"Epoch: 98\t100.00% complete. 20.14 seconds elapsed in epoch.\n",
"---------------------------------------------------------------------------------------------Epoch 98 Results---------------------------------------------------------------------------------------------\n",
"\n",
"Training Loss: 0.0280 \t Validation Loss: 0.8553\n",
"Training Accuracy: 99.24%\t Validation Accuracy: 78.19%\n",
"roc_auc_score :0.8123850703245366\n",
"\n",
"Class 0 Precision :80.58 \n",
"Class 0 Recall :94.05 \n",
"Class 0 f1_score :86.80 \n",
"\n",
"Class 1 Precision :61.82 \n",
"Class 1 Recall :29.82 \n",
"Class 1 f1_score :40.24 \n",
"Epoch: 99\t100.00% complete. 20.03 seconds elapsed in epoch.\n",
"---------------------------------------------------------------------------------------------Epoch 99 Results---------------------------------------------------------------------------------------------\n",
"\n",
"Training Loss: 0.0239 \t Validation Loss: 0.8146\n",
"Training Accuracy: 99.51%\t Validation Accuracy: 79.52%\n",
"roc_auc_score :0.820939565627951\n",
"\n",
"Class 0 Precision :81.52 \n",
"Class 0 Recall :94.33 \n",
"Class 0 f1_score :87.46 \n",
"\n",
"Class 1 Precision :65.81 \n",
"Class 1 Recall :33.77 \n",
"Class 1 f1_score :44.64 \n",
"\n",
"Best epoch: 4 with loss: 0.43 and acc: 79.52%\n",
"2144.07 total seconds elapsed. 21.66 seconds per epoch.\n",
"\n",
"\n",
"----------------\n",
"\n",
"\n",
"history df shape: (100, 5)\n"
]
}
],
"source": [
"model, history = train(\n",
" model,\n",
" device,\n",
" criterion,\n",
" optimizer_ft,\n",
" scheduler,\n",
" train_loader,\n",
" test_loader,\n",
" save_file_name=save_file_name,\n",
" max_epochs_stop=4,\n",
" n_epochs=NUM_EPOCHS,\n",
" print_every=1)"
]
},
{
"cell_type": "code",
"execution_count": 38,
"metadata": {},
"outputs": [],
"source": [
"def plot(history):\n",
" \n",
" # The history object contains results on the training and test\n",
" # sets for each epoch\n",
" acc = history['train_acc']\n",
" val_acc = history['valid_acc']\n",
" loss = history['train_loss']\n",
" val_loss = history['valid_loss']\n",
" auc = history['roc']\n",
"\n",
" # Get the number of epochs\n",
" epochs = range(len(acc))\n",
"\n",
" plt.title('Training and validation accuracy')\n",
" plt.plot(epochs, acc, color='blue', label='Train')\n",
" plt.plot(epochs, val_acc, color='orange', label='Val')\n",
" plt.xlabel('Epoch')\n",
" plt.ylabel('Accuracy')\n",
" plt.grid(True, axis = 'y')\n",
" plt.legend()\n",
"\n",
"# _ = plt.figure()\n",
"# plt.title('Training and validation loss')\n",
"# plt.plot(epochs, loss, color='blue', label='Train')\n",
"# plt.plot(epochs, val_loss, color='orange', label='Val')\n",
"# plt.xlabel('Epoch')\n",
"# plt.ylabel('Loss')\n",
"# plt.grid(True, axis = 'y')\n",
"# plt.legend()\n",
"\n",
" _ = plt.figure()\n",
" plt.title('Validation AUC')\n",
" plt.plot(epochs, auc, color='blue', label='Train')\n",
" plt.xlabel('Epoch')\n",
" plt.ylabel('Auc')\n",
" plt.grid(True, axis = 'y')\n",
" plt.legend()"
]
},
{
"cell_type": "code",
"execution_count": 35,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>train_loss</th>\n",
" <th>valid_loss</th>\n",
" <th>train_acc</th>\n",
" <th>valid_acc</th>\n",
" <th>roc</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>0.545076</td>\n",
" <td>0.497729</td>\n",
" <td>0.750336</td>\n",
" <td>0.757447</td>\n",
" <td>0.737762</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>0.423507</td>\n",
" <td>0.444942</td>\n",
" <td>0.780089</td>\n",
" <td>0.761702</td>\n",
" <td>0.804862</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>0.374174</td>\n",
" <td>0.438525</td>\n",
" <td>0.814765</td>\n",
" <td>0.772340</td>\n",
" <td>0.813727</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>0.335164</td>\n",
" <td>0.436611</td>\n",
" <td>0.855705</td>\n",
" <td>0.781383</td>\n",
" <td>0.822269</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>0.302722</td>\n",
" <td>0.430409</td>\n",
" <td>0.877852</td>\n",
" <td>0.783511</td>\n",
" <td>0.831954</td>\n",
" </tr>\n",
" <tr>\n",
" <th>...</th>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>95</th>\n",
" <td>0.028597</td>\n",
" <td>0.859924</td>\n",
" <td>0.992617</td>\n",
" <td>0.779255</td>\n",
" <td>0.813292</td>\n",
" </tr>\n",
" <tr>\n",
" <th>96</th>\n",
" <td>0.026329</td>\n",
" <td>0.835668</td>\n",
" <td>0.994407</td>\n",
" <td>0.787766</td>\n",
" <td>0.808304</td>\n",
" </tr>\n",
" <tr>\n",
" <th>97</th>\n",
" <td>0.027203</td>\n",
" <td>0.816045</td>\n",
" <td>0.993065</td>\n",
" <td>0.789362</td>\n",
" <td>0.809925</td>\n",
" </tr>\n",
" <tr>\n",
" <th>98</th>\n",
" <td>0.028048</td>\n",
" <td>0.855256</td>\n",
" <td>0.992394</td>\n",
" <td>0.781915</td>\n",
" <td>0.812385</td>\n",
" </tr>\n",
" <tr>\n",
" <th>99</th>\n",
" <td>0.023886</td>\n",
" <td>0.814558</td>\n",
" <td>0.995078</td>\n",
" <td>0.795213</td>\n",
" <td>0.820940</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"<p>100 rows × 5 columns</p>\n",
"</div>"
],
"text/plain": [
" train_loss valid_loss train_acc valid_acc roc\n",
"0 0.545076 0.497729 0.750336 0.757447 0.737762\n",
"1 0.423507 0.444942 0.780089 0.761702 0.804862\n",
"2 0.374174 0.438525 0.814765 0.772340 0.813727\n",
"3 0.335164 0.436611 0.855705 0.781383 0.822269\n",
"4 0.302722 0.430409 0.877852 0.783511 0.831954\n",
".. ... ... ... ... ...\n",
"95 0.028597 0.859924 0.992617 0.779255 0.813292\n",
"96 0.026329 0.835668 0.994407 0.787766 0.808304\n",
"97 0.027203 0.816045 0.993065 0.789362 0.809925\n",
"98 0.028048 0.855256 0.992394 0.781915 0.812385\n",
"99 0.023886 0.814558 0.995078 0.795213 0.820940\n",
"\n",
"[100 rows x 5 columns]"
]
},
"execution_count": 35,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"history"
]
},
{
"cell_type": "code",
"execution_count": 36,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"train_loss 0.302722\n",
"valid_loss 0.430409\n",
"train_acc 0.877852\n",
"valid_acc 0.783511\n",
"roc 0.831954\n",
"Name: 4, dtype: float64\n",
"train_loss 0.036840\n",
"valid_loss 0.752872\n",
"train_acc 0.990828\n",
"valid_acc 0.814362\n",
"roc 0.819455\n",
"Name: 51, dtype: float64\n"
]
}
],
"source": [
"print(history.iloc[history['roc'].argmax()])\n",
"print(history.iloc[history['valid_acc'].argmax()])"
]
},
{
"cell_type": "code",
"execution_count": 39,
"metadata": {},
"outputs": [
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEWCAYAAAB8LwAVAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nOydd3hUVfrHP29CQguhE5BeFRBFiFiwYC+rwlpWXV3bqquu29R1XXV3Xf1Zdt1iXdvadlXQxYaKBREsCAoo0jsBAiT0hA5J3t8f7x0zmcwkk5BhQvJ+nmeemXvvuee+t8z5nvOe95wrqorjOI7jRJKSbAMcx3Gc2okLhOM4jhMVFwjHcRwnKi4QjuM4TlRcIBzHcZyouEA4juM4UXGBcOJGRFJFZKuIdKnJtMlERHqJSI3HeovIySKSE7a8QESOjSdtNY71bxG5vbr7O04sGiTbACdxiMjWsMUmwC6gOFj+maq+XJX8VLUYyKjptPUBVT2wJvIRkauBS1V1WFjeV9dE3o4TiQtEHUZVvy+ggxrq1ar6caz0ItJAVYv2hW2OUxn+PCYfdzHVY0Tk/0TkVREZKSJbgEtF5CgRmSIim0VkjYg8IiJpQfoGIqIi0i1YfinY/r6IbBGRySLSvappg+1niMhCESkQkUdFZJKIXBHD7nhs/JmILBaRTSLySNi+qSLyTxHZICJLgNMruD53isioiHWPi8g/gt9Xi8i84HyWBLX7WHnlisiw4HcTEflvYNscYHCU4y4N8p0jIucE6wcAjwHHBu679WHX9q6w/a8Lzn2DiLwlIh3iuTZVuc4he0TkYxHZKCJ5InJr2HH+EFyTQhGZJiIHRHPnicgXofscXM/PguNsBO4Ukd4iMiE4l/XBdWsetn/X4BzXBdsfFpFGgc19w9J1EJHtItI61vk6UVBV/9SDD5ADnByx7v+A3cDZWGWhMXA4cATWuuwBLARuDNI3ABToFiy/BKwHsoE04FXgpWqkbQdsAYYH224C9gBXxDiXeGx8G2gOdAM2hs4duBGYA3QCWgOf2d8g6nF6AFuBpmF5rwWyg+WzgzQCnAjsAA4Jtp0M5ITllQsMC37/DZgItAS6AnMj0v4I6BDckx8HNmQF264GJkbY+RJwV/D71MDGgUAj4F/AJ/Fcmype5+ZAPvAroCGQCQwJtv0e+A7oHZzDQKAV0CvyWgNfhO5zcG5FwPVAKvY89gFOAtKD52QS8Lew85kdXM+mQfqhwbangXvDjnMz8Gay/4f72yfpBvhnH93o2ALxSSX73QL8L/gdrdB/MiztOcDsaqS9Cvg8bJsAa4ghEHHaeGTY9jeAW4Lfn2GuttC2MyMLrYi8pwA/Dn6fASysIO27wM+D3xUJxIrwewHcEJ42Sr6zgR8EvysTiBeB+8K2ZWL9Tp0quzZVvM4/AabFSLckZG/E+ngEYmklNpwPTA1+HwvkAalR0g0FlgESLM8Azq3p/1Vd/7iLyVkZviAiB4nIe4HLoBC4G2hTwf55Yb+3U3HHdKy0B4TbofaPzo2VSZw2xnUsYHkF9gK8Alwc/P4x8H3HvoicJSJfBS6WzVjtvaJrFaJDRTaIyBUi8l3gJtkMHBRnvmDn931+qloIbAI6hqWJ655Vcp07A4tj2NAZE4nqEPk8theR10RkVWDDCxE25KgFRJRBVSdhrZFjRORgoAvwXjVtqre4QDiRIZ5PYTXWXqqaCfwRq9EnkjVYDRcAERHKFmiR7I2Na7CCJURlYbivAieLSCfMBfZKYGNjYDRwP+b+aQF8FKcdebFsEJEewBOYm6V1kO/8sHwrC8ldjbmtQvk1w1xZq+KwK5KKrvNKoGeM/WJt2xbY1CRsXfuINJHn9xcs+m5AYMMVETZ0FZHUGHb8B7gUa+28pqq7YqRzYuAC4UTSDCgAtgWdfD/bB8d8FxgkImeLSAPMr902QTa+BvxaRDoGHZa/qyixquZjbpDngQWquijY1BDzi68DikXkLMxXHq8Nt4tIC7FxIjeGbcvACsl1mFZejbUgQuQDncI7iyMYCfxURA4RkYaYgH2uqjFbZBVQ0XUeA3QRkRtFJF1EMkVkSLDt38D/iUhPMQaKSCtMGPOwYIhUEbmWMDGrwIZtQIGIdMbcXCEmAxuA+8Q6/huLyNCw7f/FXFI/xsTCqSIuEE4kNwOXY53GT2E16IQSFMIXAv/A/vA9gW+xmmNN2/gEMB6YBUzFWgGV8QrWp/BKmM2bgd8Ab2IdvedjQhcPf8JaMjnA+4QVXqo6E3gE+DpIcxDwVdi+44BFQL6IhLuKQvt/gLmC3gz27wJcEqddkcS8zqpaAJwCnId1ii8Ejg82Pwi8hV3nQqzDuFHgOrwGuB0LWOgVcW7R+BMwBBOqMcDrYTYUAWcBfbHWxArsPoS252D3ebeqflnFc3co7cBxnFpD4DJYDZyvqp8n2x5n/0VE/oN1fN+VbFv2R3ygnFMrEJHTMZfBTixMsgirRTtOtQj6c4YDA5Jty/6Ku5ic2sIxwFLM9XA6MMI7FZ3qIiL3Y2Mx7lPVFcm2Z3/FXUyO4zhOVLwF4TiO40SlzvRBtGnTRrt165ZsMxzHcfYrpk+fvl5Vo4aV1xmB6NatG9OmTUu2GY7jOPsVIhJzNgF3MTmO4zhRcYFwHMdxopIwgRCR50RkrYjMjrFdgnnfF4vITBEZFLbtchFZFHwuT5SNjuM4TmwS2YJ4gQpexoJNndw7+FyLTYFAMGfLn7B56IcAfxKRlgm003Ecx4lCwgRCVT/D5qiJxXDgP2pMAVqIvfnqNGCcqm5U1U3Y3DMVCY3jOI6TAJIZxdSRsnO/5wbrYq0vRzAb5LUAWVlZTJw4MSGGOo7j1EeSKRDR5s3XCtaXX6n6NDZTJNnZ2Tps2LAaM85xHKe+k8woplzKvjSlEzaDZ6z1juM4SeWrr+CFF2D37mRbUkpxMWzalJi8kykQY4DLgmimI4ECVV0DfAicKiItg87pU4N1juM4rFkDL78MS6K81HTHjor3LS6G+++HZ56BjRX0kBYVWdoQqvDggzB0KFx5JfTtCy+9VDZNcTF8+y08+ij85jfw3/9CTo7tWx327IG334ZJk6CkJHqa5cvhxBPh3HNjp9kbEuZiEpGRwDCgjYjkYpFJaQCq+iQwFnth/GLsvbhXBts2isg92MtcAO5W1Yo6ux3HiUFxMdx2G6xYAa+8AqmxXs65H7BpE/z1r/Dww6VCcOCBcOyxdn6zZpl4tG0LhxwChx4KN94I3btbWlW47jr4979t+ec/hzPOgOxsaN4cMjNh2TL44guYMgXS0+G00+DMM+GNN6ywPu88uPRSuPtu+MlPTAgaNy61b+tW+92wIewK5iLu2hX++Ee44gpIiaNKvn07PPcc/O1vJgChPH78Yzj1VBgwAFq1svt5ww12Xo89BpKAFwPXmdlcs7Oz1afacJxStm+Hiy+GMWNs+R//sAItxOrV8PHHVki2DWbiWbXKCr8334TDD4cf/ACOP97Wz5oFCxdawdm8OTRrZgV1YSFs2QItW0KXLtCxIyxdagXtV19ZDfeZZ0oLUrACePp0K5SbNzfhKiiwz7p1sHKlFfrr1pXus3SpHevii61gnD4d3nsPvv7aRGDAAOjZ0wrVWbNg5kyz9bHHrDC/+Wb45z/hjjusxv3KK/Dqq5Ab9jLWlBQTlqFDYds2GDsW8vOhQQMrsH/5SyuIS0pg9Gj48MPSFkJGBhx1lO3bsSPMmWPX4KWXYPJku5733mstly++gGnTSkWkpMTEpbAQNm+21sMxx8Bvf2vrXn4Zxo0rbbG0bg0bNtix/vvfUhGsDiIyXVWzo25zgXCcshQWWkGRkZGY/BcssBrp4MFW+DZsWPk+qlaAFBSYfW3blhbq0VizBkaMsELo4Yfho49MDGbOhF69IC/Pat6LF1vhfOqp0KMHPPusFUJnnw0zZlihHE6bNra9oKDUpZGRYWKxcWNpgQdWWB98sAnUUUdZDbxZM/jDH6ywrajoad/exKZt29Jad6tWcNNN1jqIh+XL4bLL4LPPYOBAO59f/hIeeqhsbXvPHrumBQV2fpmZpdtKSsxtlJkJvXvHd9xIVE2Mfvtbuy8ATZqYYIQfKyPDxLJFCxPmY44pm8/69fDNNyZ+s2ebIP7ylyZee4MLhFMnKSqyWtohh9RM87qoyArTP/zBls85x5r1p59uNdFwZs60mmu/ftC/v/2xi4utJp2TY37jL76w2ukJJ9gfvn17uOce6+QM1QSbNoWTTrLCYsAAO5fI2uDkyfCjH5Wt6YLlN2AA9OljhWnnzlYovvcefPmlCc/IkTB8uLUA+ve3gvL1182mJUvM3TJzphVgK1daTfuuu8wGVROzKVPMxTFggBWgYNu2b4dGjUrdVqqltf8DDoAOHWz96NHmlunSxQrG774zV88dd1gLpKDArkfz5vZp1So+0YyH4mL4+9/hzjtNLJ5+Oj43TyLYsgXefdeE5tBDIS0tOXZEUpFAoKp14jN48GB16he/+50qqJ5yiurcudXPp6REddo01exsy++ss1Svv161TRtb7t9fdd680vSjR6s2bGjbQp+MjLLLoNqxo+rhh6umpJSuS09X/dWvVHNyVN99147Tq1fZ/U46SfXrr+1YL7xg+/ToofqXv6g+9ZTqyJGqf/+76hVXqA4erNqiRdn9DztM9c47y1+TZ5+17VlZlue4caXbiotVCwqqfw0r44svVFu1Um3XTvWddxJ3nFgUFNh9dsoDTNMY5aq3IJz9kiVLrPZ+2GFWy9261Wqlp5xiNd2uXSuvKX7+udWm333X8mvb1iJQfvQja5Hs2WPukeuvt5ruM8+Ya+amm+DII+Gpp0r93WvXlvrTO3SAo4+2GrOI+Yo//BAWLbKOyq5dy9uydau1hiZONPfL+vXWqpg61Xz4r71mfudYFBZazb1Vq9KaeySq1hoaP95cXOecE+/VrhnWr7dac/Pm+/a4TsV4C8Kpc5x7rmrTpqqrVqmuXat6zTVla+pNmqh27qx68MGqxx+v+uKLqkVFtm9hoepVV1m6Ro1UzzhD9bHHVDdsiH6s3FzVoUNL8z73XNXt2xN3boWFqn/+s2rLlqq//KXq7t01l/e2barz59dcfs7+D96CcOoSn34Kw4aZP//OO0vXFxZaLXzWLJg/38IOCwqshTF3rvngr7/eonlyciz88447zC9eGXv2WASKqoUs7otwUdXEhC46TjjeSe0knSlTYN680njz/HwryGfNso7Ezp3NJdO0aWm4Y/v2cM01ZV0rxcXmelm/3gr+8NDJWJSUmCvpzjstTLNbNwsNjIwScZz6iAuEUy1Uzbfepk3lNeaSEouYefRRK4jPOqt026xZ1lcQPuoUzB990EEWCbNihYkGWK25WTNrETRpYiJxxhnmjx8/3vz0I0fCRRdV7XyKiizU8+ijy4YXOk59xgXCqTK7d8PVV1tNu3Fj6xDu2dPCGEMDmAYMsFp4z54mDLNmWThos2YWp92+vQnHcceZy2fixNIY+latbBRseKjfzp0WR9+smXUwz5ljI2dfecUKd7C4+hEjbDCXu18cZ+9xgXCqREGBTSkwfrxNVZCWZoX/smWQlWVROC1b2gCiadPMP9+rl/UJDBhgA8BOOw3eegtefNHmrnn2WbjqqurZs2KFCczhh9txHcepOSoSiGRO9+0kkS1brBXQo0fZ9StWmHto3jwb0HV5JS983bHD+gL69y9tDdx3X+m0Bvffby6dK66ovq1dutjHcZx9SzJnc3WSxK5dNnq3Tx9z1YTcNx98AIMGWYTP2LGViwOY+2ngwLKuol//2txKN99skURPPJG80auO41Qf/9vWce65x2atDJ8G+de/tg7fYcPgT3+yOXl++1ubtfKAA8xtdMop1T9mSoq1Plq1snzjnTvHcZzahbuY6jCzZ5sAqNq8QW+9Zf0KTz4Jt94Kf/kLjBplYwOmTLG+gscei29cQGV0727z/zRqtPd5OY6THLyTug4zYgRMmGAT0P3iFzZbZEEBHHGETR0cmgVy1SrrBD7ppOTa6zjOvqeiTmp3MdVRJk+26ZVvvdU6iCdPtv6Cli2t1RA+RXDHji4OjuOUx11MdRBVm0YiK8v6G8DGD8yZY+MbfLI0x3HiwVsQ+wErVlTtvbYffGAvSfnjH23qihCNG7s4OI4TPy4QtZw337SBaeGvioxk2zabgvqII2yq5zPPtNHN11yz7+x0HKfu4S6mWszChTYWoWlT62geNsw6nsOZPx/OP99mKw29uaxrV5unqLa8scpxnP0TF4hayrZtNt1FerqFoF54oYWhHnaYCUBRkb3I/MYbLZT0gw/svcKO4zg1hQtELUTV3o42Z44V/L16wauv2ijniy4yF9LTT9s7io8+2rZ16pRsqx3HqWt4H0QtZPRoeOkl+POfS1sFvXqZKEyZYp3PffvaayM//dTFwXGcxOAtiFrG9u02h9Ghh8Ltt5fddtFFNo6hRw/o3Ts59jmOU39wgahlPPCAvXz+5Zejv6TntNP2vU2O49RP3MVUi1i2zF6Qc/HFNoGe4zhOMnGBqEXcdJNNgfHgg8m2xHEcxwUi6RQXwzvv2DuX33oL7rjD5kZyHMdJNt4HkURmzYKzz4bly+09DHffbR3UjuM4tQEXiCShCjfcYAPiRo+Gc87xkc+O49QuXCCSxOuvwxdfwFNP2Yhpx3Gc2kZC+yBE5HQRWSAii0Xktijbu4rIeBGZKSITRaRT2LZiEZkRfMYk0s59zc6dpa/i/OlPk22N4zhOdBLWghCRVOBx4BQgF5gqImNUdW5Ysr8B/1HVF0XkROB+4CfBth2qOjBR9iWThx6CnBx7/We0sQ6O4zi1gUS2IIYAi1V1qaruBkYBwyPS9APGB78nRNle58jLg/vusz6HE09MtjWO4zixSWQfREdgZdhyLnBERJrvgPOAh4EfAs1EpLWqbgAaicg0oAh4QFXfijyAiFwLXAuQlZXFxIkTa/wkapLt21O55ZZD2bkzgwsumMrEiTuSbZLjOE5MEikQEmVd5HvRbgEeE5ErgM+AVZggAHRR1dUi0gP4RERmqeqSMpmpPg08DZCdna3Dhg2rQfNrlh07bBbWhQutg3r48EitdBzHqV0kUiBygc5hy52A1eEJVHU1cC6AiGQA56lqQdg2VHWpiEwEDgPKCMT+wu7d9lKfTz+1WVqH13lHmuM4dYFE9kFMBXqLSHcRSQcuAspEI4lIGxEJ2fB74LlgfUsRaRhKAwwFwju39yvuvx/GjoUnn4Qf/zjZ1jiO48RHwgRCVYuAG4EPgXnAa6o6R0TuFpFzgmTDgAUishDIAu4N1vcFponId1jn9QMR0U/7Dbt2wb/+ZSOmr7022dY4juPET0IHyqnqWGBsxLo/hv0eDYyOst+XwIBE2raveO01WLsWfvGLZFviOI5TNXyyvgTz6KNw0EFw8snJtsRxHKdquEAkkK++gqlT4cYbQaLFdDmO49RiXCASyKOPQrNmcNllybbEcRyn6rhAJIi8POt/uPJKEwnHcZz9DReIBPHcc7BnD/z858m2xHEcp3q4QCSIjz+GQYOgT59kW+I4jlM9XCASwJ491kE9dGiyLXEcx6k+LhAJ4LvvYPt2FwjHcfZvXCASwKRJ9u0C4TjO/owLRAKYNAm6dIFOnSpP6ziOU1txgahhVE0gvPXgOM7+jgtEDbN8Oaxe7QLhOM7+jwtEDeP9D47j1BVcIGqYSZNs5PSAOjEXreM49RkXiBpm0iQ48khITU22JY7jOHuHC0QNUlAAs2a5e8lxnLqBC0QNMmWKRTG5QDiOUxdwgahBJk2ClBQ44ohkW+I4jrP3uEDUIBMmwGGH+fTejuPUDVwgaogtW8zFdMopybbEcRynZnCBqCEmToSiIjj11GRb4jiOUzO4QNQQH30ETZrA0Ucn2xLHcZyawQWihhg3Do47Dho2TLYljuM4NYMLRA2wYgUsWODuJcdx6hYuEDXAuHH27R3UjuPUJVwgaoBx46BDB+jfP9mWOI7j1BwuEHtJSQl8/LG1HkSSbY3jOE7N4QKxl3z7LWzY4O4lx3HqHi4Qe0mo/+Hkk5Nrh+M4Tk3jArGXfPYZHHwwtG+fbEscx3FqFheIvWTRIujXL9lWOI7j1DwJFQgROV1EFojIYhG5Lcr2riIyXkRmishEEekUtu1yEVkUfC5PpJ3VpagIcnKgR49kW+I4jlPzJEwgRCQVeBw4A+gHXCwikXXtvwH/UdVDgLuB+4N9WwF/Ao4AhgB/EpGWibK1uuTmmkj07JlsSxzHcWqeRLYghgCLVXWpqu4GRgHDI9L0A8YHvyeEbT8NGKeqG1V1EzAOOD2BtlaLJUvs2wXCcZy6SIME5t0RWBm2nIu1CML5DjgPeBj4IdBMRFrH2Ldj5AFE5FrgWoCsrCwmTpxYU7bHxdixHYADyc+fzMSJu/bpsR3HcRJNIgUi2rAxjVi+BXhMRK4APgNWAUVx7ouqPg08DZCdna3Dhg3bC3OrzgcfQFoaXHDBUaSm7tNDO47jJJxKXUwicmM1/f+5QOew5U7A6vAEqrpaVc9V1cOAO4J1BfHsWxtYsgS6d8fFwXGcOkk8fRDtgaki8loQlRTvhBJTgd4i0l1E0oGLgDHhCUSkjYiEbPg98Fzw+0PgVBFpGYjTqcG6WsWSJR7B5DhO3aVSgVDVO4HewLPAFcAiEblPRCrsmlXVIuBGrGCfB7ymqnNE5G4ROSdINgxYICILgSzg3mDfjcA9mMhMBe4O1tUaVE0gvIPacZy6Slx9EKqqIpIH5GF9BC2B0SIyTlVvrWC/scDYiHV/DPs9GhgdY9/nKG1R1Do2boTCQhcIx3HqLpUKhIj8ErgcWA/8G/itqu4JXEOLgJgCUZcJhbi6i8lx9k/27NlDbm4uO3fuTLYp+4RGjRrRqVMn0tLS4t4nnhZEG+BcVV0evlJVS0TkrCraWGdYutS+vQXhOPsnubm5NGvWjG7duhF/1+r+iaqyYcMGcnNz6d69e9z7xdNJPRb43v8vIs1E5IjgoPOqbGkdwVsQjrN/s3PnTlq3bl3nxQFARGjdunWVW0vxCMQTwNaw5W3BunrNkiU2g2uTJsm2xHGc6lIfxCFEdc41HoEQVf1+kJqqlpDYAXb7BR7B5DjO3rBhwwYGDhzIwIEDad++PR07dvx+effu3XHlceWVV7JgwYKE2RhPQb806KgOtRpuAJYmzKL9hKVL4cQTk22F4zj7K61bt2bGjBkA3HXXXWRkZHDLLbeUSaOqqCopKdHr8s8//3xCbYynBXEdcDQ2DUZoPqVrE2lUbWfnTli1yvsfHMepeRYvXszBBx/Mddddx6BBg1izZg3XXnst2dnZ9O/fn7vvvvv7tMcccwwzZsygqKiIFi1acNttt3HooYdy1FFHsXbt2r22pdIWhKquxUZBOwHLltlAOXcxOU7d4Ne/hqAyX2MMHAgPPVS9fefOncvzzz/Pk08+CcADDzxAq1atKCoq4oQTTuD888+nX8SbygoKCjj++ON54IEHuOmmm3juuee47bZyr+GpEvGMg2gE/BToDzQKrVfVq/bqyPsxHuLqOE4i6dmzJ4cffvj3yyNHjuTZZ5+lqKiI1atXM3fu3HIC0bhxY8444wwABg8ezOeff77XdsTTB/FfYD72joa7gUuwqTPqLR7i6jh1i+rW9BNF06ZNv/+9aNEiHn74Yb7++mtatGjBpZdeGjVcNT09/fvfqampFBUV7bUd8fRB9FLVPwDbVPVF4AfAgL0+8n7MkiXQtCm0a5dsSxzHqesUFhbSrFkzMjMzWbNmDR9+uO/mLY2nBbEn+N4sIgdj8zF1S5hF+wFLl5p7qR6FUDuOkyQGDRpEv379OPjgg+nRowdDhw7dZ8eWsCEO0ROIXA28jrUaXgAygD+o6lMJt64KZGdn67Rp0/bJsfr0gQED4PXX98nhHMdJAPPmzaNv377JNmOfEu2cRWS6qmZHS19hCyKYkK8weC/0Z0C997rv2AGLF8PFFyfbEsdxnMRSYR9EMGr6xn1ky37BvHkW4jqgXvfCOI5TH4ink3qciNwiIp1FpFXok3DLaimzZ9v3wQcn1w7HcZxEE08ndWi8w8/D1in11N00ezakp0OvXsm2xHEcJ7HEM5I6/snD6wGzZ0PfvtCg3k9X6DhOXSeekdSXRVuvqv+peXNqP7Nnw3HHJdsKx3GcxBNPH8ThYZ9jgbuAcxJoU62loABWrvT+B8dx9p5hw4aVG/T20EMPccMNN8TcJyMjI9FmlaFSgVDVX4R9rgEOA9Ir268uMmeOfbtAOI6zt1x88cWMGjWqzLpRo0ZxcS2KoY+nBRHJdqB3TRuyP+ARTI7j1BTnn38+7777Lrt27QIgJyeH1atXM3DgQE466SQGDRrEgAEDePvtt5NmYzx9EO9gUUtggtIPeC2RRtVWZs+GjAzo0iXZljiOU6NM/zVsquH5vlsOhMGxZwFs3bo1Q4YM4YMPPmD48OGMGjWKCy+8kMaNG/Pmm2+SmZnJ+vXrOfLIIznnnHOS8nrUeGJx/hb2uwhYrqq5CbKnVjN7NvTvDzFe7uQ4jlMlQm6mkEA899xzqCq33347n332GSkpKaxatYr8/Hzat2+/z+2LRyBWAGtUdSeAiDQWkW6qmpNQy2oZqjBrFowYkWxLHMepcSqo6SeSESNGcNNNN/HNN9+wY8cOBg0axAsvvMC6deuYPn06aWlpdOvWLer03vuCeOrC/wNKwpaLg3X1irVrYf16739wHKfmyMjIYNiwYVx11VXfd04XFBTQrl070tLSmDBhAsuXL0+affEIRANV3R1aCH7Xuygm76B2HCcRXHzxxXz33XdcdJG92fmSSy5h2rRpZGdn8/LLL3PQQQclzbZ4XEzrROQcVR0DICLDgfWJNav24QLhOE4i+OEPf0j4axfatGnD5MmTo6bdunXrvjILiE8grgNeFpHHguVcIOro6rrM7NnQpo2/Rc5xnPpDPHMxLQGOFJEM7AVDWxJvVu1j9mxrPfhb5BzHqS9U2gchIveJSAtV3aqqW0SkpYj8374wrjaxbJnP4Oo4Tv0ink7qM1R1c2gheCRqfhEAACAASURBVLvcmfFkLiKni8gCEVksIrdF2d5FRCaIyLciMlNEzgzWdxORHSIyI/g8Ge8JJYLiYli3Djp0SKYVjuPUNJW9crkuUZ1zjacPIlVEGqrqLrBxEEDDynYSkVTgceAUrN9iqoiMUdW5YcnuBF5T1SdEpB8wFugWbFuiqgPjP5XEsW4dlJRAEsapOI6TIBo1asSGDRto3bp1UkYp70tUlQ0bNtCoUaMq7RePQLwEjBeR54PlK4EX49hvCLBYVZcCiMgoYDgQLhAKZAa/mwOr4zF6X5Ofb98uEI5Td+jUqRO5ubmsW7cu2absExo1akSnTp2qtE88ndR/FZGZwMmAAB8AXePIuyOwMmw5FzgiIs1dwEci8gugaXCMEN1F5FugELhTVT+P45gJIS/PvrOykmWB4zg1TVpaGt27+/vQKiLe96LlYaOpfwQsA16PY59obbZIJ9jFwAuq+ncROQr4r4gcDKwBuqjqBhEZDLwlIv1VtbDMAUSuBa4FyMrKYuLEiXGeTtWYMCEL6EtOzhT27EnOkHfHcZx9TUyBEJE+wEVYIb4BeBULcz0hzrxzgc5hy50o70L6KXA6gKpOFpFGQBtVXQvsCtZPF5ElQB9gWvjOqvo08DRAdna2Dhs2LE7TqsZXX9n38OFHso/f1+E4jpM0Kopimg+cBJytqseo6qPYPEzxMhXoLSLdRSQdE5sxEWlWBMdARPoCjbCR222DTm5EpAf2/omlVTh2jZKXZ9N8uzg4jlOfqEggzsNcSxNE5BkROYnobqOoqGoRcCPwITAPi1aaIyJ3i0jolaU3A9eIyHfASOAKtVis44CZwfrRwHWqurGqJ1dT5OV5B7XjOPUPqSw2VkSaAiMwV9OJWATTm6r6UeLNi5/s7GydNm1a5QmrwQknQFERfJ60bnLHcZzEICLTVTU72rZ43km9TVVfVtWzsH6EGUC5QW91GW9BOI5TH6nSu9FUdaOqPqWqJybKoNpIfr4LhOM49Q9/eWYl7NoFmza5QDiOU/9wgaiE0ChqHyTnOE59wwWiEkKjqL0F4ThOfcMFohJcIBzHqa+4QFSCC4TjOPUVF4hKCAmEv2rUcZz6hgtEJeTlQevWkJ6ebEscx3H2LS4QleCD5BzHqa+4QFSCD5JzHKe+4gJRCXl5PgbCcZz6iQtEBai6i8lxnPqLC0QFbN0K27e7QDiOUz9xgagAHwPhOE59xgWiAlwgHMepz7hAVIALhOM49RkXiApwgXAcpz7jAlEB+fmQmmojqR3HceobLhAVEBoDkeJXyXGceogXfRXgg+Qcx6nPuEBUgA+ScxynPuMCUQEuEE5CKCmC3HdAS5JtieNUiAtEDIqLTSA6dEi2JU6dY/ko+OwcWPZSsi1xnApxgYhBfr6JROfOybbEqXOsfs++5z7grQinVuMCEYOVK+27U6fk2uHUMUqKYM2H0KQTFM6D3LeSbZHjxMQFIga5ufbtAlEPUIUlz0PhgsQfa8PXsHsTDPwrZPSC2ffa8R2nFuICEQMXiHrEnHvhq6vg83OhZE9ij7V6LEgqHHA69L8NNn0Daz5K7DEdp5q4QMQgNxcaNarno6g3TINPToPdBcm2JHEseRZm/gFaD4GCubDg0cQeb/X70OYoSG8J3X4CTTqbQDl1k6LtsPrD/baV6AIRg9xcaz2IJNuSJDLjNsj7CJY+n2xLEsOqd+Hrn0GH0+CUL6DDGTDrLtixJjHH25FnLYYDzrTl1HTo+1tY9zmMTINR6fBaBuR9kpjjJ4OJZ8PMu5JtRdXJGw9rP9u7PPZsgYlnwMTTYfmrNWPXPsYFIgYrV9Zz99L6ryB/PKSkw8LH6060jaoVwJ+fB5+NgJaHwTGjISUNsh+Bkl3w7W8rz2f3Zivwq8KaD+z7gDNK1/W6Fg6914TioFvsei95pmr5VpeSYti2InH579oIq9+FeQ/a70SgCgXzajbP4l0w6SL45qbq57G7ACacDusmQaMsmHvfftmKcIGIQagFUevI+9gKp0Qz5z5IbwXZj8HWxYnzk8/6M8y5v2bzLCmG3DH2Hc6eQvhgMHxyEqz9FPreAid8AGkZtr1ZLyuoc16uvPY49Qb4+Piq/elXj4XGHaDFoaXrUhtC/9th4H326fIjs71oW/z5VpcZt8KYHlaIJYL1X9p38XZY/FRijrHmA3ivX80+n7lvwa71UDi/ehWj3Zvgk1MsIOGY1ywgYfMsa7HGYs042LO1+jYniIQKhIicLiILRGSxiNwWZXsXEZkgIt+KyEwROTNs2++D/RaIyGmJtDOSkhJYtWovBUJLrHa2bbl9KvPjb54Db3WFGbdbQRaN9VPswUu0z3rzLFg1Bg78FXS/HBq1h4WP1fxxCheYQHx3O+RPqLl8V42Bz4bD4ifLrl/4L9j0LRz+JIzIhYEPQMOITqb+t1uNr7LzXT8FtiyErUvjs6mkyAqxDmdU7LfseqEVqKveiy/f6rJzHSx6ArQYvrwkMZWOdV9Yy6ztsbDwUSjeXfPHCAnDooh7vT0X3j3IKh9Vrbkvftq+i7ZZPlVh1wYYfxJs/g6OfQM6nwvdLoam3ex/G82W9V/DhFOtP6yWkTCBEJFU4HHgDKAfcLGI9ItIdifwmqoeBlwE/CvYt1+w3B84HfhXkN8+Ye1aKCraS4GYcRu83RXe7hZ8ulYsEstegO0rYe79MKZX8OeNqL3M/IN9r3yz4od+59q9q43MuR8aZECfG81P3utnVvvdsqT6eUZj7gOQ2ggyesBXV9dcrTlvvH3PuReKdtjvou0w/x/Q4XTo/TM7bjQaNIGsE0wAYrF7M2xbFhzr4/hsWj8Z9hSUdS9Fo+1xJsgrEuyzXvAQFO+EI5+3QnDq9TXvAlk3CVoOMtHdscZGkFeX/Ikw4/flW4VrP7XvVWPK9h3NfdAqIN/dDpMvN7dRPBQugvxPoP0pwfL82GmLd8Lm2aU27VwL40+wYIfj3oZOZ9v6lDTodyts+Cp6RWhJIEgFs+OzcR+SyBbEEGCxqi5V1d3AKGB4RBoFMoPfzYHVwe/hwChV3aWqy4DFQX77hFCIa7VHURfvhqXPQbthcMRzVlPdUwArXoueXhVWvG6F12lToXlfc2HM+H1pmrWfWWHUciBsXWIPYdS8SmDcMeZKqU70UeEiK5x63wANW9m6XtdaaOaiJ6qeXyy2LbepJnpeY4XU1qXw3R9qJu/8Tyw6aMea0lbEkmdh1zrof0fl+7c+0sR6++ro2zd9V/o7XoFY9Y4VFKGCJxYpqdDlAmtBxGpJhvPNLdYyisWOfHuWvrioVIB3F1gLqfN50OMKGPBnK7yX/bf8/iV7LAqnpKhyW8Ip3gUbpkLboRYE0Lw/zP979URo0VPwyclWoVgf5g7bvRk2zYBul1pLKBRMsXOt9eN0vxwOuQdy/gvjT4yvNbDk3/asH/agLcfq39i5Hj46GsYOgDfawRcXwsfDYMtiGPauhTGH0+NKE/4595Vdv6cQckba78Ia7kupARokMO+OwMqw5VzgiIg0dwEficgvgKbAyWH7hlfhcoN1ZRCRa4FrAbKyspg4cWJN2M3nn7cBDiYvbxoTJ1a9Jt5mx2ccvGsDM4vOYOPK7qDdOLxBV4q+fZhvc3uXS5+xeyHZ25Yxv8F55M3aCql30afJPzlg3l+Zk9eIdY2GMXDDr2mc0ppv027hSC5l6ef/ZEWzS8vl1WLXNwzcsgiA9e+cyexW94CkkFqylb6bHyC9eCNzWv6JXQ3Kz2OeXryBQzfcRCPS+WrjEewOu579Gh5LywVPMXnzyZSkxKh9V4Hemx+mg8JXBUPZNbeE3k3O4YAFD/Hthl4Upkc2NEvpsO09Om37H9+0eYLilMZRzmEjRxfOY0mza2mVPo2mM+7h69W9yF57DzvTD2HG3CKYO7F8xmFk7k5jEDD702dZ3/jYcts7bh1Nb2BDwyFk5n7EpAnjrVCJhSpD1r7CzrRDmfnltxUe247fh0Elu5g37i/kN4ktKKkl2xma9xDF0oTJuT3K3JeUkp103vY/Om8dSYruRlAK8uYxq9X9dNz2Bj32FDJt56lsnTgR9EgGph9CxlfX8c0S2J7W5ft8ehQ+RZeto8jJuJyczCvKHD9Fd1Mi6THOYQ6DSnYxO78F6z/9lPZyJgdtfpDvPvoHmxoOrvQaAIgW07PwX3Ta9gYbGx5Oi10zWDX5UZY0t5Z1651fMgDl2y2D6ZY+h0azH+WrtUfSfcuzdCneydfbTmTHri60bXkXB224H97uxcqmP2JlxkUUpzQBVdJKCilKyUAlFdE9HJX/NAUNj2LOjI0MlUzWzR/PwrxDy9iVVryJQzfcQpOilSxrdi1NilbQKnc8KbqLOa3uY/P8BjB/Yrnz6Zw+gp75TzLjw7+zObgGB2x7mz7F29mS1ptG62cxacKE2hU6qaoJ+QAXAP8OW/4J8GhEmpuAm4PfRwFzsVbN48ClYemeBc6r6HiDBw/WmuKRR1RBNT+/mhlMOEv1jQNUi4tK1835q+rLqBbML59+xu2qr6Sq7lhXuq5ol+qHR6uOaqI67x+274LHbNsHR6i+f3j0Y0/6ieprzUuPN/Mu1cLFqu/0VX2lgeqrzVRfz1Jd+2XZ/batVB3TW/XVDNX8T8vnm/+p5bfkxapdi2hsX6M6sqHqlJ+WrttdoPpmZ9WPhsbeb9dG1f+1NDvmPxo9zbJXbPv6qXaOL6P6frZ9r3o/PvuKdqqOTFf95tbo27+83K7h0pcs3w3TKs5v8xxLt/Bf8R2/pNiuxYQfVJwu913L92VUFz0Ttn+J6ien2frPzlUtWKia86o9Yx8cqTq6jeonZ5TNa9tK1dFtVd850O6Fquqajy2P/7WyZ2fTzNL0qz9SHdVIdf4j0W0LPX/b82y5aKdds3cOVN2yJL7rMPdBy2Pab+y/NOEHqm91t/NTVZ1+sz1HRTtUc0ZZ2pxXVV/LVP38grJ5bVmm+sVFlub1dqpjB9p/4WXsvzrzLnumXkZ11Vjb56OhquOOK5vP9jX2XxrV2K5P+DUv3lPx+ezZqjqmj+obHVV3brB93jvUbAn9x3esje/a1CDANI1RribSxZQLhDtpOlHqQgrxU+A1AFWdDDQC2sS5b8LIzYX0dGjTpho778iDNe9D98vMXRCi+6VWy1z6Qtn0qrBitLmjGoUdMDUdjh0N6c0t3K5JZ+h5tW3rNAI2Ti3fZN5TCCtHQ9eLLEKn++UW1/9BNuzMgxM/gtO+ggZNzVc66x5zGy38F4w7DnbmwwkfQrvjyp9X22NtaojIMRGqNnV18c74r9H8f4Lugb6/K12XlmnujvWTY7vGZt9rboWMnrDgn+X90WDupbTmFr7a9igbc7BxmvnCO8QZ65Da0PbfEKMfYtMMc/W1P8mWK3MzheZb6nhOfMeXFOuszvuo4vDQvI8hpSE072cuo5D7Zs0HNt/TYX+DY1+HzN7Q9UdwzP9g03SL0Ol/e9m8mnSy7VsWw+TLrBN78mWQeRCcMcMG9k25ylxNG7+xUefFu8zHH80Vt36SPS+Ng5ZqakOL6Nm5Fj4cEt8Yg/wJ5poa/A/7L3UaYX0/m2fZ9rWfQpsjrD+p0who2MZGxO8phH6/L5tXRjcYOhJOnQKthkDjA+x5O+xBaHGI/U+m/wKadIH2p9o+mQeVdzHN+B1sy4Fh75fef7Baf0olDpkGTWHoK7BrLXx9jbngNn9nLtzMvpamOm6mPVsSFkKbSIGYCvQWke4iko51Oo+JSLMCOAlARPpiArEuSHeRiDQUke5Ab+DrBNpahtxc6Nixmq8azXnJ/KE9riy7vnEH62NY9p+yBVvBHIuG6XJ++bwad7AY/bTmcOj99icD6BR05eRGXM7lr0HxDnvwReDwJ8yf3rg9nPqVdb427wunfW2jeWf90fzT035ufSQnjIO2R0c/LxHLd+1E2LqsdP3KN2zq6gUPx3d99my1foHOF1jBFU7WidaHEq3w2LrMImF6XG59OluXwqq3y6fLnwDtji8V50P+zwqQQ/5ctaZ76yNsJHmk7714NxTONQFp3B6aH1y5QKx8y0ZqNynnJY1NlwvN/58b5RxD5H0M7Y6FA39tBc26SWbvt7+1wrnPL8qm7/xDK9gG/hXaHVM+v6zjYdA/7JjvH2ZCMnQkNO1s4c4bp8E3N9vgr/RWcPKnZmPkuBFVs6Xt0LLr2x1nz2HDNtanMOse6+SNVripWpho6zCvdMezATHB3V1ggw7bHW/bUhva81m0zSLFWh0W/Zq1OQKGvQPD3rNxL31vgRPeh7MWQr/b4PDHS5+dzL7Wb7VrQ6lNeePs/5d1fPT8K6PVYDjkXvvffPljSG0C3S6x/yVUb0zHlCut3zEBJEwgVLUIuBH4EJiHRSvNEZG7RSRUlboZuEZEvgNGAlcErZ45WMtiLvAB8HNVjVJdTAzVHgOhajXsNkdDZp/y23tcCTtW20MWYsVoQKDTD6Pn2fZoOG89dL+kdF3mQdCsT/mZQJe9YNtCf6oGjeGUz+EHc8oWxg1bw0mfwLn58MO84LMK2lQSB9D9MrN16Yu2XFJsIgN23vHUYnJeshregb8qv63NkVaYR4v0+O4Oa4Edco/VFpt2s6ikcLYttw78rBNL17U6DM4vgI5nVW5bpC3F28tHlhTMsUKx5UBbbn8yrP28NFoqku251trrNKJqx2812MJt82OMqt6RZ7a1Pxm6/RjSWpiALn3BbBx4v7VCI2l/EvSrYCBgn1/YFCA7VpkQh86zywV2DgsfMRE64QMTp363wvJXIP/T0jy2LDRxaRul0MrsbbX49qfaszN2ALzVGRY8UjbdthzLo3XYM9k4y/4PuW+ZAGmJtbxD9L7e/heH3B37/GKR2duuWfhz8n2tfn7pee1YU/b5qg59b4ask+xZ7XaxtZ6bdDaxqGoLomSPlSfN+++dTTFI6DgIVR2rqn1Utaeq3hus+6Oqjgl+z1XVoap6qKoOVNWPwva9N9jvQFV9P5F2RpKbW80Ipg1TLbqoxxXRt3c82wrncDfNytftj9a4fKfx90Q2XUXsz5o/oTR+vXCh/Wl6XFm2ppzSwFwWkYhAo3Z23MZZscM+w2na2QqkZS/Yn3PFq3a+B5xpIYUbvqp4f1VzhbQabAVwJKmNTFwjC8UNU2H5SDjoJnOFpDSwWvO6STbiO0RIWLJOiMg3ekdqhYTsiwx33TTDvsMFomRX6aCwSEKtvKoKhIjVuNdOjC68oVDe9ieb66LnVVYrnXmHtQ47n1e144Uf94hn4KQJZUVcBA7/l8X1D3uvtMbb7zZo2hWm3Vg60WFo4F1kCyJEeguL9Bm+AoY8Y+6eGb8rK7IbAodB64hKS8fhNpZl2X8sKiz8OcroAWcvgNbZ1Tv3SCJr9aHnMvL5qiqSAkf9xwZF9r21dF3mgRWH1UZj3ZdW4aosfLqa+EjqCEpK9qIFkfMSpDa2Gx+N1HTo+mPrJ3ivP0w402qBnaO4lyqj0wjQIvtjrXjdQgglBbr/pBqGV4EeV1hNPW+8+W1bHAJHv2LnXdmcTWs/tdptnxtju3uyTjR3yc71pevm3GvC2u/W0nU9rzLX2/y/l67Ln2DuixYHV/fsSmnaDRq2LS96m2ZYgZzRy5bbHQfSILabKfdtq9VmHlR1G9oNsxbItmXlt+V/bG6eFoFQ9b7BXJs711rfw95EwqQ2hKxh5SsWjTtYn0Z4odygCQx6yJ7jCaeZIK79zGzLPLDi4zTtDL2uNvdf8U6bkyrEhq+twhB5L0NCu+JVE48GTap9mpXStKvZEKrV5wXh0xk99z7vJgfAMa+W9TRk9q26i2n12CB8+qTK01YDF4gI1q+H3burKRAbp5l7J7157DSH/Nmmc2jWG7avsFpPlwuqfqw2R1hNfPHT8MX59t3hDPsTJ5JOP7Qm8ZQrYMsii6FPb24it3yUDUiLxcLHrKDvcmHsNKHaWWgA1NYcG0PQ62d23BBpzWzditHw9XUW75//ie0frcVUVUSsICzXgvjWRDHkp05rZulWvVt+qvDdm82mTiOqV2CH/Ov5E8uuVzVByjqx1I5mPS2IodfPYvcjJYpOw63vonChjWBf9qK1BOO9D+2Ot872NR+WrtvwtQUWpKSVTZvZ2zrloax7KRFICjQ70AptLbHWXNYJiQtDbd7XyoRYg1wLF5Uf8b7mfXPlhf83ahAXiAiq/R4IVetwq8wXmN7SfLvHvQU/mA3nLLHOzqoiKXD6NLigAE7/xjqzhzxd9XyqSoPGFiW1Y7UJVKjDvOeV1tSN9Ya0bSttW8+rLY9YtD7cauih5nxooFuv68qnHfBHa40sedbmFNqeu/fN/3DaHGlN/t2bbFlLSiOYwul5tdWgJ11UOp2EltgkdVpUdfdSiOb9rEW09tOy67cstHPtEDFG4oinYUjElBP7AhE46DcwPMeew07DoXeU+xWLBk3MzRqaNqOkCDZOL+9eChG6nu2q2VFcFZr3tWdg82zrE9nb/oeKCPV5bAl7cVXxbpsJdtxx8G4f+CKsMrltpUV0hWYHTgAuEBFUWyC2r4SiLTXj3qgKaZnWEdvlPGu27gt6/cxqfIfeX1qbane8uWViuZkWPwVo9II+nJQ0m24i/xPzSS/5d9ApHaVTqEFTi0T5wRzocKq5uTqcXj5ddQl19m+Yat/bcuwet4yIkOlxublZVr5hrbktiy1KZ8591hfQJnJ8aJyI2HWNFIiQO6v9yeX3SSYpDew5PO4t6PiDqu3b/lQT2e2rzA1ZvCO2QPS+wVrh+0IgMvvafQ+9R7wmKyCRRPZ5lBTBh4dbxWPHKvsf5H1c2qIMzQ7cITH9D+ACUY5qC8TmINql+T4WiGTQahBcUFi2BispNu4ib7y5hUKoWtTTgofggLMsHr0ysk6wWtuChyzEMDJcM5LMPnDcm/CjrZDRvTpnFJ3WhwNiHYEAG4NR0JEtCICDfmWduKvegXf6mItkyDM2tmBvXF7tjg+is3JK1+V9DE27m3uyrhAao7Lmo9gd1CGadITD/lq94IOq0rwvoObCzegJTbtUuku1yehlkXqhjuoVo2HzTHuuzl5kfX2ND7A52VSt/6FJl1KXWwJI5FQb+yUrV0KDBpBVQVBRVArm2HeLxISb1Tqi/Tl7XGHz3o89xGrV3S6BeX+zSK22x1qMeTy0D5rxs/5kLrt4a4o10fcQTlqmicSce+yPKin2B45VCeh9PaQ2NVfaYQ9av8DeEjr3tZ+auG7NMV9998v2Pu/aRIsBNldR3kc2UWR6q9ohgCG3z7YcmzcskaSmmwgVzjMBmP936+jv9TN79ho0tgGO0240ccj72P5jCZyaw1sQEVQ6SK6kCJb+B9Z+UXZ9wWxT9/SWCbex1pLRDU6dbD7oxU/DR0fZLJsD/2Jhk03ibJa1GGhx/SV7Ko542hcc95aFIq773IQu86CK+1B6XAbHvVEz4gDmskxvZQKhaiNwJbX8SOj9HRFzE+aNs9H0rYfUjjmJmvUurXgk0r0UonkQybTucwt6OfA3ZSs+Pa+2SKopl0PR1oT2P4ALRDlihriq2gybYw+xmzP1+rLbN8+uH+6lymg1GI7+L4xYCUOegtOnW3hq+LQjlZGSan/GtEybqTOZNO5gA6hGrLQm/pAEvfgmFpJiobT5E61/J+9jc68k0tWRLDqcZi7Fgjmx3Uv7mtSG0DRoyewLgcjsa9GBc/9iEX+RYeupDeHgP9h1SklPbKc5LhDliCkQky+HT8+yqJTO51qLIfTKyZJim34hQaMZ90satbM5ZloMqN7+2Y/Ze6JDb3tLNqmNbNRrrMFfiaTd8TYWYvqvTCx6/Wzf27AvCO90ry0CAeZmDE2tkmgy+1oZs3qsdcZHG+fR4wpr2WSdlPD/hwtEGKoxBGLTTJtTvs8vLWIm1LwPhWJuW2YDffZ1BFNdpskB1ReXukbWMPvWIhjy75rva6ktNGpnYx8gCBCoJQx5Ck6M870fe0toQGVKQ+j98+hpUtLMlTt0ZMLNqaNPWvVYuxZ27oQuka33RY9bDXLAn+zmtBhofuFQuGF9imBy9j0tDrHpMwb9s/wEh3WNXtfYlDSN2iXbklLSmpW+PCvRNO9rI/O7X1rx9DsNW1c8ILeG8CimMJYFMxp0D4+U3L3J3nzW7ZLShyQl1Xx/eR9bsyM0oVsCw82ceoykwKkx5nqqa/S+rmqD7Ooaac3g5M9qjTfCWxBh5OTYdxmBWPqCzeoZ2dxrf7INjtuyyFoQTbvVHn+54zj7L22PMqGoBbhAhBFqQXTrFqzQElj4uHVMRs4vH+pQy/vYoi7cveQ4Th3DBSKMZcugbVvICDUE1nxoc7b3vrF84owe1mpYPdZGPtaSJqHjOE5N4QIRxrJlYa0HsNlHG7W3sNZIRKwVseZ9iy7xEFfHceoYLhBhLFsW1v+wfRWsft9GLsaa86X9yeaGAm9BOI5T53CBCCguhhUrwgRi+UhAK57zJjSKUVKq90IYx3GcWoyHuQasXg179oQJxLKXbDRnRXHnjdrazJ7FO+J7ZafjOM5+hAtEQJkxEJtn22svBz9S4T6AvaSneGdCbXMcx0kGLhABZQQi52WbMbNrBa/GDFGbpgRwHMepQbwPImDZMgtM6tK5xASiw2m1a7i/4zjOPsYFImDZMjjgAGhY8LmNkO52SbJNchzHSSouEAHfh7jmvGTvOu40PNkmOY7jJBUXiICcHDi83zJY8Rp0OtdEwnEcpx7jAgHs3g07C9Zx+1GnWef0wXck2yTHcZyk41FMwMpl23j3lh/QIm0lHD/eXhTuOI5Tz/EWRMkeMmf9iMHdpzO/9avQ9uhkW+Q4jlMrcIHYtpymO6dx/XNPkNnvnGRb4ziOU2twF1OzXjw4bwHPf96CJzom2xjHcZzag7cggPlLW9ClC6SmJtsSx3Gc2kNCBUJETheRBSKyWERui7L9nyIyqO3/OgAAB1VJREFUI/gsFJHNYduKw7aNSaSdZab5dhzHcYAEuphEJBV4HDgFyAWmisgYVZ0bSqOqvwlL/wsg/L2eO1R1YKLsC2fZMjjHux8cx3HKkMgWxBBgsaouVdXdwCigouHJFwMjE2hPVLZtg7VrvQXhOI4TSSI7qTsCK8OWc4EjoiUUka5Ad+CTsNWNRGQaUAQ8oKpvRdnvWuBagKysLCZOnFhlIzdvTuPEE3uRnp7HxImbqry/4zhOXSWRAiFR1mmMtBcBo1W1OGxdF1VdLSI9gE9EZJaqLimTmerTwNMA2dnZOmzYsGoZOmIEQFa19nUcx6mrJNLFlAt0DlvuBKyOkfYiItxLqro6+F4KTKRs/4TjOI6TYBIpEFOB3iLSXUTSMREoF40kIgcCLYHJYetaikjD4HcbYCgwN3Jfx3EcJ3EkzMWkqkUiciPwIZAKPKeqc0TkbmCaqobE4mJglKqGu5/6Ak+JSAkmYg+ERz85juM4iUfKlsv7L9nZ2Tpt2rRkm+E4jrNfISLTVTU72jYfSe04juNExQXCcRzHiYoLhOM4jhMVFwjHcRwnKnWmk1pE1gHL9yKLNsD6GjJnf6E+njPUz/Ouj+cM9fO8q3rOXVW1bbQNdUYg9hYRmRarJ7+uUh/PGernedfHc4b6ed41ec7uYnIcx3Gi4gLhOI7jRMUFopSnk21AEqiP5wz187zr4zlD/TzvGjtn74NwHMdxouItCMdxHCcqLhCO4zhOVOq9QIjI6SKyQEQWi8htybYnUYhIZxGZICLzRGSOiPwqWN9KRMaJyKLgu2Wyba1pRCRVRL4VkXeD5e4i8lVwzq8G09HXKUSkhYiMFpH5wT0/qq7faxH5TfBszxaRkSLSqC7eaxF5TkTWisjssHVR760YjwTl20wRGVSVY9VrgRCRVOBx4AygH3CxiPRLrlUJowi4WVX7AkcCPw/O9TZgvKr2BsYHy3WNXwHzwpb/AvwzOOdNwE+TYlVieRj4QFUPAg7Fzr/O3msR6Qj8EshW1YOxVwxcRN281y8Ap0esi3VvzwB6B59rgSeqcqB6LRDAEGCxqi5V1d3AKGB4km1KCKq6RlW/CX5vwQqMjtj5vhgkexEYkRwLE4OIdAJ+APw7WBbgRGB0kKQunnMmcBzwLICq7lbVzdTxe42936axiDQAmgBrqIP3WlU/AzZGrI51b4cD/1FjCtBCRDrEe6z6LhAdgZVhy7nBujqNiHTDXuH6FZClqmvARARolzzLEsJDwK1ASbDcGtisqkXBcl285z2AdcDzgWvt3yLSlDp8r1V1FfA3YAUmDAXAdOr+vQ4R697uVRlX3wVCoqyr03G/IpIBvA78WlULk21PIhGRs4C1qjo9fHWUpHXtnjcABgFPqOphwDbqkDspGoHPfTjQHTgAaIq5VyKpa/e6Mvbqea/vApELdA5b7gSsTpItCUdE0jBxeFlV3whW54eanMH32mTZlwCGAueISA7mPjwRa1G0CNwQUDfveS6Qq6pfBcujMcGoy/f6ZGCZqq5T1T3AG8DR1P17HSLWvd2rMq6+C8RUoHcQ6ZCOdWqNqWSf/ZLA9/4sME9V/xG2aQxwefD7cuDtfW1bolDV36tqJ1Xtht3bT1T1EmACcH6QrE6dM4Cq5gErReTAYNVJwFzq8L3GXEtHikiT4FkPnXOdvtdhxLq3Y4DLgmimI4GCkCsqHur9SGoROROrVaYCz6nqvUk2KSGIyDHA58AsSv3xt2P9EK8BXbA/2QWqGtkBtt8jIsOAW1T1LBHpgbUoWgHfApeq6q5k2lfTiMhArGM+HVgKXIlVCOvsvRaRPwMXYhF73wJXY/72OnWvRWQkMAyb1jsf+BPwFlHubSCWj2FRT9uBK1V1WtzHqu8C4TiO40SnvruYHMdxnBi4QDiO4zhRcYFwHMdxouIC4TiO40TFBcJxHMeJiguE41QBESkWkRlhnxoboSwi3cJn6HScZNOg8iSO44SxQ1UHJtsIx9kXeAvCcWoAEckRkb+IyNfBp1ewvquIjA/m4h8vIl2C9Vki8qaIfBd8jg6yShWRZ4L3GnwkIo2TdlJOvccFwnGqRuMIF9OFYdsKVXUINnL1oWDdY9h0y4cALwOPBOsfAT5V1UOxeZLmBOt7A4+ran9gM3Begs/HcWLiI6kdpwqIyFZVzYiyPgc4UVWXBpMi5qlqaxFZD3RQ1T3B+jWq2kZE1gGdwqd9CKZhHxe89AUR+R2Qpqr/l/gzc5zyeAvCcWoOjfE7VppohM8TVIz3EzpJxAXCcWqOC8O+Jwe/v8RmkgW4BPgi+D0euB6+f2d25r4y0nHixWsnjlM1GovIjP9v745tEIhhKIDaFQvdMpSIiuaoWIaCOViBXW4HUyRIFC5PguK9LqnS/TiO5K/1s6o+X10PmfmKcfE6zr01Ih6ZeYsx5e00968Rcc/Mc4xK4RJjEhr8DT0I2MHsQSxVtf36LLAXT0wAtFQQALRUEAC0BAQALQEBQEtAANASEAC03gXZrZHOsHbVAAAAAElFTkSuQmCC\n",
"text/plain": [
"<Figure size 432x288 with 1 Axes>"
]
},
"metadata": {
"needs_background": "light"
},
"output_type": "display_data"
},
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEWCAYAAAB8LwAVAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO2deZgU1bn/P+8Mq4ggICM7KCigosgmSiJRiRJiTFyiJC5RE5NcE2PUazR6TeKNub9oEjVxuTHRRIzXPTEiCLiNYkR2EBhkVWEEWUV2Z+H8/njrpKt7qme6Z7qmZ3k/z9NPd1VXVZ/qqjrf8y7nHHHOYRiGYRipFOS7AIZhGEbDxATCMAzDiMQEwjAMw4jEBMIwDMOIxATCMAzDiMQEwjAMw4jEBMJodohIXxFxItIiWH5JRC7LZNta/NZPReTPdSmvYeQLEwij0SEi00Xk9oj154jIx9lW5s658c65R3NQrrEiUppy7F85575d12PX8JtORG6sqSzB+mIR+XZo+SgReUZEtorIpyLyrohcJyKFcZXZaDyYQBiNkb8Cl4iIpKy/BHjcOVdR/0XKG5cB24P3rBCRI4HZwHrgOOdcB+ACYDjQPpeFNBonJhBGY+R5oBPwOb9CRA4FvgxMCpYniMhCEdkpIutF5OfpDhZuVYtIoYj8JmhRrwUmpGx7uYgsF5FdIrJWRL4brG8HvAR0F5Hdwau7iPxcRP4W2v8rIrJMRHYEvzso9N0HInJD0Ir/VESeEpE21ZT7IOB84GpggIgMz/gfVH4BvO2cu845txHAObfCOfcN59yOLI9lNEFMIIxGh3NuH/A0cGlo9deB95xzi4PlPcH3HdFK/vsi8tUMDv8dVGiGoi3p81O+3xx8fwhwOXC3iJzonNsDjAc2OOcODl4bwjuKyFHAE8C1wGHAVGCyiLRKOY+zgH7AEOBb1ZT1PGA38AwwneT/IxPOAJ7Nch+jGWECYTRWHgUuEJG2wfKlwToAnHPFzrklzrkDzrl30Yr51AyO+3XgHufceufcduB/wl8656Y459Y45Q1gBiFLpgYuBKY45152zpUDvwHaAieHtvm9c25D8NuTgROqOd5lwFPOuUrg/4CJItIyw7IAdAY2ZrG90cwwgTAaJc65t4AtwDkicgQwAq0kARCRUSLyuohsEZFPge8BXTI4dHfUJ+/5MPyliIwXkXdEZLuI7AC+lOFx/bH/fTzn3IHgt3qEtvk49HkvcHDUgUSkF/AF4PFg1T+BNiRcYhVAlFi0BMqDz9uAbhmW3WiGmEAYjZlJqOVwCTDDObcp9N3/AS8AvYLg6/8CqUHtKDYCvULLvf0HEWkNPIe2/Iuccx1RN5E/bk1DI28A+oSOJ8FvfZRBuVK5BH1+J4vIx8BaVCC8m2kd0EVE/i0wwe/1ISFSr6BuKsOIxATCaMxMQv3o3yHkXgpoD2x3zu0XkZHANzI85tPANSLSMwh83xT6rhXQGrVcKkRkPPDF0PebgM4i0qGaY08QkdMDV9D1wGfA2xmWLcylaJD5hNDrvOD4nZ1z69AMpV+LyMGBuP0nalm8ExzjZ8DJInKXiBwOICL9ReRvItKxFmUymhgmEEajxTn3AVq5tkOthTD/AdwuIruA29DKORP+hAZ8FwMLgL+Hfm8XcE1wrE9Q0Xkh9P17aKxjbZCl1D2lvCuAi4E/AFuBs4GznXNlGZYNABE5CegL3O+c+zj0egFYDUwMNr0Q6Bqs+wg4HfiSc25/UJ41wOjgWMsCV9xzwDxgVzZlMpomYhMGGYZhGFGYBWEYhmFEYgJhGIZhRGICYRiGYURiAmEYhmFEUqshjBsiXbp0cX379s13MQzDMBoV8+fP3+qcOyzquyYjEH379mXevHn5LoZhGEajQkQ+TPeduZgMwzCMSEwgDMMwjEhMIAzDMIxITCAMwzCMSEwgDMMwjEhMIAzDMIxITCAMwzCMSEwgasGsWbBgQb5LYRiGES9NpqNcffLtb8Phh8Orr+a7JIZhGPFhApElFRWwahVUVua7JIZhGPFiLqYs+fBDKC+HjRvzXRLDMIx4MYHIkhUr9H3nTtizJ79lMQyjYbN+PVx0Eezene+S1A4TiCzxAgFmRRiGUT2TJ8NTT8GcOfkuSe0wgciSlSsTnzdsyF85DMNo+Cxfru/vv5/fctQWE4gsWbECOnbUz5kKxNat8PLL8ZXJMIyGSUmJvptANBNWrIDPf14/Z+piuusuOOss2LEjvnI1Bd58E6ZOzXcpDCN3eAvigw/yWoxaYwKRBbt2qdUwahS0bp25BTF3Lhw4AEuXxlu+xs6tt8JNN+W7FIaRG3bsSDQizYJoBqxape9HHw3du2dmQRw4kOh1/e678ZWtKbB6NWzfnu9SGEZu8NbDYYeZQDQLfAaTF4hMLIg1a+DTT/XzkiXxla2xs3evCu4nn+S7JEYuueQSuPfefJciP/j4w/jxem/v35/f8tQGE4gsWLECRKB/f+jWLTMLYv58fe/c2SyI6li7Vt/37oWysvyWxcgNBw7AM8/AI4/kuyT5YflydUWfdpouf5h25ueGiwlEFqxcCX36QJs2mVsQ8+dDq1Zw3nlqQRw4EH850/HBBw1XpNasSXxuzlbEhx+Cc/kuRW7YuBE++0zvuW3b8l2a+qekBAYOhCOP1OXG6GYygciCFSvUvQRqQaT2pn74Ybj66uR95s+HIUNg+HANcuezFXHjjXDxxfn7/eowgdD764gj4Lnn8l2S3BC+pm++mb9y5Ivly2HQIOjbV5cbYyaTCUSGOKcWhBeI7t31PexmevRReOCBREvBOQ1QDx+uIgH5jUNs3AhbtuTv96tj9erE5+YaqJ4yRS3MefPyXZLc4N2GAG+8kb9y5IM9e7QxOHiw1hWtWpkF0aTZuFHHUznqKF3u1k3fvZvJuYT75skn9d0HqIcNg2OO0XW1cfHkyuWwbVvD7YuxZg20CMYWbq4WxIwZ+u6Dm42dtWuhoED7DRUX1+1Y8+bBxIk6mnJjYMUKfW4HDdL/oE8fE4gmTTiDCapaEOvXJ7KVnnhC332AetgwOPhg9UVmKxAffaQ9t//+99qX3bN9u2ZSfPZZ3Y+Va9asgeOO08/NUSD27Uu0spuSQPTuDePG6X1fF8vwsce04dVY3DQ+xXXwYH3v27fxlD2MCUSGpBMIb0H4in/iRHUjLV2aCFB762HIkOwF4m9/01jH/ffXrfzOJQKFXsgaChUVao4PH67LzVEgZs5U8R45UivWffvyXaK6s3atxlTGjtX7ry5xCD/Y3fr1OSla7JSUQGGhZjwC9OtnFkSTZuVKOOgg6NFDlzt21BQ2b0H4iv/229WkfOIJNYuHDFGRAG0hr1qV+cPvHEyapKm1r7+eeYD71lvhwQeT1+3alTDPG5qbad06LduwYbrcHAVixgy9T77/fb3u4VGDGyteIEaMgLZta+9mKi+HhQv1c2MRiOXLYcCAxLPfr5+OydbYhv02gciQFSv0ghcE/5hIcqrru+/qTdC/P5x+ugrEggWJSg9ULA4cyNyFsGCBbvuTn2il8dhjme33+ONVM2HCaYYNTSB8tsvRR0P79s1TIKZPh899LmFFNXY30+7dsGmTulVbt4aTT669QCxZknCL1pdAXHcd3HJL7fcvKdH4g6exZjKZQGTIe+8l3EuecGe5d99NZCp94xtqTvoAtcd/n6mbadIkbYHceCOceqouZxKw3rIFNm9OXhf2/zZUgejfHw49NLcCMXWq/nfhYdo9r73WMNxtH32kLskzz9RGSGFh4xcI70454gh9Hzu29nGIuXP1vUULtTbjpqIC/vxnzSqrDWVlmpXn4w+gjUdofG4mE4gM2LNHL6yPJXi8BbF/v1oYXgC+9jVtNUGiRQj6sLRtm5lAlJfD//0ffOUrWmledpm6p2bNqn6/ffu0vKkCEbYgGkKlGGb1av2/unfXc81FmuvevdonZcIE9X3/4x/J32/aBGecAf/933X/rbris5fOPFP/h/79G79A+BTXsEDUNg4xZ46ORHDccfVjQSxcmBiYszb4OevDFoQJRAQicpaIrBCR1SJSZZxOEektIq+LyEIReVdEvhSsHyci80VkSfB+WpzlrInly/XmPvbY5PXduulNVFKiriMvEB06aMXUpk2yqBQW6jEy6QsxbZr6LC+9VJfPP19jII8+Wv1+Xgi2bk3utd3QXUxHHKHuu1xYEFu3quX2wANw/fX6cKbO6DV7tl7TF16o22/lghkz9F7yWVyDBzc9gahLHGLOHA3e9+5dPwLhy7hlizbUssWP2hy2IA47TJ/fOFxMU6bU3tqpidgEQkQKgfuB8cBgYKKIDE7Z7FbgaefcUOAi4IFg/VbgbOfcccBlQIbe93jwFzxVILp31wwj36r3AgE6QNlLLyWCVJ4hQ2Dx4ppdRZMmQZcuOo8EqG/+3HN1+sLqgtxbt+p7ZWVyRdvQBcIPR5ALgXjiCXUJTpkCv/kNnHRSVYF45x19X7UqvwHhykqdTOqLX9S4FmjFsnp1w0xHzpS1a7WhdOihuty6tV6Hf/0ru+Ps3q1iOXIk9OpVvwIB8PHH2e//6KNQVJTcOBTROEQcFsSvfw3/8z+5Py7Ea0GMBFY759Y658qAJ4FzUrZxwCHB5w7ABgDn3ELnnDfwlgFtRKR1jGWtlmXL9Ab3lZjHp7pOn66to/D3PXuqWZ3KccdpJZ7qAgqzY4e2bL/xDWjZMrH+ssvUPTR5cvp9wz2lw5+926agoGEJhHNamfh0wE6dqheIf/5Tg7mVlem3efFFjRd96Uu6PHIklJYmuwxmz9YKB/JnRWzeDFdcoeJ95pmJ9ccco+fnh5dvjPgMJi96oPOoLFqUXQrvggVqCY8Yodfr00+1URYXFRWacuzvjWznnS8p0YbhD35QtXEYR6qrc9qATXV/54oW8RwWgB5AWO9LgVEp2/wcmCEiPwTaAWdEHOc8YKFzrkp7SkSuAq4CKCoqoriu3TXT8Oabx9G7dytmzpyftH7z5kOB43n55Ur69t3DzJkLajzWZ591Aobw1FMLGTIkOhgwa1YnysqG0K/fQoqLE9uIQLt2Y3j88U107Rpde7z5ZlfUYIPp0xfy8ce6/+LF/WnX7nAKCx3Llm2muLhh1D7bt7diz56TOXBgFcXFH7F79xFs29aD4uKZkdv/4Q+DeOutIp58cjY9elStafbuLeT110/h3HNLKS5WP0eLFocAJ/LII0sYM2YblZUwa9YYzjxzE61bH8Jjj1UyYsSiOE8zCefg+ed78PDD/fjsswImTiyla9e1/2657tnTDhjBM88sY+vWBjo2Sg0sWTKSI47YTXFxwlfWrl0XKiqO5eGHF3DssZnV8k8/3RPoT1nZv9i161BgMM89N4d+/fbGUu733mvPrl3D+NKX1vHUU72ZMWMpe/duzXj/u+46mtatu3Lcce9QXJzsn2rZcgCrVxdRXPxWzsq7bVsrPvnkZFq31ucn5zjnYnkBFwB/Di1fAvwhZZvrgOuDz6OBEqAg9P0xwBrgyJp+b9iwYS4uevZ07pJLqq5futQ5fdydu/LKzI61erVu//DD6be5/XbdZufOqt+dcYZzQ4em3/f3v0+U6ZlnEuu/+U3n+vVz7sgj9XND4a23tKxTp+ryHXfo8v790dsPGKDfT54c/f1zz+n3xcWJdXv3OldY6NxPf6rL776r2zz2mHO33eZcQYFzW7bk7pxq4h//0N8fN865996r+v3evVqmn/2s/sqUSyornWvVyrkbb0xev2GDnvfvfpf5sb7+def69NHPM2fq/i+9lLOiVuHOO/U3FizQ9wceyHzfjz92rnVr5773vejvf/MbPeb27bkpq3POzZihx3zttdofA5jn0tSrcbqYSoFeoeWeBC6kEFcCTwM452YBbYAuACLSE/gHcKlzbg15YscOdU+kxh8gMR4TJAKMNdGnj7qNqnMfLFigYz61b1/1u1GjNAtqb5oG1NZQYyfsYtq2TTNBOnRoWC4mP0hfOAYB0W6mHTsS/5sfyiCVyZO1E+PJJyfWtW2rsR8fh5g9W99POgnOPltdGPU5F/aLL+p1mDq1auq0L+8RRzTeQPWGDZrq6QPUnm7dNNDs4z+Z4APUoPtCvHGI4mIdonvIEHXHZpPJ9MADGje69tro7+PIZEoXH80VcQrEXGCAiPQTkVZoEDrV27sOOB1ARAahArFFRDoCU4CbnXNZhrVyy7Jl+h51AQ49NJHOGg5QV0eLFvrgROXlexYsgBNPjP5u1Cj1Ty9I483asgUOCaI64TjH9u0qEB07NiyBWLNGH0TfkcgLRFSqa3iU0/feq/r9gQMamB4/Pjl2A1rJ+LnB33lH/4sjj9T/uXv36uM6ucQ5zVAbNy4xOGEUjTmTKTWDKcxJJyUEuia2bNGsHy8Q3bvrvRJXXwgffxg7VjMOi4oyj0Hs26cCcfbZ0aIPcPzx+p46su1nn+nMe4sXZ1/mpUuha1fNkoqD2ATCOVcB/ACYDixHs5WWicjtIvKVYLPrge+IyGLgCeBbgcnzA6A/8F8isih4dY2rrNVRnUKLJKyITC0IUOsgnQWxdas+AOkEwj8sqVk54f27ddNgb1ggvAURh0DMnVt90Lg61qzRlqEP6FVnQfgOU0OGRFsQc+ZopfLlL1f9buRIDXCuWqUCMWqUXr+CAn2op02rn6yhkhLtGBcOSkcxeLA2ImqTZplvqhOIUaN0yJhMsoP89R4xQt9btNB7uzYWxPbtNU/W5fs/+OQSn8aeCXffrc/e9den3+bII1Uknn02ef3kyTrmWurwOJmwdGl81gPE3A/COTfVOXeUc+5I59wdwbrbnHMvBJ9LnHOnOOeOd86d4JybEaz/pXOuXbDOv6rJ+4mPpUvV1dOrV/T33bvrq0uXzI85YIBWVFE3rB9zJp1AFBWpmypdK2zrVm1NdO1a1cXUqVPuBWLJEq18H3ig5m2jWLEikcEENQtE//7qPnrvvaqpwi++qC0/nxocxgvrK69oJX3SSYnvzj5b0yljynFIYto0fc9EIMrLkyfdaSysXavXwbuEwowK0lQysSKinoXapLouX65jqP35z9Vv56//qafqe/fumVkQzz6r45+dd54ObV4d558Pb7+tjQTPpEn6PmVKdkP7+2F7Gq1ANAW8QofT9cJ85zs6FEY2HHWU9r7+KCLpwLuO0gkE6ENWnUB06aIi4S2IigoVhTgsCG8uP/xw9vvu26dmdbi3eadO+h4lEPPmaWty4ED9PjVVePJkOOWUxDHCDBoE7dqpkDmXqKhA5wxu2za+zkZhpk3TlMR0DQ6P72SVCzfTa6/BH/9Yu30//FDv/2yEyluFqW4+0Pu6RYvMBGLDBr1nw7G4bDvLOQfXXKPP22uvVb/d9Ol6bx1+uK7LZFrhf/1LZ2k86SQdKy1dPeE5/3x998P3b9miabF9+2qsM5vRntet04ZNXCmuYAJRLc5pC7k6hf7Wt+BHP8ruuAMG6HtUHGLBAg1m+ZZ0FN5M37Sp6ndbtqhAhC0IX9n6IPWePbmbeOWtIGNv8eJEiy9T5s/XcowenViXzoLYtEkrhhEjEkMYhOMQ69bpw3X22dG/VVioQuQrXG9RgIrDqadqh7U42bNHh5qIsnBS8RNThWfaqw2VlXDVVXDDDbWbeGrmTI3DZdNXxPeBiKJtW3WzZBKo3rxZ7+Mw3oLI9Fyee06txg4d0ouSc3DzzfDqq/DNbybWd+umZUjn5lu1SofC6d1b/5+2bWsuz8CBWp94N9NTT+kz8Kc/6fKLL2Z2XhB/gBpMIKpl82Z1zeRaob1ARMUhqgtQe9KZ6c5FWxA+4OstCMhsPKaFC3Vk2nTxEudUIPwYQo88UvMxw/ge6GF3jy9fqkCE/dEDB+rncBzCZyFNmJD+97woDBqU+B3PuHEqOKWlmZc/W4qLNbsnE4Fo314tntr05A0zZYq26HfvjrZYa8Jf+0ynDD1wQPfxGTtRjBqVWdwqnUDs35+crZeOPXt0VNbjj1cB+OCDqlanc/Bf/6W9kb/3veQRXH1H2KiGGMDvfqdxq5deys7FfP75Krwff6zupeOP13HBRozIzor1AmEWRJ6IS6F79NDWRqoF8emn2mKsSSCGDtUWcapA7NyprREfg9i2TR9CP8yGj0FAZm6ml15Ss3zs2GhrZ906rXTOPhu++lUdXDCbQO/bb2vgLlwJFBZqFlaUQBQU6Ln36qWVZ9iCmDZNzXQvHlF4gRiV2l0TfUBBW5txMW2aXvcxYzLb/vDD01dOmXL33fqfQnTmV0346z5zZs1BXlALadu26FEEPCedpIKVLlXZk04gIDM3069+pdvdd18i7Tn1mbnjDn19+9s6KVfYRZQ6rXAq77yjx00dYaEmzj9fhen//T+9ry+5RNdPmKDHzET8QOunXr3UOooLE4hqqC7FtS4UFGiwNbVlvijozFuTQBx0UHJev8e7lLyLyc8i5wUibEFkIhArVuj25eX6wKeOWeTdS2PGwOWXq6WSqSvCObUgwv0VPFEjus6dqy2ldu30IR44MFHBlJWpe+Css6r3AY8erX7xL3yh6nfHHacJAHG6maZN099u0yaz7Q8/vG4WxKJFarV4F2htBaKgQK+Hfx6q4y9/0Qrr3HPTb+MFuiY3U5RARPWFiLL6Nm/Wcbguvljvz2HDqjaqPv1UJ/i64AKN0RSk1Iap0wqH2bs3kaCRLYMH6/177736m9/4hq6fMEGfi5deyuw4cWcwgQlEtSxdmqhsc01Uqqufw3ro0Jr3HzVKBSLcqvMtD+9igoSbDGonEEOH6mx2lZUqEuEW7VtvaWv/2GO1Bd6zp1YQmfDBB3qscPzBkzpgn3MqED7dEfQB8xXev/6lLdKaXDc9eqh//OKLq34noufwyiuZtZSzZc0atQ4zcS95iorqJhD33quNiVtv1Uo7W4FwTu9R77aryc20cyc88wxcdFH1/vgBA/QaVxeoLi9XUarJgnjmGV0X7iMDem+WlemQ76D/w3HHJf/mlCn6Oz/+cVVxgOotiAUL9JmIskZrQiQRrB43LvE7J56ojYJM3EwVFXo9TSDySE0ZTHVhwACtNMLB4gULtBIrKqp5/1Gj9IEMt+q9QHgXE6hAhGMQ3hytKQbhp708+mhtub/8sh7r7rsT27z1llbwhYX6uuwyzQTJZGpUH3/IRCA+/FDPLSwQgwYlsjimTVPL4LQMBoXv2TO6MgAViM2bMxuOPVtmzkz8RqbUxYLYtEldfpdfrv9nWFCzOcauXVqJ9e5ds0A8/bRmpl1xRfXbiWg66PPPp2+o+Hs5VSAOO0z7zKxbp0L+i1/o+lTX4Ntva1ws3NhKbVT9/e9aOaer5Lt21XslyoLwQlMbCwJURAsL1bXlKSjQASanTau5/8uaNerONYHIE86pSR1XAOioo1QcwpVpJgFqT1SgOmxB+Adryxa1ILxvP1MLYssW3cb3Ch0yRFs9Dz6o4vLJJyqgYX/6t76lD/+AAVoR/u536X9n1ix1F6XroR4WiNQOU5CINaxcqSb5mDHRQ5Nkw7hx+h5HHML3GA/3+aiJww9Xcc+2A5/3b5eVaYon1CwQCxfq9SsrS6zz8YejjtIK/c03q88eeuQRdZ+Er1M6fvYzvS/TTdjkg8mpAlFQoCK/fr1OArVsmabNegH2zJqlbqXWoTGgw42qffv0vvnqV9M3GFq00N+PsiDmzNH+SJk05qI45hgVHm9JeL78ZX2+3n67+v3rI4MJTCDS8tFHejPFdQFSU1337NEHOFOB8PM3zw8NMBuOQaS6mDp10so7U4Hwlkl42IAbb9T/5I9/TFgAYYHo31/9ytdeqy3f669PtjjCzJqlra+o4SZSh/yeO1dbjeHe6j7V9ZVXtMWfjesmHT166HHjiEOsXZu+b0A6fD5+dUPDp7Jnj04ydc89WuH7dNmBAxP3dCr796sf/NFHk+Na3gU6YICmAW/enH7ujOXL9ZpecUVmFvfQoXDllfD730cfM51AgP6P69bBL3+pZbv0UnUzesugrEyfi1TrNNyomjFD4whf+1r15UzXWW727Nq5l8JEDY9xxhl6j9SU7rp0aSIWFycmEGnwAbnwrFC5xD+4/iH0rbNMBaKgQMsWDhxu3aotpoMP1kq2oCBhQXTurNu0b683Vm0EYtgwTXu95x4NCrdoUdXEHj4c7rxTb+DevaMHJtu7V/tNRLmXoKoFMXu2Vijh8fX791er6L77dHn8+OrPJ1PGjdNrsX9/bo7nef/96lM/o/ACkWkm06pV+p8+/ri2zMOdF31FElUZ33FHwroIT2m7cqVWVn36JHoXp3Mz/fWvej2i4jvp+OUvNVYRNTxFdQLRq5c2RBYtgp/+VGNjn36aaFUvXKhWV+r9NXCgWtGzZ6v10bFj9dlWEN1ZbtMmtfxr616qjvbt9b+uKQ6xdKlmTx10UO7LEMYEIg2+Q1VcAnHYYXqzrlypldG112qaZjY+6mOOqSoQXbqoABQWqij4GIQXiIKCqiO6vvmmBurC7oMVK1Rs+vRJ/s2f/ERbVPfdp4JR3Q3qW3qpzJtXtYNcmEMP1f9k3z7dbt68qq21Vq30AVm/Xh/iXFl6Z5yhv1vT3N/ZUl3nsXR4gcgkDrFwof6fGzaoD/vWW5NdJ14gUt1MixerO+qyy/T/TBUIL8T9+2t5wnNKz5sHf/iD+tH/93/VPZKNy6WoSPsgTJmisaswNQnEgQP6vHzzmzqBFCTcTN49k3p/FRSo++tf/9Jsu7PPrtmi69atqgXhray6WhDpmDBBLTI/plUqH3+sFlAcApWKCUQaSkq0Es+mA0w2eF/9qlWaardypfamzKZFMHiwPkg+9uAFwtO1a7KLydOhQ3KQ+rHH1CoIt/b9GEk+h95zxhnami8rqzmfP51A+Ac43EEuTLg3dUmJWhxRD4Ov9GpKb82GsWPVMsplHGLvXm111taCqEkg3nlH02fbtdPPX/xi1W2OPFLPKywQFRXqEurUSeNFo0erQPiGwqpVCVeoiLZs33hDr+m552ple801OsvfSSdpv4NsueYavc9+9rPk9Zs3a+UdlePvU11vuilh4fTsmRCIWbN0nU9TDTNqlLokPxOJo0MAAB+/SURBVPmkZvcS6DE2b05OJpk9W5+LTK39bPGDTaazIm66SRsxqf9ZHJhApCHOALXnqKP0gb7zTvUXZ2M9QKJ83orww2x4/HAbYRcTVB2Pybu5wmPVrFgR7d8U0RsUEm6HdPTurTnqqT1mZ83Siied+IYFwgfho1prPg6RK/cSqIk/cGBmOf+Z4oU3WwvCt56rE4iZM9Ut1qWLtu7TBcFbttTvwgLx0EOaGHH//SoSJ5+sv/XBB9pCX7064QoFDVR/9JG6HadNU0H46COtQKdPr5213bq1tuSXLEm2YH0fiCjhP/dcdYtdfrkui6gVMXNmon9NOuvU30dt29Y8YCKoBeFcsptvzhyNh8Xl3unfX//3KIF4+22NFV1/ffK1iQsTiAic05ZrXO4lz4ABGjTs0gV++9vs908VCD+Sq8cPt5GpQLz+ur6Xl6t5m25c+wsuUDM9aljtML166bFSfegLFlRvHocFYs4crbyiequOG6eVuc8+yhX9+mklmSu8QGRrQbRurf9FOoHYv1+vRY8eKg6p7sBUwplMzmlG2vDhiUwaX6nOmqWuu88+S1gQoBVqq1b6XlKiw1d07153661fP7WywqMPR3WS8xx2mMYewjGpMWPUvTZzpjZKahKIs87KrIL3VoiPQxw4oPdkXO4lz4QJ2slxz57EuspK7dfRs6e6EOsDE4gINm5UF0zcAuGzcu67L3oE0pro2VPjGGGBSLUg1q9XczSdQOzenbj5X39dK461a9WkTicQItrarKliiOr1unevPsDpjg1VLYiRI6N/6/TT1Veb66EG+vbNrUBUNz9CTVTXF+Jvf1PxfeCBaHdKKgMHamOgokIzw5YuTc7DP/ZYdVPNmpWc4uo58ki9b55/PjHBUy6ImmmtOoGIwsch7rxT36N66IPGPe66C267LbPj+k5sPg6xcqXWDXH7/ydMUIF+9dXEuoce0sD8b3+r16k+MIGIIO4AtedrX9MRSFNzoTNFJDHzWEWFVqhhgTjsMBUHSBagsED40ULHjdOHYOXK6Aym2uAFIhyH8MNGh1umqfiyrl+v4hd3ay2Vvn0TfT1ywfvva2u1NrN+pROIAwc0bnDCCdFDh0QxcKBadO+/rxlObdtqhy2Pz0pLJxCQ2Yil2ZILgTjmGG1YTJmSGDE2HTfcoP9bJqRaEHEHqD2f+5y6O32667x58J//qZ1BL7gg3t8OYwIRgW+Rxx2DKCzMbia6KHwmkx9OI1wJhR+wsAURDlJ799J3v6vvr70Wr0CEc+vT4S0IP+xFfWRrhPGt41xZET6DqTaumHQD9k2bptbTdddlflwfU5o/H554QhsmqdbX6NHaSl20SFup4XnX48L/3+GsnWwFoqBA5wIBdZtl09+kOnxv6g0btEU/aVIiThUnrVppo23qVBXr8eP12f7b3+IZ2SEdJhARlJRohRrXPK+55Jhj1HfrB65LdTF5Ul1MO3cmhmYG9Sv37KluphUrdN/UIbGzpUMHfZjCAuEtluoEwlda3rxuTAIxc2bVlOHa9IHwpLMgfvtbbd1eeGHmx/KC/6tf6RAaV15ZdZuTT1Zf93PP6TWqj8qoXTt1/XgLYs8edUVmOwaadzOliz/UBt+betUqjbm9+qq6sVKz++Lgy1/WJAAvfNOn149ghzGBiMAHqOtTqWuLd4P5DkypLiZPqkA4pyKxapXedAcfrK6K4mINZNbVegD9/1JTXVet0gfukEPS71dYqCKxc6e2vONKNU5HXQTizjs1Zdi70nxMpzbxB9CKc/dufXkWLVJL75prkgO1NdGxowrOkiWaKRM1PaZPPf7kk+pFPNf065cQiOr6QFSHzwLMZEyubOjeHZ58Uv/zRx/VeSPqA5+dt2+fWhL1kbWUiglECnGPwZRrfDmjBKI6CwI0DrFyZaIiOO00tUbeeSc3AgHRApHJeETezVTf8Qf/2+3bZy8Qe/cm+k/4jLCtW7VFXBcLApLdTL/7nba6r7oq++P51OB0Q2J07pyoiOqzQsqFQJx4oopxJumr2dC7twrxs8/qsB71xeGHa2D65ZczG98qDkwgUti0SVtPcQeoc0WPHtoa9z1gU9NcPalBalCBWLUqURH4YGdlZbwCkUnLNJ8CIVK7TKZXXtHU04KCRJ+SumQwQdXOcuXlWlFdfHH109KmY/BgLV91FZ130dS3BbFund57tRUIf5xcc889mvWVSce6XPOd7+TWZZYtJhAp1FcGU64QUSvCjx2Uaim0aKFZHeHsE+/jX7dOLQZfEfTpk3jAcikQW7dq63rPHg32ZSMQ9R1/8NRGICZPVrE+7zwVCOdq3wfCkyoQixery6G2bpSbb9ZhGnr0SL+NTxHN1T2QCf36aSZeaWndBCIO+vTR0YybIyYQKcQ9SF8ceDfTIYck+6QLCtSKCIsGJCwIP4x2uML2VkQuBQI0ZTWTFFdPp06aiZLJ5Elx4AWiuuGtwxw4oCmJZ56pnbA2b9bGhrcgcuVi8rOwpRumpCZ69ND+I9VxySU68VN9Wm/+/1m7NiEQjSFJpKkTMdhy86akRFuv/sFsDHiBiHqgDjusqq+5OoH47nd1nKVs59lNRzjV1Q81nYlAnH++xioynZ4z1/Ttq+XdsSPalXPHHdoj/Kmn1EqbN09b+Wefncimee01tSC6dq19x6YuXVTovQUxa5YGTf3ManHQtq0O/VKfeBfc+++rQLRvH0+fCyM7TCBSaEwZTB4vEFHZPmPGVJ1CM1UgwmIwcqQO3pcrwgLhh1LIJEh94YXZpXDmmnAmU6pAbN+uArFvn6aM3nabupf8jGCdO2uL+LXXEplYtaWwUAXGC8Q776j10Jjuz0zo1UvP1QtEQ3EvNXfMxZRCfYzBlGt8eaME4v77dcydMD7FdPt2rcDjbKn16KGV2bp1GqAuKqr7zG/1QXWprn/6k4rDqafqSLxz5qhAnHJKwp132mmaMrxmTd0Dp35u6s2b1QWTz6BlXLRooSJhAtGwMIEIUV6uAdU4zfc46N5dK6ZMO9G0aKH9HiD+TJWWLbV869drJ7n6zIypC+kEoqJCRfe003RMoh491B22eLG6lzynnabuqQ8/rJsFAYnOcj5Trbbxh4aOT3U1gWg4mECE8PPxhuexbQyI6Py62YwP791M9VFh+1TXTFNcGwLp+kI8/7yK3TXX6H84aZJm3kCyQITHR6qrBeEF4p13VNyHDavb8RoqJhANDxOIEOXl+p6rcVzqkxEjdKiMTKlvgSgp0cEAM4k/NATS9YW4916tyPxQ56eeqtN7jh+fnPnVrVuiU1ouLIhNm3QugBNOaLrB23799B4xgWg4mECE8BZENsMXNFbqWyD8cMmNxYKAqgKxYAG89Rb88IfJY/HccosOhZAaOPZ9FXJhQZSVqUA0xfiDx/9PBw6YQDQULIsphLcgmpNA1MdwCj6TCRqfQLzxhvaFEIG779bYzRVXZLb/1VfrMN/h868NPuW6oqLpxh8g2dIygWgYmECE8BZEY3QxZUuHDpqWGcfQBKmEK8jG4mKC5L4QpaXw+OM61WOmExQNGpSYwKYuhPvkNAcLAkwgGgomECGak4vpi1/UgGd9nKvPCvOjxjYWwplMt9yiVtdPf1r/5Sgq0veuXXM7k1tDo6hI4yv79plANBRijUGIyFkiskJEVovITRHf9xaR10VkoYi8KyJfCn13c7DfChHJ8fiM0TTmIHW2XHop/PWv9fNb3oJoTNYDJCrjhx/WLLFbbqndAHl1xVsQo0c3vQ5yYXxiAJhANBRiEwgRKQTuB8YDg4GJIpLaBe1W4Gnn3FDgIuCBYN/BwfIxwFnAA8HxYqU5WRD1SadOmjJan4O/5QJfWd1/vw7YdvXV+SlHx446LlJtp6ZtTPTrp0KROn6YkR/idDGNBFY759YCiMiTwDlASWgbB/ipYzoAwcyvnAM86Zz7DHhfRFYHx5sVY3mblQVRn4jAP/+Zu/Gd6gvfF2LXLh1aI1/jQokkBulr6gwbpr3P62PGNqNm4hSIHsD60HIpkDo+5M+BGSLyQ6AdcEZo3/AjURqsS0JErgKuAigqKqK4uLhOBX733Q7AUJYvX8xBB+VoxnoD0Epu7drkeYcbAz16nEhlpdCt23zqeHsZGfD5zwujRxdQXFyZ76IYxCsQUd7S1MGTJwJ/dc79VkRGA4+JyLEZ7otz7iHgIYDhw4e7sWPH1qnAlcE9OWLE8ZHTMRrNj1df1Z71nTuPzXdRDKPeiVMgSoHwqEY9SbiQPFeiMQacc7NEpA3QJcN9c465mIxUunfPdwkMI3/EmcU0FxggIv1EpBUadH4hZZt1wOkAIjIIaANsCba7SERai0g/YAAwJ8ayAhakNgzDCBObBeGcqxCRHwDTgULgEefcMhG5HZjnnHsBuB74k4j8GHUhfcs554BlIvI0GtCuAK52zsXulDQLwjAMI0GsHeWcc1OBqSnrbgt9LgFOSbPvHcAdcZYvFbMgDMMwEthgfSGa01hMhmEYNWECEaI5jcVkGIZREyYQIczFZBiGkcAEIoQFqQ3DMBKYQIQwC8IwDCOBCUQIC1IbhmEkMIEIYUFqwzCMBCYQIcrKdBKdpjzmvmEYRqaYQIQoLzfrwTAMw2MCEaKszOIPhmEYHhOIEGZBGIZhJDCBCGEWhGEYRgITiBDl5SYQhmEYHhOIEGVl5mIyDMPwmECEMBeTYRhGAhOIEBakNgzDSGACEcIsCMMwjAQmECEsSG0YhpHABCKEBakNwzAS1CgQItJORApCywUiclC8xcoP5mIyDMNIkIkF8SoQFoSDgFfiKU5+sSC1YRhGgkwEoo1zbrdfCD6bBWEYhtHEyUQg9ojIiX5BRIYB++IrUv6wILVhGEaCFhlscy3wjIhsCJa7ARfGV6T8YUFqwzCMBDUKhHNurogMBI4GBHjPOVcee8nygLmYDMMwEtQoECJyacqqoSKCc25STGXKGxakNgzDSJCJi2lE6HMb4HRgAdDkBMIsCMMwjASZuJh+GF4WkQ7AY7GVKI+YBWEYhpGgNj2p9wJH5bogDQGzIAzDMBJkEoOYDLhgsRAYBDwdZ6HygXNQUWECYRiG4ckkBvGb0OcKNJNpYjzFyR/lQV6WuZgMwzCUTGIQb4jICcA3gK8D7wPPxV2w+qasTN/NgjAMw1DSCoSIHAVchFoL24CnAHHOfaGeylavmAVhGIaRTHVB6vfQlNaznXNjnHN/ACqzObiInCUiK0RktYjcFPH93SKyKHitFJEdoe/uFJFlIrJcRH4vIpLNb2eLWRCGYRjJVOdiOg+1IF4XkWnAk2j8ISNEpBC4HxgHlAJzReQF51yJ38Y59+PQ9j8EhgafTwZOAYYEX78FnAoUZ/r72eItCBMIwzAMJa0F4Zz7h3PuQmAgWjH/GCgSkQdF5IsZHHsksNo5t9Y5V4YKzDnVbD8ReML/PNoprxXQGmgJbMrgN2uNtyDMxWQYhqFkEqTeAzwOPC4inYALgJuAGTXs2gNYH1ouBUZFbSgifYB+wGvBb84SkdeBjajVcp9zbnnEflcBVwEUFRVRXFxc0+mkZd26g4CRrF5dQnHx5lofxzAMo6mQSZrrv3HObQf+GLxqIsod5SLWgbqynnXOVQKISH+0v0XP4PuXReTzzrk3U8rzEPAQwPDhw93YsWMzKFY0S5bo+/HHD2bs2MG1Po5hGEZTIc45qUuBXqHlnsCGNNteRMK9BPA14B3n3O5ggqKXgJNiKWWABakNwzCSiVMg5gIDRKSfiLRCReCF1I1E5GjgUGBWaPU64FQRaSEiLdEAdRUXUy6xILVhGEYysQmEc64C+AEwHa3cn3bOLROR20XkK6FNJwJPOufC7qdngTXAEmAxsNg5NzmusoIFqQ3DMFLJKgaRLc65qcDUlHW3pSz/PGK/SuC7cZYtFXMxGYZhJBOni6lRYT2pDcMwkjGBCDALwjAMIxkTiAALUhuGYSRjAhFgQWrDMIxkTCACzIIwDMNIxgQiwCwIwzCMZEwgAixIbRiGkYwJRICluRqGYSRjAhFgFoRhGEYyJhABFqQ2DMNIxgQiwFsQLWIdfMQwDKPxYAIRUFam8Yd4Z742DMNoPJhABJSXW4DaMAwjjAlEQFmZxR8MwzDCmEAElJebQBiGYYQxgQjwMQjDMAxDMYEIMBeTYRhGMiYQARakNgzDSMYEIsAsCMMwjGRMIAIsSG0YhpGMCUSABakNwzCSMYEIMAvCMAwjGROIALMgDMMwkjGBCLAgtWEYRjImEAGW5moYhpGMCUSAWRCGYRjJmEAEWJDaMAwjGROIAAtSG4ZhJGMCEWAuJsMwjGRMIAIsSG0YhpGMCUSAWRCGYRjJmEAEWJDaMAwjGROIAAtSG4ZhJBOrQIjIWSKyQkRWi8hNEd/fLSKLgtdKEdkR+q63iMwQkeUiUiIifeMq54EDUFlpFoRhGEaYFnEdWEQKgfuBcUApMFdEXnDOlfhtnHM/Dm3/Q2Bo6BCTgDuccy+LyMHAgbjKWl6u72ZBGIZhJIjTghgJrHbOrXXOlQFPAudUs/1E4AkAERkMtHDOvQzgnNvtnNsbV0HLyvTdLAjDMIwEsVkQQA9gfWi5FBgVtaGI9AH6Aa8Fq44CdojI34P1rwA3OecqU/a7CrgKoKioiOLi4loVdOfOFsAYPvxwFcXFH9XqGIZhGE2NOAVCIta5NNteBDwbEoAWwOdQl9M64CngW8DDSQdz7iHgIYDhw4e7sWPH1qqgH3+s74MHD2Ds2AG1OoZhGEZTI04XUynQK7TcE9iQZtuLCNxLoX0XBu6pCuB54MRYSkkiBmEuJsMwjARxCsRcYICI9BORVqgIvJC6kYgcDRwKzErZ91AROSxYPg0oSd03V/gYhAWpDcMwEsQmEEHL/wfAdGA58LRzbpmI3C4iXwltOhF40jnnQvtWAjcAr4rIEtRd9ae4ympBasMwjKrEGYPAOTcVmJqy7raU5Z+n2fdlYEhshQthLibDMIyqWE9qzMVkGIYRhQkEZkEYhmFEYQKBWRCGYRhRmEBgQWrDMIwoTCCwsZgMwzCiMIHALAjDMIwoTCCwILVhGEYUJhBYkNowDCMKEwjMgjAMw4jCBAKzIAzDMKIwgcCC1IZhGFGYQGAuJsMwjChMIDAXk2EYRhQmEJgFYRiGEYUJBAkLorAwv+UwDMNoSJhAoALRqhVI1CzahmEYzRQTCNTFZO4lwzCMZEwgUAvCAtSGYRjJmEBgFoRhGEYUJhCYBWEYhhGFCQSJILVhGIaRwAQCdTGZBWEYhpGMCQRmQRiGYURhAoEFqQ3DMKIwgcCC1IZhGFGYQGAWhGEYRhQmEJgFYRiGEYUJBBakNgzDiKJFvgvQEDAXk2E0P8rLyyktLWX//v35Lkq90KZNG3r27EnLLNwlJhCYi8kwmiOlpaW0b9+evn37Ik18KGfnHNu2baO0tJR+/fplvJ+5mDALwjCaI/v376dz585NXhwARITOnTtnbS2ZQGAWhGE0V5qDOHhqc64mEFiQ2jAMI4pYBUJEzhKRFSKyWkRuivj+bhFZFLxWisiOlO8PEZGPROS+OMtpLibDMOqbbdu2ccIJJ3DCCSdw+OGH06NHj38vl/l5kGvg8ssvZ8WKFbGVMbYgtYgUAvcD44BSYK6IvOCcK/HbOOd+HNr+h8DQlMP8N/BGXGX0mIvJMIz6pnPnzixatAiAn//85xx88MHccMMNSds453DOUVAQ3Zb/y1/+EmsZ48xiGgmsds6tBRCRJ4FzgJI0208EfuYXRGQYUARMA4bHWE6zIAyjmXPttRDU1TnjhBPgnnuy32/16tV89atfZcyYMcyePZsXX3yRX/ziFyxYsIB9+/Zx4YUXcttttwEwZswY7rvvPo499li6dOnC9773PV566SUOOugg/vnPf9K1a9c6nUOcAtEDWB9aLgVGRW0oIn2AfsBrwXIB8FvgEuD0dD8gIlcBVwEUFRVRXFxcq4KWlZ3Khg3rKC5+v1b7G4bR+OjQoQO7du0CoKysNZWVufW4l5UdYNeuzzLa9rPPPqNly5bs2rWL3bt3U1JSwn333cddd90FwC233EKnTp2oqKhgwoQJjB8/noEDB1JZWcmePXvYtWsXn376KSNGjOCWW27h5ptv5sEHH+S6665L+p39+/dnVU/GKRBRIXOXZtuLgGedc5XB8n8AU51z66uLvDvnHgIeAhg+fLgbO3Zs1oWsrIQDB2DAgD6MHdsn6/0Nw2icLF++nPbt2wPwwANx/UpmronWrVvTunVr2rdvz8EHH8yRRx5JuD6bNGkSDz/8MBUVFWzYsIEPP/yQESNGUFhYSLt27Wjfvj1t27blvPPOA2D06NHMnDnz3+fnadOmDUOHpnry0xOnQJQCvULLPYENaba9CLg6tDwa+JyI/AdwMNBKRHY756oEuutKebm+m4vJMIyGQrt27f79edWqVdx7773MmTOHjh07cvHFF0f2Z2gVqsQKCwupqKiocznizGKaCwwQkX4i0goVgRdSNxKRo4FDgVl+nXPum8653s65vsANwKQ4xAE0QA0WpDYMo2Gyc+dO2rdvzyGHHMLGjRuZPn16vf12bBaEc65CRH4ATAcKgUecc8tE5HZgnnPOi8VE4EnnXDr3U6yYBWEYRkPmxBNPZPDgwRx77LEcccQRnHLKKfX225KnejnnDB8+3M2bNy/r/XbsgO9+F664As48M4aCGYbRIFm+fDmDBg3KdzHqlahzFpH5zrnITNFmP1hfx47w1FP5LoVhGEbDw4baMAzDMCIxgTAMo9nSVFzsmVCbczWBMAyjWdKmTRu2bdvWLETCzwfRpk2brPZr9jEIwzCaJz179qS0tJQtW7bkuyj1gp9RLhtMIAzDaJa0bNkyq9nVmiPmYjIMwzAiMYEwDMMwIjGBMAzDMCJpMj2pRWQL8GEdDtEF2Jqj4jQWmuM5Q/M87+Z4ztA8zzvbc+7jnDss6osmIxB1RUTmpetu3lRpjucMzfO8m+M5Q/M871yes7mYDMMwjEhMIAzDMIxITCASPJTvAuSB5njO0DzPuzmeMzTP887ZOVsMwjAMw4jELAjDMAwjEhMIwzAMI5JmLxAicpaIrBCR1SISy7zXDQER6SUir4vIchFZJiI/CtZ3EpGXRWRV8H5ovsuaa0SkUEQWisiLwXI/EZkdnPNTwZzpTQoR6Sgiz4rIe8E1H93Ur7WI/Di4t5eKyBMi0qYpXmsReURENovI0tC6yGsryu+D+u1dETkxm99q1gIhIoXA/cB4YDAwUUQG57dUsVEBXO+cGwScBFwdnOtNwKvOuQHAq8FyU+NHwPLQ8q+Bu4Nz/gS4Mi+lipd7gWnOuYHA8ej5N9lrLSI9gGuA4c65Y4FC4CKa5rX+K3BWyrp013Y8MCB4XQU8mM0PNWuBAEYCq51za51zZcCTwDl5LlMsOOc2OucWBJ93oRVGD/R8Hw02exT4an5KGA8i0hOYAPw5WBbgNODZYJOmeM6HAJ8HHgZwzpU553bQxK81Ojp1WxFpARwEbKQJXmvn3JvA9pTV6a7tOcAkp7wDdBSRbpn+VnMXiB7A+tByabCuSSMifYGhwGygyDm3EVREgK75K1ks3APcCBwIljsDO5xzFcFyU7zmRwBbgL8ErrU/i0g7mvC1ds59BPwGWIcKw6fAfJr+tfaku7Z1quOau0BIxLomnfcrIgcDzwHXOud25rs8cSIiXwY2O+fmh1dHbNrUrnkL4ETgQefcUGAPTcidFEXgcz8H6Ad0B9qh7pVUmtq1rok63e/NXSBKgV6h5Z7AhjyVJXZEpCUqDo875/4erN7kTc7gfXO+yhcDpwBfEZEPUPfhaahF0TFwQ0DTvOalQKlzbnaw/CwqGE35Wp8BvO+c2+KcKwf+DpxM07/WnnTXtk51XHMXiLnAgCDToRUa1Hohz2WKhcD3/jCw3Dn3u9BXLwCXBZ8vA/5Z32WLC+fczc65ns65vui1fc05903gdeD8YLMmdc4AzrmPgfUicnSw6nSghCZ8rVHX0kkiclBwr/tzbtLXOkS6a/sCcGmQzXQS8Kl3RWVCs+9JLSJfQluVhcAjzrk78lykWBCRMcBMYAkJf/xP0TjE00Bv9CG7wDmXGgBr9IjIWOAG59yXReQI1KLoBCwELnbOfZbP8uUaETkBDcy3AtYCl6MNwiZ7rUXkF8CFaMbeQuDbqL+9SV1rEXkCGIsO670J+BnwPBHXNhDL+9Csp73A5c65eRn/VnMXCMMwDCOa5u5iMgzDMNJgAmEYhmFEYgJhGIZhRGICYRiGYURiAmEYhmFEYgJhGFkgIpUisij0ylkPZRHpGx6h0zDyTYuaNzEMI8Q+59wJ+S6EYdQHZkEYRg4QkQ9E5NciMid49Q/W9xGRV4Ox+F8Vkd7B+iIR+YeILA5eJweHKhSRPwXzGswQkbZ5Oymj2WMCYRjZ0TbFxXRh6LudzrmRaM/Ve4J196HDLQ8BHgd+H6z/PfCGc+54dJykZcH6AcD9zrljgB3AeTGfj2GkxXpSG0YWiMhu59zBEes/AE5zzq0NBkX82DnXWUS2At2cc+XB+o3OuS4isgXoGR72IRiG/eVg0hdE5CdAS+fcL+M/M8OoilkQhpE7XJrP6baJIjxOUCUWJzTyiAmEYeSOC0Pvs4LPb6MjyQJ8E3gr+Pwq8H3495zZh9RXIQ0jU6x1YhjZ0VZEFoWWpznnfKpraxGZjTa8JgbrrgEeEZH/RGd5uzxY/yPgIRG5ErUUvo/OhGYYDQaLQRhGDghiEMOdc1vzXRbDyBXmYjIMwzAiMQvCMAzDiMQsCMMwDCMSEwjDMAwjEhMIwzAMIxITCMMwDCMSEwjDMAwjkv8PjQhuNGjcu5EAAAAASUVORK5CYII=\n",
"text/plain": [
"<Figure size 432x288 with 1 Axes>"
]
},
"metadata": {
"needs_background": "light"
},
"output_type": "display_data"
}
],
"source": [
"plot(history) "
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.3"
}
},
"nbformat": 4,
"nbformat_minor": 4
}
Datasets
Models
