--- a
+++ b/Serialized/Post Full Head Models Train .ipynb
@@ -0,0 +1,931 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 22,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "The autoreload extension is already loaded. To reload it, use:\n",
+      "  %reload_ext autoreload\n"
+     ]
+    }
+   ],
+   "source": [
+    "from __future__ import absolute_import\n",
+    "from __future__ import division\n",
+    "from __future__ import print_function\n",
+    "\n",
+    "\n",
+    "import numpy as np # linear algebra\n",
+    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
+    "import os\n",
+    "import datetime\n",
+    "import seaborn as sns\n",
+    "\n",
+    "#import pydicom\n",
+    "import time\n",
+    "from functools import partial\n",
+    "import gc\n",
+    "import operator \n",
+    "import matplotlib.pyplot as plt\n",
+    "import torch\n",
+    "import torch.nn as nn\n",
+    "import torch.utils.data as D\n",
+    "import torch.nn.functional as F\n",
+    "from sklearn.model_selection import KFold\n",
+    "from tqdm import tqdm, tqdm_notebook\n",
+    "from IPython.core.interactiveshell import InteractiveShell\n",
+    "InteractiveShell.ast_node_interactivity = \"all\"\n",
+    "import warnings\n",
+    "warnings.filterwarnings(action='once')\n",
+    "import pickle\n",
+    "%load_ext autoreload\n",
+    "%autoreload 2\n",
+    "%matplotlib inline\n",
+    "from skimage.io import imread,imshow\n",
+    "from helper import *\n",
+    "import helper\n",
+    "import torchvision.models as models\n",
+    "from torch.optim import Adam\n",
+    "from defenitions import *"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Set parameters below"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# here you should set which model parameters you want to choose (see definitions.py) and what GPU to use\n",
+    "params=parameters['se_resnext101_32x4d_3'] # se_resnet101_5, se_resnext101_32x4d_3, se_resnext101_32x4d_5\n",
+    "\n",
+    "device=device_by_name(\"Tesla\") # RTX , cpu\n",
+    "torch.cuda.set_device(device)\n",
+    "sendmeemail=Email_Progress(my_gmail,my_pass,to_email,'{} results'.format(params['model_name']))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{'model_name': 'se_resnext101_32x4d',\n",
+       " 'SEED': 8153,\n",
+       " 'n_splits': 3,\n",
+       " 'Pre_version': None,\n",
+       " 'focal': False,\n",
+       " 'version': 'classifier_splits',\n",
+       " 'train_prediction': 'predictions_train_tta',\n",
+       " 'train_features': 'features_train_tta',\n",
+       " 'test_prediction': 'predictions_test_tta',\n",
+       " 'test_features': 'features_test_tta',\n",
+       " 'num_epochs': 5,\n",
+       " 'num_pool': 8}"
+      ]
+     },
+     "execution_count": 3,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "params"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "SEED = params['SEED']\n",
+    "n_splits=params['n_splits']"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "(674252, 15)"
+      ]
+     },
+     "execution_count": 5,
+     "metadata": {},
+     "output_type": "execute_result"
+    },
+    {
+     "data": {
+      "text/plain": [
+       "(674252, 15)"
+      ]
+     },
+     "execution_count": 5,
+     "metadata": {},
+     "output_type": "execute_result"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>PatientID</th>\n",
+       "      <th>epidural</th>\n",
+       "      <th>intraparenchymal</th>\n",
+       "      <th>intraventricular</th>\n",
+       "      <th>subarachnoid</th>\n",
+       "      <th>subdural</th>\n",
+       "      <th>any</th>\n",
+       "      <th>PID</th>\n",
+       "      <th>StudyI</th>\n",
+       "      <th>SeriesI</th>\n",
+       "      <th>WindowCenter</th>\n",
+       "      <th>WindowWidth</th>\n",
+       "      <th>ImagePositionZ</th>\n",
+       "      <th>ImagePositionX</th>\n",
+       "      <th>ImagePositionY</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>63eb1e259</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>a449357f</td>\n",
+       "      <td>62d125e5b2</td>\n",
+       "      <td>0be5c0d1b3</td>\n",
+       "      <td>['00036', '00036']</td>\n",
+       "      <td>['00080', '00080']</td>\n",
+       "      <td>180.199951</td>\n",
+       "      <td>-125.0</td>\n",
+       "      <td>-8.000000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>2669954a7</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>363d5865</td>\n",
+       "      <td>a20b80c7bf</td>\n",
+       "      <td>3564d584db</td>\n",
+       "      <td>['00047', '00047']</td>\n",
+       "      <td>['00080', '00080']</td>\n",
+       "      <td>922.530821</td>\n",
+       "      <td>-156.0</td>\n",
+       "      <td>45.572849</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>52c9913b1</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>9c2b4bd7</td>\n",
+       "      <td>3e3634f8cf</td>\n",
+       "      <td>973274ffc9</td>\n",
+       "      <td>40</td>\n",
+       "      <td>150</td>\n",
+       "      <td>4.455000</td>\n",
+       "      <td>-125.0</td>\n",
+       "      <td>-115.063000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>4e6ff6126</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>3ae81c2d</td>\n",
+       "      <td>a1390c15c2</td>\n",
+       "      <td>e5ccad8244</td>\n",
+       "      <td>['00036', '00036']</td>\n",
+       "      <td>['00080', '00080']</td>\n",
+       "      <td>100.000000</td>\n",
+       "      <td>-99.5</td>\n",
+       "      <td>28.500000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>7858edd88</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>c1867feb</td>\n",
+       "      <td>c73e81ed3a</td>\n",
+       "      <td>28e0531b3a</td>\n",
+       "      <td>40</td>\n",
+       "      <td>100</td>\n",
+       "      <td>145.793000</td>\n",
+       "      <td>-125.0</td>\n",
+       "      <td>-132.190000</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "   PatientID  epidural  intraparenchymal  intraventricular  subarachnoid  \\\n",
+       "0  63eb1e259         0                 0                 0             0   \n",
+       "1  2669954a7         0                 0                 0             0   \n",
+       "2  52c9913b1         0                 0                 0             0   \n",
+       "3  4e6ff6126         0                 0                 0             0   \n",
+       "4  7858edd88         0                 0                 0             0   \n",
+       "\n",
+       "   subdural  any       PID      StudyI     SeriesI        WindowCenter  \\\n",
+       "0         0    0  a449357f  62d125e5b2  0be5c0d1b3  ['00036', '00036']   \n",
+       "1         0    0  363d5865  a20b80c7bf  3564d584db  ['00047', '00047']   \n",
+       "2         0    0  9c2b4bd7  3e3634f8cf  973274ffc9                  40   \n",
+       "3         0    0  3ae81c2d  a1390c15c2  e5ccad8244  ['00036', '00036']   \n",
+       "4         0    0  c1867feb  c73e81ed3a  28e0531b3a                  40   \n",
+       "\n",
+       "          WindowWidth  ImagePositionZ  ImagePositionX  ImagePositionY  \n",
+       "0  ['00080', '00080']      180.199951          -125.0       -8.000000  \n",
+       "1  ['00080', '00080']      922.530821          -156.0       45.572849  \n",
+       "2                 150        4.455000          -125.0     -115.063000  \n",
+       "3  ['00080', '00080']      100.000000           -99.5       28.500000  \n",
+       "4                 100      145.793000          -125.0     -132.190000  "
+      ]
+     },
+     "execution_count": 5,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "train_df = pd.read_csv(data_dir+'train.csv')\n",
+    "train_df.shape\n",
+    "train_df=train_df[~train_df.PatientID.isin(bad_images)].reset_index(drop=True)\n",
+    "train_df=train_df.drop_duplicates().reset_index(drop=True)\n",
+    "train_df.shape\n",
+    "train_df.head()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>PatientID</th>\n",
+       "      <th>epidural</th>\n",
+       "      <th>intraparenchymal</th>\n",
+       "      <th>intraventricular</th>\n",
+       "      <th>subarachnoid</th>\n",
+       "      <th>subdural</th>\n",
+       "      <th>any</th>\n",
+       "      <th>SeriesI</th>\n",
+       "      <th>PID</th>\n",
+       "      <th>StudyI</th>\n",
+       "      <th>WindowCenter</th>\n",
+       "      <th>WindowWidth</th>\n",
+       "      <th>ImagePositionZ</th>\n",
+       "      <th>ImagePositionX</th>\n",
+       "      <th>ImagePositionY</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>28fbab7eb</td>\n",
+       "      <td>0.5</td>\n",
+       "      <td>0.5</td>\n",
+       "      <td>0.5</td>\n",
+       "      <td>0.5</td>\n",
+       "      <td>0.5</td>\n",
+       "      <td>0.5</td>\n",
+       "      <td>ebfd7e4506</td>\n",
+       "      <td>cf1b6b11</td>\n",
+       "      <td>93407cadbb</td>\n",
+       "      <td>30</td>\n",
+       "      <td>80</td>\n",
+       "      <td>158.458000</td>\n",
+       "      <td>-125.0</td>\n",
+       "      <td>-135.598000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>877923b8b</td>\n",
+       "      <td>0.5</td>\n",
+       "      <td>0.5</td>\n",
+       "      <td>0.5</td>\n",
+       "      <td>0.5</td>\n",
+       "      <td>0.5</td>\n",
+       "      <td>0.5</td>\n",
+       "      <td>6d95084e15</td>\n",
+       "      <td>ad8ea58f</td>\n",
+       "      <td>a337baa067</td>\n",
+       "      <td>30</td>\n",
+       "      <td>80</td>\n",
+       "      <td>138.729050</td>\n",
+       "      <td>-125.0</td>\n",
+       "      <td>-101.797981</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>a591477cb</td>\n",
+       "      <td>0.5</td>\n",
+       "      <td>0.5</td>\n",
+       "      <td>0.5</td>\n",
+       "      <td>0.5</td>\n",
+       "      <td>0.5</td>\n",
+       "      <td>0.5</td>\n",
+       "      <td>8e06b2c9e0</td>\n",
+       "      <td>ecfb278b</td>\n",
+       "      <td>0cfe838d54</td>\n",
+       "      <td>30</td>\n",
+       "      <td>80</td>\n",
+       "      <td>60.830002</td>\n",
+       "      <td>-125.0</td>\n",
+       "      <td>-133.300003</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>42217c898</td>\n",
+       "      <td>0.5</td>\n",
+       "      <td>0.5</td>\n",
+       "      <td>0.5</td>\n",
+       "      <td>0.5</td>\n",
+       "      <td>0.5</td>\n",
+       "      <td>0.5</td>\n",
+       "      <td>e800f419cf</td>\n",
+       "      <td>e96e31f4</td>\n",
+       "      <td>c497ac5bad</td>\n",
+       "      <td>30</td>\n",
+       "      <td>80</td>\n",
+       "      <td>55.388000</td>\n",
+       "      <td>-125.0</td>\n",
+       "      <td>-146.081000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>a130c4d2f</td>\n",
+       "      <td>0.5</td>\n",
+       "      <td>0.5</td>\n",
+       "      <td>0.5</td>\n",
+       "      <td>0.5</td>\n",
+       "      <td>0.5</td>\n",
+       "      <td>0.5</td>\n",
+       "      <td>faeb7454f3</td>\n",
+       "      <td>69affa42</td>\n",
+       "      <td>854e4fbc01</td>\n",
+       "      <td>30</td>\n",
+       "      <td>80</td>\n",
+       "      <td>33.516888</td>\n",
+       "      <td>-125.0</td>\n",
+       "      <td>-118.689819</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "   PatientID  epidural  intraparenchymal  intraventricular  subarachnoid  \\\n",
+       "0  28fbab7eb       0.5               0.5               0.5           0.5   \n",
+       "1  877923b8b       0.5               0.5               0.5           0.5   \n",
+       "2  a591477cb       0.5               0.5               0.5           0.5   \n",
+       "3  42217c898       0.5               0.5               0.5           0.5   \n",
+       "4  a130c4d2f       0.5               0.5               0.5           0.5   \n",
+       "\n",
+       "   subdural  any     SeriesI       PID      StudyI WindowCenter WindowWidth  \\\n",
+       "0       0.5  0.5  ebfd7e4506  cf1b6b11  93407cadbb           30          80   \n",
+       "1       0.5  0.5  6d95084e15  ad8ea58f  a337baa067           30          80   \n",
+       "2       0.5  0.5  8e06b2c9e0  ecfb278b  0cfe838d54           30          80   \n",
+       "3       0.5  0.5  e800f419cf  e96e31f4  c497ac5bad           30          80   \n",
+       "4       0.5  0.5  faeb7454f3  69affa42  854e4fbc01           30          80   \n",
+       "\n",
+       "   ImagePositionZ  ImagePositionX  ImagePositionY  \n",
+       "0      158.458000          -125.0     -135.598000  \n",
+       "1      138.729050          -125.0     -101.797981  \n",
+       "2       60.830002          -125.0     -133.300003  \n",
+       "3       55.388000          -125.0     -146.081000  \n",
+       "4       33.516888          -125.0     -118.689819  "
+      ]
+     },
+     "execution_count": 6,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "test_df = pd.read_csv(data_dir+'test.csv')\n",
+    "test_df.head()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "split_sid = train_df.PID.unique()\n",
+    "splits=list(KFold(n_splits=n_splits,shuffle=True, random_state=SEED).split(split_sid))\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def my_loss(y_pred,y_true,weights):\n",
+    "    window=(y_true>=0).to(torch.float)\n",
+    "    loss = (F.binary_cross_entropy_with_logits(y_pred,y_true,reduction='none')*window*weights.expand_as(y_true)).mean()/(window.mean()+1e-7)\n",
+    "    return loss"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "class Metric():\n",
+    "    def __init__(self,weights,k=0.03):\n",
+    "        self.weights=weights\n",
+    "        self.k=k\n",
+    "        self.zero()\n",
+    "        \n",
+    "    def zero(self):\n",
+    "        self.loss_sum=0.\n",
+    "        self.loss_count=0.\n",
+    "        self.lossf=0.\n",
+    "        \n",
+    "    def calc(self,y_pred,y_true,prefix=\"\"):\n",
+    "        window=(y_true>=0).to(torch.float)\n",
+    "        loss = (F.binary_cross_entropy_with_logits(y_pred,y_true,reduction='none')*window*self.weights.expand_as(y_true)).mean()/(window.mean()+1e-5)\n",
+    "        self.lossf=self.lossf*(1-self.k)+loss*self.k\n",
+    "        self.loss_sum=self.loss_sum+loss*window.sum()\n",
+    "        self.loss_count=self.loss_count+window.sum()\n",
+    "        return({prefix+'mloss':self.lossf})    \n",
+    "        \n",
+    "    def calc_sums(self,prefix=\"\"):\n",
+    "        return({prefix+'mloss_tot':self.loss_sum/self.loss_count})    \n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#features=(features-features.mean())/features.std()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [],
+   "source": [
+    "%matplotlib nbagg\n",
+    "for num_split in range(params['n_splits']):\n",
+    "    multi=3\n",
+    "    model_name,version = params['model_name'] , params['version']\n",
+    "    print (model_name,version,num_split)\n",
+    "    pickle_file=open(outputs_dir+outputs_format.format(model_name,version,params['train_features'],num_split),'rb')\n",
+    "    features=pickle.load(pickle_file)\n",
+    "    pickle_file.close()\n",
+    "    features.shape\n",
+    "\n",
+    "    features=features.reshape(features.shape[0]//4,4,-1)\n",
+    "    features.shape\n",
+    "    split_train = train_df[train_df.PID.isin(set(split_sid[splits[num_split][0]]))].SeriesI.unique()\n",
+    "    split_validate =  train_df[train_df.PID.isin(set(split_sid[splits[num_split][1]]))].SeriesI.unique()\n",
+    "\n",
+    "    np.random.seed(SEED+num_split)\n",
+    "    torch.manual_seed(SEED+num_split)\n",
+    "    torch.cuda.manual_seed(SEED+num_split)\n",
+    "    torch.backends.cudnn.deterministic = True\n",
+    "    batch_size=16\n",
+    "    num_workers=18\n",
+    "    num_epochs=24\n",
+    "    klr=1\n",
+    "    weights = torch.tensor([1.,1.,1.,1.,1.,2.],device=device)\n",
+    "    train_dataset=FullHeadDataset(train_df,\n",
+    "                                  split_train,\n",
+    "                                  features,\n",
+    "                                  'SeriesI',\n",
+    "                                  'ImagePositionZ',\n",
+    "                                  hemorrhage_types,\n",
+    "                                  multi=multi)                \n",
+    "    validate_dataset=FullHeadDataset(train_df,\n",
+    "                                     split_validate,\n",
+    "                                     torch.cat([features[:,i,:] for i in range(4)],-1),\n",
+    "                                     'SeriesI',\n",
+    "                                     'ImagePositionZ',\n",
+    "                                     hemorrhage_types)                \n",
+    "\n",
+    "    model=ResModelPool(features.shape[-1])\n",
+    "    version=version+'_fullhead_resmodel_pool2_{}'.format(multi)\n",
+    "    _=model.to(device)\n",
+    "    #mixup=Mixup(device=device)\n",
+    "    loss_func=my_loss\n",
+    "    #fig,ax = plt.subplots(figsize=(10,7))\n",
+    "    #gr=loss_graph(fig,ax,num_epochs,len(train_dataset)//batch_size+1,limits=[0.02,0.06])\n",
+    "    num_train_optimization_steps = num_epochs*(len(train_dataset)//batch_size+int(len(train_dataset)%batch_size>0))\n",
+    "    sched=WarmupExpCosineWithWarmupRestartsSchedule( t_total=num_train_optimization_steps, cycles=2,tau=1)\n",
+    "    optimizer = BertAdam(model.parameters(),lr=klr*1e-3,schedule=sched)\n",
+    "    history,best_model= model_train(model,\n",
+    "                                    optimizer,\n",
+    "                                    train_dataset,\n",
+    "                                    batch_size,\n",
+    "                                    num_epochs,\n",
+    "                                    loss_func,\n",
+    "                                    weights=weights,\n",
+    "                                    do_apex=False,\n",
+    "                                    validate_dataset=validate_dataset,\n",
+    "                                    param_schedualer=None,\n",
+    "                                    weights_data=None,\n",
+    "                                    metric=Metric(torch.tensor([1.,1.,1.,1.,1.,2.])),\n",
+    "                                    return_model=True,\n",
+    "                                    best_average=3,\n",
+    "                                    num_workers=num_workers,\n",
+    "                                    sampler=None,\n",
+    "                                    graph=None)\n",
+    "    torch.save(best_model.state_dict(), models_dir+models_format.format(model_name,version,num_split))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## create submission file - for reference"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 22,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def align(arr,index1,index2):\n",
+    "    return arr[np.argsort(index2)[np.argsort(np.argsort(index1))]]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "pred_list = []\n",
+    "for num_split in range(params['n_splits']):\n",
+    "    model_name,version = params['model_name'] , params['version']\n",
+    "    pickle_file=open(outputs_dir+outputs_format.format(model_name,version,params['test_features'],num_split),'rb')\n",
+    "    features=pickle.load(pickle_file)\n",
+    "    pickle_file.close()\n",
+    "    features=features.reshape(features.shape[0]//8,8,-1)\n",
+    "\n",
+    "    model=ResModelPool(features.shape[-1])\n",
+    "    version=version+'_fullhead_resmodel_pool2_3'\n",
+    "\n",
+    "    model.load_state_dict(torch.load(models_dir+models_format.format(model_name,version,num_split),map_location=torch.device(device)))\n",
+    "    test_dataset=train_dataset=FullHeadDataset(test_df,\n",
+    "                                  test_df.SeriesI.unique(),\n",
+    "                                  features,\n",
+    "                                  'SeriesI',\n",
+    "                                  'ImagePositionZ',multi=4)\n",
+    "    for i in tqdm_notebook(range(8),leave=False):\n",
+    "        pred_list.append(torch.sigmoid(model_run(model,test_dataset,do_apex=False,batch_size=128))[...,None])\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 18,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "24"
+      ]
+     },
+     "execution_count": 18,
+     "metadata": {},
+     "output_type": "execute_result"
+    },
+    {
+     "data": {
+      "text/plain": [
+       "torch.Size([2214, 60, 6, 1])"
+      ]
+     },
+     "execution_count": 18,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "len(pred_list)\n",
+    "pred_list[0].shape"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 19,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "pred=torch.cat(pred_list,-1).mean(-1)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 20,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "367fa4dcd20f4104b6be2b333db348ba",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "HBox(children=(IntProgress(value=0, max=2214), HTML(value='')))"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/plain": [
+       "(78545,)"
+      ]
+     },
+     "execution_count": 20,
+     "metadata": {},
+     "output_type": "execute_result"
+    },
+    {
+     "data": {
+      "text/plain": [
+       "(78545, 6)"
+      ]
+     },
+     "execution_count": 20,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "images_id_list=[]\n",
+    "dummeys=[]\n",
+    "image_arr=test_df.PatientID.values\n",
+    "ref_arr=test_df.SeriesI.values\n",
+    "order_arr=test_df.ImagePositionZ.values\n",
+    "for s in tqdm_notebook(test_df.SeriesI.unique()):\n",
+    "    dumm=np.zeros(60)\n",
+    "    head_idx = np.where(ref_arr==s)[0]\n",
+    "    sorted_head_idx=head_idx[np.argsort(order_arr[head_idx])]\n",
+    "    images_id_list.append(image_arr[sorted_head_idx])\n",
+    "    dumm[0:head_idx.shape[0]]=1\n",
+    "    dummeys.append(dumm)\n",
+    "image_ids=np.concatenate(images_id_list)\n",
+    "preds=pred.reshape(pred.shape[0]*pred.shape[1],6).numpy()[np.concatenate(dummeys)==1]\n",
+    "\n",
+    "image_ids.shape\n",
+    "\n",
+    "preds.shape"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 23,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>ID</th>\n",
+       "      <th>Label</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>ID_000012eaf_any</td>\n",
+       "      <td>0.000721</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>ID_000012eaf_epidural</td>\n",
+       "      <td>0.000080</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>ID_000012eaf_intraparenchymal</td>\n",
+       "      <td>0.000067</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>ID_000012eaf_intraventricular</td>\n",
+       "      <td>0.000017</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>ID_000012eaf_subarachnoid</td>\n",
+       "      <td>0.000069</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>5</th>\n",
+       "      <td>ID_000012eaf_subdural</td>\n",
+       "      <td>0.000702</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>6</th>\n",
+       "      <td>ID_0000ca2f6_any</td>\n",
+       "      <td>0.001115</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>7</th>\n",
+       "      <td>ID_0000ca2f6_epidural</td>\n",
+       "      <td>0.000038</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8</th>\n",
+       "      <td>ID_0000ca2f6_intraparenchymal</td>\n",
+       "      <td>0.000144</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>9</th>\n",
+       "      <td>ID_0000ca2f6_intraventricular</td>\n",
+       "      <td>0.000022</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>10</th>\n",
+       "      <td>ID_0000ca2f6_subarachnoid</td>\n",
+       "      <td>0.000203</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>11</th>\n",
+       "      <td>ID_0000ca2f6_subdural</td>\n",
+       "      <td>0.000839</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                               ID     Label\n",
+       "0                ID_000012eaf_any  0.000721\n",
+       "1           ID_000012eaf_epidural  0.000080\n",
+       "2   ID_000012eaf_intraparenchymal  0.000067\n",
+       "3   ID_000012eaf_intraventricular  0.000017\n",
+       "4       ID_000012eaf_subarachnoid  0.000069\n",
+       "5           ID_000012eaf_subdural  0.000702\n",
+       "6                ID_0000ca2f6_any  0.001115\n",
+       "7           ID_0000ca2f6_epidural  0.000038\n",
+       "8   ID_0000ca2f6_intraparenchymal  0.000144\n",
+       "9   ID_0000ca2f6_intraventricular  0.000022\n",
+       "10      ID_0000ca2f6_subarachnoid  0.000203\n",
+       "11          ID_0000ca2f6_subdural  0.000839"
+      ]
+     },
+     "execution_count": 23,
+     "metadata": {},
+     "output_type": "execute_result"
+    },
+    {
+     "data": {
+      "text/plain": [
+       "(471270, 2)"
+      ]
+     },
+     "execution_count": 23,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "submission_df=get_submission_ids(image_ids,torch.tensor(preds),do_sigmoid=False)\n",
+    "submission_df.head(12)\n",
+    "submission_df.shape\n",
+    "sub_num=999\n",
+    "submission_df.to_csv('/media/hd/notebooks/data/RSNA/submissions/submission{}.csv'.format(sub_num),\n",
+    "                                                                  index=False, columns=['ID','Label'])\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.6.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}