{

 "cells": [

  {

   "cell_type": "code",

   "execution_count": 22,

   "metadata": {},

   "outputs": [

    {

     "name": "stdout",

     "output_type": "stream",

     "text": [

      "The autoreload extension is already loaded. To reload it, use:\n",

      "  %reload_ext autoreload\n"

     ]

    }

   ],

   "source": [

    "from __future__ import absolute_import\n",

    "from __future__ import division\n",

    "from __future__ import print_function\n",

    "\n",

    "\n",

    "import numpy as np # linear algebra\n",

    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",

    "import os\n",

    "import datetime\n",

    "import seaborn as sns\n",

    "\n",

    "#import pydicom\n",

    "import time\n",

    "from functools import partial\n",

    "import gc\n",

    "import operator \n",

    "import matplotlib.pyplot as plt\n",

    "import torch\n",

    "import torch.nn as nn\n",

    "import torch.utils.data as D\n",

    "import torch.nn.functional as F\n",

    "from sklearn.model_selection import KFold\n",

    "from tqdm import tqdm, tqdm_notebook\n",

    "from IPython.core.interactiveshell import InteractiveShell\n",

    "InteractiveShell.ast_node_interactivity = \"all\"\n",

    "import warnings\n",

    "warnings.filterwarnings(action='once')\n",

    "import pickle\n",

    "%load_ext autoreload\n",

    "%autoreload 2\n",

    "%matplotlib inline\n",

    "from skimage.io import imread,imshow\n",

    "from helper import *\n",

    "import helper\n",

    "import torchvision.models as models\n",

    "from torch.optim import Adam\n",

    "from defenitions import *"

   ]

  },

  {

   "cell_type": "markdown",

   "metadata": {},

   "source": [

    "## Set parameters below"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": 2,

   "metadata": {},

   "outputs": [],

   "source": [

    "# here you should set which model parameters you want to choose (see definitions.py) and what GPU to use\n",

    "params=parameters['se_resnext101_32x4d_3'] # se_resnet101_5, se_resnext101_32x4d_3, se_resnext101_32x4d_5\n",

    "\n",

    "device=device_by_name(\"Tesla\") # RTX , cpu\n",

    "torch.cuda.set_device(device)\n",

    "sendmeemail=Email_Progress(my_gmail,my_pass,to_email,'{} results'.format(params['model_name']))"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": 3,

   "metadata": {},

   "outputs": [

    {

     "data": {

      "text/plain": [

       "{'model_name': 'se_resnext101_32x4d',\n",

       " 'SEED': 8153,\n",

       " 'n_splits': 3,\n",

       " 'Pre_version': None,\n",

       " 'focal': False,\n",

       " 'version': 'classifier_splits',\n",

       " 'train_prediction': 'predictions_train_tta',\n",

       " 'train_features': 'features_train_tta',\n",

       " 'test_prediction': 'predictions_test_tta',\n",

       " 'test_features': 'features_test_tta',\n",

       " 'num_epochs': 5,\n",

       " 'num_pool': 8}"

      ]

     },

     "execution_count": 3,

     "metadata": {},

     "output_type": "execute_result"

    }

   ],

   "source": [

    "params"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": 4,

   "metadata": {},

   "outputs": [],

   "source": [

    "SEED = params['SEED']\n",

    "n_splits=params['n_splits']"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": 5,

   "metadata": {},

   "outputs": [

    {

     "data": {

      "text/plain": [

       "(674252, 15)"

      ]

     },

     "execution_count": 5,

     "metadata": {},

     "output_type": "execute_result"

    },

    {

     "data": {

      "text/plain": [

       "(674252, 15)"

      ]

     },

     "execution_count": 5,

     "metadata": {},

     "output_type": "execute_result"

    },

    {

     "data": {

      "text/html": [

       "<div>\n",

       "<style scoped>\n",

       "    .dataframe tbody tr th:only-of-type {\n",

       "        vertical-align: middle;\n",

       "    }\n",

       "\n",

       "    .dataframe tbody tr th {\n",

       "        vertical-align: top;\n",

       "    }\n",

       "\n",

       "    .dataframe thead th {\n",

       "        text-align: right;\n",

       "    }\n",

       "</style>\n",

       "<table border=\"1\" class=\"dataframe\">\n",

       "  <thead>\n",

       "    <tr style=\"text-align: right;\">\n",

       "      <th></th>\n",

       "      <th>PatientID</th>\n",

       "      <th>epidural</th>\n",

       "      <th>intraparenchymal</th>\n",

       "      <th>intraventricular</th>\n",

       "      <th>subarachnoid</th>\n",

       "      <th>subdural</th>\n",

       "      <th>any</th>\n",

       "      <th>PID</th>\n",

       "      <th>StudyI</th>\n",

       "      <th>SeriesI</th>\n",

       "      <th>WindowCenter</th>\n",

       "      <th>WindowWidth</th>\n",

       "      <th>ImagePositionZ</th>\n",

       "      <th>ImagePositionX</th>\n",

       "      <th>ImagePositionY</th>\n",

       "    </tr>\n",

       "  </thead>\n",

       "  <tbody>\n",

       "    <tr>\n",

       "      <th>0</th>\n",

       "      <td>63eb1e259</td>\n",

       "      <td>0</td>\n",

       "      <td>0</td>\n",

       "      <td>0</td>\n",

       "      <td>0</td>\n",

       "      <td>0</td>\n",

       "      <td>0</td>\n",

       "      <td>a449357f</td>\n",

       "      <td>62d125e5b2</td>\n",

       "      <td>0be5c0d1b3</td>\n",

       "      <td>['00036', '00036']</td>\n",

       "      <td>['00080', '00080']</td>\n",

       "      <td>180.199951</td>\n",

       "      <td>-125.0</td>\n",

       "      <td>-8.000000</td>\n",

       "    </tr>\n",

       "    <tr>\n",

       "      <th>1</th>\n",

       "      <td>2669954a7</td>\n",

       "      <td>0</td>\n",

       "      <td>0</td>\n",

       "      <td>0</td>\n",

       "      <td>0</td>\n",

       "      <td>0</td>\n",

       "      <td>0</td>\n",

       "      <td>363d5865</td>\n",

       "      <td>a20b80c7bf</td>\n",

       "      <td>3564d584db</td>\n",

       "      <td>['00047', '00047']</td>\n",

       "      <td>['00080', '00080']</td>\n",

       "      <td>922.530821</td>\n",

       "      <td>-156.0</td>\n",

       "      <td>45.572849</td>\n",

       "    </tr>\n",

       "    <tr>\n",

       "      <th>2</th>\n",

       "      <td>52c9913b1</td>\n",

       "      <td>0</td>\n",

       "      <td>0</td>\n",

       "      <td>0</td>\n",

       "      <td>0</td>\n",

       "      <td>0</td>\n",

       "      <td>0</td>\n",

       "      <td>9c2b4bd7</td>\n",

       "      <td>3e3634f8cf</td>\n",

       "      <td>973274ffc9</td>\n",

       "      <td>40</td>\n",

       "      <td>150</td>\n",

       "      <td>4.455000</td>\n",

       "      <td>-125.0</td>\n",

       "      <td>-115.063000</td>\n",

       "    </tr>\n",

       "    <tr>\n",

       "      <th>3</th>\n",

       "      <td>4e6ff6126</td>\n",

       "      <td>0</td>\n",

       "      <td>0</td>\n",

       "      <td>0</td>\n",

       "      <td>0</td>\n",

       "      <td>0</td>\n",

       "      <td>0</td>\n",

       "      <td>3ae81c2d</td>\n",

       "      <td>a1390c15c2</td>\n",

       "      <td>e5ccad8244</td>\n",

       "      <td>['00036', '00036']</td>\n",

       "      <td>['00080', '00080']</td>\n",

       "      <td>100.000000</td>\n",

       "      <td>-99.5</td>\n",

       "      <td>28.500000</td>\n",

       "    </tr>\n",

       "    <tr>\n",

       "      <th>4</th>\n",

       "      <td>7858edd88</td>\n",

       "      <td>0</td>\n",

       "      <td>0</td>\n",

       "      <td>0</td>\n",

       "      <td>0</td>\n",

       "      <td>0</td>\n",

       "      <td>0</td>\n",

       "      <td>c1867feb</td>\n",

       "      <td>c73e81ed3a</td>\n",

       "      <td>28e0531b3a</td>\n",

       "      <td>40</td>\n",

       "      <td>100</td>\n",

       "      <td>145.793000</td>\n",

       "      <td>-125.0</td>\n",

       "      <td>-132.190000</td>\n",

       "    </tr>\n",

       "  </tbody>\n",

       "</table>\n",

       "</div>"

      ],

      "text/plain": [

       "   PatientID  epidural  intraparenchymal  intraventricular  subarachnoid  \\\n",

       "0  63eb1e259         0                 0                 0             0   \n",

       "1  2669954a7         0                 0                 0             0   \n",

       "2  52c9913b1         0                 0                 0             0   \n",

       "3  4e6ff6126         0                 0                 0             0   \n",

       "4  7858edd88         0                 0                 0             0   \n",

       "\n",

       "   subdural  any       PID      StudyI     SeriesI        WindowCenter  \\\n",

       "0         0    0  a449357f  62d125e5b2  0be5c0d1b3  ['00036', '00036']   \n",

       "1         0    0  363d5865  a20b80c7bf  3564d584db  ['00047', '00047']   \n",

       "2         0    0  9c2b4bd7  3e3634f8cf  973274ffc9                  40   \n",

       "3         0    0  3ae81c2d  a1390c15c2  e5ccad8244  ['00036', '00036']   \n",

       "4         0    0  c1867feb  c73e81ed3a  28e0531b3a                  40   \n",

       "\n",

       "          WindowWidth  ImagePositionZ  ImagePositionX  ImagePositionY  \n",

       "0  ['00080', '00080']      180.199951          -125.0       -8.000000  \n",

       "1  ['00080', '00080']      922.530821          -156.0       45.572849  \n",

       "2                 150        4.455000          -125.0     -115.063000  \n",

       "3  ['00080', '00080']      100.000000           -99.5       28.500000  \n",

       "4                 100      145.793000          -125.0     -132.190000  "

      ]

     },

     "execution_count": 5,

     "metadata": {},

     "output_type": "execute_result"

    }

   ],

   "source": [

    "train_df = pd.read_csv(data_dir+'train.csv')\n",

    "train_df.shape\n",

    "train_df=train_df[~train_df.PatientID.isin(bad_images)].reset_index(drop=True)\n",

    "train_df=train_df.drop_duplicates().reset_index(drop=True)\n",

    "train_df.shape\n",

    "train_df.head()"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": 6,

   "metadata": {},

   "outputs": [

    {

     "data": {

      "text/html": [

       "<div>\n",

       "<style scoped>\n",

       "    .dataframe tbody tr th:only-of-type {\n",

       "        vertical-align: middle;\n",

       "    }\n",

       "\n",

       "    .dataframe tbody tr th {\n",

       "        vertical-align: top;\n",

       "    }\n",

       "\n",

       "    .dataframe thead th {\n",

       "        text-align: right;\n",

       "    }\n",

       "</style>\n",

       "<table border=\"1\" class=\"dataframe\">\n",

       "  <thead>\n",

       "    <tr style=\"text-align: right;\">\n",

       "      <th></th>\n",

       "      <th>PatientID</th>\n",

       "      <th>epidural</th>\n",

       "      <th>intraparenchymal</th>\n",

       "      <th>intraventricular</th>\n",

       "      <th>subarachnoid</th>\n",

       "      <th>subdural</th>\n",

       "      <th>any</th>\n",

       "      <th>SeriesI</th>\n",

       "      <th>PID</th>\n",

       "      <th>StudyI</th>\n",

       "      <th>WindowCenter</th>\n",

       "      <th>WindowWidth</th>\n",

       "      <th>ImagePositionZ</th>\n",

       "      <th>ImagePositionX</th>\n",

       "      <th>ImagePositionY</th>\n",

       "    </tr>\n",

       "  </thead>\n",

       "  <tbody>\n",

       "    <tr>\n",

       "      <th>0</th>\n",

       "      <td>28fbab7eb</td>\n",

       "      <td>0.5</td>\n",

       "      <td>0.5</td>\n",

       "      <td>0.5</td>\n",

       "      <td>0.5</td>\n",

       "      <td>0.5</td>\n",

       "      <td>0.5</td>\n",

       "      <td>ebfd7e4506</td>\n",

       "      <td>cf1b6b11</td>\n",

       "      <td>93407cadbb</td>\n",

       "      <td>30</td>\n",

       "      <td>80</td>\n",

       "      <td>158.458000</td>\n",

       "      <td>-125.0</td>\n",

       "      <td>-135.598000</td>\n",

       "    </tr>\n",

       "    <tr>\n",

       "      <th>1</th>\n",

       "      <td>877923b8b</td>\n",

       "      <td>0.5</td>\n",

       "      <td>0.5</td>\n",

       "      <td>0.5</td>\n",

       "      <td>0.5</td>\n",

       "      <td>0.5</td>\n",

       "      <td>0.5</td>\n",

       "      <td>6d95084e15</td>\n",

       "      <td>ad8ea58f</td>\n",

       "      <td>a337baa067</td>\n",

       "      <td>30</td>\n",

       "      <td>80</td>\n",

       "      <td>138.729050</td>\n",

       "      <td>-125.0</td>\n",

       "      <td>-101.797981</td>\n",

       "    </tr>\n",

       "    <tr>\n",

       "      <th>2</th>\n",

       "      <td>a591477cb</td>\n",

       "      <td>0.5</td>\n",

       "      <td>0.5</td>\n",

       "      <td>0.5</td>\n",

       "      <td>0.5</td>\n",

       "      <td>0.5</td>\n",

       "      <td>0.5</td>\n",

       "      <td>8e06b2c9e0</td>\n",

       "      <td>ecfb278b</td>\n",

       "      <td>0cfe838d54</td>\n",

       "      <td>30</td>\n",

       "      <td>80</td>\n",

       "      <td>60.830002</td>\n",

       "      <td>-125.0</td>\n",

       "      <td>-133.300003</td>\n",

       "    </tr>\n",

       "    <tr>\n",

       "      <th>3</th>\n",

       "      <td>42217c898</td>\n",

       "      <td>0.5</td>\n",

       "      <td>0.5</td>\n",

       "      <td>0.5</td>\n",

       "      <td>0.5</td>\n",

       "      <td>0.5</td>\n",

       "      <td>0.5</td>\n",

       "      <td>e800f419cf</td>\n",

       "      <td>e96e31f4</td>\n",

       "      <td>c497ac5bad</td>\n",

       "      <td>30</td>\n",

       "      <td>80</td>\n",

       "      <td>55.388000</td>\n",

       "      <td>-125.0</td>\n",

       "      <td>-146.081000</td>\n",

       "    </tr>\n",

       "    <tr>\n",

       "      <th>4</th>\n",

       "      <td>a130c4d2f</td>\n",

       "      <td>0.5</td>\n",

       "      <td>0.5</td>\n",

       "      <td>0.5</td>\n",

       "      <td>0.5</td>\n",

       "      <td>0.5</td>\n",

       "      <td>0.5</td>\n",

       "      <td>faeb7454f3</td>\n",

       "      <td>69affa42</td>\n",

       "      <td>854e4fbc01</td>\n",

       "      <td>30</td>\n",

       "      <td>80</td>\n",

       "      <td>33.516888</td>\n",

       "      <td>-125.0</td>\n",

       "      <td>-118.689819</td>\n",

       "    </tr>\n",

       "  </tbody>\n",

       "</table>\n",

       "</div>"

      ],

      "text/plain": [

       "   PatientID  epidural  intraparenchymal  intraventricular  subarachnoid  \\\n",

       "0  28fbab7eb       0.5               0.5               0.5           0.5   \n",

       "1  877923b8b       0.5               0.5               0.5           0.5   \n",

       "2  a591477cb       0.5               0.5               0.5           0.5   \n",

       "3  42217c898       0.5               0.5               0.5           0.5   \n",

       "4  a130c4d2f       0.5               0.5               0.5           0.5   \n",

       "\n",

       "   subdural  any     SeriesI       PID      StudyI WindowCenter WindowWidth  \\\n",

       "0       0.5  0.5  ebfd7e4506  cf1b6b11  93407cadbb           30          80   \n",

       "1       0.5  0.5  6d95084e15  ad8ea58f  a337baa067           30          80   \n",

       "2       0.5  0.5  8e06b2c9e0  ecfb278b  0cfe838d54           30          80   \n",

       "3       0.5  0.5  e800f419cf  e96e31f4  c497ac5bad           30          80   \n",

       "4       0.5  0.5  faeb7454f3  69affa42  854e4fbc01           30          80   \n",

       "\n",

       "   ImagePositionZ  ImagePositionX  ImagePositionY  \n",

       "0      158.458000          -125.0     -135.598000  \n",

       "1      138.729050          -125.0     -101.797981  \n",

       "2       60.830002          -125.0     -133.300003  \n",

       "3       55.388000          -125.0     -146.081000  \n",

       "4       33.516888          -125.0     -118.689819  "

      ]

     },

     "execution_count": 6,

     "metadata": {},

     "output_type": "execute_result"

    }

   ],

   "source": [

    "test_df = pd.read_csv(data_dir+'test.csv')\n",

    "test_df.head()"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": 7,

   "metadata": {},

   "outputs": [],

   "source": [

    "split_sid = train_df.PID.unique()\n",

    "splits=list(KFold(n_splits=n_splits,shuffle=True, random_state=SEED).split(split_sid))\n"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": 8,

   "metadata": {},

   "outputs": [],

   "source": [

    "def my_loss(y_pred,y_true,weights):\n",

    "    window=(y_true>=0).to(torch.float)\n",

    "    loss = (F.binary_cross_entropy_with_logits(y_pred,y_true,reduction='none')*window*weights.expand_as(y_true)).mean()/(window.mean()+1e-7)\n",

    "    return loss"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": 9,

   "metadata": {},

   "outputs": [],

   "source": [

    "class Metric():\n",

    "    def __init__(self,weights,k=0.03):\n",

    "        self.weights=weights\n",

    "        self.k=k\n",

    "        self.zero()\n",

    "        \n",

    "    def zero(self):\n",

    "        self.loss_sum=0.\n",

    "        self.loss_count=0.\n",

    "        self.lossf=0.\n",

    "        \n",

    "    def calc(self,y_pred,y_true,prefix=\"\"):\n",

    "        window=(y_true>=0).to(torch.float)\n",

    "        loss = (F.binary_cross_entropy_with_logits(y_pred,y_true,reduction='none')*window*self.weights.expand_as(y_true)).mean()/(window.mean()+1e-5)\n",

    "        self.lossf=self.lossf*(1-self.k)+loss*self.k\n",

    "        self.loss_sum=self.loss_sum+loss*window.sum()\n",

    "        self.loss_count=self.loss_count+window.sum()\n",

    "        return({prefix+'mloss':self.lossf})    \n",

    "        \n",

    "    def calc_sums(self,prefix=\"\"):\n",

    "        return({prefix+'mloss_tot':self.loss_sum/self.loss_count})    \n",

    "\n"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": 10,

   "metadata": {},

   "outputs": [],

   "source": [

    "#features=(features-features.mean())/features.std()"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": null,

   "metadata": {

    "scrolled": true

   },

   "outputs": [],

   "source": [

    "%matplotlib nbagg\n",

    "for num_split in range(params['n_splits']):\n",

    "    multi=3\n",

    "    model_name,version = params['model_name'] , params['version']\n",

    "    print (model_name,version,num_split)\n",

    "    pickle_file=open(outputs_dir+outputs_format.format(model_name,version,params['train_features'],num_split),'rb')\n",

    "    features=pickle.load(pickle_file)\n",

    "    pickle_file.close()\n",

    "    features.shape\n",

    "\n",

    "    features=features.reshape(features.shape[0]//4,4,-1)\n",

    "    features.shape\n",

    "    split_train = train_df[train_df.PID.isin(set(split_sid[splits[num_split][0]]))].SeriesI.unique()\n",

    "    split_validate =  train_df[train_df.PID.isin(set(split_sid[splits[num_split][1]]))].SeriesI.unique()\n",

    "\n",

    "    np.random.seed(SEED+num_split)\n",

    "    torch.manual_seed(SEED+num_split)\n",

    "    torch.cuda.manual_seed(SEED+num_split)\n",

    "    torch.backends.cudnn.deterministic = True\n",

    "    batch_size=16\n",

    "    num_workers=18\n",

    "    num_epochs=24\n",

    "    klr=1\n",

    "    weights = torch.tensor([1.,1.,1.,1.,1.,2.],device=device)\n",

    "    train_dataset=FullHeadDataset(train_df,\n",

    "                                  split_train,\n",

    "                                  features,\n",

    "                                  'SeriesI',\n",

    "                                  'ImagePositionZ',\n",

    "                                  hemorrhage_types,\n",

    "                                  multi=multi)                \n",

    "    validate_dataset=FullHeadDataset(train_df,\n",

    "                                     split_validate,\n",

    "                                     torch.cat([features[:,i,:] for i in range(4)],-1),\n",

    "                                     'SeriesI',\n",

    "                                     'ImagePositionZ',\n",

    "                                     hemorrhage_types)                \n",

    "\n",

    "    model=ResModelPool(features.shape[-1])\n",

    "    version=version+'_fullhead_resmodel_pool2_{}'.format(multi)\n",

    "    _=model.to(device)\n",

    "    #mixup=Mixup(device=device)\n",

    "    loss_func=my_loss\n",

    "    #fig,ax = plt.subplots(figsize=(10,7))\n",

    "    #gr=loss_graph(fig,ax,num_epochs,len(train_dataset)//batch_size+1,limits=[0.02,0.06])\n",

    "    num_train_optimization_steps = num_epochs*(len(train_dataset)//batch_size+int(len(train_dataset)%batch_size>0))\n",

    "    sched=WarmupExpCosineWithWarmupRestartsSchedule( t_total=num_train_optimization_steps, cycles=2,tau=1)\n",

    "    optimizer = BertAdam(model.parameters(),lr=klr*1e-3,schedule=sched)\n",

    "    history,best_model= model_train(model,\n",

    "                                    optimizer,\n",

    "                                    train_dataset,\n",

    "                                    batch_size,\n",

    "                                    num_epochs,\n",

    "                                    loss_func,\n",

    "                                    weights=weights,\n",

    "                                    do_apex=False,\n",

    "                                    validate_dataset=validate_dataset,\n",

    "                                    param_schedualer=None,\n",

    "                                    weights_data=None,\n",

    "                                    metric=Metric(torch.tensor([1.,1.,1.,1.,1.,2.])),\n",

    "                                    return_model=True,\n",

    "                                    best_average=3,\n",

    "                                    num_workers=num_workers,\n",

    "                                    sampler=None,\n",

    "                                    graph=None)\n",

    "    torch.save(best_model.state_dict(), models_dir+models_format.format(model_name,version,num_split))"

   ]

  },

  {

   "cell_type": "markdown",

   "metadata": {},

   "source": [

    "## create submission file - for reference"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": 22,

   "metadata": {},

   "outputs": [],

   "source": [

    "def align(arr,index1,index2):\n",

    "    return arr[np.argsort(index2)[np.argsort(np.argsort(index1))]]"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": null,

   "metadata": {},

   "outputs": [],

   "source": [

    "pred_list = []\n",

    "for num_split in range(params['n_splits']):\n",

    "    model_name,version = params['model_name'] , params['version']\n",

    "    pickle_file=open(outputs_dir+outputs_format.format(model_name,version,params['test_features'],num_split),'rb')\n",

    "    features=pickle.load(pickle_file)\n",

    "    pickle_file.close()\n",

    "    features=features.reshape(features.shape[0]//8,8,-1)\n",

    "\n",

    "    model=ResModelPool(features.shape[-1])\n",

    "    version=version+'_fullhead_resmodel_pool2_3'\n",

    "\n",

    "    model.load_state_dict(torch.load(models_dir+models_format.format(model_name,version,num_split),map_location=torch.device(device)))\n",

    "    test_dataset=train_dataset=FullHeadDataset(test_df,\n",

    "                                  test_df.SeriesI.unique(),\n",

    "                                  features,\n",

    "                                  'SeriesI',\n",

    "                                  'ImagePositionZ',multi=4)\n",

    "    for i in tqdm_notebook(range(8),leave=False):\n",

    "        pred_list.append(torch.sigmoid(model_run(model,test_dataset,do_apex=False,batch_size=128))[...,None])\n"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": 18,

   "metadata": {},

   "outputs": [

    {

     "data": {

      "text/plain": [

       "24"

      ]

     },

     "execution_count": 18,

     "metadata": {},

     "output_type": "execute_result"

    },

    {

     "data": {

      "text/plain": [

       "torch.Size([2214, 60, 6, 1])"

      ]

     },

     "execution_count": 18,

     "metadata": {},

     "output_type": "execute_result"

    }

   ],

   "source": [

    "len(pred_list)\n",

    "pred_list[0].shape"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": 19,

   "metadata": {},

   "outputs": [],

   "source": [

    "pred=torch.cat(pred_list,-1).mean(-1)"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": 20,

   "metadata": {},

   "outputs": [

    {

     "data": {

      "application/vnd.jupyter.widget-view+json": {

       "model_id": "367fa4dcd20f4104b6be2b333db348ba",

       "version_major": 2,

       "version_minor": 0

      },

      "text/plain": [

       "HBox(children=(IntProgress(value=0, max=2214), HTML(value='')))"

      ]

     },

     "metadata": {},

     "output_type": "display_data"

    },

    {

     "data": {

      "text/plain": [

       "(78545,)"

      ]

     },

     "execution_count": 20,

     "metadata": {},

     "output_type": "execute_result"

    },

    {

     "data": {

      "text/plain": [

       "(78545, 6)"

      ]

     },

     "execution_count": 20,

     "metadata": {},

     "output_type": "execute_result"

    }

   ],

   "source": [

    "images_id_list=[]\n",

    "dummeys=[]\n",

    "image_arr=test_df.PatientID.values\n",

    "ref_arr=test_df.SeriesI.values\n",

    "order_arr=test_df.ImagePositionZ.values\n",

    "for s in tqdm_notebook(test_df.SeriesI.unique()):\n",

    "    dumm=np.zeros(60)\n",

    "    head_idx = np.where(ref_arr==s)[0]\n",

    "    sorted_head_idx=head_idx[np.argsort(order_arr[head_idx])]\n",

    "    images_id_list.append(image_arr[sorted_head_idx])\n",

    "    dumm[0:head_idx.shape[0]]=1\n",

    "    dummeys.append(dumm)\n",

    "image_ids=np.concatenate(images_id_list)\n",

    "preds=pred.reshape(pred.shape[0]*pred.shape[1],6).numpy()[np.concatenate(dummeys)==1]\n",

    "\n",

    "image_ids.shape\n",

    "\n",

    "preds.shape"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": 23,

   "metadata": {},

   "outputs": [

    {

     "data": {

      "text/html": [

       "<div>\n",

       "<style scoped>\n",

       "    .dataframe tbody tr th:only-of-type {\n",

       "        vertical-align: middle;\n",

       "    }\n",

       "\n",

       "    .dataframe tbody tr th {\n",

       "        vertical-align: top;\n",

       "    }\n",

       "\n",

       "    .dataframe thead th {\n",

       "        text-align: right;\n",

       "    }\n",

       "</style>\n",

       "<table border=\"1\" class=\"dataframe\">\n",

       "  <thead>\n",

       "    <tr style=\"text-align: right;\">\n",

       "      <th></th>\n",

       "      <th>ID</th>\n",

       "      <th>Label</th>\n",

       "    </tr>\n",

       "  </thead>\n",

       "  <tbody>\n",

       "    <tr>\n",

       "      <th>0</th>\n",

       "      <td>ID_000012eaf_any</td>\n",

       "      <td>0.000721</td>\n",

       "    </tr>\n",

       "    <tr>\n",

       "      <th>1</th>\n",

       "      <td>ID_000012eaf_epidural</td>\n",

       "      <td>0.000080</td>\n",

       "    </tr>\n",

       "    <tr>\n",

       "      <th>2</th>\n",

       "      <td>ID_000012eaf_intraparenchymal</td>\n",

       "      <td>0.000067</td>\n",

       "    </tr>\n",

       "    <tr>\n",

       "      <th>3</th>\n",

       "      <td>ID_000012eaf_intraventricular</td>\n",

       "      <td>0.000017</td>\n",

       "    </tr>\n",

       "    <tr>\n",

       "      <th>4</th>\n",

       "      <td>ID_000012eaf_subarachnoid</td>\n",

       "      <td>0.000069</td>\n",

       "    </tr>\n",

       "    <tr>\n",

       "      <th>5</th>\n",

       "      <td>ID_000012eaf_subdural</td>\n",

       "      <td>0.000702</td>\n",

       "    </tr>\n",

       "    <tr>\n",

       "      <th>6</th>\n",

       "      <td>ID_0000ca2f6_any</td>\n",

       "      <td>0.001115</td>\n",

       "    </tr>\n",

       "    <tr>\n",

       "      <th>7</th>\n",

       "      <td>ID_0000ca2f6_epidural</td>\n",

       "      <td>0.000038</td>\n",

       "    </tr>\n",

       "    <tr>\n",

       "      <th>8</th>\n",

       "      <td>ID_0000ca2f6_intraparenchymal</td>\n",

       "      <td>0.000144</td>\n",

       "    </tr>\n",

       "    <tr>\n",

       "      <th>9</th>\n",

       "      <td>ID_0000ca2f6_intraventricular</td>\n",

       "      <td>0.000022</td>\n",

       "    </tr>\n",

       "    <tr>\n",

       "      <th>10</th>\n",

       "      <td>ID_0000ca2f6_subarachnoid</td>\n",

       "      <td>0.000203</td>\n",

       "    </tr>\n",

       "    <tr>\n",

       "      <th>11</th>\n",

       "      <td>ID_0000ca2f6_subdural</td>\n",

       "      <td>0.000839</td>\n",

       "    </tr>\n",

       "  </tbody>\n",

       "</table>\n",

       "</div>"

      ],

      "text/plain": [

       "                               ID     Label\n",

       "0                ID_000012eaf_any  0.000721\n",

       "1           ID_000012eaf_epidural  0.000080\n",

       "2   ID_000012eaf_intraparenchymal  0.000067\n",

       "3   ID_000012eaf_intraventricular  0.000017\n",

       "4       ID_000012eaf_subarachnoid  0.000069\n",

       "5           ID_000012eaf_subdural  0.000702\n",

       "6                ID_0000ca2f6_any  0.001115\n",

       "7           ID_0000ca2f6_epidural  0.000038\n",

       "8   ID_0000ca2f6_intraparenchymal  0.000144\n",

       "9   ID_0000ca2f6_intraventricular  0.000022\n",

       "10      ID_0000ca2f6_subarachnoid  0.000203\n",

       "11          ID_0000ca2f6_subdural  0.000839"

      ]

     },

     "execution_count": 23,

     "metadata": {},

     "output_type": "execute_result"

    },

    {

     "data": {

      "text/plain": [

       "(471270, 2)"

      ]

     },

     "execution_count": 23,

     "metadata": {},

     "output_type": "execute_result"

    }

   ],

   "source": [

    "submission_df=get_submission_ids(image_ids,torch.tensor(preds),do_sigmoid=False)\n",

    "submission_df.head(12)\n",

    "submission_df.shape\n",

    "sub_num=999\n",

    "submission_df.to_csv('/media/hd/notebooks/data/RSNA/submissions/submission{}.csv'.format(sub_num),\n",

    "                                                                  index=False, columns=['ID','Label'])\n"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": null,

   "metadata": {},

   "outputs": [],

   "source": []

  }

 ],

 "metadata": {

  "kernelspec": {

   "display_name": "Python 3",

   "language": "python",

   "name": "python3"

  },

  "language_info": {

   "codemirror_mode": {

    "name": "ipython",

    "version": 3

   },

   "file_extension": ".py",

   "mimetype": "text/x-python",

   "name": "python",

   "nbconvert_exporter": "python",

   "pygments_lexer": "ipython3",

   "version": "3.6.6"

  }

 },

 "nbformat": 4,

 "nbformat_minor": 2

}