Datasets
Models
Downloads: 1
Diff of
/Serialized/prepare_ensembling.ipynb
[000000]
..
[53d15f]
Switch to side-by-side view
--- a +++ b/Serialized/prepare_ensembling.ipynb @@ -0,0 +1,733 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/reina/anaconda3/envs/RSNA/lib/python3.6/importlib/_bootstrap.py:219: ImportWarning: can't resolve package from __spec__ or __package__, falling back on __name__ and __path__\n", + " return f(*args, **kwds)\n", + "/home/reina/anaconda3/envs/RSNA/lib/python3.6/importlib/_bootstrap.py:219: ImportWarning: can't resolve package from __spec__ or __package__, falling back on __name__ and __path__\n", + " return f(*args, **kwds)\n" + ] + } + ], + "source": [ + "from __future__ import absolute_import\n", + "from __future__ import division\n", + "from __future__ import print_function\n", + "\n", + "\n", + "import numpy as np # linear algebra\n", + "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n", + "import os\n", + "import datetime\n", + "import seaborn as sns\n", + "\n", + "#import pydicom\n", + "import time\n", + "from functools import partial\n", + "import gc\n", + "import operator \n", + "import matplotlib.pyplot as plt\n", + "import torch\n", + "import torch.nn as nn\n", + "import torch.utils.data as D\n", + "import torch.nn.functional as F\n", + "from sklearn.model_selection import KFold\n", + "from tqdm import tqdm, tqdm_notebook\n", + "from IPython.core.interactiveshell import InteractiveShell\n", + "InteractiveShell.ast_node_interactivity = \"all\"\n", + "import warnings\n", + "warnings.filterwarnings(action='once')\n", + "import pickle\n", + "%load_ext autoreload\n", + "%autoreload 2\n", + "%matplotlib inline\n", + "from skimage.io import imread,imshow\n", + "from helper import *\n", + "import helper\n", + "import torchvision.models as models\n", + "from torch.optim import Adam\n", + "from defenitions import *" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "# here you should set which model parameters you want to choose (see definitions.py) and what GPU to use\n", + "\n", + "device=device_by_name(\"Tesla\") # RTX , cpu\n", + "torch.cuda.set_device(device)" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(674252, 15)" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "text/plain": [ + "(674252, 15)" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "text/html": [ + "<div>\n", + "<style scoped>\n", + " .dataframe tbody tr th:only-of-type {\n", + " vertical-align: middle;\n", + " }\n", + "\n", + " .dataframe tbody tr th {\n", + " vertical-align: top;\n", + " }\n", + "\n", + " .dataframe thead th {\n", + " text-align: right;\n", + " }\n", + "</style>\n", + "<table border=\"1\" class=\"dataframe\">\n", + " <thead>\n", + " <tr style=\"text-align: right;\">\n", + " <th></th>\n", + " <th>PatientID</th>\n", + " <th>epidural</th>\n", + " <th>intraparenchymal</th>\n", + " <th>intraventricular</th>\n", + " <th>subarachnoid</th>\n", + " <th>subdural</th>\n", + " <th>any</th>\n", + " <th>PID</th>\n", + " <th>StudyI</th>\n", + " <th>SeriesI</th>\n", + " <th>WindowCenter</th>\n", + " <th>WindowWidth</th>\n", + " <th>ImagePositionZ</th>\n", + " <th>ImagePositionX</th>\n", + " <th>ImagePositionY</th>\n", + " </tr>\n", + " </thead>\n", + " <tbody>\n", + " <tr>\n", + " <th>0</th>\n", + " <td>63eb1e259</td>\n", + " <td>0</td>\n", + " <td>0</td>\n", + " <td>0</td>\n", + " <td>0</td>\n", + " <td>0</td>\n", + " <td>0</td>\n", + " <td>a449357f</td>\n", + " <td>62d125e5b2</td>\n", + " <td>0be5c0d1b3</td>\n", + " <td>['00036', '00036']</td>\n", + " <td>['00080', '00080']</td>\n", + " <td>180.199951</td>\n", + " <td>-125.0</td>\n", + " <td>-8.000000</td>\n", + " </tr>\n", + " <tr>\n", + " <th>1</th>\n", + " <td>2669954a7</td>\n", + " <td>0</td>\n", + " <td>0</td>\n", + " <td>0</td>\n", + " <td>0</td>\n", + " <td>0</td>\n", + " <td>0</td>\n", + " <td>363d5865</td>\n", + " <td>a20b80c7bf</td>\n", + " <td>3564d584db</td>\n", + " <td>['00047', '00047']</td>\n", + " <td>['00080', '00080']</td>\n", + " <td>922.530821</td>\n", + " <td>-156.0</td>\n", + " <td>45.572849</td>\n", + " </tr>\n", + " <tr>\n", + " <th>2</th>\n", + " <td>52c9913b1</td>\n", + " <td>0</td>\n", + " <td>0</td>\n", + " <td>0</td>\n", + " <td>0</td>\n", + " <td>0</td>\n", + " <td>0</td>\n", + " <td>9c2b4bd7</td>\n", + " <td>3e3634f8cf</td>\n", + " <td>973274ffc9</td>\n", + " <td>40</td>\n", + " <td>150</td>\n", + " <td>4.455000</td>\n", + " <td>-125.0</td>\n", + " <td>-115.063000</td>\n", + " </tr>\n", + " <tr>\n", + " <th>3</th>\n", + " <td>4e6ff6126</td>\n", + " <td>0</td>\n", + " <td>0</td>\n", + " <td>0</td>\n", + " <td>0</td>\n", + " <td>0</td>\n", + " <td>0</td>\n", + " <td>3ae81c2d</td>\n", + " <td>a1390c15c2</td>\n", + " <td>e5ccad8244</td>\n", + " <td>['00036', '00036']</td>\n", + " <td>['00080', '00080']</td>\n", + " <td>100.000000</td>\n", + " <td>-99.5</td>\n", + " <td>28.500000</td>\n", + " </tr>\n", + " <tr>\n", + " <th>4</th>\n", + " <td>7858edd88</td>\n", + " <td>0</td>\n", + " <td>0</td>\n", + " <td>0</td>\n", + " <td>0</td>\n", + " <td>0</td>\n", + " <td>0</td>\n", + " <td>c1867feb</td>\n", + " <td>c73e81ed3a</td>\n", + " <td>28e0531b3a</td>\n", + " <td>40</td>\n", + " <td>100</td>\n", + " <td>145.793000</td>\n", + " <td>-125.0</td>\n", + " <td>-132.190000</td>\n", + " </tr>\n", + " </tbody>\n", + "</table>\n", + "</div>" + ], + "text/plain": [ + " PatientID epidural intraparenchymal intraventricular subarachnoid \\\n", + "0 63eb1e259 0 0 0 0 \n", + "1 2669954a7 0 0 0 0 \n", + "2 52c9913b1 0 0 0 0 \n", + "3 4e6ff6126 0 0 0 0 \n", + "4 7858edd88 0 0 0 0 \n", + "\n", + " subdural any PID StudyI SeriesI WindowCenter \\\n", + "0 0 0 a449357f 62d125e5b2 0be5c0d1b3 ['00036', '00036'] \n", + "1 0 0 363d5865 a20b80c7bf 3564d584db ['00047', '00047'] \n", + "2 0 0 9c2b4bd7 3e3634f8cf 973274ffc9 40 \n", + "3 0 0 3ae81c2d a1390c15c2 e5ccad8244 ['00036', '00036'] \n", + "4 0 0 c1867feb c73e81ed3a 28e0531b3a 40 \n", + "\n", + " WindowWidth ImagePositionZ ImagePositionX ImagePositionY \n", + "0 ['00080', '00080'] 180.199951 -125.0 -8.000000 \n", + "1 ['00080', '00080'] 922.530821 -156.0 45.572849 \n", + "2 150 4.455000 -125.0 -115.063000 \n", + "3 ['00080', '00080'] 100.000000 -99.5 28.500000 \n", + "4 100 145.793000 -125.0 -132.190000 " + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "train_df = pd.read_csv(data_dir+'train.csv')\n", + "train_df.shape\n", + "train_df=train_df[~train_df.PatientID.isin(bad_images)].reset_index(drop=True)\n", + "train_df=train_df.drop_duplicates().reset_index(drop=True)\n", + "train_df.shape\n", + "train_df.head()" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "<div>\n", + "<style scoped>\n", + " .dataframe tbody tr th:only-of-type {\n", + " vertical-align: middle;\n", + " }\n", + "\n", + " .dataframe tbody tr th {\n", + " vertical-align: top;\n", + " }\n", + "\n", + " .dataframe thead th {\n", + " text-align: right;\n", + " }\n", + "</style>\n", + "<table border=\"1\" class=\"dataframe\">\n", + " <thead>\n", + " <tr style=\"text-align: right;\">\n", + " <th></th>\n", + " <th>PatientID</th>\n", + " <th>epidural</th>\n", + " <th>intraparenchymal</th>\n", + " <th>intraventricular</th>\n", + " <th>subarachnoid</th>\n", + " <th>subdural</th>\n", + " <th>any</th>\n", + " <th>SeriesI</th>\n", + " <th>PID</th>\n", + " <th>StudyI</th>\n", + " <th>WindowCenter</th>\n", + " <th>WindowWidth</th>\n", + " <th>ImagePositionZ</th>\n", + " <th>ImagePositionX</th>\n", + " <th>ImagePositionY</th>\n", + " </tr>\n", + " </thead>\n", + " <tbody>\n", + " <tr>\n", + " <th>0</th>\n", + " <td>28fbab7eb</td>\n", + " <td>0.5</td>\n", + " <td>0.5</td>\n", + " <td>0.5</td>\n", + " <td>0.5</td>\n", + " <td>0.5</td>\n", + " <td>0.5</td>\n", + " <td>ebfd7e4506</td>\n", + " <td>cf1b6b11</td>\n", + " <td>93407cadbb</td>\n", + " <td>30</td>\n", + " <td>80</td>\n", + " <td>158.458000</td>\n", + " <td>-125.0</td>\n", + " <td>-135.598000</td>\n", + " </tr>\n", + " <tr>\n", + " <th>1</th>\n", + " <td>877923b8b</td>\n", + " <td>0.5</td>\n", + " <td>0.5</td>\n", + " <td>0.5</td>\n", + " <td>0.5</td>\n", + " <td>0.5</td>\n", + " <td>0.5</td>\n", + " <td>6d95084e15</td>\n", + " <td>ad8ea58f</td>\n", + " <td>a337baa067</td>\n", + " <td>30</td>\n", + " <td>80</td>\n", + " <td>138.729050</td>\n", + " <td>-125.0</td>\n", + " <td>-101.797981</td>\n", + " </tr>\n", + " <tr>\n", + " <th>2</th>\n", + " <td>a591477cb</td>\n", + " <td>0.5</td>\n", + " <td>0.5</td>\n", + " <td>0.5</td>\n", + " <td>0.5</td>\n", + " <td>0.5</td>\n", + " <td>0.5</td>\n", + " <td>8e06b2c9e0</td>\n", + " <td>ecfb278b</td>\n", + " <td>0cfe838d54</td>\n", + " <td>30</td>\n", + " <td>80</td>\n", + " <td>60.830002</td>\n", + " <td>-125.0</td>\n", + " <td>-133.300003</td>\n", + " </tr>\n", + " <tr>\n", + " <th>3</th>\n", + " <td>42217c898</td>\n", + " <td>0.5</td>\n", + " <td>0.5</td>\n", + " <td>0.5</td>\n", + " <td>0.5</td>\n", + " <td>0.5</td>\n", + " <td>0.5</td>\n", + " <td>e800f419cf</td>\n", + " <td>e96e31f4</td>\n", + " <td>c497ac5bad</td>\n", + " <td>30</td>\n", + " <td>80</td>\n", + " <td>55.388000</td>\n", + " <td>-125.0</td>\n", + " <td>-146.081000</td>\n", + " </tr>\n", + " <tr>\n", + " <th>4</th>\n", + " <td>a130c4d2f</td>\n", + " <td>0.5</td>\n", + " <td>0.5</td>\n", + " <td>0.5</td>\n", + " <td>0.5</td>\n", + " <td>0.5</td>\n", + " <td>0.5</td>\n", + " <td>faeb7454f3</td>\n", + " <td>69affa42</td>\n", + " <td>854e4fbc01</td>\n", + " <td>30</td>\n", + " <td>80</td>\n", + " <td>33.516888</td>\n", + " <td>-125.0</td>\n", + " <td>-118.689819</td>\n", + " </tr>\n", + " </tbody>\n", + "</table>\n", + "</div>" + ], + "text/plain": [ + " PatientID epidural intraparenchymal intraventricular subarachnoid \\\n", + "0 28fbab7eb 0.5 0.5 0.5 0.5 \n", + "1 877923b8b 0.5 0.5 0.5 0.5 \n", + "2 a591477cb 0.5 0.5 0.5 0.5 \n", + "3 42217c898 0.5 0.5 0.5 0.5 \n", + "4 a130c4d2f 0.5 0.5 0.5 0.5 \n", + "\n", + " subdural any SeriesI PID StudyI WindowCenter WindowWidth \\\n", + "0 0.5 0.5 ebfd7e4506 cf1b6b11 93407cadbb 30 80 \n", + "1 0.5 0.5 6d95084e15 ad8ea58f a337baa067 30 80 \n", + "2 0.5 0.5 8e06b2c9e0 ecfb278b 0cfe838d54 30 80 \n", + "3 0.5 0.5 e800f419cf e96e31f4 c497ac5bad 30 80 \n", + "4 0.5 0.5 faeb7454f3 69affa42 854e4fbc01 30 80 \n", + "\n", + " ImagePositionZ ImagePositionX ImagePositionY \n", + "0 158.458000 -125.0 -135.598000 \n", + "1 138.729050 -125.0 -101.797981 \n", + "2 60.830002 -125.0 -133.300003 \n", + "3 55.388000 -125.0 -146.081000 \n", + "4 33.516888 -125.0 -118.689819 " + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "test_df = pd.read_csv(data_dir+'test.csv')\n", + "test_df.head()" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "def my_loss(y_pred,y_true,weights):\n", + " window=(y_true>=0).to(torch.float)\n", + " loss = (F.binary_cross_entropy_with_logits(y_pred,y_true,reduction='none')*window*weights.expand_as(y_true)).mean()/(window.mean()+1e-7)\n", + " return loss" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "class Metric():\n", + " def __init__(self,weights,k=0.03):\n", + " self.weights=weights\n", + " self.k=k\n", + " self.zero()\n", + " \n", + " def zero(self):\n", + " self.loss_sum=0.\n", + " self.loss_count=0.\n", + " self.lossf=0.\n", + " \n", + " def calc(self,y_pred,y_true,prefix=\"\"):\n", + " window=(y_true>=0).to(torch.float)\n", + " loss = (F.binary_cross_entropy_with_logits(y_pred,y_true,reduction='none')*window*self.weights.expand_as(y_true)).mean()/(window.mean()+1e-5)\n", + " self.lossf=self.lossf*(1-self.k)+loss*self.k\n", + " self.loss_sum=self.loss_sum+loss*window.sum()\n", + " self.loss_count=self.loss_count+window.sum()\n", + " return({prefix+'mloss':self.lossf}) \n", + " \n", + " def calc_sums(self,prefix=\"\"):\n", + " return({prefix+'mloss_tot':self.loss_sum/self.loss_count}) \n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "model_names=[]\n", + "types_train=[]\n", + "types_test=[]\n", + "versions=[]\n", + "num_splits =[]\n", + "seeds=[]\n", + "for key in parameters.keys():\n", + " model_names.append(parameters[key]['model_name'])\n", + " types_train.append(parameters[key]['train_features'])\n", + " types_test.append(parameters[key]['test_features'])\n", + " versions.append(parameters[key]['version'])\n", + " num_splits.append(parameters[key]['n_splits'])\n", + " seeds.append(parameters[key]['SEED']) " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "multi=3\n", + "for model_name,type_,version_,n,SEED in zip(model_names,types_train,versions,num_splits,seeds):\n", + " for num_split in tqdm_notebook(range(n)):\n", + " pickle_file=open(outputs_dir+'PID_splits_{}.pkl'.format(n_splits),'rb')\n", + " split_sid,splits=pickle.load(pickle_file)\n", + " pickle_file.close()\n", + " pred_list=[]\n", + " print(model_name,version_,type_,num_split) \n", + " pickle_file=open(outputs_dir+outputs_format.format(model_name,version_,type_,num_split),'rb')\n", + " features=pickle.load(pickle_file)\n", + " pickle_file.close()\n", + " features=features.reshape(features.shape[0]//4,4,-1)\n", + " split_validate = train_df[train_df.PID.isin(set(split_sid[splits[num_split][1]]))].SeriesI.unique()\n", + " model=ResModelPool(features.shape[-1])\n", + " version=version_+'_fullhead_resmodel_pool2_{}'.format(multi)\n", + "\n", + " model.load_state_dict(torch.load(models_dir+models_format.format(model_name,version,num_split),map_location=torch.device(device)))\n", + "\n", + " valid_dataset=FullHeadDataset(train_df,\n", + " split_validate,\n", + " features,\n", + " 'SeriesI',\n", + " 'ImagePositionZ',\n", + " multi =3)\n", + "\n", + " win_dataset=FullHeadDataset(train_df,\n", + " split_validate,\n", + " features,\n", + " 'SeriesI',\n", + " 'ImagePositionZ',\n", + " target_columns=hemorrhage_types)\n", + " win_list=[]\n", + " dl = D.DataLoader(win_dataset,batch_size=128,num_workers=16)\n", + " for _,win in tqdm_notebook(dl):\n", + " win_list.append(win.reshape(win.shape[0]*win.shape[1],-1)) \n", + " wins = torch.cat(win_list,0).sum(1)>=0\n", + " wins.sum()\n", + " for i in tqdm_notebook(range(32),leave=False):\n", + " pr = model_run(model,valid_dataset,do_apex=False,batch_size=128)\n", + " pred_list.append(pr.reshape(pr.shape[0]*pr.shape[1],-1)[wins])\n", + " pickle_file=open(outputs_dir+outputs_format.format(model_name,version,'OOF_pred',num_split),'wb')\n", + " pickle.dump(pred_list,pickle_file,protocol=4)\n", + " pickle_file.close()\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "multi=3\n", + "for model_name,type_,version_,n,SEED in zip(model_names,types_test,versions,num_splits,seeds):\n", + " for num_split in tqdm_notebook(range(n)):\n", + " pred_list=[]\n", + " print(model_name,version_,type_,num_split) \n", + " pickle_file=open(outputs_dir+outputs_format.format(model_name,version_,type_,num_split),'rb')\n", + " features=pickle.load(pickle_file)\n", + " pickle_file.close()\n", + " features=features.reshape(features.shape[0]//8,8,-1)\n", + " print(features.shape)\n", + " model=ResModelPool(features.shape[-1])\n", + " version=version_+'_fullhead_resmodel_pool2_{}'.format(multi)\n", + "\n", + " model.load_state_dict(torch.load(models_dir+models_format.format(model_name,version,num_split),map_location=torch.device(device)))\n", + "\n", + " valid_dataset=FullHeadDataset(test_df,\n", + " test_df.SeriesI.unique(),\n", + " features,\n", + " 'SeriesI',\n", + " 'ImagePositionZ',\n", + " multi =4)\n", + "\n", + " win_dataset=FullHeadDataset(test_df,\n", + " test_df.SeriesI.unique(),\n", + " features,\n", + " 'SeriesI',\n", + " 'ImagePositionZ',\n", + " target_columns=hemorrhage_types)\n", + " win_list=[]\n", + " dl = D.DataLoader(win_dataset,batch_size=128,num_workers=16)\n", + " for _,win in tqdm_notebook(dl):\n", + " win_list.append(win.reshape(win.shape[0]*win.shape[1],-1)) \n", + " wins = torch.cat(win_list,0).sum(1)>=0\n", + " wins.sum()\n", + " for i in tqdm_notebook(range(32),leave=False):\n", + " pr = model_run(model,valid_dataset,do_apex=False,batch_size=128)\n", + " pred_list.append(pr.reshape(pr.shape[0]*pr.shape[1],-1)[wins])\n", + " pickle_file=open(outputs_dir+outputs_format.format(model_name,version,'test_pred_ensemble',num_split),'wb')\n", + " pickle.dump(pred_list,pickle_file,protocol=4)\n", + " pickle_file.close()\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "OOF_ids={}\n", + "SEED=8153\n", + "n_splits=3\n", + "pickle_file=open(outputs_dir+'PID_splits_{}.pkl'.format(n_splits),'rb')\n", + "split_sid,splits=pickle.load(pickle_file)\n", + "pickle_file.close()\n", + "for i in range( n_splits):\n", + " images_id_list=[]\n", + " split_validate = train_df[train_df.PID.isin(set(split_sid[splits[i][1]]))].SeriesI.unique()\n", + " image_arr=train_df.PatientID.values\n", + " ref_arr=train_df.SeriesI.values\n", + " order_arr=train_df.ImagePositionZ.values\n", + " for s in tqdm_notebook(split_validate):\n", + " head_idx = np.where(ref_arr==s)[0]\n", + " sorted_head_idx=head_idx[np.argsort(order_arr[head_idx])]\n", + " images_id_list.append(image_arr[sorted_head_idx])\n", + " image_ids=np.concatenate(images_id_list)\n", + " print(image_ids.shape,train_df[train_df.PID.isin(set(split_sid[splits[i][1]]))].shape[0])\n", + " OOF_ids[i]=image_ids\n", + "\n", + "\n", + "pickle_file=open(outputs_dir+'OOF_validation_image_ids_{}.pkl'.format(n_splits),'wb')\n", + "pickle.dump(OOF_ids,pickle_file,protocol=4)\n", + "pickle_file.close()\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "OOF_ids={}\n", + "SEED=432\n", + "n_splits=5\n", + "pickle_file=open(outputs_dir+'PID_splits_{}.pkl'.format(n_splits),'rb')\n", + "split_sid,splits=pickle.load(pickle_file)\n", + "pickle_file.close()\n", + "for i in range( n_splits):\n", + " images_id_list=[]\n", + " split_validate = train_df[train_df.PID.isin(set(split_sid[splits[i][1]]))].SeriesI.unique()\n", + " image_arr=train_df.PatientID.values\n", + " ref_arr=train_df.SeriesI.values\n", + " order_arr=train_df.ImagePositionZ.values\n", + " for s in tqdm_notebook(split_validate):\n", + " head_idx = np.where(ref_arr==s)[0]\n", + " sorted_head_idx=head_idx[np.argsort(order_arr[head_idx])]\n", + " images_id_list.append(image_arr[sorted_head_idx])\n", + " image_ids=np.concatenate(images_id_list)\n", + " print(image_ids.shape,train_df[train_df.PID.isin(set(split_sid[splits[i][1]]))].shape[0])\n", + " OOF_ids[i]=image_ids\n", + "\n", + "\n", + "pickle_file=open(outputs_dir+'OOF_validation_image_ids_{}.pkl'.format(n_splits),'wb')\n", + "pickle.dump(OOF_ids,pickle_file,protocol=4)\n", + "pickle_file.close()\n" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "05087bea7c1b4380a9c6bd7ff81eb4bd", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(IntProgress(value=0, max=2214), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "images_id_list=[]\n", + "dummeys=[]\n", + "image_arr=test_df.PatientID.values\n", + "ref_arr=test_df.SeriesI.values\n", + "order_arr=test_df.ImagePositionZ.values\n", + "for s in tqdm_notebook(test_df.SeriesI.unique()):\n", + " dumm=np.zeros(60)\n", + " head_idx = np.where(ref_arr==s)[0]\n", + " sorted_head_idx=head_idx[np.argsort(order_arr[head_idx])]\n", + " images_id_list.append(image_arr[sorted_head_idx])\n", + " dumm[0:head_idx.shape[0]]=1\n", + " dummeys.append(dumm)\n", + "image_ids=np.concatenate(images_id_list)\n", + "select=np.concatenate(dummeys)==1\n", + "\n", + "pickle_file=open(outputs_dir+'ensemble_test_image_ids.pkl','wb')\n", + "pickle.dump(image_ids,pickle_file,protocol=4)\n", + "pickle_file.close()\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.6" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +}