Datasets
Models
Downloads: 1
1967 lines (1966 with data), 83.0 kB
{
"cells": [
{
"cell_type": "code",
"execution_count": 28,
"metadata": {},
"outputs": [],
"source": [
"#conda activate torch-xla-nightly\n",
"#export XRT_TPU_CONFIG=\"tpu_worker;0;$10.0.101.2:8470\"\n",
"#git init\n",
"#git remote add origin https://github.com/nosound2/RSNA-Hemorrhage\n",
"#git config remote.origin.push HEAD\n",
"#git config credential.helper store\n",
"#git pull origin master\n",
"#git config --global branch.master.remote origin\n",
"#git config --global branch.master.merge refs/heads/master\n",
"#gcloud config set compute/zone europe-west4-a\n",
"#gcloud config set compute/zone us-central1-c\n",
"#gcloud auth login\n",
"#gcloud config set project endless-empire-239015\n",
"#pip install kaggle --user\n",
"#PATH=$PATH:~/.local/bin\n",
"#mkdir .kaggle\n",
"#gsutil cp gs://recursion-double-strand/kaggle-keys/kaggle.json ~/.kaggle\n",
"#chmod 600 /home/zahar_chikishev/.kaggle/kaggle.json\n",
"#kaggle competitions download rsna-intracranial-hemorrhage-detection -f stage_2_train.csv\n",
"#sudo apt install unzip\n",
"#unzip stage_1_train.csv.zip\n",
"#kaggle kernels output xhlulu/rsna-generate-metadata-csvs -p .\n",
"#kaggle kernels output zaharch/dicom-test-metadata-to-csv -p .\n",
"#kaggle kernels output zaharch/dicom-metadata-to-csv -p .\n",
"#gsutil -m cp gs://rsna-hemorrhage/yuvals/* .\n",
"\n",
"#export XRT_TPU_CONFIG=\"tpu_worker;0;10.0.101.2:8470\"; conda activate torch-xla-nightly; jupyter notebook\n",
"\n",
"# 35.188.114.109\n",
"\n",
"# lsblk\n",
"# sudo mkfs.ext4 -m 0 -F -E lazy_itable_init=0,lazy_journal_init=0,discard /dev/sdb\n",
"# sudo mkdir /mnt/edisk\n",
"# sudo mount -o discard,defaults /dev/sdb /mnt/edisk\n",
"# sudo chmod a+w /mnt/edisk\n",
"# df -H\n",
"# echo UUID=`sudo blkid -s UUID -o value /dev/sdb` /mnt/edisk ext4 discard,defaults,nofail 0 2 | sudo tee -a /etc/fstab\n",
"# cat /etc/fstab\n",
"\n",
"#jupyter notebook --generate-config\n",
"#vi ~/.jupyter/jupyter_notebook_config.py\n",
"\n",
"#c = get_config()\n",
"#c.NotebookApp.ip = '*'\n",
"#c.NotebookApp.open_browser = False\n",
"#c.NotebookApp.port = 5000"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"import sys\n",
"\n",
"from pathlib import Path\n",
"from PIL import ImageDraw, ImageFont, Image\n",
"from matplotlib import patches, patheffects\n",
"import time\n",
"from random import randint\n",
"import numpy as np\n",
"import pandas as pd\n",
"import pickle\n",
"\n",
"from sklearn.model_selection import KFold, StratifiedKFold, GroupKFold\n",
"from sklearn.preprocessing import LabelEncoder, OneHotEncoder, LabelBinarizer\n",
"from sklearn.preprocessing import StandardScaler\n",
"from sklearn.metrics import mean_squared_error,log_loss,roc_auc_score\n",
"from scipy.stats import ks_2samp\n",
"\n",
"import pdb\n",
"\n",
"import scipy as sp\n",
"from tqdm import tqdm, tqdm_notebook\n",
"\n",
"import os\n",
"import glob\n",
"\n",
"import torch\n",
"\n",
"#CLOUD = not torch.cuda.is_available()\n",
"CLOUD = (torch.cuda.get_device_name(0) in ['Tesla P4', 'Tesla K80', 'Tesla P100-PCIE-16GB'])\n",
"CLOUD_SINGLE = True\n",
"TPU = False\n",
"\n",
"if not CLOUD:\n",
" torch.cuda.current_device()\n",
"\n",
"import torch.nn as nn\n",
"import torch.utils.data as D\n",
"import torch.nn.functional as F\n",
"import torch.utils as U\n",
"\n",
"import torchvision\n",
"from torchvision import transforms as T\n",
"from torchvision import models as M\n",
"\n",
"import matplotlib.pyplot as plt\n",
"\n",
"if CLOUD:\n",
" PATH = Path('/home/zahar_chikishev/Hemorrhage')\n",
" PATH_WORK = Path('/mnt/edisk/running')\n",
" PATH_DISK = Path('/mnt/edisk/running')\n",
"else:\n",
" PATH = Path('C:/StudioProjects/Hemorrhage')\n",
" PATH_WORK = Path('C:/StudioProjects/Hemorrhage/running')\n",
" PATH_DISK = PATH_WORK\n",
"\n",
"from collections import defaultdict, Counter\n",
"import random\n",
"import seaborn as sn\n",
"\n",
"pd.set_option(\"display.max_columns\", 100)\n",
"\n",
"all_ich = ['any','epidural','intraparenchymal','intraventricular','subarachnoid','subdural']\n",
"class_weights = 6.0*np.array([2,1,1,1,1,1])/7.0\n",
"\n",
"if CLOUD and TPU:\n",
" import torch_xla\n",
" import torch_xla.distributed.data_parallel as dp\n",
" import torch_xla.utils as xu\n",
" import torch_xla.core.xla_model as xm\n",
"\n",
"from typing import Collection"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
"all_black = '006d4432e'\n",
"all_black = '00bd6c59c'\n",
"\n",
"if CLOUD:\n",
" if TPU:\n",
" device = xm.xla_device()\n",
" else:\n",
" device = 'cuda'\n",
" #device = 'cpu'\n",
" MAX_DEVICES = 1 if CLOUD_SINGLE else 8\n",
" NUM_WORKERS = 16\n",
" bs = 32\n",
"else:\n",
" device = 'cuda'\n",
" #device = 'cpu'\n",
" MAX_DEVICES = 1\n",
" NUM_WORKERS = 0\n",
" bs = 16\n",
"\n",
"#if CLOUD and (not CLOUD_SINGLE):\n",
"# devices = xm.get_xla_supported_devices(max_devices=MAX_DEVICES)"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [],
"source": [
"SEED = 2351\n",
"\n",
"def setSeeds(seed):\n",
" np.random.seed(seed)\n",
" torch.manual_seed(seed)\n",
" torch.cuda.manual_seed(seed)\n",
"\n",
"setSeeds(SEED)\n",
"torch.backends.cudnn.deterministic = True"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [],
"source": [
"def getDSName(dataset):\n",
" if dataset == 6:\n",
" return 'Densenet201_F3'\n",
" elif dataset == 7:\n",
" return 'Densenet161_F3'\n",
" elif dataset == 8:\n",
" return 'Densenet169_F3'\n",
" elif dataset == 9:\n",
" return 'se_resnext101_32x4d_F3'\n",
" elif dataset == 10:\n",
" return 'se_resnet101_F3'\n",
" elif dataset == 11:\n",
" return 'se_resnext101_32x4d_F5'\n",
" elif dataset == 12:\n",
" return 'se_resnet101_F5'\n",
" elif dataset == 13:\n",
" return 'se_resnet101_focal_F5'\n",
" elif dataset == 14:\n",
" return 'se_resnet101_F5'\n",
" else: assert False"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [],
"source": [
"def getDSParams(dataset):\n",
" if dataset == 6:\n",
" return 'Densenet201','_3','_v5',240,'','','classifier',''\n",
" elif dataset == 7:\n",
" return 'Densenet161','_3','_v4',552,'2','','classifier',''\n",
" elif dataset == 8:\n",
" return 'Densenet169','_3','_v3',208,'2','','classifier',''\n",
" elif dataset == 9:\n",
" return 'se_resnext101_32x4d','','',256,'','_tta','classifier',''\n",
" elif dataset == 10:\n",
" return 'se_resnet101','','',256,'','','classifier',''\n",
" elif dataset == 11:\n",
" return 'se_resnext101_32x4d_5','','',256,'','','new',''\n",
" elif dataset == 12:\n",
" return 'se_resnet101_5','','',256,'','','new',''\n",
" elif dataset == 13:\n",
" return 'se_resnet101_5f','','',256,'','','new','focal_'\n",
" elif dataset == 14:\n",
" return 'se_resnet101_5n','','',256,'','','new','stage2_'\n",
" else: assert False\n",
"\n",
"def getNFolds(dataset):\n",
" if dataset <= 10:\n",
" return 3\n",
" elif dataset <= 14:\n",
" return 5\n",
" else: assert False\n",
"\n",
"my_datasets3 = [7,9]\n",
"my_datasets5 = [11,12,13]\n",
"my_len = len(my_datasets3) + len(my_datasets5)"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [],
"source": [
"cols_le,cols_float,cols_bool = pickle.load(open(PATH_WORK/'covs','rb'))\n",
"meta_cols = cols_bool + cols_float\n",
"\n",
"#meta_cols = [c for c in meta_cols if c not in ['SeriesPP']]\n",
"#cols_le = cols_le[:-1]"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Pre-processing"
]
},
{
"cell_type": "code",
"execution_count": 34,
"metadata": {},
"outputs": [],
"source": [
"def loadMetadata(rerun=False):\n",
" if rerun:\n",
" train_dedup = pd.read_csv(PATH_WORK/'yuval'/'train_dedup.csv')\n",
" pids, folding = pickle.load(open(PATH_WORK/'yuval'/'PID_splits.pkl','rb'))\n",
"\n",
" assert len(pids) == 17079\n",
" assert len(np.unique(pids)) == 17079\n",
"\n",
" for fol in folding:\n",
" assert len(fol[0]) + len(fol[1]) == 17079\n",
"\n",
" assert len(folding[0][1]) + len(folding[1][1]) + len(folding[2][1]) == 17079\n",
"\n",
" assert len(train_dedup.PID.unique()) == 17079\n",
"\n",
" train_dedup['fold'] = np.nan\n",
"\n",
" for fold in range(3):\n",
" train_dedup.loc[train_dedup.PID.isin(pids[folding[fold][1]]),'fold'] = fold\n",
"\n",
" assert train_dedup.fold.isnull().sum() == 0\n",
" \n",
" oof5 = pickle.load(open(PATH_DISK/'yuval'/'OOF_validation_image_ids_5.pkl','rb'))\n",
" for fold in range(5):\n",
" train_dedup.loc[train_dedup.PatientID.isin(oof5[fold]),'fold5'] = fold\n",
" assert train_dedup.fold5.isnull().sum() == 0\n",
" \n",
" train_md = pd.read_csv(PATH_WORK/'train_md.csv').sort_values(['SeriesInstanceUID','pos_idx1'])\n",
" train_md['img_id'] = train_md.SOPInstanceUID.str.split('_').apply(lambda x: x[1])\n",
"\n",
" ids_df = train_dedup[['fold','PatientID','fold5']]\n",
" ids_df.columns = ['fold','img_id','fold5']\n",
"\n",
" train_md = ids_df.join(train_md.set_index('img_id'), on = 'img_id')\n",
"\n",
" pickle.dump(train_md, open(PATH_WORK/'train.ids.df','wb'))\n",
"\n",
" #test_md = pickle.load(open(PATH_WORK/'test.post.processed.1','rb'))\n",
" test_md = pd.read_csv(PATH_WORK/'test_md.csv').sort_values(['SeriesInstanceUID','pos_idx1'])\n",
" test_md['img_id'] = test_md.SOPInstanceUID.str.split('_').apply(lambda x: x[1])\n",
"\n",
" filename = PATH_WORK/'yuval'/'test_indexes.pkl'\n",
" test_ids = pickle.load(open(filename,'rb'))\n",
"\n",
" test_ids_df = pd.DataFrame(test_ids, columns = ['img_id'])\n",
" test_md = test_ids_df.join(test_md.set_index('img_id'), on = 'img_id')\n",
" \n",
" assert len(test_md.SeriesInstanceUID.unique()) == 2214\n",
" \n",
" #train_md = pd.concat([train_md, test_md], axis = 0, sort=False).reset_index(drop=True)\n",
" \n",
" #pickle.dump(train_md, open(PATH_WORK/'train.ids.df','wb'))\n",
" pickle.dump(test_md, open(PATH_WORK/'test.ids.df','wb'))\n",
" else:\n",
" train_md = pickle.load(open(PATH_WORK/'train.ids.df','rb'))\n",
" test_md = pickle.load(open(PATH_WORK/'test.ids.df','rb'))\n",
" \n",
" return train_md, test_md"
]
},
{
"cell_type": "code",
"execution_count": 17,
"metadata": {},
"outputs": [],
"source": [
"def loadMetadata2(rerun=False):\n",
" if rerun:\n",
" train_dedup2 = pd.read_csv(PATH_WORK/'yuval'/'train_stage2.csv')\n",
"\n",
" assert train_dedup2.PatientID.nunique() == 752797\n",
" assert train_dedup2.PID.nunique() == 18938\n",
"\n",
" pids, folding = pickle.load(open(PATH_WORK/'yuval'/'PID_splits.pkl','rb'))\n",
" pids3, folding3 = pickle.load(open(PATH_WORK/'yuval'/'PID_splits_3_stage_2.pkl','rb'))\n",
" pids5, folding5 = pickle.load(open(PATH_WORK/'yuval'/'PID_splits_5_stage_2.pkl','rb'))\n",
"\n",
" assert np.array([len(folding3[i][1]) for i in range(3)]).sum() == 18938\n",
" assert np.array([len(folding3[i][0]) for i in range(3)]).sum() == 2*18938\n",
" \n",
" for i in range(3):\n",
" assert set(pids[folding[i][1]]).issubset(set(pids3[folding3[i][1]]))\n",
"\n",
" for i in range(3):\n",
" train_dedup2.loc[train_dedup2.PID.isin(pids3[folding3[i][1]]),'fold'] = i\n",
" assert train_dedup2.fold.isnull().sum() == 0\n",
"\n",
" for i in range(5):\n",
" train_dedup2.loc[train_dedup2.PID.isin(pids5[folding5[i][1]]),'fold5'] = i\n",
" assert train_dedup2.fold5.isnull().sum() == 0\n",
" \n",
" oof5 = pickle.load(open(PATH_DISK/'yuval'/'OOF_validation_image_ids_5.pkl','rb'))\n",
" for i in range(5):\n",
" assert set(oof5[i]).issubset(set(train_dedup2.loc[train_dedup2.fold5==i,'PatientID']))\n",
" \n",
" train_csv = pd.read_csv(PATH/'stage_2_train.csv')\n",
" train_csv = train_csv.loc[~train_csv.ID.duplicated()].sort_values('ID').reset_index(drop=True) \n",
" all_sop_ids = train_csv.ID.str.split('_').apply(lambda x: x[0]+'_'+x[1]).unique()\n",
" train_df = pd.DataFrame(train_csv.Label.values.reshape((-1,6)), columns = all_ich)\n",
" train_df['sop_id'] = all_sop_ids\n",
" \n",
" old_test = pd.read_csv(PATH_WORK/'test_md.csv')\n",
" old_test = old_test.drop(all_ich,axis=1)\n",
" old_test = old_test.join(train_df.set_index('sop_id'), on = 'SOPInstanceUID')\n",
" \n",
" train_md2 = pd.concat([pd.read_csv(PATH_WORK/'train_md.csv'), old_test],\\\n",
" axis=0,sort=False)\n",
" train_md2['img_id'] = train_md2.SOPInstanceUID.str.split('_').apply(lambda x: x[1])\n",
" \n",
" ids_df = train_dedup2[['fold','PatientID','fold5']]\n",
" ids_df.columns = ['fold','img_id','fold5']\n",
"\n",
" train_md2 = ids_df.join(train_md2.set_index('img_id'), on = 'img_id')\n",
" assert train_md2.test.sum() == 78545\n",
" \n",
" pickle.dump(train_md2, open(PATH_WORK/'train2.ids.df','wb'))\n",
" \n",
" test_md = pd.read_csv(PATH_WORK/'test2_md.csv').sort_values(['SeriesInstanceUID','pos_idx1'])\n",
" test_md['img_id'] = test_md.SOPInstanceUID.str.split('_').apply(lambda x: x[1])\n",
"\n",
" filename = PATH_WORK/'yuval/test_indexes_stage2.pkl'\n",
" test_ids = pickle.load(open(filename,'rb'))\n",
"\n",
" test_ids_df = pd.DataFrame(test_ids, columns = ['img_id'])\n",
" test_md = test_ids_df.join(test_md.set_index('img_id'), on = 'img_id')\n",
"\n",
" assert len(test_md.SeriesInstanceUID.unique()) == 3518\n",
"\n",
" pickle.dump(test_md, open(PATH_WORK/'test2.ids.df','wb'))\n",
"\n",
" else:\n",
" train_md2 = pickle.load(open(PATH_WORK/'train2.ids.df','rb'))\n",
" test_md = pickle.load(open(PATH_WORK/'test2.ids.df','rb'))\n",
" \n",
" return train_md2, test_md"
]
},
{
"cell_type": "code",
"execution_count": 34,
"metadata": {},
"outputs": [],
"source": [
"def loadMetadata3(rerun=False):\n",
" if rerun:\n",
" train_md = pickle.load(open(PATH_WORK/'train.ids.df','rb'))\n",
" test_md = pickle.load(open(PATH_WORK/'test.ids.df','rb'))\n",
" train_md = pd.concat([train_md, test_md], axis=0,sort=False).reset_index(drop=True)\n",
" \n",
" train_md.fold = np.nan\n",
" train_md['PID'] = train_md.PatientID.str.split('_').apply(lambda x: x[1])\n",
" \n",
" pids, folding = pickle.load(open(PATH_WORK/'yuval'/'PID_splits.pkl','rb'))\n",
" pids3, folding3 = pickle.load(open(PATH_WORK/'yuval'/'PID_splits_3_stage_2.pkl','rb'))\n",
" pids5, folding5 = pickle.load(open(PATH_WORK/'yuval'/'PID_splits_5_stage_2.pkl','rb'))\n",
"\n",
" for i in range(3):\n",
" train_md.loc[train_md.PID.isin(pids3[folding3[i][1]]),'fold'] = i\n",
" assert train_md.fold.isnull().sum() == 0\n",
"\n",
" for i in range(5):\n",
" train_md.loc[train_md.PID.isin(pids5[folding5[i][1]]),'fold5'] = i\n",
" assert train_md.fold5.isnull().sum() == 0\n",
" \n",
" train_md.weights = train_md.weights.fillna(1)\n",
" \n",
" pickle.dump(train_md, open(PATH_WORK/'train3.ids.df','wb'))\n",
" \n",
" test_md = pd.read_csv(PATH_WORK/'test2_md.csv').sort_values(['SeriesInstanceUID','pos_idx1'])\n",
" test_md['img_id'] = test_md.SOPInstanceUID.str.split('_').apply(lambda x: x[1])\n",
"\n",
" filename = PATH_WORK/'yuval/test_indexes_stage2.pkl'\n",
" test_ids = pickle.load(open(filename,'rb'))\n",
"\n",
" test_ids_df = pd.DataFrame(test_ids, columns = ['img_id'])\n",
" test_md = test_ids_df.join(test_md.set_index('img_id'), on = 'img_id')\n",
"\n",
" assert len(test_md.SeriesInstanceUID.unique()) == 3518\n",
"\n",
" pickle.dump(test_md, open(PATH_WORK/'test3.ids.df','wb'))\n",
" else:\n",
" train_md = pickle.load(open(PATH_WORK/'train3.ids.df','rb'))\n",
" test_md = pickle.load(open(PATH_WORK/'test3.ids.df','rb'))\n",
" \n",
" return train_md, test_md"
]
},
{
"cell_type": "code",
"execution_count": 66,
"metadata": {},
"outputs": [],
"source": [
"#train_md, test_md = loadMetadata(True)"
]
},
{
"cell_type": "code",
"execution_count": 18,
"metadata": {},
"outputs": [],
"source": [
"#train_md = loadMetadata2(True)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [],
"source": [
"def preprocessedData(dataset, folds = range(3), fold_col=None, do_test=True, do_test2=True, do_train=True):\n",
" assert dataset >= 6\n",
" \n",
" dataset_name, filename_add, filename_add2, feat_sz, ds_num, test_fix, dsft, focal = getDSParams(dataset)\n",
" \n",
" #if 'train_md' not in globals() or 'test_md' not in globals():\n",
" # train_md, test_md = loadMetadata(False)\n",
" \n",
" PATH_DS = PATH_DISK/'features/{}{}'.format(dataset_name,filename_add2)\n",
" if not PATH_DS.is_dir():\n",
" PATH_DS.mkdir()\n",
" if not (PATH_DS/'train').is_dir():\n",
" (PATH_DS/'train').mkdir()\n",
" if not (PATH_DS/'test').is_dir():\n",
" (PATH_DS/'test').mkdir()\n",
" if not (PATH_DS/'test2').is_dir():\n",
" (PATH_DS/'test2').mkdir()\n",
" \n",
" if fold_col is None:\n",
" fold_col = 'fold'\n",
" if len(folds) == 5:\n",
" fold_col = 'fold5'\n",
" \n",
" for fold in folds:\n",
" \n",
" if do_train:\n",
" filename = PATH_DISK/'yuval'/\\\n",
" 'model_{}{}_version_{}_splits_{}type_features_train_tta{}_split_{}.pkl'\\\n",
" .format(dataset_name.replace('_5n','').replace('_5f','').replace('_5',''), \\\n",
" filename_add, dsft, focal, ds_num, fold)\n",
" feats = pickle.load(open(filename,'rb'))\n",
"\n",
" print('dataset',dataset,'fold',fold,'feats size', feats.shape)\n",
" if dataset <= 13:\n",
" train_md_loc = train_md.loc[~train_md.test]\n",
" else:\n",
" train_md_loc = train_md\n",
" \n",
" assert len(feats) == 4*len(train_md_loc)\n",
" assert feats.shape[1] == feat_sz\n",
" means = feats.mean(0,keepdim=True)\n",
" stds = feats.std(0,keepdim=True)\n",
"\n",
" feats = feats - means\n",
" feats = torch.where(stds > 0, feats/stds, feats)\n",
"\n",
" for i in range(4):\n",
" feats_sub1 = feats[torch.BoolTensor(np.arange(len(feats))%4 == i)]\n",
" feats_sub2 = feats_sub1[torch.BoolTensor(train_md_loc[fold_col] != fold)]\n",
" pickle.dump(feats_sub2, open(PATH_DS/'train/train.f{}.a{}'.format(fold,i),'wb'))\n",
"\n",
" feats_sub2 = feats_sub1[torch.BoolTensor(train_md_loc[fold_col] == fold)]\n",
" pickle.dump(feats_sub2, open(PATH_DS/'train/valid.f{}.a{}'.format(fold,i),'wb'))\n",
"\n",
" if i==0:\n",
" black_feats = feats_sub1[torch.BoolTensor(train_md_loc.img_id == all_black)].squeeze()\n",
" pickle.dump(black_feats, open(PATH_DS/'train/black.f{}'.format(fold),'wb'))\n",
" \n",
" if do_test:\n",
" filename = PATH_DISK/'yuval'/\\\n",
" 'model_{}{}_version_{}_splits_{}type_features_test{}{}_split_{}.pkl'\\\n",
" .format(dataset_name.replace('_5n','').replace('_5f','').replace('_5',''),\n",
" filename_add,dsft,focal,ds_num,test_fix,fold)\n",
" feats = pickle.load(open(filename,'rb'))\n",
"\n",
" assert len(feats) == 8*len(test_md)\n",
" assert feats.shape[1] == feat_sz\n",
"\n",
" feats = feats - means\n",
" feats = torch.where(stds > 0, feats/stds, feats)\n",
"\n",
" for i in range(8):\n",
" feats_sub = feats[torch.BoolTensor(np.arange(len(feats))%8 == i)]\n",
" pickle.dump(feats_sub, open(PATH_DS/'test/test.f{}.a{}'.format(fold,i),'wb'))\n",
" assert len(feats_sub) == len(test_md)\n",
" \n",
" if do_test2:\n",
" filename = PATH_DISK/'yuval'/\\\n",
" 'model_{}{}_version_{}_splits_{}type_features_test_stage2_split_{}.pkl'\\\n",
" .format(dataset_name.replace('_5n','').replace('_5f','').replace('_5',''),\n",
" filename_add,dsft,focal,fold)\n",
" feats = pickle.load(open(filename,'rb'))\n",
"\n",
" assert len(feats) == 8*len(test_md)\n",
" assert feats.shape[1] == feat_sz\n",
"\n",
" feats = feats - means\n",
" feats = torch.where(stds > 0, feats/stds, feats)\n",
"\n",
" for i in range(8):\n",
" feats_sub = feats[torch.BoolTensor(np.arange(len(feats))%8 == i)]\n",
" pickle.dump(feats_sub, open(PATH_DS/'test2/test.f{}.a{}'.format(fold,i),'wb'))\n",
" assert len(feats_sub) == len(test_md)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Dataset"
]
},
{
"cell_type": "code",
"execution_count": 86,
"metadata": {},
"outputs": [],
"source": [
"if False:\n",
" path = PATH_WORK/'features/densenet161_v3/train/ID_992b567eb6'\n",
" black_feats = pickle.load(open(path,'rb'))[41]"
]
},
{
"cell_type": "code",
"execution_count": 87,
"metadata": {},
"outputs": [],
"source": [
"class RSNA_DataSet(D.Dataset):\n",
" def __init__(self, metadata, dataset, mode='train', bs=None, fold=0):\n",
" \n",
" super(RSNA_DataSet, self).__init__()\n",
" \n",
" dataset_name, filename_add, filename_add2, feat_sz,_,_,_,_ = getDSParams(dataset)\n",
" num_folds = getNFolds(dataset)\n",
" \n",
" self.dataset_name = dataset_name\n",
" self.filename_add2 = filename_add2\n",
" \n",
" folds_col = 'fold'\n",
" if num_folds == 5:\n",
" folds_col = 'fold5'\n",
" \n",
" if (dataset <= 13) and (mode in ['train','valid']) and TRAIN_ON_STAGE_1:\n",
" if mode == 'train':\n",
" self.test_mask = torch.BoolTensor((metadata.loc[metadata.test, folds_col] != fold).values)\n",
" else:\n",
" self.test_mask = torch.BoolTensor((metadata.loc[metadata.test, folds_col] == fold).values)\n",
" \n",
" if mode == 'train':\n",
" md = metadata.loc[metadata[folds_col] != fold].copy().reset_index(drop=True)\n",
" elif mode == 'valid':\n",
" md = metadata.loc[metadata[folds_col] == fold].copy().reset_index(drop=True)\n",
" else:\n",
" md = metadata.copy().reset_index(drop=True)\n",
" \n",
" series = np.sort(md.SeriesInstanceUID.unique())\n",
" md = md.set_index('SeriesInstanceUID', drop=True)\n",
" \n",
" samples_add = 0\n",
" if (mode != 'train') and not DATA_SMALL:\n",
" batch_num = -((-len(series))//(bs*MAX_DEVICES))\n",
" samples_add = batch_num*bs*MAX_DEVICES - len(series)\n",
" print('adding dummy serieses', samples_add)\n",
" \n",
" #self.records = df.to_records(index=False)\n",
" self.mode = mode\n",
" self.real = np.concatenate([np.repeat(True,len(series)),np.repeat(False,samples_add)])\n",
" self.series = np.concatenate([series, random.sample(list(series),samples_add)])\n",
" self.metadata = md\n",
" self.dataset = dataset\n",
" self.fold = fold\n",
" \n",
" print('DataSet', dataset, mode, 'size', len(self.series), 'fold', fold)\n",
" \n",
" path = PATH_DISK/'features/{}{}/train/black.f{}'.format(self.dataset_name, self.filename_add2, fold)\n",
" self.black_feats = pickle.load(open(path,'rb')).squeeze()\n",
" \n",
" if WEIGHTED and (self.mode == 'train'):\n",
" tt = self.metadata['weights'].groupby(self.metadata.index).mean()\n",
" self.weights = tt.loc[self.series].values\n",
" print(pd.value_counts(self.weights))\n",
" \n",
" def setFeats(self, anum, epoch=0):\n",
" def getAPath(an,mode):\n",
" folder = 'test2' if mode == 'test' else 'train'\n",
" return PATH_DISK/'features/{}{}/{}/{}.f{}.a{}'\\\n",
" .format(self.dataset_name,self.filename_add2,folder,mode,self.fold,an)\n",
" \n",
" def getAPathFeats(mode, sz, mask=None):\n",
" max_a = 8 if mode == 'test' else 4\n",
" feats2 = torch.stack([pickle.load(open(getAPath(an,mode),'rb')) for an in range(max_a)])\n",
" if mask is not None:\n",
" feats2 = feats2[:,mask]\n",
" assert feats2.shape[1] == sz\n",
" feats = feats2[torch.randint(max_a,(sz,)), torch.arange(sz, dtype=torch.long)].squeeze()\n",
" return feats\n",
" \n",
" if self.dataset == 1: return\n",
" print('setFeats, augmentation', anum)\n",
" self.anum = anum\n",
" sz = len(self.metadata)\n",
" \n",
" assert anum <= 0\n",
" \n",
" if anum == -1:\n",
" if (self.dataset <= 13) and (self.mode in ['train','valid']) and TRAIN_ON_STAGE_1:\n",
" feats = torch.cat([getAPathFeats(self.mode, sz - self.metadata.test.sum()), \n",
" getAPathFeats('test', self.metadata.test.sum(), self.test_mask)] ,axis=0)\n",
" else:\n",
" feats = getAPathFeats(self.mode, sz)\n",
" else:\n",
" if (self.dataset <= 13) and (self.mode in ['train','valid']) and TRAIN_ON_STAGE_1:\n",
" feats = torch.cat([pickle.load(open(getAPath(anum,self.mode),'rb')),\n",
" pickle.load(open(getAPath(anum,'test'),'rb'))[self.test_mask]], axis=0)\n",
" else:\n",
" feats = pickle.load(open(getAPath(anum,self.mode),'rb'))\n",
" \n",
" self.feats = feats\n",
" self.epoch = epoch\n",
" assert len(feats) == sz\n",
" \n",
" def __getitem__(self, index):\n",
" \n",
" series_id = self.series[index]\n",
" #df = self.metadata.loc[self.metadata.SeriesInstanceUID == series_id].reset_index(drop=True)\n",
" df = self.metadata.loc[series_id].reset_index(drop=True)\n",
" \n",
" if self.dataset >= 6:\n",
" feats = self.feats[torch.BoolTensor(self.metadata.index.values == series_id)]\n",
" else: assert False\n",
" \n",
" order = np.argsort(df.pos_idx1.values)\n",
" df = df.sort_values(['pos_idx1'])\n",
" feats = feats[torch.LongTensor(order)]\n",
" \n",
" #if WEIGHTED:\n",
" # non_black = torch.Tensor(df['weights'])\n",
" #else:\n",
" non_black = torch.ones(len(feats))\n",
" \n",
" feats = torch.cat([feats, torch.Tensor(df[meta_cols].values)], dim=1)\n",
" feats_le = torch.LongTensor(df[cols_le].values)\n",
" \n",
" target = torch.Tensor(df[all_ich].values)\n",
" \n",
" PAD = 12\n",
" \n",
" np.random.seed(1234 + index + 10000*self.epoch)\n",
" offset = np.random.randint(0, 61 - feats.shape[0])\n",
" \n",
" #offset = 0\n",
" bk_add = 0\n",
" top_pad = PAD + offset\n",
" if top_pad > 0:\n",
" dummy_row = torch.cat([self.black_feats, torch.Tensor(df.head(1)[meta_cols].values).squeeze()])\n",
" feats = torch.cat([dummy_row.repeat(top_pad,1), feats], dim=0)\n",
" feats_le = torch.cat([torch.LongTensor(df.head(1)[cols_le].values).squeeze().repeat(top_pad,1), feats_le])\n",
" if offset > 0:\n",
" non_black = torch.cat([bk_add + torch.zeros(offset), non_black])\n",
" target = torch.cat([torch.zeros((offset, len(all_ich))), target], dim=0)\n",
" bot_pad = 60 - len(df) - offset + PAD\n",
" if bot_pad > 0:\n",
" dummy_row = torch.cat([self.black_feats, torch.Tensor(df.tail(1)[meta_cols].values).squeeze()])\n",
" feats = torch.cat([feats, dummy_row.repeat(bot_pad,1)], dim=0)\n",
" feats_le = torch.cat([feats_le, torch.LongTensor(df.tail(1)[cols_le].values).squeeze().repeat(bot_pad,1)])\n",
" if (60 - len(df) - offset) > 0:\n",
" non_black = torch.cat([non_black, bk_add + torch.zeros(60 - len(df) - offset)])\n",
" target = torch.cat([target, torch.zeros((60 - len(df) - offset, len(all_ich)))], dim=0)\n",
" \n",
" assert feats_le.shape[0] == (60 + 2*PAD)\n",
" assert feats.shape[0] == (60 + 2*PAD)\n",
" assert target.shape[0] == 60\n",
" \n",
" idx = index\n",
" if not self.real[index]: idx = -1\n",
" \n",
" if self.mode == 'train':\n",
" return feats, feats_le, target, non_black\n",
" else:\n",
" return feats, feats_le, target, idx, offset\n",
" \n",
" def __len__(self):\n",
" return len(self.series) if not DATA_SMALL else int(0.01*len(self.series))"
]
},
{
"cell_type": "code",
"execution_count": 88,
"metadata": {},
"outputs": [],
"source": [
"def getCurrentBatch(dataset, fold=0, ver=VERSION):\n",
" sel_batch = None\n",
" for filename in os.listdir(PATH_DISK/'models'):\n",
" splits = filename.split('.')\n",
" if int(splits[2][1]) != fold: continue\n",
" if int(splits[3][1:]) != dataset: continue\n",
" if int(splits[4][1:]) != ver: continue\n",
" if sel_batch is None:\n",
" sel_batch = int(splits[1][1:])\n",
" else:\n",
" sel_batch = max(sel_batch, int(splits[1][1:]))\n",
" return sel_batch\n",
"\n",
"def modelFileName(dataset, fold=0, batch = 1, return_last = False, return_next = False, ver=VERSION):\n",
" sel_batch = batch\n",
" if return_last or return_next:\n",
" sel_batch = getCurrentBatch(fold=fold, dataset=dataset, ver=ver)\n",
" if return_last and sel_batch is None:\n",
" return None\n",
" if return_next:\n",
" if sel_batch is None: sel_batch = 1\n",
" else: sel_batch += 1\n",
" \n",
" return 'model.b{}.f{}.d{}.v{}'.format(sel_batch, fold, dataset, ver)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Model"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [],
"source": [
"class BCEWithLogitsLoss(nn.Module):\n",
" def __init__(self, weight=None):\n",
" super().__init__()\n",
" self.weight = weight\n",
" \n",
" def forward(self, input, target, batch_weights = None, focal=0):\n",
" if focal == 0:\n",
" loss = (torch.log(1+torch.exp(input)) - target*input)*self.weight\n",
" else:\n",
" loss = torch.pow(1+torch.exp(input), -focal) * \\\n",
" (((1-target)*torch.exp(focal*input) + target) * torch.log(1+torch.exp(input)) - target*input)\n",
" loss = loss*self.weight\n",
" if batch_weights is not None:\n",
" loss = batch_weights*loss\n",
" return loss.mean()"
]
},
{
"cell_type": "code",
"execution_count": 90,
"metadata": {},
"outputs": [],
"source": [
"def bn_drop_lin(n_in:int, n_out:int, bn:bool=True, p:float=0., actn=None):\n",
" \"Sequence of batchnorm (if `bn`), dropout (with `p`) and linear (`n_in`,`n_out`) layers followed by `actn`.\"\n",
" layers = [nn.BatchNorm1d(n_in)] if bn else []\n",
" if p != 0: layers.append(nn.Dropout(p))\n",
" layers.append(nn.Linear(n_in, n_out))\n",
" if actn is not None: layers.append(actn)\n",
" return layers"
]
},
{
"cell_type": "code",
"execution_count": 91,
"metadata": {},
"outputs": [],
"source": [
"def noop(x): return x\n",
"act_fun = nn.ReLU(inplace=True)\n",
"\n",
"def conv_layer(ni, nf, ks=3, act=True):\n",
" bn = nn.BatchNorm1d(nf)\n",
" layers = [nn.Conv1d(ni, nf, ks), bn]\n",
" if act: layers.append(act_fun)\n",
" return nn.Sequential(*layers)\n",
"\n",
"class ResBlock(nn.Module):\n",
" def __init__(self, ni, nh):\n",
" super().__init__()\n",
" layers = [conv_layer(ni, nh, 1),\n",
" conv_layer(nh, nh, 5, act=False)]\n",
" self.convs = nn.Sequential(*layers)\n",
" self.idconv = noop if (ni == nh) else conv_layer(ni, nh, 1, act=False)\n",
" \n",
" def forward(self, x): return act_fun(self.convs(x) + self.idconv(x[:,:,2:-2]))"
]
},
{
"cell_type": "code",
"execution_count": 77,
"metadata": {},
"outputs": [],
"source": [
"class ResNetModel(nn.Module):\n",
" def __init__(self, n_cont:int, feat_sz=2208):\n",
" super().__init__()\n",
" \n",
" self.le_sz = 10\n",
" assert self.le_sz == (len(cols_le))\n",
" le_in_sizes = np.array([5,5,7,4,4,11,4,6,3,3])\n",
" le_out_sizes = np.array([3,3,4,2,2,6,2,4,2,2])\n",
" le_out_sz = le_out_sizes.sum()\n",
" self.embeddings = nn.ModuleList([embedding(le_in_sizes[i], le_out_sizes[i]) for i in range(self.le_sz)])\n",
" \n",
" self.feat_sz = feat_sz\n",
" \n",
" self.n_cont = n_cont\n",
" scale = 4\n",
" \n",
" self.conv2D = nn.Conv2d(1,scale*16,(feat_sz + n_cont + le_out_sz,1))\n",
" self.bn1 = nn.BatchNorm1d(scale*16)\n",
" \n",
" self.res1 = ResBlock(scale*16,scale*16)\n",
" self.res2 = ResBlock(scale*16,scale*8)\n",
" \n",
" self.res3 = ResBlock(scale*24,scale*16)\n",
" self.res4 = ResBlock(scale*16,scale*8)\n",
" \n",
" self.res5 = ResBlock(scale*32,scale*16)\n",
" self.res6 = ResBlock(scale*16,scale*8)\n",
" \n",
" self.conv1D = nn.Conv1d(scale*40,6,1)\n",
" \n",
" def forward(self, x, x_le, x_le_mix = None, lambd = None) -> torch.Tensor:\n",
" x_le = [e(x_le[:,:,i]) for i,e in enumerate(self.embeddings)]\n",
" x_le = torch.cat(x_le, 2)\n",
" \n",
" x = torch.cat([x, x_le], 2)\n",
" x = x.transpose(1,2)\n",
" \n",
" #x = torch.cat([x[:,:self.feat_sz],self.bn_cont(x[:,self.feat_sz:])], dim=1)\n",
" x = x.reshape(x.shape[0],1,x.shape[1],x.shape[2])\n",
" \n",
" x = self.conv2D(x).squeeze()\n",
" x = self.bn1(x)\n",
" x = act_fun(x)\n",
" \n",
" x2 = self.res1(x)\n",
" x2 = self.res2(x2)\n",
" \n",
" x3 = torch.cat([x[:,:,4:-4], x2], 1)\n",
" \n",
" x3 = self.res3(x3)\n",
" x3 = self.res4(x3)\n",
" \n",
" x4 = torch.cat([x[:,:,8:-8], x2[:,:,4:-4], x3], 1)\n",
" \n",
" x4 = self.res5(x4)\n",
" x4 = self.res6(x4)\n",
" \n",
" x5 = torch.cat([x[:,:,12:-12], x2[:,:,8:-8], x3[:,:,4:-4], x4], 1)\n",
" x5 = self.conv1D(x5)\n",
" x5 = x5.transpose(1,2)\n",
" \n",
" return x5"
]
},
{
"cell_type": "code",
"execution_count": 93,
"metadata": {},
"outputs": [],
"source": [
"def trunc_normal_(x, mean:float=0., std:float=1.):\n",
" \"Truncated normal initialization.\"\n",
" # From https://discuss.pytorch.org/t/implementing-truncated-normal-initializer/4778/12\n",
" return x.normal_().fmod_(2).mul_(std).add_(mean)"
]
},
{
"cell_type": "code",
"execution_count": 94,
"metadata": {},
"outputs": [],
"source": [
"def embedding(ni:int,nf:int) -> nn.Module:\n",
" \"Create an embedding layer.\"\n",
" emb = nn.Embedding(ni, nf)\n",
" # See https://arxiv.org/abs/1711.09160\n",
" with torch.no_grad(): trunc_normal_(emb.weight, std=0.01)\n",
" return emb"
]
},
{
"cell_type": "code",
"execution_count": 95,
"metadata": {},
"outputs": [],
"source": [
"class TabularModel(nn.Module):\n",
" \"Basic model for tabular data.\"\n",
" def __init__(self, n_cont:int, feat_sz=2208, fc_drop_p=0.3):\n",
" super().__init__()\n",
" self.le_sz = 10\n",
" assert len(cols_le) == self.le_sz\n",
" le_in_sizes = np.array([5,5,7,4,4,11,4,6,3,3])\n",
" le_out_sizes = np.array([3,3,4,2,2,6,2,4,2,2])\n",
" le_out_sz = le_out_sizes.sum()\n",
" self.embeddings = nn.ModuleList([embedding(le_in_sizes[i], le_out_sizes[i]) for i in range(self.le_sz)])\n",
" #self.bn_cont = nn.BatchNorm1d(feat_sz + n_cont)\n",
" \n",
" self.feat_sz = feat_sz\n",
" self.bn_cont = nn.BatchNorm1d(n_cont + le_out_sz)\n",
" self.n_cont = n_cont\n",
" self.fc_drop = nn.Dropout(p=fc_drop_p)\n",
" self.relu = nn.ReLU(inplace=True)\n",
" \n",
" scale = 4\n",
" \n",
" self.conv2D_1 = nn.Conv2d(1,16*scale,(feat_sz + n_cont + le_out_sz,1))\n",
" self.conv2D_2 = nn.Conv2d(1,16*scale,(feat_sz + n_cont + le_out_sz,5))\n",
" self.bn_cont1 = nn.BatchNorm1d(32*scale)\n",
" self.conv1D_1 = nn.Conv1d(32*scale,16*scale,3)\n",
" self.conv1D_3 = nn.Conv1d(32*scale,16*scale,5,dilation=5)\n",
" self.conv1D_2 = nn.Conv1d(32*scale,6,3)\n",
" self.bn_cont2 = nn.BatchNorm1d(32*scale)\n",
" self.bn_cont3 = nn.BatchNorm1d(6)\n",
"\n",
" self.conv1D_4 = nn.Conv1d(32*scale,32*scale,3)\n",
" self.bn_cont4 = nn.BatchNorm1d(32*scale)\n",
"\n",
" def forward(self, x, x_le, x_le_mix = None, lambd = None) -> torch.Tensor:\n",
" x_le = [e(x_le[:,:,i]) for i,e in enumerate(self.embeddings)]\n",
" x_le = torch.cat(x_le, 2)\n",
" \n",
" if MIXUP and x_le_mix is not None:\n",
" x_le_mix = [e(x_le_mix[:,:,i]) for i,e in enumerate(self.embeddings)]\n",
" x_le_mix = torch.cat(x_le_mix, 2)\n",
" x_le = lambd * x_le + (1-lambd) * x_le_mix\n",
" \n",
" #assert torch.isnan(x_le).any().cpu() == False\n",
" x = torch.cat([x, x_le], 2)\n",
" x = x.transpose(1,2)\n",
" \n",
" #x = torch.cat([x[:,:self.feat_sz],self.bn_cont(x[:,self.feat_sz:])], dim=1)\n",
" \n",
" x = x.reshape(x.shape[0],1,x.shape[1],x.shape[2])\n",
" x = self.fc_drop(x)\n",
" x = torch.cat([self.conv2D_1(x[:,:,:,2:(-2)]).squeeze(), \n",
" self.conv2D_2(x).squeeze()], dim=1)\n",
" x = self.relu(x)\n",
" x = self.bn_cont1(x)\n",
" x = self.fc_drop(x)\n",
" \n",
" x = torch.cat([self.conv1D_1(x[:,:,9:(-9)]),\n",
" self.conv1D_3(x)], dim=1)\n",
" x = self.relu(x)\n",
" x = self.bn_cont2(x)\n",
" x = self.fc_drop(x)\n",
" \n",
" x = self.conv1D_4(x)\n",
" x = self.relu(x)\n",
" x = self.bn_cont4(x)\n",
" \n",
" x = self.conv1D_2(x)\n",
" x = x.transpose(1,2)\n",
" \n",
" return x"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Training"
]
},
{
"cell_type": "code",
"execution_count": 96,
"metadata": {},
"outputs": [],
"source": [
"def train_loop_fn(model, loader, device, context = None):\n",
" \n",
" if CLOUD and (not CLOUD_SINGLE):\n",
" tlen = len(loader._loader._loader)\n",
" OUT_LOSS = 1000\n",
" OUT_TIME = 4\n",
" generator = loader\n",
" device_num = int(str(device)[-1])\n",
" dataset = loader._loader._loader.dataset\n",
" else:\n",
" tlen = len(loader)\n",
" OUT_LOSS = 50\n",
" OUT_TIME = 50\n",
" generator = enumerate(loader)\n",
" device_num = 1\n",
" dataset = loader.dataset\n",
" \n",
" #print('Start training {}'.format(device), 'batches', tlen)\n",
" \n",
" criterion = BCEWithLogitsLoss(weight = torch.Tensor(class_weights).to(device))\n",
" optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate, weight_decay=weight_decay, betas=(0.9, 0.99))\n",
" \n",
" model.train()\n",
" \n",
" if CLOUD and TPU:\n",
" tracker = xm.RateTracker()\n",
"\n",
" tloss = 0\n",
" tloss_count = 0\n",
" \n",
" st = time.time()\n",
" mixup_collected = False\n",
" x_le_mix = None\n",
" lambd = None\n",
" for i, (x, x_le, y, non_black) in generator:\n",
" if (not CLOUD) or CLOUD_SINGLE:\n",
" x = x.to(device)\n",
" x_le = x_le.to(device)\n",
" y = y.to(device)\n",
" non_black = non_black.to(device)\n",
" \n",
" if MIXUP:\n",
" if mixup_collected:\n",
" lambd = np.random.beta(0.4, 0.4, y.size(0))\n",
" lambd = torch.Tensor(lambd).to(device)[:,None,None]\n",
" #shuffle = torch.randperm(y.size(0)).to(device)\n",
" x = lambd * x + (1-lambd) * x_mix #x[shuffle]\n",
" #x_le = lambd * x_le + (1-lambd) * x_le_mix #x[shuffle]\n",
" mixup_collected = False\n",
" else:\n",
" x_mix = x\n",
" x_le_mix = x_le\n",
" y_mix = y\n",
" mixup_collected = True\n",
" continue\n",
" \n",
" optimizer.zero_grad()\n",
" output = model(x, x_le, x_le_mix, lambd)\n",
" \n",
" if MIXUP:\n",
" if NO_BLACK_LOSS:\n",
" loss = criterion(output, y, lambd*non_black[:,:,None]) \\\n",
" + criterion(output, y_mix, (1-lambd)*non_black[:,:,None])\n",
" else:\n",
" loss = criterion(output, y, lambd) + criterion(output, y_mix, 1-lambd)\n",
" del x_mix, y_mix\n",
" else:\n",
" if NO_BLACK_LOSS:\n",
" loss = criterion(output, y, non_black[:,:,None])\n",
" else:\n",
" loss = criterion(output, y)\n",
" \n",
" loss.backward()\n",
" \n",
" tloss += len(y)*loss.cpu().detach().item()\n",
" tloss_count += len(y)\n",
" \n",
" if (CLOUD or CLOUD_SINGLE) and TPU:\n",
" xm.optimizer_step(optimizer)\n",
" if CLOUD_SINGLE:\n",
" xm.mark_step()\n",
" else:\n",
" optimizer.step()\n",
" \n",
" if CLOUD and TPU:\n",
" tracker.add(len(y))\n",
" \n",
" st_passed = time.time() - st\n",
" if (i+1)%OUT_TIME == 0 and device_num == 1:\n",
" #print(torch_xla._XLAC._xla_metrics_report())\n",
" print('Batch {} device: {} time passed: {:.3f} time per batch: {:.3f}'\n",
" .format(i+1, device, st_passed, st_passed/(i+1)))\n",
" \n",
" del loss, output, y, x, x_le\n",
" \n",
" return tloss, tloss_count"
]
},
{
"cell_type": "code",
"execution_count": 97,
"metadata": {},
"outputs": [],
"source": [
"@torch.no_grad()\n",
"def val_loop_fn(model, loader, device, context = None):\n",
" \n",
" if CLOUD and (not CLOUD_SINGLE):\n",
" tlen = len(loader._loader._loader)\n",
" OUT_LOSS = 1000\n",
" OUT_TIME = 4\n",
" generator = loader\n",
" device_num = int(str(device)[-1])\n",
" else:\n",
" tlen = len(loader)\n",
" OUT_LOSS = 1000\n",
" OUT_TIME = 50\n",
" generator = enumerate(loader)\n",
" device_num = 1\n",
" \n",
" #print('Start validating {}'.format(device), 'batches', tlen)\n",
" \n",
" st = time.time()\n",
" model.eval()\n",
" \n",
" results = []\n",
" indices = []\n",
" offsets = []\n",
" \n",
" for i, (x, x_le, y, idx, offset) in generator:\n",
" \n",
" if (not CLOUD) or CLOUD_SINGLE:\n",
" x = x.to(device)\n",
" x_le = x_le.to(device)\n",
" \n",
" output = model(x, x_le)\n",
" assert torch.isnan(output).any().cpu() == False\n",
" output = torch.sigmoid(output)\n",
" assert torch.isnan(output).any().cpu() == False\n",
" \n",
" mask = (idx >= 0)\n",
" results.append(output[mask].cpu().detach().numpy())\n",
" indices.append(idx[mask].cpu().detach().numpy())\n",
" offsets.append(offset[mask].cpu().detach().numpy())\n",
" \n",
" st_passed = time.time() - st\n",
" if (i+1)%OUT_TIME == 0 and device_num == 1:\n",
" print('Batch {} device: {} time passed: {:.3f} time per batch: {:.3f}'\n",
" .format(i+1, device, st_passed, st_passed/(i+1)))\n",
" \n",
" del output, y, x, x_le, idx, offset\n",
" \n",
" results = np.concatenate(results)\n",
" indices = np.concatenate(indices)\n",
" offsets = np.concatenate(offsets)\n",
" \n",
" return results, indices, offsets"
]
},
{
"cell_type": "code",
"execution_count": 98,
"metadata": {},
"outputs": [],
"source": [
"@torch.no_grad()\n",
"def test_loop_fn(model, loader, device, context = None):\n",
" \n",
" if CLOUD and (not CLOUD_SINGLE):\n",
" tlen = len(loader._loader._loader)\n",
" OUT_LOSS = 1000\n",
" OUT_TIME = 100\n",
" generator = loader\n",
" device_num = int(str(device)[-1])\n",
" else:\n",
" tlen = len(loader)\n",
" OUT_LOSS = 1000\n",
" OUT_TIME = 10\n",
" generator = enumerate(loader)\n",
" device_num = 1\n",
" \n",
" #print('Start testing {}'.format(device), 'batches', tlen)\n",
" \n",
" st = time.time()\n",
" model.eval()\n",
" \n",
" results = []\n",
" indices = []\n",
" offsets = []\n",
" \n",
" for i, (x, x_le, y, idx, offset) in generator:\n",
" \n",
" if (not CLOUD) or CLOUD_SINGLE:\n",
" x = x.to(device)\n",
" x_le = x_le.to(device)\n",
" \n",
" output = torch.sigmoid(model(x, x_le))\n",
" \n",
" mask = (idx >= 0)\n",
" results.append(output[mask].cpu().detach().numpy())\n",
" indices.append(idx[mask].cpu().detach().numpy())\n",
" offsets.append(offset[mask].cpu().detach().numpy())\n",
" \n",
" st_passed = time.time() - st\n",
" if (i+1)%OUT_TIME == 0 and device_num == 1:\n",
" print('B{} -> time passed: {:.3f} time per batch: {:.3f}'.format(i+1, st_passed, st_passed/(i+1)))\n",
" \n",
" del output, x, y, idx, offset\n",
" \n",
" return np.concatenate(results), np.concatenate(indices), np.concatenate(offsets)"
]
},
{
"cell_type": "code",
"execution_count": 76,
"metadata": {},
"outputs": [],
"source": [
"def train_one(dataset, weight=None, load_model=True, epochs=1, bs=100, fold=0, init_ver=None):\n",
" \n",
" st0 = time.time()\n",
" dataset_name, filename_add, filename_add2, feat_sz,_,_,_,_ = getDSParams(dataset)\n",
" \n",
" cur_epoch = getCurrentBatch(fold=fold, dataset=dataset)\n",
" if cur_epoch is None: cur_epoch = 0\n",
" print('completed epochs:', cur_epoch, 'starting now:', epochs)\n",
" \n",
" setSeeds(SEED + cur_epoch)\n",
" \n",
" assert train_md.weights.isnull().sum() == 0\n",
" \n",
" if dataset >= 6:\n",
" trn_ds = RSNA_DataSet(train_md, mode='train', bs=bs, fold=fold, dataset=dataset)\n",
" val_ds = RSNA_DataSet(train_md, mode='valid', bs=bs, fold=fold, dataset=dataset)\n",
" else: assert False\n",
" val_ds.setFeats(0)\n",
" \n",
" if WEIGHTED:\n",
" print('WeightedRandomSampler')\n",
" sampler = D.sampler.WeightedRandomSampler(trn_ds.weights, len(trn_ds))\n",
" loader = D.DataLoader(trn_ds, num_workers=NUM_WORKERS, batch_size=bs, \n",
" shuffle=False, drop_last=True, sampler=sampler)\n",
" else:\n",
" loader = D.DataLoader(trn_ds, num_workers=NUM_WORKERS, batch_size=bs, \n",
" shuffle=True, drop_last=True)\n",
" loader_val = D.DataLoader(val_ds, num_workers=NUM_WORKERS, batch_size=bs, \n",
" shuffle=True)\n",
" print('dataset train:', len(trn_ds), 'valid:', len(val_ds), 'loader train:', len(loader), 'valid:', len(loader_val))\n",
" \n",
" #model = TabularModel(n_cont = len(meta_cols), feat_sz=feat_sz, fc_drop_p=0)\n",
" model = ResNetModel(n_cont = len(meta_cols), feat_sz=feat_sz)\n",
" \n",
" model_file_name = modelFileName(return_last=True, fold=fold, dataset=dataset)\n",
" \n",
" if (model_file_name is None) and (init_ver is not None):\n",
" model_file_name = modelFileName(return_last=True, fold=fold, dataset=dataset, ver=init_ver)\n",
" \n",
" if model_file_name is not None:\n",
" print('loading model', model_file_name)\n",
" state_dict = torch.load(PATH_DISK/'models'/model_file_name)\n",
" model.load_state_dict(state_dict)\n",
" else:\n",
" print('starting from scratch')\n",
" \n",
" if (not CLOUD) or CLOUD_SINGLE:\n",
" model = model.to(device)\n",
" else:\n",
" model_parallel = dp.DataParallel(model, device_ids=devices)\n",
" \n",
" loc_data = val_ds.metadata.copy()\n",
" \n",
" if DATA_SMALL:\n",
" val_sz = len(val_ds)\n",
" val_series = val_ds.series[:val_sz]\n",
" loc_data = loc_data.loc[loc_data.index.isin(val_series)]\n",
" \n",
" series_counts = loc_data.index.value_counts()\n",
" \n",
" loc_data['orig_idx'] = np.arange(len(loc_data))\n",
" loc_data = loc_data.sort_values(['SeriesInstanceUID','pos_idx1'])\n",
" loc_data['my_order'] = np.arange(len(loc_data))\n",
" loc_data = loc_data.sort_values(['orig_idx'])\n",
" \n",
" ww_val = loc_data.weights\n",
" \n",
" for i in range(cur_epoch+1, cur_epoch+epochs+1):\n",
" st = time.time()\n",
" \n",
" #trn_ds.setFeats((i-1) % 4)\n",
" trn_ds.setFeats(-1, epoch=i)\n",
" \n",
" if CLOUD and (not CLOUD_SINGLE):\n",
" results = model_parallel(train_loop_fn, loader)\n",
" tloss, tloss_count = np.stack(results).sum(0)\n",
" state_dict = model_parallel._models[0].state_dict()\n",
" else:\n",
" tloss, tloss_count = train_loop_fn(model, loader, device)\n",
" state_dict = model.state_dict()\n",
" \n",
" state_dict = {k:v.to('cpu') for k,v in state_dict.items()}\n",
" tr_ll = tloss / tloss_count\n",
" \n",
" train_time = time.time()-st\n",
" \n",
" model_file_name = modelFileName(return_next=True, fold=fold, dataset=dataset)\n",
" if not DATA_SMALL:\n",
" torch.save(state_dict, PATH_DISK/'models'/model_file_name)\n",
" \n",
" st = time.time()\n",
" if CLOUD and (not CLOUD_SINGLE):\n",
" results = model_parallel(val_loop_fn, loader_val)\n",
" predictions = np.concatenate([results[i][0] for i in range(MAX_DEVICES)])\n",
" indices = np.concatenate([results[i][1] for i in range(MAX_DEVICES)])\n",
" offsets = np.concatenate([results[i][2] for i in range(MAX_DEVICES)])\n",
" else:\n",
" predictions, indices, offsets = val_loop_fn(model, loader_val, device)\n",
" \n",
" predictions = predictions[np.argsort(indices)]\n",
" offsets = offsets[np.argsort(indices)]\n",
" assert len(predictions) == len(loc_data.index.unique())\n",
" assert len(predictions) == len(offsets)\n",
" assert np.all(indices[np.argsort(indices)] == np.array(range(len(predictions))))\n",
" \n",
" #val_results = np.zeros((len(loc_data),6))\n",
" val_results = []\n",
" for k, series in enumerate(np.sort(loc_data.index.unique())):\n",
" cnt = series_counts[series]\n",
" #mask = loc_data.SeriesInstanceUID == series\n",
" assert (offsets[k] + cnt) <= 60\n",
" #val_results[mask] = predictions[k,offsets[k]:(offsets[k] + cnt)]\n",
" val_results.append(predictions[k,offsets[k]:(offsets[k] + cnt)])\n",
" \n",
" val_results = np.concatenate(val_results)\n",
" assert np.isnan(val_results).sum() == 0\n",
" val_results = val_results[loc_data.my_order]\n",
" assert np.isnan(val_results).sum() == 0\n",
" assert len(val_results) == len(loc_data)\n",
" \n",
" lls = [log_loss(loc_data.loc[~loc_data.test, all_ich[k]].values, val_results[~loc_data.test,k], \\\n",
" eps=1e-7, labels=[0,1]) for k in range(6)]\n",
" ll = (class_weights * np.array(lls)).mean()\n",
" cor = np.corrcoef(loc_data.loc[~loc_data.test,all_ich].values.reshape(-1), \\\n",
" val_results[~loc_data.test].reshape(-1))[0,1]\n",
" auc = roc_auc_score(loc_data.loc[~loc_data.test, all_ich].values.reshape(-1), \\\n",
" val_results[~loc_data.test].reshape(-1))\n",
" \n",
" lls_w = [log_loss(loc_data.loc[~loc_data.test, all_ich[k]].values, val_results[~loc_data.test,k], eps=1e-7, \n",
" labels=[0,1], sample_weight=ww_val[~loc_data.test]) for k in range(6)]\n",
" ll_w = (class_weights * np.array(lls_w)).mean()\n",
" \n",
" if TRAIN_ON_STAGE_1:\n",
" lls = [log_loss(loc_data.loc[loc_data.test, all_ich[k]].values, val_results[loc_data.test,k], \\\n",
" eps=1e-7, labels=[0,1]) for k in range(6)]\n",
" ll2 = (class_weights * np.array(lls)).mean()\n",
"\n",
" lls_w = [log_loss(loc_data.loc[loc_data.test, all_ich[k]].values, val_results[loc_data.test,k], eps=1e-7, \n",
" labels=[0,1], sample_weight=ww_val[loc_data.test]) for k in range(6)]\n",
" ll2_w = (class_weights * np.array(lls_w)).mean()\n",
" else:\n",
" ll2 = 0\n",
" ll2_w = 0\n",
" \n",
" print('v{}, d{}, e{}, f{}, trn ll: {:.4f}, val ll: {:.4f}, ll_w: {:.4f}, cor: {:.4f}, auc: {:.4f}, lr: {}'\\\n",
" .format(VERSION, dataset, i, fold, tr_ll, ll, ll_w, cor, auc, learning_rate))\n",
" valid_time = time.time()-st\n",
" \n",
" epoch_stats = pd.DataFrame([[VERSION, dataset, i, fold, tr_ll, ll, ll_w, ll2, ll2_w, cor, \n",
" lls[0], lls[1], lls[2], lls[3], \n",
" lls[4], lls[5], len(trn_ds), len(val_ds), bs, train_time, valid_time,\n",
" learning_rate, weight_decay]],\n",
" columns = \n",
" ['ver','dataset','epoch','fold','train_loss','val_loss','val_w_loss',\n",
" 'val_loss2','val_w_loss2','cor',\n",
" 'any','epidural','intraparenchymal','intraventricular','subarachnoid','subdural',\n",
" 'train_sz','val_sz','bs','train_time','valid_time','lr','wd'\n",
" ])\n",
"\n",
" stats_filename = PATH_WORK/'stats.f{}.v{}'.format(fold,VERSION)\n",
" if stats_filename.is_file():\n",
" epoch_stats = pd.concat([pd.read_csv(stats_filename), epoch_stats], sort=False)\n",
" #if not DATA_SMALL:\n",
" epoch_stats.to_csv(stats_filename, index=False)\n",
" \n",
" print('total running time', time.time() - st0)\n",
" \n",
" return model, predictions, val_results"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# OOF"
]
},
{
"cell_type": "code",
"execution_count": 100,
"metadata": {},
"outputs": [],
"source": [
"def oof_one(dataset, num_iter=1, bs=100, fold=0):\n",
" \n",
" st0 = time.time()\n",
" dataset_name, filename_add, filename_add2, feat_sz,_,_,_,_ = getDSParams(dataset)\n",
" \n",
" cur_epoch = getCurrentBatch(fold=fold, dataset=dataset)\n",
" if cur_epoch is None: cur_epoch = 0\n",
" print('completed epochs:', cur_epoch, 'iters starting now:', num_iter)\n",
" \n",
" setSeeds(SEED + cur_epoch)\n",
" \n",
" val_ds = RSNA_DataSet(train_md, mode='valid', bs=bs, fold=fold, dataset=dataset)\n",
" \n",
" loader_val = D.DataLoader(val_ds, num_workers=NUM_WORKERS, batch_size=bs, \n",
" shuffle=True)\n",
" print('dataset valid:', len(val_ds), 'loader valid:', len(loader_val))\n",
" \n",
" #model = TabularModel(n_cont = len(meta_cols), feat_sz=feat_sz, fc_drop_p=0)\n",
" model = ResNetModel(n_cont = len(meta_cols), feat_sz=feat_sz)\n",
" \n",
" model_file_name = modelFileName(return_last=True, fold=fold, dataset=dataset)\n",
" if model_file_name is not None:\n",
" print('loading model', model_file_name)\n",
" state_dict = torch.load(PATH_DISK/'models'/model_file_name)\n",
" model.load_state_dict(state_dict)\n",
" else:\n",
" print('starting from scratch')\n",
" \n",
" if (not CLOUD) or CLOUD_SINGLE:\n",
" model = model.to(device)\n",
" else:\n",
" model_parallel = dp.DataParallel(model, device_ids=devices)\n",
" \n",
" loc_data = val_ds.metadata.copy()\n",
" series_counts = loc_data.index.value_counts()\n",
" \n",
" loc_data['orig_idx'] = np.arange(len(loc_data))\n",
" loc_data = loc_data.sort_values(['SeriesInstanceUID','pos_idx1'])\n",
" loc_data['my_order'] = np.arange(len(loc_data))\n",
" loc_data = loc_data.sort_values(['orig_idx'])\n",
" \n",
" preds = []\n",
" \n",
" for i in range(num_iter):\n",
" \n",
" val_ds.setFeats(-1, i+100)\n",
" \n",
" if CLOUD and (not CLOUD_SINGLE):\n",
" results = model_parallel(val_loop_fn, loader_val)\n",
" predictions = np.concatenate([results[i][0] for i in range(MAX_DEVICES)])\n",
" indices = np.concatenate([results[i][1] for i in range(MAX_DEVICES)])\n",
" offsets = np.concatenate([results[i][2] for i in range(MAX_DEVICES)])\n",
" else:\n",
" predictions, indices, offsets = val_loop_fn(model, loader_val, device)\n",
"\n",
" predictions = predictions[np.argsort(indices)]\n",
" offsets = offsets[np.argsort(indices)]\n",
" assert len(predictions) == len(loc_data.index.unique())\n",
" assert len(predictions) == len(offsets)\n",
" assert np.all(indices[np.argsort(indices)] == np.array(range(len(predictions))))\n",
"\n",
" val_results = []\n",
" for k, series in enumerate(np.sort(loc_data.index.unique())):\n",
" cnt = series_counts[series]\n",
" assert (offsets[k] + cnt) <= 60\n",
" val_results.append(predictions[k,offsets[k]:(offsets[k] + cnt)])\n",
"\n",
" val_results = np.concatenate(val_results)\n",
" assert np.isnan(val_results).sum() == 0\n",
" val_results = val_results[loc_data.my_order]\n",
" assert np.isnan(val_results).sum() == 0\n",
" assert len(val_results) == len(loc_data)\n",
" \n",
" preds.append(val_results)\n",
"\n",
" lls = [log_loss(loc_data[all_ich[k]].values, val_results[:,k], eps=1e-7, labels=[0,1])\\\n",
" for k in range(6)]\n",
" ll = (class_weights * np.array(lls)).mean()\n",
" cor = np.corrcoef(loc_data.loc[:,all_ich].values.reshape(-1), val_results.reshape(-1))[0,1]\n",
" auc = roc_auc_score(loc_data.loc[:,all_ich].values.reshape(-1), val_results.reshape(-1))\n",
"\n",
" print('ver {}, iter {}, fold {}, val ll: {:.4f}, cor: {:.4f}, auc: {:.4f}'\n",
" .format(VERSION, i, fold, ll, cor, auc))\n",
" \n",
" print('total running time', time.time() - st0)\n",
" \n",
" return np.stack(preds)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"def predBounding(pp, target=None):\n",
" if target is not None:\n",
" ll = ((- target * np.log(pp.mean(0)) - (1 - target) * np.log(np.clip(1 - pp.mean(0),1e-15,1-1e-15)))\n",
" * class_weights).mean()\n",
" print('initial score', ll)\n",
" \n",
" print('any too low inconsistencies')\n",
" for i in range(1,6):\n",
" print(i, 'class:', (pp[...,0] < pp[...,i]).mean())\n",
" print('total', (pp[...,0] < pp[...,1:].max(-1)).mean())\n",
" \n",
" max_vals = pp[...,1:].max(-1)\n",
" mask = pp[...,0] < max_vals\n",
" pp[mask,0] = max_vals[mask]\n",
" #mask_vals = 0.5*(preds_all[:,:,:,0] + max_vals)[mask]\n",
" #preds_all[mask,0] = mask_vals\n",
" #preds_all[mask] = np.clip(preds_all[mask],0,np.expand_dims(mask_vals,1))\n",
"\n",
" assert (pp[...,0] < pp[...,1:].max(-1)).sum() == 0\n",
"\n",
" if target is not None:\n",
" ll = ((- target * np.log(pp.mean(0)) - (1 - target) * np.log(np.clip(1 - pp.mean(0),1e-15,1-1e-15)))\n",
" * class_weights).mean()\n",
" print('any too low corrected score', ll)\n",
" \n",
" print('any too high inconsistencies')\n",
" mask = pp[...,0] > pp[...,1:].sum(-1)\n",
" print('total', mask.mean())\n",
"\n",
" mask_val = 0.5*(pp[mask,0] + pp[...,1:].sum(-1)[mask])\n",
" scaler = mask_val / pp[...,1:].sum(-1)[mask]\n",
" pp[mask,1:] = pp[mask,1:] * np.expand_dims(scaler,1)\n",
" pp[mask,0] = mask_val\n",
"\n",
" if target is not None:\n",
" ll = ((- target * np.log(pp.mean(0)) - (1 - target) * np.log(np.clip(1 - pp.mean(0),1e-15,1-1e-15)))\n",
" * class_weights).mean()\n",
" print('any too high corrected score', ll)\n",
" \n",
" return pp"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Selecting runs aggregation"
]
},
{
"cell_type": "code",
"execution_count": 26,
"metadata": {},
"outputs": [],
"source": [
"def scalePreds(x, power = 2, center=0.5):\n",
" res = x.copy()\n",
" res[x > center] = center + (1 - center) * ((res[x > center] - center)/(1 - center))**power\n",
" res[x < center] = center - center * ((center - res[x < center])/center)**power\n",
" return res\n",
"\n",
"def getPredsOOF(ver,aug=32,datasets=range(6,11),datasets5=range(11,13)):\n",
" preds_all = np.zeros((len(datasets) + len(datasets5),aug,len(train_md),6))\n",
" \n",
" if len(datasets) > 0:\n",
" for fold in range(3):\n",
" preds = np.stack([pickle.load(open(PATH_DISK/'ensemble/oof_d{}_f{}_v{}'\n",
" .format(ds, fold, ver),'rb')) for ds in datasets])\n",
" preds_all[:len(datasets),:,train_md.fold == fold,:] = preds\n",
" \n",
" if len(datasets5) > 0:\n",
" for fold in range(5):\n",
" preds = np.stack([pickle.load(open(PATH_DISK/'ensemble/oof_d{}_f{}_v{}'\n",
" .format(ds, fold, ver),'rb')) for ds in datasets5])\n",
" preds_all[len(datasets):,:,train_md.fold5 == fold,:] = preds\n",
" \n",
" preds_all = np.clip(preds_all, 1e-15, 1-1e-15)\n",
" return preds_all"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"def getYuvalOOF(names, train_md, names5=None, aug=32):\n",
" \n",
" for num_folds in [3,5]:\n",
" if ('yuval_idx' not in train_md.columns) or ('yuval_idx5' not in train_md.columns):\n",
" print('adding yuval_idx')\n",
" fn = 'OOF_validation_image_ids'\n",
" if num_folds == 5:\n",
" fn += '_5_stage2'\n",
" if num_folds == 3:\n",
" fn += '_3_stage2'\n",
" yuval_oof = pickle.load(open(PATH_DISK/'yuval/OOF_stage2/{}.pkl'.format(fn),'rb'))\n",
" \n",
" df2 = pd.DataFrame()\n",
" col_name = 'yuval_idx'\n",
" fold_col = 'fold'\n",
" if num_folds == 5:\n",
" col_name += '5'\n",
" fold_col += '5'\n",
" \n",
" for fold in range(num_folds):\n",
" \n",
" assert np.all(train_md.loc[train_md.img_id.isin(yuval_oof[fold])][fold_col].values == fold)\n",
" \n",
" df = pd.DataFrame(np.arange(len(yuval_oof[fold])), columns=[col_name])\n",
" df.index = yuval_oof[fold]\n",
" df2 = pd.concat([df2,df],sort=False)\n",
" \n",
" train_md = train_md.join(df2, on = 'img_id')\n",
" assert train_md[col_name].isnull().sum() == 0\n",
" \n",
" preds_y = np.zeros((len(names),len(train_md),6))\n",
" \n",
" for fold in range(3):\n",
" preds = np.stack([torch.sigmoid(torch.stack(\n",
" pickle.load(open(PATH_DISK/'yuval/OOF_stage2'/name.format(fold),'rb')))).numpy() for name in names])\n",
" preds = preds[:,:,train_md.loc[train_md.fold == fold, 'yuval_idx']]\n",
" \n",
" preds_y[:,train_md.fold == fold] = preds.mean(1)\n",
" \n",
" if names5 is not None:\n",
" preds_y5 = np.zeros((len(names5),len(train_md),6))\n",
" for fold in range(5):\n",
" preds = np.stack([torch.sigmoid(torch.stack(\n",
" pickle.load(open(PATH_DISK/'yuval/OOF_stage2'/name.format(fold),'rb')))).numpy() for name in names5])\n",
" preds = preds[:,:,train_md.loc[train_md.fold5 == fold, 'yuval_idx5']]\n",
"\n",
" preds_y5[:,train_md.fold5 == fold] = preds.mean(1)\n",
" \n",
" preds_y = np.concatenate([preds_y, preds_y5], axis=0)\n",
" \n",
" \n",
" preds_y = np.clip(preds_y, 1e-15, 1-1e-15)[:,:,np.array([5,0,1,2,3,4])]\n",
" return preds_y"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Inference"
]
},
{
"cell_type": "code",
"execution_count": 102,
"metadata": {},
"outputs": [],
"source": [
"def inference_one(dataset, bs = 100, add_seed = 0, fold = 0, anum = 0):\n",
" \n",
" st = time.time()\n",
" dataset_name, filename_add, filename_add2, feat_sz,_,_,_,_ = getDSParams(dataset)\n",
"\n",
" cur_epoch = getCurrentBatch(fold=fold, dataset=dataset)\n",
" if cur_epoch is None: cur_epoch = 0\n",
" print('completed epochs:', cur_epoch)\n",
"\n",
" #model = TabularModel(n_cont = len(meta_cols), feat_sz=feat_sz)\n",
" model = ResNetModel(n_cont = len(meta_cols), feat_sz=feat_sz)\n",
" \n",
" model_file_name = modelFileName(return_last=True, fold=fold, dataset=dataset)\n",
" if model_file_name is not None:\n",
" print('loading model', model_file_name)\n",
" state_dict = torch.load(PATH_DISK/'models'/model_file_name)\n",
" model.load_state_dict(state_dict)\n",
" \n",
" if (not CLOUD) or CLOUD_SINGLE:\n",
" model = model.to(device)\n",
" else:\n",
" model_parallel = dp.DataParallel(model, device_ids=devices)\n",
"\n",
" setSeeds(SEED + cur_epoch + anum + 100*fold)\n",
"\n",
" tst_ds = RSNA_DataSet(test_md, mode='test', bs=bs, fold=fold, dataset=dataset)\n",
" loader_tst = D.DataLoader(tst_ds, num_workers=NUM_WORKERS, batch_size=bs, shuffle=False)\n",
" print('dataset test:', len(tst_ds), 'loader test:', len(loader_tst), 'anum:', anum)\n",
" \n",
" tst_ds.setFeats(-1, epoch = anum+100)\n",
"\n",
" loc_data = tst_ds.metadata.copy()\n",
" series_counts = loc_data.index.value_counts()\n",
"\n",
" loc_data['orig_idx'] = np.arange(len(loc_data))\n",
" loc_data = loc_data.sort_values(['SeriesInstanceUID','pos_idx1'])\n",
" loc_data['my_order'] = np.arange(len(loc_data))\n",
" loc_data = loc_data.sort_values(['orig_idx'])\n",
" \n",
" if CLOUD and (not CLOUD_SINGLE):\n",
" results = model_parallel(test_loop_fn, loader_tst)\n",
" predictions = np.concatenate([results[i][0] for i in range(MAX_DEVICES)])\n",
" indices = np.concatenate([results[i][1] for i in range(MAX_DEVICES)])\n",
" offsets = np.concatenate([results[i][2] for i in range(MAX_DEVICES)])\n",
" else:\n",
" predictions, indices, offsets = test_loop_fn(model, loader_tst, device)\n",
"\n",
" predictions = predictions[np.argsort(indices)]\n",
" offsets = offsets[np.argsort(indices)]\n",
" assert len(predictions) == len(test_md.SeriesInstanceUID.unique())\n",
" assert np.all(indices[np.argsort(indices)] == np.array(range(len(predictions))))\n",
" \n",
" val_results = []\n",
" for k, series in enumerate(np.sort(loc_data.index.unique())):\n",
" cnt = series_counts[series]\n",
" assert (offsets[k] + cnt) <= 60\n",
" val_results.append(predictions[k,offsets[k]:(offsets[k] + cnt)])\n",
"\n",
" val_results = np.concatenate(val_results)\n",
" assert np.isnan(val_results).sum() == 0\n",
" val_results = val_results[loc_data.my_order]\n",
" assert len(val_results) == len(loc_data)\n",
"\n",
" print('test processing time:', time.time() - st)\n",
" \n",
" return val_results"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Ensembling"
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {
"scrolled": true
},
"outputs": [],
"source": [
"def getStepX(train_md, preds_all, fold=0, target=0, mode='train'):\n",
" \n",
" print('my_len', my_len)\n",
" X = np.stack([preds_all[:my_len,:,target].mean(0), \n",
" preds_all[my_len:,:,target].mean(0)], axis=0)\n",
" \n",
" if mode == 'train':\n",
" X = X[:,train_md.fold != fold]\n",
" y = train_md.loc[train_md.fold != fold, all_ich[target]].values\n",
" ww = train_md.loc[train_md.fold != fold, 'weights'].values\n",
" elif mode == 'valid':\n",
" X = X[:,train_md.fold == fold]\n",
" y = train_md.loc[train_md.fold == fold, all_ich[target]].values\n",
" ww = train_md.loc[train_md.fold == fold, 'weights'].values\n",
" else:\n",
" X = X\n",
" y = None\n",
" \n",
" ll = None\n",
" llw = None\n",
" auc = None\n",
" if y is not None:\n",
" ll = log_loss(y, X.mean(0), eps=1e-7, labels=[0,1])\n",
" llw = log_loss(y, X.mean(0), eps=1e-7, labels=[0,1], sample_weight=ww)\n",
" auc = roc_auc_score(y, X.mean(0))\n",
" \n",
" return X, y, ww, ll, llw, auc\n",
"\n",
"\n",
"def train_ensemble(train_md, preds_all, fold = 0, target = 0, weighted = False):\n",
" \n",
" print('starting fold',fold,'target',target)\n",
" \n",
" st = time.time()\n",
" \n",
" limit_low = 1e-15\n",
" limit_high = 1 - 1e-5\n",
" \n",
" prior = train_md.loc[train_md.fold != fold, all_ich[target]].mean()\n",
" \n",
" def my_objective(x,preds,vals,ww):\n",
" preds_sum = np.clip((preds * x[:,None]).sum(0), limit_low, limit_high)\n",
" res = np.average(- vals * np.log(preds_sum) - (1 - vals) * np.log(1 - preds_sum), weights=ww)\n",
" #print('x ',x, x.sum())\n",
" print('obj ',res)\n",
" return res\n",
"\n",
" def my_grad(x,preds,vals,ww):\n",
" preds_sum = np.clip((preds * x[:,None]).sum(0), limit_low, limit_high)\n",
" res = np.average(- vals * preds / preds_sum + (1 - vals) * preds / (1 - preds_sum), weights=ww, axis=1)\n",
" #print('grad',res)\n",
" return res\n",
"\n",
" def my_hess(x,preds,vals,ww):\n",
" preds_sum = np.clip((preds * x[:,None]).sum(0), limit_low, limit_high)\n",
" res = np.average(preds * np.expand_dims(preds, axis=1) * \n",
" (vals / preds_sum**2 + (1 - vals) / (1 - preds_sum)**2), weights=ww, axis=2)\n",
" return res\n",
" \n",
" X,y,ww,ll_train,llw_train,auc_train = getStepX(train_md, preds_all, fold=fold, target=target)\n",
" \n",
" bnds_low = np.zeros(X.shape[0])\n",
" bnds_high = np.ones(X.shape[0])\n",
" \n",
" initial_sol = np.ones(X.shape[0])/X.shape[0]\n",
" \n",
" bnds = sp.optimize.Bounds(bnds_low, bnds_high)\n",
" cons = sp.optimize.LinearConstraint(np.ones((1,X.shape[0])), 0.98, 1.00)\n",
" \n",
" if weighted:\n",
" ww_in = ww\n",
" else:\n",
" ww_in = np.ones(len(y))\n",
" \n",
" model = sp.optimize.minimize(my_objective, initial_sol, jac=my_grad, hess=my_hess, args=(X, y, ww_in),\n",
" bounds=bnds, method='trust-constr', constraints=cons,\n",
" options={'gtol': 1e-11, 'initial_tr_radius': 0.1, 'initial_barrier_parameter': 0.01})\n",
" \n",
" pickle.dump(model, open(PATH_DISK/'ensemble'/'model.f{}.t{}.v{}'\n",
" .format(fold,target,VERSION),'wb'))\n",
"\n",
" print('model', model.x, 'sum', model.x.sum())\n",
" \n",
" train_preds = (X*np.expand_dims(model.x, axis=1)).sum(0)\n",
" ll_train2 = log_loss(y, train_preds, eps=1e-7, labels=[0,1])\n",
" llw_train2 = log_loss(y, train_preds, eps=1e-7, labels=[0,1], sample_weight=ww)\n",
" auc_train2 = roc_auc_score(y, train_preds)\n",
" \n",
" X,y,ww,ll_val,llw_val,auc_val = getStepX(train_md, preds_all, fold=fold, target=target, mode='valid')\n",
" \n",
" val_preds = (X*np.expand_dims(model.x, axis=1)).sum(0)\n",
" \n",
" ll_val2 = log_loss(y, val_preds, eps=1e-7, labels=[0,1])\n",
" llw_val2 = log_loss(y, val_preds, eps=1e-7, labels=[0,1], sample_weight=ww)\n",
" auc_val2 = roc_auc_score(y, val_preds)\n",
" \n",
" print('v{} f{} t{}: original ll {:.4f}/{:.4f}, ensemble ll {:.4f}/{:.4f}'\n",
" .format(VERSION,fold,target,ll_val,llw_val,ll_val2,llw_val2))\n",
" \n",
" run_time = time.time() - st\n",
" print('running time', run_time)\n",
" \n",
" stats = pd.DataFrame([[VERSION,fold,target,\n",
" ll_train,auc_train,ll_train2,auc_train2,\n",
" ll_val,auc_val,ll_val2,auc_val2,\n",
" llw_train,llw_train2,llw_val,llw_val2,\n",
" run_time,weighted]],\n",
" columns = \n",
" ['version','fold','target',\n",
" 'train_loss','train_auc','train_loss_ens','train_auc_ens', \n",
" 'valid_loss','valid_auc','valid_loss_ens','valid_auc_ens',\n",
" 'train_w_loss','train_w_loss_ens','valid_w_loss','valid_w_loss_ens',\n",
" 'run_time','weighted'\n",
" ])\n",
" \n",
" stats_filename = PATH_DISK/'ensemble'/'stats.v{}'.format(VERSION)\n",
" if stats_filename.is_file():\n",
" stats = pd.concat([pd.read_csv(stats_filename), stats], sort=False)\n",
" stats.to_csv(stats_filename, index=False)\n",
"\n",
"#model.cols = Xt.columns\n",
"#predictions[data_filt['fold'] == i] = (Xv*model.x).sum(1)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.4"
}
},
"nbformat": 4,
"nbformat_minor": 2
}