Datasets
Models
Downloads: 1
Diff of
/3a-L1-train-and-generate-predictions-fastai_v1.ipynb
[000000]
..
[fc9ccf]
Switch to side-by-side view
--- a +++ b/3a-L1-train-and-generate-predictions-fastai_v1.ipynb @@ -0,0 +1,815 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "#os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"2\"" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "stage = \"stage_2\"" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "experiment_name = 'NONAMED'" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "import fastai\n", + "from fastai.vision import *\n", + "from fastai.callbacks import *\n", + "from fastai.distributed import *\n", + "from fastai.utils.mem import *\n", + "import re\n", + "import pydicom\n", + "import pdb\n", + "import pickle\n", + "import torch\n" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "densenet121-53_train_cpu_window_uint8_sz512_cv0.0386_subdural_loss_fold5_of_5.pth\r\n", + "densenet121-53_train_cpu_window_uint8_sz512_cv0.0399_subdural_loss_fold4_of_5.pth\r\n", + "densenet121-53_train_cpu_window_uint8_sz512_cv0.0403_subdural_loss_fold3_of_5.pth\r\n", + "densenet121-53_train_cpu_window_uint8_sz512_cv0.0406_subdural_loss_fold2_of_5.pth\r\n", + "densenet121-53_train_cpu_window_uint8_sz512_cv0.0413_subdural_loss_fold1_of_5.pth\r\n", + "densenet121_sz512_cv0.0738_weighted_loss_fold4_of_5.pth\r\n", + "densenet121_sz512_cv0.0739_weighted_loss_fold5_of_5.pth\r\n", + "densenet121_sz512_cv0.0743_weighted_loss_fold2_of_5.pth\r\n", + "densenet121_sz512_cv0.0748_weighted_loss_fold3_of_5.pth\r\n", + "densenet121_sz512_cv0.0749_weighted_loss_fold1_of_5.pth\r\n", + "resnet101-53_train_cpu_window_uint8_sz512_cv0.0365_subdural_loss_fold5_of_5.pth\r\n", + "resnet101-53_train_cpu_window_uint8_sz512_cv0.0366_subdural_loss_fold4_of_5.pth\r\n", + "resnet101-53_train_cpu_window_uint8_sz512_cv0.0369_subdural_loss_fold2_of_5.pth\r\n", + "resnet101-53_train_cpu_window_uint8_sz512_cv0.0371_subdural_loss_fold3_of_5.pth\r\n", + "resnet101-53_train_cpu_window_uint8_sz512_cv0.0375_subdural_loss_fold1_of_5.pth\r\n", + "resnet101_sz512_cv0.0749_weighted_loss_fold5_of_5.pth\r\n", + "resnet101_sz512_cv0.0751_weighted_loss_fold4_of_5.pth\r\n", + "resnet101_sz512_cv0.0754_weighted_loss_fold2_of_5.pth\r\n", + "resnet101_sz512_cv0.0758_weighted_loss_fold1_of_5.pth\r\n", + "resnet101_sz512_cv0.0761_weighted_loss_fold3_of_5.pth\r\n", + "resnet18-53_train_cpu_window_uint8_sz512_cv0.0522_subdural_loss_fold5_of_5.pth\r\n", + "resnet18-53_train_cpu_window_uint8_sz512_cv0.0528_subdural_loss_fold3_of_5.pth\r\n", + "resnet18-53_train_cpu_window_uint8_sz512_cv0.0530_subdural_loss_fold4_of_5.pth\r\n", + "resnet18-53_train_cpu_window_uint8_sz512_cv0.0550_subdural_loss_fold2_of_5.pth\r\n", + "resnet18-53_train_cpu_window_uint8_sz512_cv0.0556_subdural_loss_fold1_of_5.pth\r\n", + "resnet18_sz512_cv0.0762_weighted_loss_fold4_of_5.pth\r\n", + "resnet18_sz512_cv0.0768_weighted_loss_fold5_of_5.pth\r\n", + "resnet18_sz512_cv0.0773_weighted_loss_fold2_of_5.pth\r\n", + "resnet18_sz512_cv0.0786_weighted_loss_fold1_of_5.pth\r\n", + "resnet18_sz512_cv0.0786_weighted_loss_fold3_of_5.pth\r\n", + "resnet34-53_train_cpu_window_uint8_sz512_cv0.0453_subdural_loss_fold5_of_5.pth\r\n", + "resnet34-53_train_cpu_window_uint8_sz512_cv0.0466_subdural_loss_fold4_of_5.pth\r\n", + "resnet34-53_train_cpu_window_uint8_sz512_cv0.0474_subdural_loss_fold3_of_5.pth\r\n", + "resnet34-53_train_cpu_window_uint8_sz512_cv0.0484_subdural_loss_fold2_of_5.pth\r\n", + "resnet34-53_train_cpu_window_uint8_sz512_cv0.0496_subdural_loss_fold1_of_5.pth\r\n", + "resnet34_sz512_cv0.0750_weighted_loss_fold4_of_5.pth\r\n", + "resnet34_sz512_cv0.0759_weighted_loss_fold5_of_5.pth\r\n", + "resnet34_sz512_cv0.0763_weighted_loss_fold2_of_5.pth\r\n", + "resnet34_sz512_cv0.0765_weighted_loss_fold1_of_5.pth\r\n", + "resnet34_sz512_cv0.0782_weighted_loss_fold3_of_5.pth\r\n", + "resnet50_sz512_cv0.0751_weighted_loss_fold4_of_5.pth\r\n", + "resnet50_sz512_cv0.0751_weighted_loss_fold5_of_5.pth\r\n", + "resnet50_sz512_cv0.0754_weighted_loss_fold2_of_5.pth\r\n", + "resnet50_sz512_cv0.0761_weighted_loss_fold1_of_5.pth\r\n", + "resnet50_sz512_cv0.0764_weighted_loss_fold3_of_5.pth\r\n", + "resnext50_32x4d-53_train_cpu_window_uint8_sz512_cv0.0372_subdural_loss_fold5_of_5.pth\r\n", + "resnext50_32x4d-53_train_cpu_window_uint8_sz512_cv0.0376_subdural_loss_fold4_of_5.pth\r\n", + "resnext50_32x4d-53_train_cpu_window_uint8_sz512_cv0.0378_subdural_loss_fold2_of_5.pth\r\n", + "resnext50_32x4d-53_train_cpu_window_uint8_sz512_cv0.0381_subdural_loss_fold3_of_5.pth\r\n", + "resnext50_32x4d-53_train_cpu_window_uint8_sz512_cv0.0387_subdural_loss_fold1_of_5.pth\r\n" + ] + } + ], + "source": [ + "!ls models" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "arch = 'resnet18'\n", + "fold=4\n", + "lr = 1e-3\n", + "\n", + "model_fn = None\n", + "SZ = 512\n", + "n_folds=5\n", + "n_epochs = 15\n", + "n_tta = 10\n", + "\n", + "#model_fn = 'resnet34_sz512_cv0.0821_weighted_loss_fold1_of_5'\n", + "\n", + "if model_fn is not None:\n", + " model_fn_fold = int(model_fn[-6])-1\n", + " assert model_fn_fold == fold" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "data_dir = Path('data/unzip')\n", + "train_dir = data_dir / f'{stage}_train_images'\n", + "test_dir = data_dir / f'{stage}_test_images'" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "ename": "FileNotFoundError", + "evalue": "[Errno 2] No such file or directory: 'data/stage_2_fn_to_study_ix.pickle'", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mFileNotFoundError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m<ipython-input-8-6207314e915b>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mfn_to_study_ix\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpickle\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf'data/{stage}_fn_to_study_ix.pickle'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'rb'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 2\u001b[0m \u001b[0mstudy_ix_to_fn\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpickle\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf'data/{stage}_study_ix_to_fn.pickle'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'rb'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 3\u001b[0m \u001b[0mfn_to_labels\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpickle\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload\u001b[0m\u001b[0;34m(\u001b[0m \u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf'data/{stage}_train_fn_to_labels.pickle'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'rb'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 4\u001b[0m \u001b[0mstudy_to_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpickle\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf'data/{stage}_study_to_data.pickle'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'rb'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: 'data/stage_2_fn_to_study_ix.pickle'" + ] + } + ], + "source": [ + "fn_to_study_ix = pickle.load(open(f'data/{stage}_fn_to_study_ix.pickle', 'rb'))\n", + "study_ix_to_fn = pickle.load(open(f'data/{stage}_study_ix_to_fn.pickle', 'rb'))\n", + "fn_to_labels = pickle.load( open(f'data/{stage}_train_fn_to_labels.pickle', 'rb'))\n", + "study_to_data = pickle.load (open(f'data/{stage}_study_to_data.pickle', 'rb'))\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# gets a pathlib.Path, returns labels\n", + "labels = [\n", + " 'any0', 'epidural0', 'intraparenchymal0', 'intraventricular0', 'subarachnoid0', 'subdural0',\n", + " 'any1', 'epidural1', 'intraparenchymal1', 'intraventricular1', 'subarachnoid1', 'subdural1',\n", + " 'any2', 'epidural2', 'intraparenchymal2', 'intraventricular2', 'subarachnoid2', 'subdural2',\n", + "]\n", + "\n", + "def get_labels(center_p):\n", + "\n", + " study, center_ix = fn_to_study_ix[center_p.stem]\n", + " total = len(study_ix_to_fn[study])\n", + " \n", + " # Only 1 study has 2 different RescaleIntercept values (0.0 and 1.0) in the same series, so assume they're all the\n", + " # same and do everything in the big tensor\n", + " \n", + " pixels = {}\n", + " img_labels = []\n", + " ixs = [ max(0, min(ix, total-1)) for ix in range(center_ix-1, center_ix+2) ]\n", + " for i, ix in enumerate(ixs):\n", + " fn = study_ix_to_fn[study][ix]\n", + " img_labels += [ x + str(i) for x in fn_to_labels[fn] ]\n", + " \n", + " return img_labels\n", + "\n", + "get_labels(train_dir / 'ID_ac7c8fe8b.dcm')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "sample_submission = pd.read_csv(f'data/unzip/{stage}_sample_submission.csv')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "sample_submission['fn'] = sample_submission.ID.apply(lambda x: '_'.join(x.split('_')[:2]) + '.dcm')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "sample_submission.head()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "test_fns = sample_submission.fn.unique()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "wc = 50\n", + "ww = 100" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def open_dicom(p_or_fn): # when called by .from_folder it's a Path; when called from .add_test it's a string\n", + " #pdb.set_trace()\n", + " center_p = Path(p_or_fn)\n", + "\n", + " study, center_ix = fn_to_study_ix[center_p.stem]\n", + " total = len(study_ix_to_fn[study])\n", + " \n", + " # Only 1 study has 2 different RescaleIntercept values (0.0 and 1.0) in the same series, so assume they're all the\n", + " # same and do everything in the big tensor\n", + " \n", + " pixels = {}\n", + " ixs = [ max(0, min(ix, total-1)) for ix in range(center_ix-1, center_ix+2) ]\n", + " for ix in ixs:\n", + " if ix not in pixels:\n", + " p = center_p.parent / f'{study_ix_to_fn[study][ix]}.dcm'\n", + " dcm = pydicom.dcmread(str(p))\n", + " if ix == center_ix:\n", + " rescale_slope, rescale_intercept = float(dcm.RescaleSlope), float(dcm.RescaleIntercept)\n", + " pixels[ix] = torch.FloatTensor(dcm.pixel_array.astype(np.float))\n", + " \n", + " t = torch.stack([pixels[ix] for ix in ixs], dim=0) # stack chans together\n", + " if (t.shape[1:] != (SZ,SZ)):\n", + " t = torch.nn.functional.interpolate(\n", + " t.unsqueeze_(0), size=SZ, mode='bilinear', align_corners=True).squeeze_(0) # resize\n", + " t = t * rescale_slope + rescale_intercept # rescale\n", + " t = torch.clamp(t, wc-ww/2, wc+ww/2) # window\n", + " t = (t - (wc-ww/2)) / ww # normalize\n", + " \n", + " return Image(t)\n", + "\n", + "\n", + "class DicomList(ImageList):\n", + " def open(self, fn): return open_dicom(fn)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "my_stats = ([0.45, 0.45, 0.45], [0.225, 0.225, 0.225])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "class OverSamplingCallback(LearnerCallback):\n", + " def __init__(self,learn:Learner):\n", + " super().__init__(learn)\n", + " self.labels = self.learn.data.train_dl.dataset.y.items\n", + " _, counts = np.unique(self.labels,return_counts=True)\n", + " self.weights = torch.DoubleTensor((1/counts)[self.labels])\n", + " self.label_counts = np.bincount([self.learn.data.train_dl.dataset.y[i].data for i in range(len(self.learn.data.train_dl.dataset))])\n", + " self.total_len_oversample = int(self.learn.data.c*np.max(self.label_counts))\n", + " \n", + " def on_train_begin(self, **kwargs):\n", + " self.learn.data.train_dl.dl.batch_sampler = BatchSampler(WeightedRandomSampler(weights,self.total_len_oversample), self.learn.data.train_dl.batch_size,False)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# From: https://forums.fast.ai/t/is-there-any-built-in-method-to-oversample-minority-classes/46838/5\n", + "class ImbalancedDatasetSampler(torch.utils.data.sampler.Sampler):\n", + " def __init__(self, dataset, indices=None, num_samples=None):\n", + " \n", + " # if indices is not provided, \n", + " # all elements in the dataset will be considered\n", + " self.indices = list(range(len(dataset))) if indices is None else indices\n", + " \n", + " # if num_samples is not provided, \n", + " # draw `len(indices)` samples in each iteration\n", + " self.num_samples = len(self.indices) if num_samples is None else num_samples\n", + " \n", + " # distribution of classes in the dataset \n", + " label_to_count = defaultdict(int)\n", + " for idx in self.indices:\n", + " label = self._get_label(dataset, idx)\n", + " label_to_count[label] += 1\n", + " \n", + " # weight for each sample\n", + " weights = [1.0 / label_to_count[self._get_label(dataset, idx)] for idx in self.indices]\n", + " self.weights = torch.DoubleTensor(weights)\n", + "\n", + " def _get_label(self, dataset, idx):\n", + " return 'any1' in dataset.y[idx].obj\n", + " \n", + " def __iter__(self):\n", + " return (self.indices[i] for i in torch.multinomial(\n", + " self.weights, self.num_samples, replacement=True))\n", + "\n", + " def __len__(self):\n", + " return self.num_samples" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "folds = np.array_split(list({v['fold'] for v in study_to_data.values()}), n_folds)\n", + "folds" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df = pd.read_csv(f\"data/{stage}_train_dicom_diags_norm.csv\")\n", + "# split train/val\n", + "np.random.seed(666)\n", + "studies_in_fold = [k for k,v in study_to_data.items() if np.isin(v['fold'],folds[fold])]\n", + "study_id_val_set = set(df[ df['SeriesInstanceUID'].isin(studies_in_fold)]['SOPInstanceUID'] + '.dcm')\n", + "\n", + "def is_val(p_or_fn):\n", + " p = Path(p_or_fn)\n", + " return p.name in study_id_val_set" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "tfms = (\n", + " [ flip_lr(p=0.5), rotate(degrees=(-180,180), p=1.) ], # train\n", + " [] # val\n", + ")\n", + "data = (DicomList.from_folder(f'data/unzip/{stage}_train_images/', extensions=['.dcm'], presort=True)\n", + " .split_by_valid_func(is_val)\n", + " .label_from_func(get_labels, classes=labels)\n", + " .transform(tfms, mode='nearest')\n", + " .add_test(f'data/unzip/{stage}_test_images/' + test_fns))\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "db = data.databunch().normalize()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "balsampler = ImbalancedDatasetSampler(db.train_ds)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "db.train_dl.batch_sampler = balsampler\n", + "db.train_dl.batch_sampler\n", + "db" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "db.show_batch() # AttributeError: 'list' object has no attribute 'pixel' <-- because using torch datasets or whatever" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# yuval reina's loss\n", + "yuval_weights = FloatTensor([ 2, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1 ]).cuda() \n", + "\n", + "def yuval_loss(y_pred,y_true):\n", + " return F.binary_cross_entropy_with_logits(y_pred,\n", + " y_true,\n", + " yuval_weights.repeat(y_pred.shape[0],1))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "real_lb_weights = FloatTensor([ 2, 1, 1, 1, 1, 1 ])\n", + "\n", + "def real_lb_loss(pred:Tensor, targ:Tensor)->Rank0Tensor:\n", + " pred,targ = flatten_check(pred,targ)\n", + " tp = pred.view(-1,18)[:,6:12]\n", + " tt = targ.view(-1,18)[:,6:12]\n", + " return F.binary_cross_entropy_with_logits(tp, tt, real_lb_weights.to(device=pred.device))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "w_loss = 0.1\n", + "weighted_lb_weights = FloatTensor([ 2.*w_loss, 1.*w_loss, 1.*w_loss, 1.*w_loss, 1.*w_loss, 1.*w_loss, \n", + " 2., 1., 1., 1., 1., 1., \n", + " 2.*w_loss, 1.*w_loss, 1.*w_loss, 1.*w_loss, 1.*w_loss, 1.*w_loss, ])\n", + "\n", + "def weighted_loss(pred:Tensor,targ:Tensor)->Rank0Tensor:\n", + " return F.binary_cross_entropy_with_logits(pred,targ,weighted_lb_weights.to(device=pred.device))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "lb_weights = FloatTensor([ 2, 1, 1, 1, 1, 1 ]).cuda()\n", + "\n", + "def lb_loss(pred:Tensor, targ:Tensor)->Rank0Tensor:\n", + " pred,targ = flatten_check(pred,targ)\n", + " tp = pred.view(-1,6)\n", + " tt = targ.view(-1,6)\n", + " return torch.nn.functional.binary_cross_entropy_with_logits(tp, tt, lb_weights.to(device=pred.device))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "learn = cnn_learner(\n", + " db, getattr(models, arch), loss_func=weighted_loss, metrics=[real_lb_loss],\n", + " pretrained=True, lin_ftrs=[],ps=0.,\n", + " model_dir = Path('./models/').resolve())" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "try:\n", + " learn.load(model_fn,strict=False)\n", + " print(f\"Loaded {model_fn}\")\n", + "except Exception as e:\n", + " print(e)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "learn = learn.to_parallel().to_fp16()\n", + "\n", + "arch_to_batch_factor = {\n", + " 'resnet18' : 4.3,\n", + " 'resnet34' : 3.4,\n", + " 'resnet50' : 1.5,\n", + " 'resnet101' : 1.0,\n", + " 'resnext50_32x4d': 1.2,\n", + " 'densenet121': 1.1,\n", + " 'squeezenet1_0': 2.6,\n", + " 'vgg16': 1.5\n", + "}\n", + "\n", + "bs = arch_to_batch_factor[arch] * 512*512 * gpu_mem_get()[0] * 1e-3 / (SZ*SZ)\n", + "bs *= int(int(any([isinstance(cb, MixedPrecision) for cb in learn.callbacks])))+1 # 2x if fp16\n", + "if any([isinstance(cb, fastai.distributed.ParallelTrainer) for cb in learn.callbacks]):\n", + " bs *= torch.cuda.device_count()\n", + " bs = (bs // torch.cuda.device_count()) * torch.cuda.device_count()\n", + "bs = int(bs)\n", + "learn.data.batch_size = bs\n", + "bs" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "learn" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "learn.unfreeze()\n", + "learn.summary()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "try:\n", + " if lr is None:\n", + " !rm {learn.model_dir}/tmp.pth\n", + " learn.lr_find()\n", + " learn.recorder.plot(suggestion=True)\n", + "except Exception as e:\n", + " print(e)\n", + "# set lr " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "learn.fit_one_cycle(n_epochs, lr)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Save" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "v = learn.validate()\n", + "cv = float(v[-1])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "model_fn = f'{arch}_sz{SZ}_cv{cv:0.4f}_{learn.loss_func.__name__}_fold{fold+1}_of_{n_folds}'\n", + "learn.save(model_fn)\n", + "model_fn" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Predictions" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "learn.data.test_ds.tfms = learn.data.train_ds.tfms\n", + "test_preds = []\n", + "# Fastai WTF: it figures out to run outputs through sigmoid if a standard loss error is used \n", + "# (see loss_func_name2activ and related stuff) but on custom loss funcs sigmoid must be executed explicitly:\n", + "# https://forums.fast.ai/t/shouldnt-we-able-to-pass-an-activ-function-to-learner-get-preds/50492\n", + "for _ in progress_bar(range(n_tta)):\n", + " preds, _ = learn.get_preds(DatasetType.Test, activ=torch.sigmoid)\n", + " test_preds.append(preds)\n", + "tta_test_preds = torch.cat([p.unsqueeze(0) for p in test_preds],dim=0)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "learn.data.valid_ds.tfms = learn.data.train_ds.tfms\n", + "valid_preds = []\n", + "# Fastai WTF: it figures out to run outputs through sigmoid if a standard loss error is used \n", + "# (see loss_func_name2activ and related stuff) but on custom loss funcs sigmoid must be executed explicitly:\n", + "# https://forums.fast.ai/t/shouldnt-we-able-to-pass-an-activ-function-to-learner-get-preds/50492\n", + "for _ in progress_bar(range(n_tta)):\n", + " preds, _ = learn.get_preds(DatasetType.Valid, activ=torch.sigmoid)\n", + " valid_preds.append(preds)\n", + "tta_valid_preds = torch.cat([p.unsqueeze(0) for p in valid_preds],dim=0)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "PREDS_DIR = 'data/predictions'\n", + "!mkdir -p {PREDS_DIR}" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "torch.save(tta_test_preds, f'{PREDS_DIR}/{model_fn}_test.pth')\n", + "torch.save(tta_valid_preds, f'{PREDS_DIR}/{model_fn}_valid.pth')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "pd.DataFrame({'fn': [p.stem for p in learn.data.valid_ds.x.items]}).to_csv(\n", + " f'{PREDS_DIR}/{model_fn}_valid_fns.csv',index=False)\n", + "pd.DataFrame({'fn': [Path(p).stem for p in learn.data.test_ds.x.items]}).to_csv(\n", + " f'{PREDS_DIR}/{model_fn}_test_fns.csv',index=False)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Submit to Kaggle" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "tta_test_preds_mean = tta_test_preds.mean(dim=0)\n", + "tta_test_preds_geomean = torch.expm1(torch.log1p(tta_test_preds).mean(dim=0))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ids = []\n", + "labels = []\n", + "\n", + "for fn, pred in zip(test_fns, tta_test_preds_mean):\n", + " for i, label in enumerate(db.train_ds.classes):\n", + " if label.endswith('1'):\n", + " ids.append(f\"{fn.split('.')[0]}_{label.strip('1')}\")\n", + " predicted_probability = '{0:1.10f}'.format(pred[i].item())\n", + " labels.append(predicted_probability)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "mkdir -p data/submissions" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "sub_name = experiment_name + \"_\" + model_fn\n", + "sub_path = f'data/submissions/{sub_name}.csv.zip'\n", + "sub_name, sub_path" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "pd.DataFrame({'ID': ids, 'Label': labels}).to_csv(sub_path, compression='zip', index=False)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!ls data/submissions/" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!echo {sub_path}" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!kaggle competitions submit -c rsna-intracranial-hemorrhage-detection -f {sub_path} -m {model_fn}" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +}