{

 "cells": [

  {

   "cell_type": "code",

   "execution_count": 1,

   "metadata": {},

   "outputs": [],

   "source": [

    "import os\n",

    "#os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"2\""

   ]

  },

  {

   "cell_type": "code",

   "execution_count": 2,

   "metadata": {},

   "outputs": [],

   "source": [

    "stage = \"stage_2\""

   ]

  },

  {

   "cell_type": "code",

   "execution_count": 3,

   "metadata": {},

   "outputs": [],

   "source": [

    "experiment_name = 'NONAMED'"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": 4,

   "metadata": {},

   "outputs": [],

   "source": [

    "import fastai\n",

    "from fastai.vision import *\n",

    "from fastai.callbacks import *\n",

    "from fastai.distributed import *\n",

    "from fastai.utils.mem import *\n",

    "import re\n",

    "import pydicom\n",

    "import pdb\n",

    "import pickle\n",

    "import torch\n"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": 5,

   "metadata": {},

   "outputs": [

    {

     "name": "stdout",

     "output_type": "stream",

     "text": [

      "densenet121-53_train_cpu_window_uint8_sz512_cv0.0386_subdural_loss_fold5_of_5.pth\r\n",

      "densenet121-53_train_cpu_window_uint8_sz512_cv0.0399_subdural_loss_fold4_of_5.pth\r\n",

      "densenet121-53_train_cpu_window_uint8_sz512_cv0.0403_subdural_loss_fold3_of_5.pth\r\n",

      "densenet121-53_train_cpu_window_uint8_sz512_cv0.0406_subdural_loss_fold2_of_5.pth\r\n",

      "densenet121-53_train_cpu_window_uint8_sz512_cv0.0413_subdural_loss_fold1_of_5.pth\r\n",

      "densenet121_sz512_cv0.0738_weighted_loss_fold4_of_5.pth\r\n",

      "densenet121_sz512_cv0.0739_weighted_loss_fold5_of_5.pth\r\n",

      "densenet121_sz512_cv0.0743_weighted_loss_fold2_of_5.pth\r\n",

      "densenet121_sz512_cv0.0748_weighted_loss_fold3_of_5.pth\r\n",

      "densenet121_sz512_cv0.0749_weighted_loss_fold1_of_5.pth\r\n",

      "resnet101-53_train_cpu_window_uint8_sz512_cv0.0365_subdural_loss_fold5_of_5.pth\r\n",

      "resnet101-53_train_cpu_window_uint8_sz512_cv0.0366_subdural_loss_fold4_of_5.pth\r\n",

      "resnet101-53_train_cpu_window_uint8_sz512_cv0.0369_subdural_loss_fold2_of_5.pth\r\n",

      "resnet101-53_train_cpu_window_uint8_sz512_cv0.0371_subdural_loss_fold3_of_5.pth\r\n",

      "resnet101-53_train_cpu_window_uint8_sz512_cv0.0375_subdural_loss_fold1_of_5.pth\r\n",

      "resnet101_sz512_cv0.0749_weighted_loss_fold5_of_5.pth\r\n",

      "resnet101_sz512_cv0.0751_weighted_loss_fold4_of_5.pth\r\n",

      "resnet101_sz512_cv0.0754_weighted_loss_fold2_of_5.pth\r\n",

      "resnet101_sz512_cv0.0758_weighted_loss_fold1_of_5.pth\r\n",

      "resnet101_sz512_cv0.0761_weighted_loss_fold3_of_5.pth\r\n",

      "resnet18-53_train_cpu_window_uint8_sz512_cv0.0522_subdural_loss_fold5_of_5.pth\r\n",

      "resnet18-53_train_cpu_window_uint8_sz512_cv0.0528_subdural_loss_fold3_of_5.pth\r\n",

      "resnet18-53_train_cpu_window_uint8_sz512_cv0.0530_subdural_loss_fold4_of_5.pth\r\n",

      "resnet18-53_train_cpu_window_uint8_sz512_cv0.0550_subdural_loss_fold2_of_5.pth\r\n",

      "resnet18-53_train_cpu_window_uint8_sz512_cv0.0556_subdural_loss_fold1_of_5.pth\r\n",

      "resnet18_sz512_cv0.0762_weighted_loss_fold4_of_5.pth\r\n",

      "resnet18_sz512_cv0.0768_weighted_loss_fold5_of_5.pth\r\n",

      "resnet18_sz512_cv0.0773_weighted_loss_fold2_of_5.pth\r\n",

      "resnet18_sz512_cv0.0786_weighted_loss_fold1_of_5.pth\r\n",

      "resnet18_sz512_cv0.0786_weighted_loss_fold3_of_5.pth\r\n",

      "resnet34-53_train_cpu_window_uint8_sz512_cv0.0453_subdural_loss_fold5_of_5.pth\r\n",

      "resnet34-53_train_cpu_window_uint8_sz512_cv0.0466_subdural_loss_fold4_of_5.pth\r\n",

      "resnet34-53_train_cpu_window_uint8_sz512_cv0.0474_subdural_loss_fold3_of_5.pth\r\n",

      "resnet34-53_train_cpu_window_uint8_sz512_cv0.0484_subdural_loss_fold2_of_5.pth\r\n",

      "resnet34-53_train_cpu_window_uint8_sz512_cv0.0496_subdural_loss_fold1_of_5.pth\r\n",

      "resnet34_sz512_cv0.0750_weighted_loss_fold4_of_5.pth\r\n",

      "resnet34_sz512_cv0.0759_weighted_loss_fold5_of_5.pth\r\n",

      "resnet34_sz512_cv0.0763_weighted_loss_fold2_of_5.pth\r\n",

      "resnet34_sz512_cv0.0765_weighted_loss_fold1_of_5.pth\r\n",

      "resnet34_sz512_cv0.0782_weighted_loss_fold3_of_5.pth\r\n",

      "resnet50_sz512_cv0.0751_weighted_loss_fold4_of_5.pth\r\n",

      "resnet50_sz512_cv0.0751_weighted_loss_fold5_of_5.pth\r\n",

      "resnet50_sz512_cv0.0754_weighted_loss_fold2_of_5.pth\r\n",

      "resnet50_sz512_cv0.0761_weighted_loss_fold1_of_5.pth\r\n",

      "resnet50_sz512_cv0.0764_weighted_loss_fold3_of_5.pth\r\n",

      "resnext50_32x4d-53_train_cpu_window_uint8_sz512_cv0.0372_subdural_loss_fold5_of_5.pth\r\n",

      "resnext50_32x4d-53_train_cpu_window_uint8_sz512_cv0.0376_subdural_loss_fold4_of_5.pth\r\n",

      "resnext50_32x4d-53_train_cpu_window_uint8_sz512_cv0.0378_subdural_loss_fold2_of_5.pth\r\n",

      "resnext50_32x4d-53_train_cpu_window_uint8_sz512_cv0.0381_subdural_loss_fold3_of_5.pth\r\n",

      "resnext50_32x4d-53_train_cpu_window_uint8_sz512_cv0.0387_subdural_loss_fold1_of_5.pth\r\n"

     ]

    }

   ],

   "source": [

    "!ls models"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": 6,

   "metadata": {},

   "outputs": [],

   "source": [

    "arch = 'resnet18'\n",

    "fold=4\n",

    "lr = 1e-3\n",

    "\n",

    "model_fn = None\n",

    "SZ = 512\n",

    "n_folds=5\n",

    "n_epochs = 15\n",

    "n_tta = 10\n",

    "\n",

    "#model_fn = 'resnet34_sz512_cv0.0821_weighted_loss_fold1_of_5'\n",

    "\n",

    "if model_fn is not None:\n",

    "    model_fn_fold = int(model_fn[-6])-1\n",

    "    assert model_fn_fold == fold"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": 7,

   "metadata": {},

   "outputs": [],

   "source": [

    "data_dir = Path('data/unzip')\n",

    "train_dir = data_dir / f'{stage}_train_images'\n",

    "test_dir = data_dir / f'{stage}_test_images'"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": 8,

   "metadata": {},

   "outputs": [

    {

     "ename": "FileNotFoundError",

     "evalue": "[Errno 2] No such file or directory: 'data/stage_2_fn_to_study_ix.pickle'",

     "output_type": "error",

     "traceback": [

      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",

      "\u001b[0;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",

      "\u001b[0;32m<ipython-input-8-6207314e915b>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mfn_to_study_ix\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpickle\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf'data/{stage}_fn_to_study_ix.pickle'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'rb'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0mstudy_ix_to_fn\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpickle\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf'data/{stage}_study_ix_to_fn.pickle'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'rb'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0mfn_to_labels\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpickle\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload\u001b[0m\u001b[0;34m(\u001b[0m  \u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf'data/{stage}_train_fn_to_labels.pickle'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'rb'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0mstudy_to_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpickle\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf'data/{stage}_study_to_data.pickle'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'rb'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",

      "\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: 'data/stage_2_fn_to_study_ix.pickle'"

     ]

    }

   ],

   "source": [

    "fn_to_study_ix = pickle.load(open(f'data/{stage}_fn_to_study_ix.pickle', 'rb'))\n",

    "study_ix_to_fn = pickle.load(open(f'data/{stage}_study_ix_to_fn.pickle', 'rb'))\n",

    "fn_to_labels = pickle.load(  open(f'data/{stage}_train_fn_to_labels.pickle', 'rb'))\n",

    "study_to_data = pickle.load (open(f'data/{stage}_study_to_data.pickle', 'rb'))\n"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": null,

   "metadata": {},

   "outputs": [],

   "source": [

    "# gets a pathlib.Path, returns labels\n",

    "labels = [\n",

    "    'any0', 'epidural0', 'intraparenchymal0', 'intraventricular0', 'subarachnoid0', 'subdural0',\n",

    "    'any1', 'epidural1', 'intraparenchymal1', 'intraventricular1', 'subarachnoid1', 'subdural1',\n",

    "    'any2', 'epidural2', 'intraparenchymal2', 'intraventricular2', 'subarachnoid2', 'subdural2',\n",

    "]\n",

    "\n",

    "def get_labels(center_p):\n",

    "\n",

    "    study, center_ix = fn_to_study_ix[center_p.stem]\n",

    "    total = len(study_ix_to_fn[study])\n",

    "    \n",

    "    # Only 1 study has 2 different RescaleIntercept values (0.0 and 1.0) in the same series, so assume they're all the\n",

    "    # same and do everything in the big tensor\n",

    "    \n",

    "    pixels = {}\n",

    "    img_labels = []\n",

    "    ixs = [ max(0, min(ix, total-1)) for ix in range(center_ix-1, center_ix+2) ]\n",

    "    for i, ix in enumerate(ixs):\n",

    "        fn = study_ix_to_fn[study][ix]\n",

    "        img_labels += [ x + str(i) for x in fn_to_labels[fn] ]\n",

    "            \n",

    "    return img_labels\n",

    "\n",

    "get_labels(train_dir / 'ID_ac7c8fe8b.dcm')"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": null,

   "metadata": {},

   "outputs": [],

   "source": [

    "sample_submission = pd.read_csv(f'data/unzip/{stage}_sample_submission.csv')"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": null,

   "metadata": {},

   "outputs": [],

   "source": [

    "sample_submission['fn'] = sample_submission.ID.apply(lambda x: '_'.join(x.split('_')[:2]) + '.dcm')"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": null,

   "metadata": {},

   "outputs": [],

   "source": [

    "sample_submission.head()"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": null,

   "metadata": {},

   "outputs": [],

   "source": [

    "test_fns = sample_submission.fn.unique()"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": null,

   "metadata": {},

   "outputs": [],

   "source": [

    "wc = 50\n",

    "ww = 100"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": null,

   "metadata": {},

   "outputs": [],

   "source": [

    "def open_dicom(p_or_fn): # when called by .from_folder it's a Path; when called from .add_test it's a string\n",

    "    #pdb.set_trace()\n",

    "    center_p = Path(p_or_fn)\n",

    "\n",

    "    study, center_ix = fn_to_study_ix[center_p.stem]\n",

    "    total = len(study_ix_to_fn[study])\n",

    "    \n",

    "    # Only 1 study has 2 different RescaleIntercept values (0.0 and 1.0) in the same series, so assume they're all the\n",

    "    # same and do everything in the big tensor\n",

    "    \n",

    "    pixels = {}\n",

    "    ixs = [ max(0, min(ix, total-1)) for ix in range(center_ix-1, center_ix+2) ]\n",

    "    for ix in ixs:\n",

    "        if ix not in pixels:\n",

    "            p = center_p.parent / f'{study_ix_to_fn[study][ix]}.dcm'\n",

    "            dcm = pydicom.dcmread(str(p))\n",

    "            if ix == center_ix:\n",

    "                rescale_slope, rescale_intercept = float(dcm.RescaleSlope), float(dcm.RescaleIntercept)\n",

    "            pixels[ix] = torch.FloatTensor(dcm.pixel_array.astype(np.float))\n",

    "        \n",

    "    t = torch.stack([pixels[ix] for ix in ixs], dim=0) # stack chans together\n",

    "    if (t.shape[1:] != (SZ,SZ)):\n",

    "        t = torch.nn.functional.interpolate(\n",

    "            t.unsqueeze_(0), size=SZ, mode='bilinear', align_corners=True).squeeze_(0) # resize\n",

    "    t = t * rescale_slope + rescale_intercept # rescale\n",

    "    t = torch.clamp(t, wc-ww/2, wc+ww/2) # window\n",

    "    t = (t - (wc-ww/2)) / ww # normalize\n",

    "    \n",

    "    return Image(t)\n",

    "\n",

    "\n",

    "class DicomList(ImageList):\n",

    "    def open(self, fn): return open_dicom(fn)"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": null,

   "metadata": {},

   "outputs": [],

   "source": [

    "my_stats = ([0.45, 0.45, 0.45], [0.225, 0.225, 0.225])"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": null,

   "metadata": {},

   "outputs": [],

   "source": [

    "class OverSamplingCallback(LearnerCallback):\n",

    "    def __init__(self,learn:Learner):\n",

    "        super().__init__(learn)\n",

    "        self.labels = self.learn.data.train_dl.dataset.y.items\n",

    "        _, counts = np.unique(self.labels,return_counts=True)\n",

    "        self.weights = torch.DoubleTensor((1/counts)[self.labels])\n",

    "        self.label_counts = np.bincount([self.learn.data.train_dl.dataset.y[i].data for i in range(len(self.learn.data.train_dl.dataset))])\n",

    "        self.total_len_oversample = int(self.learn.data.c*np.max(self.label_counts))\n",

    "        \n",

    "    def on_train_begin(self, **kwargs):\n",

    "        self.learn.data.train_dl.dl.batch_sampler = BatchSampler(WeightedRandomSampler(weights,self.total_len_oversample), self.learn.data.train_dl.batch_size,False)"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": null,

   "metadata": {},

   "outputs": [],

   "source": [

    "# From: https://forums.fast.ai/t/is-there-any-built-in-method-to-oversample-minority-classes/46838/5\n",

    "class ImbalancedDatasetSampler(torch.utils.data.sampler.Sampler):\n",

    "    def __init__(self, dataset, indices=None, num_samples=None):\n",

    "                \n",

    "        # if indices is not provided, \n",

    "        # all elements in the dataset will be considered\n",

    "        self.indices = list(range(len(dataset))) if indices is None else indices\n",

    "            \n",

    "        # if num_samples is not provided, \n",

    "        # draw `len(indices)` samples in each iteration\n",

    "        self.num_samples = len(self.indices) if num_samples is None else num_samples\n",

    "            \n",

    "        # distribution of classes in the dataset \n",

    "        label_to_count = defaultdict(int)\n",

    "        for idx in self.indices:\n",

    "            label = self._get_label(dataset, idx)\n",

    "            label_to_count[label] += 1\n",

    "                \n",

    "        # weight for each sample\n",

    "        weights = [1.0 / label_to_count[self._get_label(dataset, idx)] for idx in self.indices]\n",

    "        self.weights = torch.DoubleTensor(weights)\n",

    "\n",

    "    def _get_label(self, dataset, idx):\n",

    "        return 'any1' in dataset.y[idx].obj\n",

    "                \n",

    "    def __iter__(self):\n",

    "        return (self.indices[i] for i in torch.multinomial(\n",

    "            self.weights, self.num_samples, replacement=True))\n",

    "\n",

    "    def __len__(self):\n",

    "        return self.num_samples"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": null,

   "metadata": {},

   "outputs": [],

   "source": [

    "folds = np.array_split(list({v['fold'] for v in study_to_data.values()}), n_folds)\n",

    "folds"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": null,

   "metadata": {},

   "outputs": [],

   "source": [

    "df = pd.read_csv(f\"data/{stage}_train_dicom_diags_norm.csv\")\n",

    "# split train/val\n",

    "np.random.seed(666)\n",

    "studies_in_fold = [k for k,v in study_to_data.items() if np.isin(v['fold'],folds[fold])]\n",

    "study_id_val_set = set(df[ df['SeriesInstanceUID'].isin(studies_in_fold)]['SOPInstanceUID'] + '.dcm')\n",

    "\n",

    "def is_val(p_or_fn):\n",

    "    p = Path(p_or_fn)\n",

    "    return p.name in study_id_val_set"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": null,

   "metadata": {},

   "outputs": [],

   "source": [

    "tfms = (\n",

    "    [ flip_lr(p=0.5), rotate(degrees=(-180,180), p=1.) ], # train\n",

    "    [] # val\n",

    ")\n",

    "data = (DicomList.from_folder(f'data/unzip/{stage}_train_images/', extensions=['.dcm'], presort=True)\n",

    "    .split_by_valid_func(is_val)\n",

    "    .label_from_func(get_labels, classes=labels)\n",

    "    .transform(tfms, mode='nearest')\n",

    "    .add_test(f'data/unzip/{stage}_test_images/' + test_fns))\n"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": null,

   "metadata": {},

   "outputs": [],

   "source": [

    "db = data.databunch().normalize()"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": null,

   "metadata": {},

   "outputs": [],

   "source": [

    "balsampler = ImbalancedDatasetSampler(db.train_ds)"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": null,

   "metadata": {

    "scrolled": true

   },

   "outputs": [],

   "source": [

    "db.train_dl.batch_sampler = balsampler\n",

    "db.train_dl.batch_sampler\n",

    "db"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": null,

   "metadata": {},

   "outputs": [],

   "source": [

    "db.show_batch() # AttributeError: 'list' object has no attribute 'pixel' <-- because using torch datasets or whatever"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": null,

   "metadata": {},

   "outputs": [],

   "source": [

    "# yuval reina's loss\n",

    "yuval_weights = FloatTensor([ 2, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1 ]).cuda() \n",

    "\n",

    "def yuval_loss(y_pred,y_true):\n",

    "    return F.binary_cross_entropy_with_logits(y_pred,\n",

    "                                  y_true,\n",

    "                                  yuval_weights.repeat(y_pred.shape[0],1))"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": null,

   "metadata": {},

   "outputs": [],

   "source": [

    "real_lb_weights = FloatTensor([ 2, 1, 1, 1, 1, 1 ])\n",

    "\n",

    "def real_lb_loss(pred:Tensor, targ:Tensor)->Rank0Tensor:\n",

    "    pred,targ = flatten_check(pred,targ)\n",

    "    tp = pred.view(-1,18)[:,6:12]\n",

    "    tt = targ.view(-1,18)[:,6:12]\n",

    "    return F.binary_cross_entropy_with_logits(tp, tt, real_lb_weights.to(device=pred.device))"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": null,

   "metadata": {},

   "outputs": [],

   "source": [

    "w_loss = 0.1\n",

    "weighted_lb_weights = FloatTensor([ 2.*w_loss, 1.*w_loss, 1.*w_loss, 1.*w_loss, 1.*w_loss, 1.*w_loss, \n",

    "                                    2.,        1.,        1.,        1.,        1.,        1., \n",

    "                                    2.*w_loss, 1.*w_loss, 1.*w_loss, 1.*w_loss, 1.*w_loss, 1.*w_loss, ])\n",

    "\n",

    "def weighted_loss(pred:Tensor,targ:Tensor)->Rank0Tensor:\n",

    "    return F.binary_cross_entropy_with_logits(pred,targ,weighted_lb_weights.to(device=pred.device))"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": null,

   "metadata": {},

   "outputs": [],

   "source": [

    "lb_weights = FloatTensor([ 2, 1, 1, 1, 1, 1 ]).cuda()\n",

    "\n",

    "def lb_loss(pred:Tensor, targ:Tensor)->Rank0Tensor:\n",

    "    pred,targ = flatten_check(pred,targ)\n",

    "    tp = pred.view(-1,6)\n",

    "    tt = targ.view(-1,6)\n",

    "    return torch.nn.functional.binary_cross_entropy_with_logits(tp, tt, lb_weights.to(device=pred.device))"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": null,

   "metadata": {},

   "outputs": [],

   "source": [

    "learn = cnn_learner(\n",

    "    db, getattr(models, arch), loss_func=weighted_loss, metrics=[real_lb_loss],\n",

    "    pretrained=True, lin_ftrs=[],ps=0.,\n",

    "    model_dir = Path('./models/').resolve())"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": null,

   "metadata": {},

   "outputs": [],

   "source": [

    "try:\n",

    "    learn.load(model_fn,strict=False)\n",

    "    print(f\"Loaded {model_fn}\")\n",

    "except Exception as e:\n",

    "    print(e)"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": null,

   "metadata": {},

   "outputs": [],

   "source": [

    "learn = learn.to_parallel().to_fp16()\n",

    "\n",

    "arch_to_batch_factor = {\n",

    "    'resnet18' : 4.3,\n",

    "    'resnet34' : 3.4,\n",

    "    'resnet50' : 1.5,\n",

    "    'resnet101' : 1.0,\n",

    "    'resnext50_32x4d': 1.2,\n",

    "    'densenet121': 1.1,\n",

    "    'squeezenet1_0': 2.6,\n",

    "    'vgg16': 1.5\n",

    "}\n",

    "\n",

    "bs = arch_to_batch_factor[arch] * 512*512 * gpu_mem_get()[0] * 1e-3 / (SZ*SZ)\n",

    "bs *= int(int(any([isinstance(cb, MixedPrecision) for cb in learn.callbacks])))+1 # 2x if fp16\n",

    "if any([isinstance(cb, fastai.distributed.ParallelTrainer) for cb in learn.callbacks]):\n",

    "    bs *= torch.cuda.device_count()\n",

    "    bs = (bs // torch.cuda.device_count()) * torch.cuda.device_count()\n",

    "bs = int(bs)\n",

    "learn.data.batch_size = bs\n",

    "bs"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": null,

   "metadata": {},

   "outputs": [],

   "source": [

    "learn"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": null,

   "metadata": {},

   "outputs": [],

   "source": [

    "learn.unfreeze()\n",

    "learn.summary()"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": null,

   "metadata": {},

   "outputs": [],

   "source": [

    "try:\n",

    "    if lr is None:\n",

    "        !rm {learn.model_dir}/tmp.pth\n",

    "        learn.lr_find()\n",

    "        learn.recorder.plot(suggestion=True)\n",

    "except Exception as e:\n",

    "    print(e)\n",

    "# set lr "

   ]

  },

  {

   "cell_type": "code",

   "execution_count": null,

   "metadata": {},

   "outputs": [],

   "source": [

    "learn.fit_one_cycle(n_epochs, lr)"

   ]

  },

  {

   "cell_type": "markdown",

   "metadata": {},

   "source": [

    "# Save"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": null,

   "metadata": {},

   "outputs": [],

   "source": [

    "v = learn.validate()\n",

    "cv = float(v[-1])"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": null,

   "metadata": {},

   "outputs": [],

   "source": [

    "model_fn = f'{arch}_sz{SZ}_cv{cv:0.4f}_{learn.loss_func.__name__}_fold{fold+1}_of_{n_folds}'\n",

    "learn.save(model_fn)\n",

    "model_fn"

   ]

  },

  {

   "cell_type": "markdown",

   "metadata": {},

   "source": [

    "# Predictions"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": null,

   "metadata": {},

   "outputs": [],

   "source": [

    "learn.data.test_ds.tfms = learn.data.train_ds.tfms\n",

    "test_preds = []\n",

    "# Fastai WTF: it figures out to run outputs through sigmoid if a standard loss error is used \n",

    "# (see loss_func_name2activ and related stuff) but on custom loss funcs sigmoid must be executed explicitly:\n",

    "# https://forums.fast.ai/t/shouldnt-we-able-to-pass-an-activ-function-to-learner-get-preds/50492\n",

    "for _ in progress_bar(range(n_tta)):\n",

    "    preds, _ = learn.get_preds(DatasetType.Test, activ=torch.sigmoid)\n",

    "    test_preds.append(preds)\n",

    "tta_test_preds = torch.cat([p.unsqueeze(0) for p in test_preds],dim=0)"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": null,

   "metadata": {},

   "outputs": [],

   "source": [

    "learn.data.valid_ds.tfms = learn.data.train_ds.tfms\n",

    "valid_preds = []\n",

    "# Fastai WTF: it figures out to run outputs through sigmoid if a standard loss error is used \n",

    "# (see loss_func_name2activ and related stuff) but on custom loss funcs sigmoid must be executed explicitly:\n",

    "# https://forums.fast.ai/t/shouldnt-we-able-to-pass-an-activ-function-to-learner-get-preds/50492\n",

    "for _ in progress_bar(range(n_tta)):\n",

    "    preds, _ = learn.get_preds(DatasetType.Valid, activ=torch.sigmoid)\n",

    "    valid_preds.append(preds)\n",

    "tta_valid_preds = torch.cat([p.unsqueeze(0) for p in valid_preds],dim=0)"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": null,

   "metadata": {},

   "outputs": [],

   "source": [

    "PREDS_DIR = 'data/predictions'\n",

    "!mkdir -p {PREDS_DIR}"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": null,

   "metadata": {},

   "outputs": [],

   "source": [

    "torch.save(tta_test_preds,  f'{PREDS_DIR}/{model_fn}_test.pth')\n",

    "torch.save(tta_valid_preds, f'{PREDS_DIR}/{model_fn}_valid.pth')"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": null,

   "metadata": {},

   "outputs": [],

   "source": [

    "pd.DataFrame({'fn': [p.stem for p in learn.data.valid_ds.x.items]}).to_csv(\n",

    "    f'{PREDS_DIR}/{model_fn}_valid_fns.csv',index=False)\n",

    "pd.DataFrame({'fn': [Path(p).stem for p in learn.data.test_ds.x.items]}).to_csv(\n",

    "    f'{PREDS_DIR}/{model_fn}_test_fns.csv',index=False)"

   ]

  },

  {

   "cell_type": "markdown",

   "metadata": {},

   "source": [

    "# Submit to Kaggle"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": null,

   "metadata": {},

   "outputs": [],

   "source": [

    "tta_test_preds_mean = tta_test_preds.mean(dim=0)\n",

    "tta_test_preds_geomean = torch.expm1(torch.log1p(tta_test_preds).mean(dim=0))"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": null,

   "metadata": {},

   "outputs": [],

   "source": [

    "ids = []\n",

    "labels = []\n",

    "\n",

    "for fn, pred in zip(test_fns, tta_test_preds_mean):\n",

    "    for i, label in enumerate(db.train_ds.classes):\n",

    "        if label.endswith('1'):\n",

    "            ids.append(f\"{fn.split('.')[0]}_{label.strip('1')}\")\n",

    "            predicted_probability = '{0:1.10f}'.format(pred[i].item())\n",

    "            labels.append(predicted_probability)"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": null,

   "metadata": {},

   "outputs": [],

   "source": [

    "mkdir -p data/submissions"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": null,

   "metadata": {},

   "outputs": [],

   "source": [

    "sub_name = experiment_name + \"_\" + model_fn\n",

    "sub_path = f'data/submissions/{sub_name}.csv.zip'\n",

    "sub_name, sub_path"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": null,

   "metadata": {},

   "outputs": [],

   "source": [

    "pd.DataFrame({'ID': ids, 'Label': labels}).to_csv(sub_path, compression='zip', index=False)"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": null,

   "metadata": {},

   "outputs": [],

   "source": [

    "!ls data/submissions/"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": null,

   "metadata": {},

   "outputs": [],

   "source": [

    "!echo {sub_path}"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": null,

   "metadata": {},

   "outputs": [],

   "source": [

    "!kaggle competitions submit -c rsna-intracranial-hemorrhage-detection -f {sub_path} -m {model_fn}"

   ]

  }

 ],

 "metadata": {

  "kernelspec": {

   "display_name": "Python 3",

   "language": "python",

   "name": "python3"

  },

  "language_info": {

   "codemirror_mode": {

    "name": "ipython",

    "version": 3

   },

   "file_extension": ".py",

   "mimetype": "text/x-python",

   "name": "python",

   "nbconvert_exporter": "python",

   "pygments_lexer": "ipython3",

   "version": "3.7.3"

  }

 },

 "nbformat": 4,

 "nbformat_minor": 2

}