Datasets
Models
Downloads: 1
Diff of
/.ipynb_checkpoints/test_preds-checkpoint.ipynb
[000000]
..
[d735e2]
Switch to side-by-side view
--- a +++ b/.ipynb_checkpoints/test_preds-checkpoint.ipynb @@ -0,0 +1,406 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Using TensorFlow backend.\n" + ] + } + ], + "source": [ + "import numpy as np\n", + "import pandas as pd\n", + "import pydicom\n", + "import os\n", + "import collections\n", + "import sys\n", + "import glob\n", + "import random\n", + "import cv2\n", + "import tensorflow as tf\n", + "import multiprocessing\n", + "\n", + "from math import ceil, floor\n", + "from copy import deepcopy\n", + "from tqdm import tqdm\n", + "from imgaug import augmenters as iaa\n", + "\n", + "import keras\n", + "import keras.backend as K\n", + "from keras.callbacks import Callback, ModelCheckpoint\n", + "from keras.layers import Dense, Flatten, Dropout\n", + "from keras.models import Model, load_model\n", + "from keras.utils import Sequence\n", + "from keras.losses import binary_crossentropy\n", + "from keras.optimizers import Adam" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Collecting efficientnet\n", + " Downloading https://files.pythonhosted.org/packages/97/82/f3ae07316f0461417dc54affab6e86ab188a5a22f33176d35271628b96e0/efficientnet-1.0.0-py3-none-any.whl\n", + "Requirement already satisfied: scikit-image in /opt/conda/lib/python3.6/site-packages (from efficientnet) (0.16.2)\n", + "Requirement already satisfied: keras-applications<=1.0.8,>=1.0.7 in /opt/conda/lib/python3.6/site-packages (from efficientnet) (1.0.8)\n", + "Requirement already satisfied: pillow>=4.3.0 in /opt/conda/lib/python3.6/site-packages (from scikit-image->efficientnet) (5.4.1)\n", + "Requirement already satisfied: networkx>=2.0 in /opt/conda/lib/python3.6/site-packages (from scikit-image->efficientnet) (2.4)\n", + "Requirement already satisfied: scipy>=0.19.0 in /opt/conda/lib/python3.6/site-packages (from scikit-image->efficientnet) (1.2.1)\n", + "Requirement already satisfied: matplotlib!=3.0.0,>=2.0.0 in /opt/conda/lib/python3.6/site-packages (from scikit-image->efficientnet) (3.0.3)\n", + "Requirement already satisfied: imageio>=2.3.0 in /opt/conda/lib/python3.6/site-packages (from scikit-image->efficientnet) (2.6.1)\n", + "Requirement already satisfied: PyWavelets>=0.4.0 in /opt/conda/lib/python3.6/site-packages (from scikit-image->efficientnet) (1.1.1)\n", + "Requirement already satisfied: h5py in /opt/conda/lib/python3.6/site-packages (from keras-applications<=1.0.8,>=1.0.7->efficientnet) (2.9.0)\n", + "Requirement already satisfied: numpy>=1.9.1 in /opt/conda/lib/python3.6/site-packages (from keras-applications<=1.0.8,>=1.0.7->efficientnet) (1.16.4)\n", + "Requirement already satisfied: decorator>=4.3.0 in /opt/conda/lib/python3.6/site-packages (from networkx>=2.0->scikit-image->efficientnet) (4.4.0)\n", + "Requirement already satisfied: cycler>=0.10 in /opt/conda/lib/python3.6/site-packages (from matplotlib!=3.0.0,>=2.0.0->scikit-image->efficientnet) (0.10.0)\n", + "Requirement already satisfied: kiwisolver>=1.0.1 in /opt/conda/lib/python3.6/site-packages (from matplotlib!=3.0.0,>=2.0.0->scikit-image->efficientnet) (1.1.0)\n", + "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /opt/conda/lib/python3.6/site-packages (from matplotlib!=3.0.0,>=2.0.0->scikit-image->efficientnet) (2.4.2)\n", + "Requirement already satisfied: python-dateutil>=2.1 in /opt/conda/lib/python3.6/site-packages (from matplotlib!=3.0.0,>=2.0.0->scikit-image->efficientnet) (2.8.0)\n", + "Requirement already satisfied: six in /opt/conda/lib/python3.6/site-packages (from h5py->keras-applications<=1.0.8,>=1.0.7->efficientnet) (1.12.0)\n", + "Requirement already satisfied: setuptools in /opt/conda/lib/python3.6/site-packages (from kiwisolver>=1.0.1->matplotlib!=3.0.0,>=2.0.0->scikit-image->efficientnet) (41.4.0)\n", + "Installing collected packages: efficientnet\n", + "Successfully installed efficientnet-1.0.0\n", + "Collecting iterative-stratification\n", + " Downloading https://files.pythonhosted.org/packages/9d/79/9ba64c8c07b07b8b45d80725b2ebd7b7884701c1da34f70d4749f7b45f9a/iterative_stratification-0.1.6-py3-none-any.whl\n", + "Requirement already satisfied: scipy in /opt/conda/lib/python3.6/site-packages (from iterative-stratification) (1.2.1)\n", + "Requirement already satisfied: numpy in /opt/conda/lib/python3.6/site-packages (from iterative-stratification) (1.16.4)\n", + "Requirement already satisfied: scikit-learn in /opt/conda/lib/python3.6/site-packages (from iterative-stratification) (0.21.3)\n", + "Requirement already satisfied: joblib>=0.11 in /opt/conda/lib/python3.6/site-packages (from scikit-learn->iterative-stratification) (0.13.2)\n", + "Installing collected packages: iterative-stratification\n", + "Successfully installed iterative-stratification-0.1.6\n" + ] + } + ], + "source": [ + "# Install Modules from internet\n", + "!pip install efficientnet\n", + "!pip install iterative-stratification" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "# Import Custom Modules\n", + "import efficientnet.keras as efn \n", + "from iterstrat.ml_stratifiers import MultilabelStratifiedShuffleSplit" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": { + "_cell_guid": "79c7e3d0-c299-4dcb-8224-4455121ee9b0", + "_uuid": "d629ff2d2480ee46fbb7e2d37f6b5fab8052498a" + }, + "outputs": [], + "source": [ + "# Seed\n", + "SEED = 12345\n", + "np.random.seed(SEED)\n", + "# tf.set_random_seed(SEED)\n", + "\n", + "# Constants\n", + "TEST_SIZE = 0.01\n", + "HEIGHT = 256\n", + "WIDTH = 256\n", + "CHANNELS = 3\n", + "TRAIN_BATCH_SIZE = 32\n", + "VALID_BATCH_SIZE = 64\n", + "SHAPE = (HEIGHT, WIDTH, CHANNELS)\n", + "\n", + "# Folders\n", + "DATA_DIR = '/kaggle/input/rsna-intracranial-hemorrhage-detection/rsna-intracranial-hemorrhage-detection/'\n", + "TEST_IMAGES_DIR = DATA_DIR + 'stage_2_test/'\n", + "TRAIN_IMAGES_DIR = DATA_DIR + 'stage_2_train/'" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "stage_2_sample_submission.csv stage_2_test stage_2_train stage_2_train.csv\r\n" + ] + } + ], + "source": [ + "!ls '/kaggle/input/rsna-intracranial-hemorrhage-detection/rsna-intracranial-hemorrhage-detection'" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "def correct_dcm(dcm):\n", + " x = dcm.pixel_array + 1000\n", + " px_mode = 4096\n", + " x[x>=px_mode] = x[x>=px_mode] - px_mode\n", + " dcm.PixelData = x.tobytes()\n", + " dcm.RescaleIntercept = -1000\n", + "\n", + "def window_image(dcm, window_center, window_width): \n", + " if (dcm.BitsStored == 12) and (dcm.PixelRepresentation == 0) and (int(dcm.RescaleIntercept) > -100):\n", + " correct_dcm(dcm)\n", + " img = dcm.pixel_array * dcm.RescaleSlope + dcm.RescaleIntercept\n", + " \n", + " # Resize\n", + " img = cv2.resize(img, SHAPE[:2], interpolation = cv2.INTER_LINEAR)\n", + " \n", + " img_min = window_center - window_width // 2\n", + " img_max = window_center + window_width // 2\n", + " img = np.clip(img, img_min, img_max)\n", + " return img\n", + "\n", + "def bsb_window(dcm):\n", + " brain_img = window_image(dcm, 40, 80)\n", + " subdural_img = window_image(dcm, 80, 200)\n", + " soft_img = window_image(dcm, 40, 380)\n", + " \n", + " brain_img = (brain_img - 0) / 80\n", + " subdural_img = (subdural_img - (-20)) / 200\n", + " soft_img = (soft_img - (-150)) / 380\n", + " bsb_img = np.array([brain_img, subdural_img, soft_img]).transpose(1,2,0)\n", + " return bsb_img\n", + "\n", + "def _read(path, SHAPE):\n", + " dcm = pydicom.dcmread(path)\n", + " try:\n", + " img = bsb_window(dcm)\n", + " except:\n", + " img = np.zeros(SHAPE)\n", + " return img" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Import the training and test datasets." + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Downloading data from https://github.com/Callidior/keras-applications/releases/download/efficientnet/efficientnet-b0_weights_tf_dim_ordering_tf_kernels_autoaugment_notop.h5\n", + "16809984/16804768 [==============================] - 0s 0us/step\n" + ] + } + ], + "source": [ + "base_model = efn.EfficientNetB0(weights = 'imagenet', include_top = False, \\\n", + " pooling = 'avg', input_shape = (HEIGHT, WIDTH, 3))\n", + "\n", + "x = base_model.output\n", + "x = Dropout(0.125)(x)\n", + "output_layer = Dense(6, activation = 'sigmoid')(x)\n", + "model = Model(inputs=base_model.input, outputs=output_layer)\n", + "model.compile(optimizer = Adam(learning_rate = 0.0001), \n", + " loss = 'binary_crossentropy',\n", + " metrics = ['acc'])" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Collecting gdown\n", + " Downloading https://files.pythonhosted.org/packages/b0/b4/a8e9d0b02bca6aa53087001abf064cc9992bda11bd6840875b8098d93573/gdown-3.8.3.tar.gz\n", + "Requirement already satisfied: filelock in /opt/conda/lib/python3.6/site-packages (from gdown) (3.0.12)\n", + "Requirement already satisfied: requests in /opt/conda/lib/python3.6/site-packages (from gdown) (2.22.0)\n", + "Requirement already satisfied: six in /opt/conda/lib/python3.6/site-packages (from gdown) (1.12.0)\n", + "Requirement already satisfied: tqdm in /opt/conda/lib/python3.6/site-packages (from gdown) (4.36.1)\n", + "Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /opt/conda/lib/python3.6/site-packages (from requests->gdown) (3.0.4)\n", + "Requirement already satisfied: idna<2.9,>=2.5 in /opt/conda/lib/python3.6/site-packages (from requests->gdown) (2.8)\n", + "Requirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.6/site-packages (from requests->gdown) (2019.9.11)\n", + "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /opt/conda/lib/python3.6/site-packages (from requests->gdown) (1.24.2)\n", + "Building wheels for collected packages: gdown\n", + " Building wheel for gdown (setup.py) ... \u001b[?25ldone\n", + "\u001b[?25h Created wheel for gdown: filename=gdown-3.8.3-cp36-none-any.whl size=8850 sha256=60eafdb73efc62d7de20834e8aece4a416db33f8fa7b5c0536f594bc52163a15\n", + " Stored in directory: /tmp/.cache/pip/wheels/a7/9d/16/9e0bda9a327ff2cddaee8de48a27553fb1efce73133593d066\n", + "Successfully built gdown\n", + "Installing collected packages: gdown\n", + "Successfully installed gdown-3.8.3\n" + ] + } + ], + "source": [ + "!pip install gdown" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Downloading...\n", + "From: https://drive.google.com/uc?id=1FXF1HymYbRf3OlThMTXAa74TRup3AhD_\n", + "To: /kaggle/working/model_effnet_bo_087.h5\n", + "16.7MB [00:00, 151MB/s]\n" + ] + } + ], + "source": [ + "!gdown https://drive.google.com/uc?id=1FXF1HymYbRf3OlThMTXAa74TRup3AhD_" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [], + "source": [ + "model.load_weights('model_effnet_bo_087.h5')" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(256, 256, 3)" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "img_data = _read(TRAIN_IMAGES_DIR + '/ID_12cadc6af.dcm', (256, 256))\n", + "img_data.shape" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(1, 256, 256, 3)" + ] + }, + "execution_count": 35, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "X = np.empty((1, 256,256, 3))\n", + "X[0] = img_data\n", + "X.shape" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "array([[0.02540314, 0.00152788, 0.0010443 , 0.0023855 , 0.00326541,\n", + " 0.00998502]], dtype=float32)" + ] + }, + "execution_count": 36, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "preds = model.predict(X)\n", + "preds" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(1, 6)" + ] + }, + "execution_count": 37, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "preds.shape" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.6" + } + }, + "nbformat": 4, + "nbformat_minor": 1 +}