Downloads: 1
Diff of
/4DetectNodules.ipynb
[000000]
..
[467f44]
Switch to unified view
| a | b/4DetectNodules.ipynb | ||
|---|---|---|---|
| 1 | { |
||
| 2 | "cells": [ |
||
| 3 | { |
||
| 4 | "cell_type": "markdown", |
||
| 5 | "metadata": {}, |
||
| 6 | "source": [ |
||
| 7 | "# 4. Detect Nodules from Kaggle Dataset\n", |
||
| 8 | "\n", |
||
| 9 | "## Summary\n", |
||
| 10 | "* load and process kaggle dataset\n", |
||
| 11 | "* Generate prediction masks with trained unet\n", |
||
| 12 | "* Reduce false positives with trained CNN" |
||
| 13 | ] |
||
| 14 | }, |
||
| 15 | { |
||
| 16 | "cell_type": "code", |
||
| 17 | "execution_count": 5, |
||
| 18 | "metadata": {}, |
||
| 19 | "outputs": [ |
||
| 20 | { |
||
| 21 | "name": "stderr", |
||
| 22 | "output_type": "stream", |
||
| 23 | "text": [ |
||
| 24 | "c:\\programdata\\anaconda3\\envs\\tensorflow\\lib\\site-packages\\ipykernel_launcher.py:64: UserWarning: Update your `Conv2D` call to the Keras 2 API: `Conv2D(32, (3, 3), activation=\"relu\", padding=\"same\")`\n", |
||
| 25 | "c:\\programdata\\anaconda3\\envs\\tensorflow\\lib\\site-packages\\ipykernel_launcher.py:66: UserWarning: Update your `Conv2D` call to the Keras 2 API: `Conv2D(32, (3, 3), activation=\"relu\", padding=\"same\")`\n", |
||
| 26 | "c:\\programdata\\anaconda3\\envs\\tensorflow\\lib\\site-packages\\ipykernel_launcher.py:69: UserWarning: Update your `Conv2D` call to the Keras 2 API: `Conv2D(64, (3, 3), activation=\"relu\", padding=\"same\")`\n", |
||
| 27 | "c:\\programdata\\anaconda3\\envs\\tensorflow\\lib\\site-packages\\ipykernel_launcher.py:71: UserWarning: Update your `Conv2D` call to the Keras 2 API: `Conv2D(64, (3, 3), activation=\"relu\", padding=\"same\")`\n", |
||
| 28 | "c:\\programdata\\anaconda3\\envs\\tensorflow\\lib\\site-packages\\ipykernel_launcher.py:74: UserWarning: Update your `Conv2D` call to the Keras 2 API: `Conv2D(128, (3, 3), activation=\"relu\", padding=\"same\")`\n", |
||
| 29 | "c:\\programdata\\anaconda3\\envs\\tensorflow\\lib\\site-packages\\ipykernel_launcher.py:76: UserWarning: Update your `Conv2D` call to the Keras 2 API: `Conv2D(128, (3, 3), activation=\"relu\", padding=\"same\")`\n", |
||
| 30 | "c:\\programdata\\anaconda3\\envs\\tensorflow\\lib\\site-packages\\ipykernel_launcher.py:79: UserWarning: Update your `Conv2D` call to the Keras 2 API: `Conv2D(256, (3, 3), activation=\"relu\", padding=\"same\")`\n", |
||
| 31 | "c:\\programdata\\anaconda3\\envs\\tensorflow\\lib\\site-packages\\ipykernel_launcher.py:81: UserWarning: Update your `Conv2D` call to the Keras 2 API: `Conv2D(256, (3, 3), activation=\"relu\", padding=\"same\")`\n", |
||
| 32 | "c:\\programdata\\anaconda3\\envs\\tensorflow\\lib\\site-packages\\ipykernel_launcher.py:84: UserWarning: Update your `Conv2D` call to the Keras 2 API: `Conv2D(512, (3, 3), activation=\"relu\", padding=\"same\")`\n", |
||
| 33 | "c:\\programdata\\anaconda3\\envs\\tensorflow\\lib\\site-packages\\ipykernel_launcher.py:86: UserWarning: Update your `Conv2D` call to the Keras 2 API: `Conv2D(512, (3, 3), activation=\"relu\", padding=\"same\")`\n", |
||
| 34 | "c:\\programdata\\anaconda3\\envs\\tensorflow\\lib\\site-packages\\ipykernel_launcher.py:88: UserWarning: The `merge` function is deprecated and will be removed after 08/2017. Use instead layers from `keras.layers.merge`, e.g. `add`, `concatenate`, etc.\n", |
||
| 35 | "c:\\programdata\\anaconda3\\envs\\tensorflow\\lib\\site-packages\\keras\\legacy\\layers.py:458: UserWarning: The `Merge` layer is deprecated and will be removed after 08/2017. Use instead layers from `keras.layers.merge`, e.g. `add`, `concatenate`, etc.\n", |
||
| 36 | " name=name)\n", |
||
| 37 | "c:\\programdata\\anaconda3\\envs\\tensorflow\\lib\\site-packages\\ipykernel_launcher.py:90: UserWarning: Update your `Conv2D` call to the Keras 2 API: `Conv2D(256, (3, 3), activation=\"relu\", padding=\"same\")`\n", |
||
| 38 | "c:\\programdata\\anaconda3\\envs\\tensorflow\\lib\\site-packages\\ipykernel_launcher.py:91: UserWarning: Update your `Conv2D` call to the Keras 2 API: `Conv2D(256, (3, 3), activation=\"relu\", padding=\"same\")`\n", |
||
| 39 | "c:\\programdata\\anaconda3\\envs\\tensorflow\\lib\\site-packages\\ipykernel_launcher.py:93: UserWarning: The `merge` function is deprecated and will be removed after 08/2017. Use instead layers from `keras.layers.merge`, e.g. `add`, `concatenate`, etc.\n", |
||
| 40 | "c:\\programdata\\anaconda3\\envs\\tensorflow\\lib\\site-packages\\ipykernel_launcher.py:95: UserWarning: Update your `Conv2D` call to the Keras 2 API: `Conv2D(128, (3, 3), activation=\"relu\", padding=\"same\")`\n", |
||
| 41 | "c:\\programdata\\anaconda3\\envs\\tensorflow\\lib\\site-packages\\ipykernel_launcher.py:96: UserWarning: Update your `Conv2D` call to the Keras 2 API: `Conv2D(128, (3, 3), activation=\"relu\", padding=\"same\")`\n", |
||
| 42 | "c:\\programdata\\anaconda3\\envs\\tensorflow\\lib\\site-packages\\ipykernel_launcher.py:98: UserWarning: The `merge` function is deprecated and will be removed after 08/2017. Use instead layers from `keras.layers.merge`, e.g. `add`, `concatenate`, etc.\n", |
||
| 43 | "c:\\programdata\\anaconda3\\envs\\tensorflow\\lib\\site-packages\\ipykernel_launcher.py:100: UserWarning: Update your `Conv2D` call to the Keras 2 API: `Conv2D(64, (3, 3), activation=\"relu\", padding=\"same\")`\n", |
||
| 44 | "c:\\programdata\\anaconda3\\envs\\tensorflow\\lib\\site-packages\\ipykernel_launcher.py:101: UserWarning: Update your `Conv2D` call to the Keras 2 API: `Conv2D(64, (3, 3), activation=\"relu\", padding=\"same\")`\n", |
||
| 45 | "c:\\programdata\\anaconda3\\envs\\tensorflow\\lib\\site-packages\\ipykernel_launcher.py:103: UserWarning: The `merge` function is deprecated and will be removed after 08/2017. Use instead layers from `keras.layers.merge`, e.g. `add`, `concatenate`, etc.\n", |
||
| 46 | "c:\\programdata\\anaconda3\\envs\\tensorflow\\lib\\site-packages\\ipykernel_launcher.py:105: UserWarning: Update your `Conv2D` call to the Keras 2 API: `Conv2D(32, (3, 3), activation=\"relu\", padding=\"same\")`\n", |
||
| 47 | "c:\\programdata\\anaconda3\\envs\\tensorflow\\lib\\site-packages\\ipykernel_launcher.py:106: UserWarning: Update your `Conv2D` call to the Keras 2 API: `Conv2D(32, (3, 3), activation=\"relu\", padding=\"same\")`\n", |
||
| 48 | "c:\\programdata\\anaconda3\\envs\\tensorflow\\lib\\site-packages\\ipykernel_launcher.py:107: UserWarning: Update your `Conv2D` call to the Keras 2 API: `Conv2D(1, (1, 1), activation=\"sigmoid\")`\n", |
||
| 49 | "c:\\programdata\\anaconda3\\envs\\tensorflow\\lib\\site-packages\\ipykernel_launcher.py:109: UserWarning: Update your `Model` call to the Keras 2 API: `Model(outputs=Tensor(\"co..., inputs=Tensor(\"in...)`\n" |
||
| 50 | ] |
||
| 51 | } |
||
| 52 | ], |
||
| 53 | "source": [ |
||
| 54 | "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n", |
||
| 55 | "### EDIT HERE ###\n", |
||
| 56 | "\n", |
||
| 57 | "unetweightspath=\"modelweights/unet-weights-improvement.hdf5\"\n", |
||
| 58 | "truenoduleweightspath=\"modelweights/truenodule-cnn-weights-improvement.hdf5\"\n", |
||
| 59 | "INPUT_FOLDER = 'stage1/' #path to kaggle stage1 dataset\n", |
||
| 60 | "datafolder=\"processeddata/\"\n", |
||
| 61 | "\n", |
||
| 62 | "####################\n", |
||
| 63 | "import numpy as np # linear algebra\n", |
||
| 64 | "\n", |
||
| 65 | "import matplotlib.pyplot as plt\n", |
||
| 66 | "import dicom\n", |
||
| 67 | "import os\n", |
||
| 68 | "import scipy.ndimage\n", |
||
| 69 | "import time\n", |
||
| 70 | "from keras.callbacks import ModelCheckpoint\n", |
||
| 71 | "import h5py\n", |
||
| 72 | "from sklearn.cluster import KMeans\n", |
||
| 73 | "from skimage import measure, morphology\n", |
||
| 74 | "import cell_magic_wand as cmw\n", |
||
| 75 | "import numpy as np\n", |
||
| 76 | "import csv\n", |
||
| 77 | "import random\n", |
||
| 78 | "from sklearn.model_selection import train_test_split\n", |
||
| 79 | "from sklearn.ensemble import RandomForestClassifier as RF\n", |
||
| 80 | "from sklearn.metrics import confusion_matrix, classification_report\n", |
||
| 81 | "from scipy.ndimage.measurements import center_of_mass, label\n", |
||
| 82 | "from skimage.measure import regionprops\n", |
||
| 83 | "\n", |
||
| 84 | "import keras\n", |
||
| 85 | "from keras.models import Sequential,load_model,Model\n", |
||
| 86 | "from keras.layers import Dense, Dropout, Activation, Flatten\n", |
||
| 87 | "from keras.layers import Conv2D, MaxPooling2D, SpatialDropout2D\n", |
||
| 88 | "from keras.layers import Input, merge, UpSampling2D, BatchNormalization\n", |
||
| 89 | "from keras.optimizers import Adam\n", |
||
| 90 | "from keras.callbacks import EarlyStopping, ModelCheckpoint\n", |
||
| 91 | "from keras import backend as K\n", |
||
| 92 | "from keras.utils import to_categorical\n", |
||
| 93 | "from keras.datasets import mnist\n", |
||
| 94 | "from keras.models import Sequential\n", |
||
| 95 | "from keras import backend as K\n", |
||
| 96 | "from keras.optimizers import Adam\n", |
||
| 97 | "# Some constants \n", |
||
| 98 | "\n", |
||
| 99 | "patients = os.listdir(INPUT_FOLDER)\n", |
||
| 100 | "#patients=patients.sort()\n", |
||
| 101 | "K.set_image_dim_ordering('th') \n", |
||
| 102 | "\n", |
||
| 103 | "#Code sourced from https://www.kaggle.com/c/data-science-bowl-2017#tutorial\n", |
||
| 104 | "smooth = 1.0\n", |
||
| 105 | "width = 32\n", |
||
| 106 | "\n", |
||
| 107 | "def dice_coef(y_true, y_pred):\n", |
||
| 108 | " y_true_f = K.flatten(y_true)\n", |
||
| 109 | " y_pred_f = K.flatten(y_pred)\n", |
||
| 110 | " intersection = K.sum(y_true_f * y_pred_f)\n", |
||
| 111 | " return (2. * intersection + smooth) / (K.sum(y_true_f) + K.sum(y_pred_f) + smooth)\n", |
||
| 112 | "def dice_coef_loss(y_true, y_pred):\n", |
||
| 113 | " return -dice_coef(y_true, y_pred)\n", |
||
| 114 | "\n", |
||
| 115 | "def unet_model():\n", |
||
| 116 | " inputs = Input((1, 512, 512))\n", |
||
| 117 | " conv1 = Conv2D(width, 3, 3, activation='relu', border_mode='same')(inputs)\n", |
||
| 118 | " conv1 = BatchNormalization(axis = 1)(conv1)\n", |
||
| 119 | " conv1 = Conv2D(width, 3, 3, activation='relu', border_mode='same')(conv1)\n", |
||
| 120 | " pool1 = MaxPooling2D(pool_size=(2, 2))(conv1)\n", |
||
| 121 | "\n", |
||
| 122 | " conv2 = Conv2D(width*2, 3, 3, activation='relu', border_mode='same')(pool1)\n", |
||
| 123 | " conv2 = BatchNormalization(axis = 1)(conv2)\n", |
||
| 124 | " conv2 = Conv2D(width*2, 3, 3, activation='relu', border_mode='same')(conv2)\n", |
||
| 125 | " pool2 = MaxPooling2D(pool_size=(2, 2))(conv2)\n", |
||
| 126 | "\n", |
||
| 127 | " conv3 = Conv2D(width*4, 3, 3, activation='relu', border_mode='same')(pool2)\n", |
||
| 128 | " conv3 = BatchNormalization(axis = 1)(conv3)\n", |
||
| 129 | " conv3 = Conv2D(width*4, 3, 3, activation='relu', border_mode='same')(conv3)\n", |
||
| 130 | " pool3 = MaxPooling2D(pool_size=(2, 2))(conv3)\n", |
||
| 131 | "\n", |
||
| 132 | " conv4 = Conv2D(width*8, 3, 3, activation='relu', border_mode='same')(pool3)\n", |
||
| 133 | " conv4 = BatchNormalization(axis = 1)(conv4)\n", |
||
| 134 | " conv4 = Conv2D(width*8, 3, 3, activation='relu', border_mode='same')(conv4)\n", |
||
| 135 | " pool4 = MaxPooling2D(pool_size=(2, 2))(conv4)\n", |
||
| 136 | "\n", |
||
| 137 | " conv5 = Conv2D(width*16, 3, 3, activation='relu', border_mode='same')(pool4)\n", |
||
| 138 | " conv5 = BatchNormalization(axis = 1)(conv5)\n", |
||
| 139 | " conv5 = Conv2D(width*16, 3, 3, activation='relu', border_mode='same')(conv5)\n", |
||
| 140 | "\n", |
||
| 141 | " up6 = merge([UpSampling2D(size=(2, 2))(conv5), conv4], mode='concat', concat_axis=1)\n", |
||
| 142 | " conv6 = SpatialDropout2D(0.35)(up6)\n", |
||
| 143 | " conv6 = Conv2D(width*8, 3, 3, activation='relu', border_mode='same')(conv6)\n", |
||
| 144 | " conv6 = Conv2D(width*8, 3, 3, activation='relu', border_mode='same')(conv6)\n", |
||
| 145 | "\n", |
||
| 146 | " up7 = merge([UpSampling2D(size=(2, 2))(conv6), conv3], mode='concat', concat_axis=1)\n", |
||
| 147 | " conv7 = SpatialDropout2D(0.35)(up7)\n", |
||
| 148 | " conv7 = Conv2D(width*4, 3, 3, activation='relu', border_mode='same')(conv7)\n", |
||
| 149 | " conv7 = Conv2D(width*4, 3, 3, activation='relu', border_mode='same')(conv7)\n", |
||
| 150 | "\n", |
||
| 151 | " up8 = merge([UpSampling2D(size=(2, 2))(conv7), conv2], mode='concat', concat_axis=1)\n", |
||
| 152 | " conv8 = SpatialDropout2D(0.35)(up8)\n", |
||
| 153 | " conv8 = Conv2D(width*2, 3, 3, activation='relu', border_mode='same')(conv8)\n", |
||
| 154 | " conv8 = Conv2D(width*2, 3, 3, activation='relu', border_mode='same')(conv8)\n", |
||
| 155 | "\n", |
||
| 156 | " up9 = merge([UpSampling2D(size=(2, 2))(conv8), conv1], mode='concat', concat_axis=1)\n", |
||
| 157 | " conv9 = SpatialDropout2D(0.35)(up9)\n", |
||
| 158 | " conv9 = Conv2D(width, 3, 3, activation='relu', border_mode='same')(conv9)\n", |
||
| 159 | " conv9 = Conv2D(width, 3, 3, activation='relu', border_mode='same')(conv9)\n", |
||
| 160 | " conv10 = Conv2D(1, 1, 1, activation='sigmoid')(conv9)\n", |
||
| 161 | "\n", |
||
| 162 | " model = Model(input=inputs, output=conv10)\n", |
||
| 163 | " model.compile(optimizer=Adam(lr=1e-5), loss=dice_coef_loss, metrics=[dice_coef])\n", |
||
| 164 | " return model\n", |
||
| 165 | "\n", |
||
| 166 | "unet_model=unet_model()\n", |
||
| 167 | "unet_model.load_weights(unetweightspath)" |
||
| 168 | ] |
||
| 169 | }, |
||
| 170 | { |
||
| 171 | "cell_type": "code", |
||
| 172 | "execution_count": 4, |
||
| 173 | "metadata": { |
||
| 174 | "collapsed": true |
||
| 175 | }, |
||
| 176 | "outputs": [], |
||
| 177 | "source": [ |
||
| 178 | "#classify as nodule or non-nodule\n", |
||
| 179 | "input_shape=(1,512,512)\n", |
||
| 180 | "num_classes=2\n", |
||
| 181 | "model = Sequential()\n", |
||
| 182 | "model.add(Conv2D(8, kernel_size=(3, 3),\n", |
||
| 183 | " activation='relu',\n", |
||
| 184 | " input_shape=input_shape))\n", |
||
| 185 | "model.add(Conv2D(16, (3, 3), activation='relu'))\n", |
||
| 186 | "model.add(MaxPooling2D(pool_size=(2, 2)))\n", |
||
| 187 | "model.add(Dropout(0.25))\n", |
||
| 188 | "model.add(Flatten())\n", |
||
| 189 | "model.add(Dense(32, activation='relu'))\n", |
||
| 190 | "model.add(Dropout(0.5))\n", |
||
| 191 | "model.add(Dense(num_classes, activation='softmax'))\n", |
||
| 192 | "\n", |
||
| 193 | "model.compile(loss=keras.losses.binary_crossentropy,\n", |
||
| 194 | " optimizer=Adam(lr=1e-5),\n", |
||
| 195 | " metrics=['accuracy'])\n", |
||
| 196 | "\n", |
||
| 197 | "\n", |
||
| 198 | "model.load_weights(truenoduleweightspath)\n", |
||
| 199 | "#os.environ[\"PATH\"] += os.pathsep + 'C:/Program Files (x86)/Graphviz2.38/bin/'\n", |
||
| 200 | "#plot_model(model, to_file=\"CNNdiagram.png\")" |
||
| 201 | ] |
||
| 202 | }, |
||
| 203 | { |
||
| 204 | "cell_type": "code", |
||
| 205 | "execution_count": 24, |
||
| 206 | "metadata": { |
||
| 207 | "collapsed": true |
||
| 208 | }, |
||
| 209 | "outputs": [], |
||
| 210 | "source": [ |
||
| 211 | "# Load the scans in given folder path\n", |
||
| 212 | "def load_scan(path):\n", |
||
| 213 | " # code sourced from https://www.kaggle.com/gzuidhof/full-preprocessing-tutorial\n", |
||
| 214 | " slices = [dicom.read_file(path + '/' + s, force=True) for s in os.listdir(path) if s.endswith('.dcm')]\n", |
||
| 215 | " slices.sort(key = lambda x: float(x.ImagePositionPatient[2]), reverse=True)\n", |
||
| 216 | " try:\n", |
||
| 217 | " slice_thickness = np.abs(slices[0].ImagePositionPatient[2] - slices[1].ImagePositionPatient[2])\n", |
||
| 218 | " except:\n", |
||
| 219 | " slice_thickness = np.abs(slices[0].SliceLocation - slices[1].SliceLocation)\n", |
||
| 220 | " \n", |
||
| 221 | " for s in slices:\n", |
||
| 222 | " s.SliceThickness = slice_thickness\n", |
||
| 223 | " \n", |
||
| 224 | " return slices\n", |
||
| 225 | "\n", |
||
| 226 | "def get_pixels_hu(slices):\n", |
||
| 227 | " #code sourced from https://www.kaggle.com/gzuidhof/full-preprocessing-tutorial\n", |
||
| 228 | " image = np.stack([s.pixel_array for s in slices])\n", |
||
| 229 | " # Convert to int16 (from sometimes int16), \n", |
||
| 230 | " # should be possible as values should always be low enough (<32k)\n", |
||
| 231 | " image = image.astype(np.int16)\n", |
||
| 232 | "\n", |
||
| 233 | " # Set outside-of-scan pixels to 0\n", |
||
| 234 | " # The intercept is usually -1024, so air is approximately 0\n", |
||
| 235 | " image[image == -2000] = 0\n", |
||
| 236 | " \n", |
||
| 237 | " # Convert to Hounsfield units (HU)\n", |
||
| 238 | " for slice_number in range(len(slices)):\n", |
||
| 239 | " \n", |
||
| 240 | " intercept = slices[slice_number].RescaleIntercept\n", |
||
| 241 | " slope = slices[slice_number].RescaleSlope\n", |
||
| 242 | " \n", |
||
| 243 | " if slope != 1:\n", |
||
| 244 | " image[slice_number] = slope * image[slice_number].astype(np.float64)\n", |
||
| 245 | " image[slice_number] = image[slice_number].astype(np.int16)\n", |
||
| 246 | " \n", |
||
| 247 | " image[slice_number] += np.int16(intercept)\n", |
||
| 248 | " \n", |
||
| 249 | " return np.array(image, dtype=np.int16)\n", |
||
| 250 | "\n", |
||
| 251 | "def processimage(img):\n", |
||
| 252 | " #function sourced from https://www.kaggle.com/c/data-science-bowl-2017#tutorial\n", |
||
| 253 | " #Standardize the pixel values\n", |
||
| 254 | " mean = np.mean(img)\n", |
||
| 255 | " std = np.std(img)\n", |
||
| 256 | " img = img-mean\n", |
||
| 257 | " img = img/std\n", |
||
| 258 | " #plt.hist(img.flatten(),bins=200)\n", |
||
| 259 | " #plt.show()\n", |
||
| 260 | " #print(thresh_img[366][280:450])\n", |
||
| 261 | " middle = img[100:400,100:400] \n", |
||
| 262 | " mean = np.mean(middle) \n", |
||
| 263 | " max = np.max(img)\n", |
||
| 264 | " min = np.min(img)\n", |
||
| 265 | " #move the underflow bins\n", |
||
| 266 | " img[img==max]=mean\n", |
||
| 267 | " img[img==min]=mean\n", |
||
| 268 | " kmeans = KMeans(n_clusters=2).fit(np.reshape(middle,[np.prod(middle.shape),1]))\n", |
||
| 269 | " centers = sorted(kmeans.cluster_centers_.flatten())\n", |
||
| 270 | " threshold = np.mean(centers)\n", |
||
| 271 | " thresh_img = np.where(img<threshold,1.0,0.0) # threshold the image\n", |
||
| 272 | " eroded = morphology.erosion(thresh_img,np.ones([4,4]))\n", |
||
| 273 | " dilation = morphology.dilation(eroded,np.ones([10,10]))\n", |
||
| 274 | " labels = measure.label(dilation)\n", |
||
| 275 | " label_vals = np.unique(labels)\n", |
||
| 276 | " #plt.imshow(labels)\n", |
||
| 277 | " #plt.show()\n", |
||
| 278 | " labels = measure.label(dilation)\n", |
||
| 279 | " label_vals = np.unique(labels)\n", |
||
| 280 | " regions = measure.regionprops(labels)\n", |
||
| 281 | " good_labels = []\n", |
||
| 282 | " for prop in regions:\n", |
||
| 283 | " B = prop.bbox\n", |
||
| 284 | " if B[2]-B[0]<475 and B[3]-B[1]<475 and B[0]>40 and B[2]<472:\n", |
||
| 285 | " good_labels.append(prop.label)\n", |
||
| 286 | " mask = np.ndarray([512,512],dtype=np.int8)\n", |
||
| 287 | " mask[:] = 0\n", |
||
| 288 | " #\n", |
||
| 289 | " # The mask here is the mask for the lungs--not the nodes\n", |
||
| 290 | " # After just the lungs are left, we do another large dilation\n", |
||
| 291 | " # in order to fill in and out the lung mask \n", |
||
| 292 | " #\n", |
||
| 293 | " for N in good_labels:\n", |
||
| 294 | " mask = mask + np.where(labels==N,1,0)\n", |
||
| 295 | " mask = morphology.dilation(mask,np.ones([10,10])) # one last dilation\n", |
||
| 296 | " return mask*img\n", |
||
| 297 | "\n", |
||
| 298 | "def processimagenomask(img):\n", |
||
| 299 | " #Standardize the pixel values\n", |
||
| 300 | " mean = np.mean(img)\n", |
||
| 301 | " std = np.std(img)\n", |
||
| 302 | " img = img-mean\n", |
||
| 303 | " img = img/std\n", |
||
| 304 | " #plt.hist(img.flatten(),bins=200)\n", |
||
| 305 | " #plt.show()\n", |
||
| 306 | " #print(thresh_img[366][280:450])\n", |
||
| 307 | " middle = img[100:400,100:400] \n", |
||
| 308 | " mean = np.mean(middle) \n", |
||
| 309 | " max = np.max(img)\n", |
||
| 310 | " min = np.min(img)\n", |
||
| 311 | " #move the underflow bins\n", |
||
| 312 | " img[img==max]=mean\n", |
||
| 313 | " img[img==min]=mean\n", |
||
| 314 | " return img\n", |
||
| 315 | "\n", |
||
| 316 | "def processimagefromfile(ppix):\n", |
||
| 317 | " processpix=np.ndarray([ppix.shape[0],1,512,512])\n", |
||
| 318 | " for i in range(ppix.shape[0]):\n", |
||
| 319 | " processpix[i,0]=processimage(ppix[i])\n", |
||
| 320 | " return processpix\n", |
||
| 321 | "\n", |
||
| 322 | "#predict mask from images\n", |
||
| 323 | "def predictmask(images):\n", |
||
| 324 | " num_test=images.shape[0]\n", |
||
| 325 | " imgs_mask_test = np.ndarray([num_test,1,512,512],dtype=np.float32)\n", |
||
| 326 | " for i in range(num_test):\n", |
||
| 327 | " imgs_mask_test[i] = unet_model.predict([images[i:i+1]], verbose=0)[0]\n", |
||
| 328 | " return imgs_mask_test\n", |
||
| 329 | "\n", |
||
| 330 | "#find number of slices where a nodule is detected\n", |
||
| 331 | "def getnoduleindex(imgs_mask_test):\n", |
||
| 332 | " masksum=[np.sum(maskslice[0]) for maskslice in imgs_mask_test]\n", |
||
| 333 | " return [i for i in range(len(masksum)) if masksum[i]>5]\n", |
||
| 334 | "\n", |
||
| 335 | "def trueindicies(processed_pix, noduleindex):\n", |
||
| 336 | " noduleimgs=[processed_pix[ind] for ind in noduleindex]\n", |
||
| 337 | " noduleimgs=np.array(noduleimgs)\n", |
||
| 338 | " predictions=model.predict(noduleimgs)\n", |
||
| 339 | " predictions=predictions[:len(predictions),1]\n", |
||
| 340 | " predictions[predictions>0.5]=True\n", |
||
| 341 | " predictions[predictions<0.5]=False\n", |
||
| 342 | " trueindicies=[ind for i,ind in enumerate(noduleindex) if predictions[i]==1]\n", |
||
| 343 | " return trueindicies\n", |
||
| 344 | "\n", |
||
| 345 | "def thresholdnodules(noduleindices,mask):\n", |
||
| 346 | " nodulearea=[]\n", |
||
| 347 | " for ind in trueindicies:\n", |
||
| 348 | " nodulearea.append(np.sum(mask[ind]))\n", |
||
| 349 | " return nodulearea\n", |
||
| 350 | "\n", |
||
| 351 | "def largestnodulecoordinates(mask):\n", |
||
| 352 | " #mask=nodulemasks[indx,0][0]\n", |
||
| 353 | " mask[mask>0.5]=1\n", |
||
| 354 | " mask[mask<0.5]=0\n", |
||
| 355 | " labeled_array,nf=label(mask)\n", |
||
| 356 | " areasinslice=[]\n", |
||
| 357 | " if nf>1:\n", |
||
| 358 | " for n in range(nf):\n", |
||
| 359 | " lab=np.array(labeled_array)\n", |
||
| 360 | " lab[lab!=(n+1)]=0\n", |
||
| 361 | " lab[lab==(n+1)]=1\n", |
||
| 362 | " areasinslice.append(np.sum(lab))\n", |
||
| 363 | " nlargest=areasinslice.index(max(areasinslice))\n", |
||
| 364 | " labeled_array[labeled_array!=(nlargest+1)]=0\n", |
||
| 365 | " com=center_of_mass(labeled_array)\n", |
||
| 366 | " else:\n", |
||
| 367 | " com=center_of_mass(mask)\n", |
||
| 368 | " return [int(com[0]),int(com[1])]\n", |
||
| 369 | "\n", |
||
| 370 | "def largestnodulearea(mask,table,i):\n", |
||
| 371 | " #mask=nodulemasks[indx,0][0]\n", |
||
| 372 | " mask[mask>0.5]=1\n", |
||
| 373 | " mask[mask<0.5]=0\n", |
||
| 374 | " labeled_array,nf=label(mask)\n", |
||
| 375 | " areasinslice=[]\n", |
||
| 376 | " if nf>1:\n", |
||
| 377 | " for n in range(nf):\n", |
||
| 378 | " lab=np.array(labeled_array)\n", |
||
| 379 | " lab[lab!=(n+1)]=0\n", |
||
| 380 | " lab[lab==(n+1)]=1\n", |
||
| 381 | " areasinslice.append(np.sum(lab))\n", |
||
| 382 | " #nlargest=areasinslice.index(max(areasinslice))\n", |
||
| 383 | " #labeled_array[labeled_array!=(nlargest+1)]=0\n", |
||
| 384 | " return max(areasinslice)\n", |
||
| 385 | " else:\n", |
||
| 386 | " return table[\"Area\"][i]\n", |
||
| 387 | "\n", |
||
| 388 | "def crop_nodule(coord,image):\n", |
||
| 389 | " dim=32\n", |
||
| 390 | " return image[coord[0]-dim:coord[0]+dim,coord[1]-dim:coord[1]+dim]\n", |
||
| 391 | "#output: 64x64 images of the nodules with malignancy labels from the patient\n", |
||
| 392 | "\n", |
||
| 393 | "def largestnoduleproperties(mask):\n", |
||
| 394 | " mask[mask>0.5]=1\n", |
||
| 395 | " mask[mask<0.5]=0\n", |
||
| 396 | " mask=mask.astype(np.int8)\n", |
||
| 397 | " labeled_array,nf=label(mask)\n", |
||
| 398 | " areasinslice=[]\n", |
||
| 399 | " if nf>1:\n", |
||
| 400 | " for n in range(nf):\n", |
||
| 401 | " lab=np.array(labeled_array)\n", |
||
| 402 | " lab[lab!=(n+1)]=0\n", |
||
| 403 | " lab[lab==(n+1)]=1\n", |
||
| 404 | " areasinslice.append(np.sum(lab))\n", |
||
| 405 | " nlargest=areasinslice.index(max(areasinslice))\n", |
||
| 406 | " labeled_array[labeled_array!=(nlargest+1)]=0\n", |
||
| 407 | " noduleprops=regionprops(labeled_array)\n", |
||
| 408 | " else:\n", |
||
| 409 | " noduleprops=regionprops(mask)\n", |
||
| 410 | " area=noduleprops[0].area\n", |
||
| 411 | " eccentricity=noduleprops[0].eccentricity\n", |
||
| 412 | " diam=noduleprops[0].equivalent_diameter\n", |
||
| 413 | " diammajor=noduleprops[0].major_axis_length\n", |
||
| 414 | " spiculation=noduleprops[0].solidity\n", |
||
| 415 | " return area, eccentricity, diam, diammajor, spiculation\n", |
||
| 416 | "\n", |
||
| 417 | "def generatefeaturetable(nodulemasks):\n", |
||
| 418 | " meannoduleHU=[]\n", |
||
| 419 | " nodulecount=[]\n", |
||
| 420 | " largestarealist=[]\n", |
||
| 421 | " eccentricitylist=[]\n", |
||
| 422 | " diamlist=[]\n", |
||
| 423 | " diammajorlist=[]\n", |
||
| 424 | " spiculationlist=[]\n", |
||
| 425 | "\n", |
||
| 426 | " for i in range(nodulemasks.shape[0]):\n", |
||
| 427 | " mask=nodulemasks[i,0]\n", |
||
| 428 | " mask[mask>0.5]=1\n", |
||
| 429 | " mask[mask<0.5]=0\n", |
||
| 430 | " meannoduleHU.append(np.sum(noduleimages[i,0]*mask)/np.sum(mask))\n", |
||
| 431 | " labeled_array,features=label(mask)\n", |
||
| 432 | " nodulecount.append(features)\n", |
||
| 433 | " area, eccentricity, diam, diammajor, spiculation = largestnoduleproperties(nodulemasks[i,0])\n", |
||
| 434 | " largestarealist.append(area)\n", |
||
| 435 | " eccentricitylist.append(eccentricity)\n", |
||
| 436 | " diamlist.append(diam)\n", |
||
| 437 | " diammajorlist.append(diammajor)\n", |
||
| 438 | " spiculationlist.append(spiculation)\n", |
||
| 439 | " table=pd.DataFrame({\"Patient\":sample,\"NoduleIndex\":noduleindicies,\"Area\":area,\"MeanHU\":meannoduleHU, \"LargestNoduleArea\":largestarealist,\n", |
||
| 440 | " \"Eccentricity\":eccentricitylist, \"Diameter\":diamlist, \"DiameterMajor\":diammajorlist, \"Spiculation\":spiculationlist})\n", |
||
| 441 | " return table" |
||
| 442 | ] |
||
| 443 | }, |
||
| 444 | { |
||
| 445 | "cell_type": "code", |
||
| 446 | "execution_count": 6, |
||
| 447 | "metadata": { |
||
| 448 | "scrolled": true |
||
| 449 | }, |
||
| 450 | "outputs": [ |
||
| 451 | { |
||
| 452 | "name": "stdout", |
||
| 453 | "output_type": "stream", |
||
| 454 | "text": [ |
||
| 455 | "Processing patient# 1200 ETA: 13.88888888888889 hrs\n", |
||
| 456 | "Processing patient# 1201 ETA: 12.623715912898382 hrs\n", |
||
| 457 | "Processing patient# 1202 ETA: 10.018525106410186 hrs\n", |
||
| 458 | "Processing patient# 1203 ETA: 18.164322644339666 hrs\n", |
||
| 459 | "Processing patient# 1204 ETA: 15.95403834571441 hrs\n", |
||
| 460 | "Processing patient# 1205 ETA: 14.166520277659098 hrs\n", |
||
| 461 | "Processing patient# 1206 ETA: 13.91482162181978 hrs\n", |
||
| 462 | "Processing patient# 1208 ETA: 13.044705638507056 hrs\n", |
||
| 463 | "Processing patient# 1209 ETA: 11.268210861565155 hrs\n", |
||
| 464 | "Processing patient# 1210 ETA: 11.153029166426924 hrs\n", |
||
| 465 | "Processing patient# 1211 ETA: 10.874260760220615 hrs\n", |
||
| 466 | "Processing patient# 1212 ETA: 10.507523334208463 hrs\n", |
||
| 467 | "Processing patient# 1213 ETA: 10.314581797112767 hrs\n", |
||
| 468 | "Processing patient# 1214 ETA: 10.08706813660406 hrs\n", |
||
| 469 | "Processing patient# 1216 ETA: 9.996492877094832 hrs\n", |
||
| 470 | "Processing patient# 1217 ETA: 9.102958305653404 hrs\n", |
||
| 471 | "Processing patient# 1219 ETA: 8.925979718978024 hrs\n", |
||
| 472 | "Processing patient# 1220 ETA: 8.507819953064123 hrs\n", |
||
| 473 | "Processing patient# 1221 ETA: 8.645915125431838 hrs\n", |
||
| 474 | "Processing patient# 1222 ETA: 8.543059910430449 hrs\n", |
||
| 475 | "Processing patient# 1223 ETA: 8.726262301500293 hrs\n", |
||
| 476 | "Processing patient# 1225 ETA: 8.727355263285615 hrs\n", |
||
| 477 | "Processing patient# 1226 ETA: 8.455341982405919 hrs\n", |
||
| 478 | "Processing patient# 1227 ETA: 8.351120784155135 hrs\n", |
||
| 479 | "Processing patient# 1228 ETA: 8.316320655887562 hrs\n", |
||
| 480 | "Processing patient# 1229 ETA: 8.488186760167965 hrs\n", |
||
| 481 | "Processing patient# 1230 ETA: 8.393290068418892 hrs\n", |
||
| 482 | "Processing patient# 1231 ETA: 8.380940078535387 hrs\n", |
||
| 483 | "Processing patient# 1232 ETA: 8.325024080611765 hrs\n", |
||
| 484 | "Processing patient# 1233 ETA: 8.342425880022724 hrs\n", |
||
| 485 | "Processing patient# 1234 ETA: 8.32099210145232 hrs\n", |
||
| 486 | "Processing patient# 1235 ETA: 8.283116624014719 hrs\n", |
||
| 487 | "Processing patient# 1236 ETA: 8.308585265549246 hrs\n", |
||
| 488 | "Processing patient# 1237 ETA: 8.243522517545086 hrs\n", |
||
| 489 | "Processing patient# 1238 ETA: 8.686181035109781 hrs\n", |
||
| 490 | "Processing patient# 1239 ETA: 8.866050753437216 hrs\n", |
||
| 491 | "Processing patient# 1240 ETA: 8.860589005996784 hrs\n", |
||
| 492 | "Processing patient# 1241 ETA: 8.904041662632933 hrs\n", |
||
| 493 | "Processing patient# 1242 ETA: 8.81039298651395 hrs\n", |
||
| 494 | "Processing patient# 1244 ETA: 8.987819228849979 hrs\n", |
||
| 495 | "Processing patient# 1245 ETA: 8.728126824490818 hrs\n", |
||
| 496 | "Processing patient# 1247 ETA: 8.675649444095177 hrs\n", |
||
| 497 | "Processing patient# 1248 ETA: 8.429746265785287 hrs\n", |
||
| 498 | "Processing patient# 1249 ETA: 8.358244928213475 hrs\n", |
||
| 499 | "Processing patient# 1251 ETA: 8.380413904567561 hrs\n", |
||
| 500 | "Processing patient# 1252 ETA: 8.127052396800783 hrs\n", |
||
| 501 | "Processing patient# 1253 ETA: 8.06759037937758 hrs\n", |
||
| 502 | "Processing patient# 1255 ETA: 8.08903314394838 hrs\n", |
||
| 503 | "Processing patient# 1256 ETA: 7.984299782819692 hrs\n", |
||
| 504 | "Processing patient# 1257 ETA: 7.922037844325599 hrs\n", |
||
| 505 | "Processing patient# 1258 ETA: 7.861369710803259 hrs\n", |
||
| 506 | "Processing patient# 1259 ETA: 7.8176956071153185 hrs\n", |
||
| 507 | "Processing patient# 1260 ETA: 7.7597010760285245 hrs\n", |
||
| 508 | "Processing patient# 1261 ETA: 7.840702676859926 hrs\n", |
||
| 509 | "Processing patient# 1262 ETA: 7.801467700129554 hrs\n", |
||
| 510 | "Processing patient# 1263 ETA: 7.790705240697878 hrs\n", |
||
| 511 | "Processing patient# 1264 ETA: 7.734633472519409 hrs\n", |
||
| 512 | "Processing patient# 1265 ETA: 7.703382088404434 hrs\n", |
||
| 513 | "Processing patient# 1266 ETA: 7.695265196908001 hrs\n", |
||
| 514 | "Processing patient# 1267 ETA: 7.699010734739984 hrs\n", |
||
| 515 | "Processing patient# 1268 ETA: 7.6863632827208335 hrs\n", |
||
| 516 | "Processing patient# 1270 ETA: 7.630932073109392 hrs\n", |
||
| 517 | "Processing patient# 1272 ETA: 7.533154250803129 hrs\n", |
||
| 518 | "Processing patient# 1273 ETA: 7.3653022840142786 hrs\n", |
||
| 519 | "Processing patient# 1274 ETA: 7.319828108404134 hrs\n", |
||
| 520 | "Processing patient# 1276 ETA: 7.362176086990922 hrs\n", |
||
| 521 | "Processing patient# 1277 ETA: 7.265992607554316 hrs\n", |
||
| 522 | "Processing patient# 1278 ETA: 7.218131187677043 hrs\n", |
||
| 523 | "Processing patient# 1279 ETA: 7.247058704429203 hrs\n", |
||
| 524 | "Processing patient# 1280 ETA: 7.2205354031175375 hrs\n", |
||
| 525 | "Processing patient# 1284 ETA: 7.229128276611714 hrs\n", |
||
| 526 | "Processing patient# 1285 ETA: 6.863715362470914 hrs\n", |
||
| 527 | "Processing patient# 1286 ETA: 6.868215039321618 hrs\n", |
||
| 528 | "Processing patient# 1287 ETA: 6.872994949424861 hrs\n", |
||
| 529 | "Processing patient# 1288 ETA: 6.876982837402911 hrs\n", |
||
| 530 | "Processing patient# 1289 ETA: 6.8752770751245915 hrs\n", |
||
| 531 | "Processing patient# 1290 ETA: 6.840804919596807 hrs\n", |
||
| 532 | "Processing patient# 1291 ETA: 6.811198504381564 hrs\n", |
||
| 533 | "Processing patient# 1292 ETA: 6.77358079140385 hrs\n", |
||
| 534 | "Processing patient# 1293 ETA: 6.830506550602018 hrs\n", |
||
| 535 | "Processing patient# 1294 ETA: 6.802673126880441 hrs\n", |
||
| 536 | "Processing patient# 1297 ETA: 6.911592493140907 hrs\n", |
||
| 537 | "Processing patient# 1298 ETA: 6.715936135601024 hrs\n", |
||
| 538 | "Processing patient# 1299 ETA: 6.67262925623777 hrs\n", |
||
| 539 | "Processing patient# 1300 ETA: 6.629177303820848 hrs\n", |
||
| 540 | "Processing patient# 1301 ETA: 6.606589229476728 hrs\n", |
||
| 541 | "Processing patient# 1302 ETA: 6.584290691885554 hrs\n", |
||
| 542 | "Processing patient# 1304 ETA: 6.567115247555663 hrs\n", |
||
| 543 | "Processing patient# 1306 ETA: 6.466728328926854 hrs\n", |
||
| 544 | "Processing patient# 1307 ETA: 6.3809614388296545 hrs\n", |
||
| 545 | "Processing patient# 1308 ETA: 6.346115140000612 hrs\n", |
||
| 546 | "Processing patient# 1309 ETA: 6.356010770753983 hrs\n", |
||
| 547 | "Processing patient# 1310 ETA: 6.314931866174394 hrs\n", |
||
| 548 | "Processing patient# 1311 ETA: 6.307182563506328 hrs\n", |
||
| 549 | "Processing patient# 1313 ETA: 6.279791420257635 hrs\n", |
||
| 550 | "Processing patient# 1314 ETA: 6.292742651619061 hrs\n", |
||
| 551 | "Processing patient# 1315 ETA: 6.269811408600369 hrs\n", |
||
| 552 | "Processing patient# 1316 ETA: 6.250402978571325 hrs\n", |
||
| 553 | "Processing patient# 1318 ETA: 6.2339090946913895 hrs\n", |
||
| 554 | "Processing patient# 1319 ETA: 6.160530481745883 hrs\n", |
||
| 555 | "Processing patient# 1320 ETA: 6.130471623537166 hrs\n", |
||
| 556 | "Processing patient# 1321 ETA: 6.121893712554752 hrs\n", |
||
| 557 | "Processing patient# 1323 ETA: 6.089000249715125 hrs\n", |
||
| 558 | "Processing patient# 1324 ETA: 5.980023680922165 hrs\n", |
||
| 559 | "Processing patient# 1325 ETA: 5.956943183755874 hrs\n", |
||
| 560 | "Processing patient# 1327 ETA: 5.946134261675508 hrs\n", |
||
| 561 | "Processing patient# 1328 ETA: 5.850976662812754 hrs\n", |
||
| 562 | "Processing patient# 1329 ETA: 5.820975284885477 hrs\n", |
||
| 563 | "Processing patient# 1331 ETA: 5.8138491802821815 hrs\n", |
||
| 564 | "Processing patient# 1333 ETA: 5.725956927912203 hrs\n", |
||
| 565 | "Processing patient# 1334 ETA: 5.628774382734477 hrs\n", |
||
| 566 | "Processing patient# 1335 ETA: 5.606204076225375 hrs\n", |
||
| 567 | "Processing patient# 1336 ETA: 5.579955613982735 hrs\n", |
||
| 568 | "Processing patient# 1337 ETA: 5.590139669641091 hrs\n", |
||
| 569 | "Processing patient# 1338 ETA: 5.566476827514825 hrs\n", |
||
| 570 | "Processing patient# 1339 ETA: 5.543931497014302 hrs\n", |
||
| 571 | "Processing patient# 1341 ETA: 5.524890386738948 hrs\n", |
||
| 572 | "Processing patient# 1342 ETA: 5.448634859161664 hrs\n", |
||
| 573 | "Processing patient# 1343 ETA: 5.417805575740921 hrs\n", |
||
| 574 | "Processing patient# 1344 ETA: 5.400687092230369 hrs\n", |
||
| 575 | "Processing patient# 1345 ETA: 5.3883224553303695 hrs\n", |
||
| 576 | "Processing patient# 1346 ETA: 5.361380771469307 hrs\n", |
||
| 577 | "Processing patient# 1348 ETA: 5.347543323521171 hrs\n", |
||
| 578 | "Processing patient# 1349 ETA: 5.270577565466265 hrs\n", |
||
| 579 | "Processing patient# 1350 ETA: 5.250531795351592 hrs\n", |
||
| 580 | "Processing patient# 1351 ETA: 5.227737224219562 hrs\n", |
||
| 581 | "Processing patient# 1352 ETA: 5.204326420002862 hrs\n", |
||
| 582 | "Processing patient# 1353 ETA: 5.186513844776084 hrs\n", |
||
| 583 | "Processing patient# 1354 ETA: 5.1599767990571594 hrs\n", |
||
| 584 | "Processing patient# 1355 ETA: 5.1349731775509415 hrs\n", |
||
| 585 | "Processing patient# 1356 ETA: 5.108270640927247 hrs\n", |
||
| 586 | "Processing patient# 1357 ETA: 5.115111228089255 hrs\n", |
||
| 587 | "Processing patient# 1358 ETA: 5.096474325358868 hrs\n", |
||
| 588 | "Processing patient# 1359 ETA: 5.068261158612955 hrs\n", |
||
| 589 | "Processing patient# 1360 ETA: 5.042548690814938 hrs\n", |
||
| 590 | "Processing patient# 1361 ETA: 5.01235181745535 hrs\n", |
||
| 591 | "Processing patient# 1362 ETA: 4.998160559473705 hrs\n", |
||
| 592 | "Processing patient# 1363 ETA: 4.974829404645009 hrs\n", |
||
| 593 | "Processing patient# 1364 ETA: 4.952033427223199 hrs\n", |
||
| 594 | "Processing patient# 1365 ETA: 4.928263342958509 hrs\n", |
||
| 595 | "Processing patient# 1366 ETA: 4.906512433510309 hrs\n", |
||
| 596 | "Processing patient# 1368 ETA: 4.884942292143961 hrs\n", |
||
| 597 | "Processing patient# 1369 ETA: 4.82783218746433 hrs\n", |
||
| 598 | "Processing patient# 1370 ETA: 4.8466363209135395 hrs\n", |
||
| 599 | "Processing patient# 1371 ETA: 4.8318471228541915 hrs\n", |
||
| 600 | "Processing patient# 1372 ETA: 4.821795012823449 hrs\n", |
||
| 601 | "Processing patient# 1373 ETA: 4.803408985386005 hrs\n", |
||
| 602 | "Processing patient# 1374 ETA: 4.777427026608132 hrs\n", |
||
| 603 | "Processing patient# 1375 ETA: 4.763656514637054 hrs\n", |
||
| 604 | "Processing patient# 1376 ETA: 4.743936037966472 hrs\n", |
||
| 605 | "Processing patient# 1377 ETA: 4.719618169200248 hrs\n", |
||
| 606 | "Processing patient# 1378 ETA: 4.690835314464851 hrs\n", |
||
| 607 | "Processing patient# 1379 ETA: 4.666089715291667 hrs\n", |
||
| 608 | "Processing patient# 1380 ETA: 4.675125468286835 hrs\n", |
||
| 609 | "Processing patient# 1381 ETA: 4.658687751179754 hrs\n", |
||
| 610 | "Processing patient# 1382 ETA: 4.635416509547295 hrs\n", |
||
| 611 | "Processing patient# 1383 ETA: 4.618964403185181 hrs\n", |
||
| 612 | "Processing patient# 1384 ETA: 4.5948020707891475 hrs\n" |
||
| 613 | ] |
||
| 614 | }, |
||
| 615 | { |
||
| 616 | "name": "stdout", |
||
| 617 | "output_type": "stream", |
||
| 618 | "text": [ |
||
| 619 | "Processing patient# 1385 ETA: 4.570799427429835 hrs\n", |
||
| 620 | "Processing patient# 1386 ETA: 4.54748295711004 hrs\n", |
||
| 621 | "Processing patient# 1387 ETA: 4.527965035849359 hrs\n", |
||
| 622 | "Processing patient# 1388 ETA: 4.499852827242081 hrs\n", |
||
| 623 | "Processing patient# 1389 ETA: 4.477454396751192 hrs\n", |
||
| 624 | "Processing patient# 1390 ETA: 4.456050586798038 hrs\n", |
||
| 625 | "Processing patient# 1391 ETA: 4.434381366932787 hrs\n", |
||
| 626 | "Processing patient# 1392 ETA: 4.407015767265111 hrs\n", |
||
| 627 | "Processing patient# 1394 ETA: 4.37906760837885 hrs\n", |
||
| 628 | "Processing patient# 1395 ETA: 4.30689841369618 hrs\n", |
||
| 629 | "Processing patient# 1396 ETA: 4.285369354544922 hrs\n", |
||
| 630 | "Processing patient# 1397 ETA: 4.268326153216628 hrs\n", |
||
| 631 | "Processing patient# 1398 ETA: 4.263744453107467 hrs\n", |
||
| 632 | "Processing patient# 1399 ETA: 4.243092987329717 hrs\n", |
||
| 633 | "Processing patient# 1400 ETA: 4.222136327758431 hrs\n", |
||
| 634 | "Processing patient# 1401 ETA: 4.198838181790889 hrs\n", |
||
| 635 | "Processing patient# 1402 ETA: 4.1793994686896 hrs\n", |
||
| 636 | "Processing patient# 1403 ETA: 4.165873683685152 hrs\n", |
||
| 637 | "Processing patient# 1404 ETA: 4.145204392498549 hrs\n", |
||
| 638 | "Processing patient# 1405 ETA: 4.127004841931144 hrs\n", |
||
| 639 | "Processing patient# 1406 ETA: 4.1010385624992045 hrs\n", |
||
| 640 | "Processing patient# 1407 ETA: 4.089603952259641 hrs\n", |
||
| 641 | "Processing patient# 1410 ETA: 4.0657218732484255 hrs\n", |
||
| 642 | "Processing patient# 1411 ETA: 3.9614282328307855 hrs\n", |
||
| 643 | "Processing patient# 1412 ETA: 3.935801912909784 hrs\n", |
||
| 644 | "Processing patient# 1414 ETA: 3.9153825925978505 hrs\n", |
||
| 645 | "Processing patient# 1415 ETA: 3.8567507741617604 hrs\n", |
||
| 646 | "Processing patient# 1417 ETA: 3.831965497984502 hrs\n", |
||
| 647 | "Processing patient# 1418 ETA: 3.776563055347595 hrs\n", |
||
| 648 | "Processing patient# 1419 ETA: 3.7698241562507033 hrs\n", |
||
| 649 | "Processing patient# 1420 ETA: 3.7446089217382843 hrs\n", |
||
| 650 | "Processing patient# 1421 ETA: 3.7240907096557123 hrs\n", |
||
| 651 | "Processing patient# 1422 ETA: 3.7038961130109156 hrs\n", |
||
| 652 | "Processing patient# 1423 ETA: 3.682920000244269 hrs\n", |
||
| 653 | "Processing patient# 1424 ETA: 3.6590023383365153 hrs\n", |
||
| 654 | "Processing patient# 1425 ETA: 3.6355938848919345 hrs\n", |
||
| 655 | "Processing patient# 1426 ETA: 3.6216272131716254 hrs\n", |
||
| 656 | "Processing patient# 1427 ETA: 3.6168983707203903 hrs\n", |
||
| 657 | "Processing patient# 1428 ETA: 3.5952787640760513 hrs\n", |
||
| 658 | "Processing patient# 1429 ETA: 3.5846346884957216 hrs\n", |
||
| 659 | "Processing patient# 1430 ETA: 3.564002736003503 hrs\n", |
||
| 660 | "Processing patient# 1431 ETA: 3.5397180199405214 hrs\n", |
||
| 661 | "Processing patient# 1432 ETA: 3.5171189450058673 hrs\n", |
||
| 662 | "Processing patient# 1433 ETA: 3.495935878656453 hrs\n", |
||
| 663 | "Processing patient# 1436 ETA: 3.490441271786393 hrs\n", |
||
| 664 | "Processing patient# 1437 ETA: 3.3961545779969957 hrs\n", |
||
| 665 | "Processing patient# 1438 ETA: 3.3754478486011674 hrs\n", |
||
| 666 | "Processing patient# 1439 ETA: 3.3569018378051583 hrs\n", |
||
| 667 | "Processing patient# 1440 ETA: 3.3322680721583744 hrs\n", |
||
| 668 | "Processing patient# 1441 ETA: 3.3100857617839687 hrs\n", |
||
| 669 | "Processing patient# 1443 ETA: 3.291559807825679 hrs\n", |
||
| 670 | "Processing patient# 1444 ETA: 3.2368162911495224 hrs\n", |
||
| 671 | "Processing patient# 1445 ETA: 3.219285455204192 hrs\n", |
||
| 672 | "Processing patient# 1446 ETA: 3.1963537533862594 hrs\n", |
||
| 673 | "Processing patient# 1447 ETA: 3.1745961232535005 hrs\n", |
||
| 674 | "Processing patient# 1448 ETA: 3.161650590983931 hrs\n", |
||
| 675 | "Processing patient# 1449 ETA: 3.1433429676488482 hrs\n", |
||
| 676 | "Processing patient# 1450 ETA: 3.1198210242682034 hrs\n", |
||
| 677 | "Processing patient# 1451 ETA: 3.0976603277366 hrs\n", |
||
| 678 | "Processing patient# 1452 ETA: 3.0779698065120384 hrs\n", |
||
| 679 | "Processing patient# 1453 ETA: 3.0560144474137805 hrs\n", |
||
| 680 | "Processing patient# 1454 ETA: 3.035682783060343 hrs\n", |
||
| 681 | "Processing patient# 1455 ETA: 3.01512198534926 hrs\n", |
||
| 682 | "Processing patient# 1457 ETA: 3.001986470494109 hrs\n", |
||
| 683 | "Processing patient# 1460 ETA: 2.952160390772817 hrs\n", |
||
| 684 | "Processing patient# 1461 ETA: 2.8634344177440965 hrs\n", |
||
| 685 | "Processing patient# 1462 ETA: 2.8440811097260443 hrs\n", |
||
| 686 | "Processing patient# 1464 ETA: 2.8232880296876375 hrs\n", |
||
| 687 | "Processing patient# 1465 ETA: 2.7719246257226184 hrs\n", |
||
| 688 | "Processing patient# 1466 ETA: 2.7521925461187706 hrs\n", |
||
| 689 | "Processing patient# 1468 ETA: 2.7326375998926817 hrs\n", |
||
| 690 | "Processing patient# 1469 ETA: 2.6889556477504155 hrs\n", |
||
| 691 | "Processing patient# 1471 ETA: 2.6669628266321768 hrs\n", |
||
| 692 | "Processing patient# 1472 ETA: 2.61743253900286 hrs\n", |
||
| 693 | "Processing patient# 1473 ETA: 2.59755473644354 hrs\n", |
||
| 694 | "Processing patient# 1474 ETA: 2.580403991718651 hrs\n", |
||
| 695 | "Processing patient# 1475 ETA: 2.5592905984358345 hrs\n", |
||
| 696 | "Processing patient# 1476 ETA: 2.537063879300142 hrs\n", |
||
| 697 | "Processing patient# 1477 ETA: 2.5267651822019 hrs\n", |
||
| 698 | "Processing patient# 1478 ETA: 2.5045537118088426 hrs\n", |
||
| 699 | "Processing patient# 1479 ETA: 2.481165919057026 hrs\n", |
||
| 700 | "Processing patient# 1480 ETA: 2.4596286582390934 hrs\n", |
||
| 701 | "Processing patient# 1481 ETA: 2.4357531235283094 hrs\n", |
||
| 702 | "Processing patient# 1482 ETA: 2.414465554013931 hrs\n", |
||
| 703 | "Processing patient# 1483 ETA: 2.397971823392403 hrs\n", |
||
| 704 | "Processing patient# 1484 ETA: 2.3773148383053253 hrs\n", |
||
| 705 | "Processing patient# 1485 ETA: 2.359427685907246 hrs\n", |
||
| 706 | "Processing patient# 1486 ETA: 2.3374206903567583 hrs\n", |
||
| 707 | "Processing patient# 1487 ETA: 2.316515679359436 hrs\n", |
||
| 708 | "Processing patient# 1488 ETA: 2.294826008774524 hrs\n", |
||
| 709 | "Processing patient# 1489 ETA: 2.2790878480482633 hrs\n", |
||
| 710 | "Processing patient# 1490 ETA: 2.2603696225018344 hrs\n", |
||
| 711 | "Processing patient# 1491 ETA: 2.239299383374652 hrs\n", |
||
| 712 | "Processing patient# 1492 ETA: 2.217807365984541 hrs\n", |
||
| 713 | "Processing patient# 1494 ETA: 2.1953265507487028 hrs\n", |
||
| 714 | "Processing patient# 1495 ETA: 2.154147859638919 hrs\n", |
||
| 715 | "Processing patient# 1496 ETA: 2.1373596109892876 hrs\n", |
||
| 716 | "Processing patient# 1498 ETA: 2.1154642674823054 hrs\n", |
||
| 717 | "Processing patient# 1499 ETA: 2.0658858120215506 hrs\n", |
||
| 718 | "Processing patient# 1500 ETA: 2.0452432717780265 hrs\n", |
||
| 719 | "Processing patient# 1501 ETA: 2.025089469167218 hrs\n", |
||
| 720 | "Processing patient# 1502 ETA: 2.006434742403083 hrs\n", |
||
| 721 | "Processing patient# 1503 ETA: 1.9848314131735696 hrs\n", |
||
| 722 | "Processing patient# 1504 ETA: 1.9642185280659386 hrs\n", |
||
| 723 | "Processing patient# 1505 ETA: 1.943846245320117 hrs\n", |
||
| 724 | "Processing patient# 1506 ETA: 1.922791921977514 hrs\n", |
||
| 725 | "Processing patient# 1507 ETA: 1.900297880433324 hrs\n", |
||
| 726 | "Processing patient# 1508 ETA: 1.8807857255689255 hrs\n", |
||
| 727 | "Processing patient# 1509 ETA: 1.8639425218906893 hrs\n", |
||
| 728 | "Processing patient# 1510 ETA: 1.8451056155743986 hrs\n", |
||
| 729 | "Processing patient# 1511 ETA: 1.8219857168580944 hrs\n", |
||
| 730 | "Processing patient# 1512 ETA: 1.7994089303411904 hrs\n", |
||
| 731 | "Processing patient# 1513 ETA: 1.7763814680467143 hrs\n", |
||
| 732 | "Processing patient# 1514 ETA: 1.7547966026234776 hrs\n", |
||
| 733 | "Processing patient# 1515 ETA: 1.7381065940352345 hrs\n", |
||
| 734 | "Processing patient# 1516 ETA: 1.7198453679184118 hrs\n", |
||
| 735 | "Processing patient# 1517 ETA: 1.6968657202587767 hrs\n", |
||
| 736 | "Processing patient# 1518 ETA: 1.6766352182999515 hrs\n", |
||
| 737 | "Processing patient# 1519 ETA: 1.6640672508136443 hrs\n", |
||
| 738 | "Processing patient# 1520 ETA: 1.6443906538188457 hrs\n", |
||
| 739 | "Processing patient# 1521 ETA: 1.6265579485526016 hrs\n", |
||
| 740 | "Processing patient# 1522 ETA: 1.6064969870503643 hrs\n", |
||
| 741 | "Processing patient# 1523 ETA: 1.5878849256592276 hrs\n", |
||
| 742 | "Processing patient# 1524 ETA: 1.5670396502469308 hrs\n", |
||
| 743 | "Processing patient# 1526 ETA: 1.5448135244235013 hrs\n", |
||
| 744 | "Processing patient# 1527 ETA: 1.5007179911218453 hrs\n", |
||
| 745 | "Processing patient# 1528 ETA: 1.4779989854163593 hrs\n", |
||
| 746 | "Processing patient# 1529 ETA: 1.455040394503777 hrs\n", |
||
| 747 | "Processing patient# 1530 ETA: 1.4348017759756608 hrs\n", |
||
| 748 | "Processing patient# 1531 ETA: 1.412959918495631 hrs\n", |
||
| 749 | "Processing patient# 1532 ETA: 1.3895584875255462 hrs\n", |
||
| 750 | "Processing patient# 1534 ETA: 1.3681563406241992 hrs\n", |
||
| 751 | "Processing patient# 1535 ETA: 1.32147174191119 hrs\n", |
||
| 752 | "Processing patient# 1536 ETA: 1.2993491290633876 hrs\n", |
||
| 753 | "Processing patient# 1537 ETA: 1.2765257389907825 hrs\n", |
||
| 754 | "Processing patient# 1538 ETA: 1.2545430914596223 hrs\n", |
||
| 755 | "Processing patient# 1539 ETA: 1.2330312886358439 hrs\n", |
||
| 756 | "Processing patient# 1540 ETA: 1.2134439415684322 hrs\n", |
||
| 757 | "Processing patient# 1541 ETA: 1.1903008280926082 hrs\n", |
||
| 758 | "Processing patient# 1542 ETA: 1.167653547794189 hrs\n", |
||
| 759 | "Processing patient# 1543 ETA: 1.1457156837554199 hrs\n", |
||
| 760 | "Processing patient# 1544 ETA: 1.1241135936051372 hrs\n", |
||
| 761 | "Processing patient# 1545 ETA: 1.102846443816275 hrs\n", |
||
| 762 | "Processing patient# 1546 ETA: 1.080637407885313 hrs\n", |
||
| 763 | "Processing patient# 1547 ETA: 1.0583609296631968 hrs\n", |
||
| 764 | "Processing patient# 1549 ETA: 1.0358784646273498 hrs\n", |
||
| 765 | "Processing patient# 1550 ETA: 0.9894958379524097 hrs\n", |
||
| 766 | "Processing patient# 1551 ETA: 0.9702932011284904 hrs\n", |
||
| 767 | "Processing patient# 1552 ETA: 0.9487030035562142 hrs\n", |
||
| 768 | "Processing patient# 1553 ETA: 0.927032014874493 hrs\n", |
||
| 769 | "Processing patient# 1555 ETA: 0.9052574532595503 hrs\n", |
||
| 770 | "Processing patient# 1556 ETA: 0.8583338576951499 hrs\n", |
||
| 771 | "Processing patient# 1557 ETA: 0.8364724836388824 hrs\n", |
||
| 772 | "Processing patient# 1558 ETA: 0.8143605039141544 hrs\n", |
||
| 773 | "Processing patient# 1559 ETA: 0.7939183002461303 hrs\n", |
||
| 774 | "Processing patient# 1560 ETA: 0.7714008516864275 hrs\n" |
||
| 775 | ] |
||
| 776 | }, |
||
| 777 | { |
||
| 778 | "name": "stdout", |
||
| 779 | "output_type": "stream", |
||
| 780 | "text": [ |
||
| 781 | "Processing patient# 1561 ETA: 0.7496390868227617 hrs\n", |
||
| 782 | "Processing patient# 1562 ETA: 0.7277910622262829 hrs\n", |
||
| 783 | "Processing patient# 1563 ETA: 0.70534063768343 hrs\n", |
||
| 784 | "Processing patient# 1564 ETA: 0.6836017272867927 hrs\n", |
||
| 785 | "Processing patient# 1566 ETA: 0.6618160449642033 hrs\n", |
||
| 786 | "Processing patient# 1568 ETA: 0.6160384095808674 hrs\n", |
||
| 787 | "Processing patient# 1570 ETA: 0.5702290712717382 hrs\n", |
||
| 788 | "Processing patient# 1571 ETA: 0.5249562931103638 hrs\n", |
||
| 789 | "Processing patient# 1572 ETA: 0.5033148116076286 hrs\n", |
||
| 790 | "Processing patient# 1573 ETA: 0.48112341017214527 hrs\n", |
||
| 791 | "Processing patient# 1574 ETA: 0.4594317513380257 hrs\n", |
||
| 792 | "Processing patient# 1575 ETA: 0.43778636997364123 hrs\n", |
||
| 793 | "Processing patient# 1576 ETA: 0.415779529424036 hrs\n", |
||
| 794 | "Processing patient# 1577 ETA: 0.3949588738670405 hrs\n", |
||
| 795 | "Processing patient# 1578 ETA: 0.37282882327230255 hrs\n", |
||
| 796 | "Processing patient# 1580 ETA: 0.3515907668564129 hrs\n", |
||
| 797 | "Processing patient# 1581 ETA: 0.30684781963013846 hrs\n", |
||
| 798 | "Processing patient# 1582 ETA: 0.28561606805205886 hrs\n", |
||
| 799 | "Processing patient# 1583 ETA: 0.26367890904528596 hrs\n", |
||
| 800 | "Processing patient# 1584 ETA: 0.241728710744323 hrs\n", |
||
| 801 | "Processing patient# 1585 ETA: 0.2196190606033997 hrs\n", |
||
| 802 | "Processing patient# 1587 ETA: 0.19766932272849366 hrs\n", |
||
| 803 | "Processing patient# 1588 ETA: 0.1533620524758064 hrs\n", |
||
| 804 | "Processing patient# 1589 ETA: 0.1314311540085944 hrs\n", |
||
| 805 | "Processing patient# 1590 ETA: 0.10954693088589212 hrs\n", |
||
| 806 | "Processing patient# 1591 ETA: 0.08776483735769842 hrs\n", |
||
| 807 | "Processing patient# 1592 ETA: 0.06588565785057654 hrs\n", |
||
| 808 | "Processing patient# 1594 ETA: 0.04392167075191941 hrs\n", |
||
| 809 | "31161.49111223221\n" |
||
| 810 | ] |
||
| 811 | } |
||
| 812 | ], |
||
| 813 | "source": [ |
||
| 814 | "start_time=time.time()\n", |
||
| 815 | "\n", |
||
| 816 | "elapsed_time=0\n", |
||
| 817 | "totaltime=94000\n", |
||
| 818 | "thresh=-500 #lower HU threshold for nodule segmentation\n", |
||
| 819 | "noduleimages=np.ndarray([5000,1,512,512],dtype=np.float32)\n", |
||
| 820 | "nodulemasks=np.ndarray([5000,1,512,512],dtype=np.float32)\n", |
||
| 821 | "sample=[]\n", |
||
| 822 | "area=[]\n", |
||
| 823 | "noduleindicies=[]\n", |
||
| 824 | "index=0\n", |
||
| 825 | "start=1\n", |
||
| 826 | "end=400\n", |
||
| 827 | "for i in range(len(patients)):\n", |
||
| 828 | " print(\"Processing patient#\",i,\"ETA:\",(totaltime-elapsed_time)/3600,\"hrs\")\n", |
||
| 829 | " if (i-1)/400-np.floor((i-1)/400)==0:\n", |
||
| 830 | " noduleimages=noduleimages[:index]\n", |
||
| 831 | " nodulemasks=nodulemasks[:index]\n", |
||
| 832 | " table=generatefeaturetable(nodulemasks)\n", |
||
| 833 | " print(\"Saving data for patients\"+str(start)+\"-\"+str(end))\n", |
||
| 834 | " np.save(datafolder+\"DSBNoduleImages\"+str(start)+\"-\"+str(end)+\".npy\",noduleimages)\n", |
||
| 835 | " np.save(datafolder+\"DSBNoduleMasks\"+str(start)+\"-\"+str(end)+\".npy\",nodulemasks)\n", |
||
| 836 | " table.to_csv(datafolder+\"DSBNoduleFeatures\"+str(start)+\"-\"+str(end)+\".csv\")\n", |
||
| 837 | " del noduleimages, nodulemasks\n", |
||
| 838 | " noduleimages=np.ndarray([5000,1,512,512],dtype=np.float32)\n", |
||
| 839 | " nodulemasks=np.ndarray([5000,1,512,512],dtype=np.float32)\n", |
||
| 840 | " sample=[]\n", |
||
| 841 | " area=[]\n", |
||
| 842 | " noduleindicies=[]\n", |
||
| 843 | " index=0 \n", |
||
| 844 | " patient_scan=load_scan(INPUT_FOLDER+patients[i])\n", |
||
| 845 | " patient_pix=get_pixels_hu(patient_scan)\n", |
||
| 846 | " processed_pix = processimagefromfile(patient_pix)\n", |
||
| 847 | " mask = predictmask(processed_pix)\n", |
||
| 848 | " noduleindex = getnoduleindex(mask)\n", |
||
| 849 | " trueinds=trueindicies(processed_pix,noduleindex)\n", |
||
| 850 | "\n", |
||
| 851 | " for ind in trueinds:\n", |
||
| 852 | " noduleimages[index,0]=patient_pix[ind]\n", |
||
| 853 | " nodulemasks[index,0]=mask[ind]\n", |
||
| 854 | " sample.append(patients[i])\n", |
||
| 855 | " area.append(np.sum(mask[ind]))\n", |
||
| 856 | " noduleindicies.append(ind)\n", |
||
| 857 | " index+=1\n", |
||
| 858 | "\n", |
||
| 859 | " elapsed_time=time.time()-start_time\n", |
||
| 860 | " totaltime=elapsed_time/(i-start+1)*(end-start)\n", |
||
| 861 | "\n", |
||
| 862 | "\n", |
||
| 863 | "\n", |
||
| 864 | "print(elapsed_time)" |
||
| 865 | ] |
||
| 866 | } |
||
| 867 | ], |
||
| 868 | "metadata": { |
||
| 869 | "kernelspec": { |
||
| 870 | "display_name": "Python 3", |
||
| 871 | "language": "python", |
||
| 872 | "name": "python3" |
||
| 873 | }, |
||
| 874 | "language_info": { |
||
| 875 | "codemirror_mode": { |
||
| 876 | "name": "ipython", |
||
| 877 | "version": 3 |
||
| 878 | }, |
||
| 879 | "file_extension": ".py", |
||
| 880 | "mimetype": "text/x-python", |
||
| 881 | "name": "python", |
||
| 882 | "nbconvert_exporter": "python", |
||
| 883 | "pygments_lexer": "ipython3", |
||
| 884 | "version": "3.6.1" |
||
| 885 | } |
||
| 886 | }, |
||
| 887 | "nbformat": 4, |
||
| 888 | "nbformat_minor": 2 |
||
| 889 | } |