Downloads: 1
Switch to side-by-side view
--- a +++ b/1. Applying AI to 2D Medical Imaging Data/10. Fine-tuning CNNs for Classification Exercise/solution.ipynb @@ -0,0 +1,559 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Using TensorFlow backend.\n" + ] + } + ], + "source": [ + "import numpy as np \n", + "import pandas as pd \n", + "import os\n", + "from glob import glob\n", + "%matplotlib inline\n", + "import matplotlib.pyplot as plt\n", + "import tensorflow as tf\n", + "from skimage import io\n", + "\n", + "from keras.preprocessing.image import ImageDataGenerator\n", + "from keras.layers import GlobalAveragePooling2D, Dense, Dropout, Flatten, Conv2D, MaxPooling2D\n", + "from keras.models import Sequential, Model\n", + "from keras.applications.vgg16 import VGG16\n", + "from keras.applications.resnet import ResNet50 \n", + "from keras.optimizers import Adam\n", + "from keras.callbacks import ModelCheckpoint, LearningRateScheduler, EarlyStopping, ReduceLROnPlateau" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "train_df = pd.read_csv('train.csv')\n", + "valid_df = pd.read_csv('test.csv')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Setting up the image augmentation from last Lesson: " + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "IMG_SIZE = (224, 224)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Found 20 validated image filenames belonging to 2 classes.\n", + "Found 6 validated image filenames belonging to 2 classes.\n" + ] + } + ], + "source": [ + "train_idg = ImageDataGenerator(rescale=1. / 255.0,\n", + " horizontal_flip = True, \n", + " vertical_flip = False, \n", + " height_shift_range= 0.1, \n", + " width_shift_range=0.1, \n", + " rotation_range=20, \n", + " shear_range = 0.1,\n", + " zoom_range=0.1)\n", + "\n", + "train_gen = train_idg.flow_from_dataframe(dataframe=train_df, \n", + " directory=None, \n", + " x_col = 'img_path',\n", + " y_col = 'class',\n", + " class_mode = 'binary',\n", + " target_size = IMG_SIZE, \n", + " batch_size = 9\n", + " )\n", + "\n", + "# Note that the validation data should not be augmented! We only want to do some basic intensity rescaling here\n", + "val_idg = ImageDataGenerator(rescale=1. / 255.0\n", + " )\n", + "\n", + "val_gen = val_idg.flow_from_dataframe(dataframe=valid_df, \n", + " directory=None, \n", + " x_col = 'img_path',\n", + " y_col = 'class',\n", + " class_mode = 'binary',\n", + " target_size = IMG_SIZE, \n", + " batch_size = 6) ## We've only been provided with 6 validation images" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "## Pull a single large batch of random validation data for testing after each epoch\n", + "testX, testY = val_gen.next()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Now we'll load in VGG16 with pre-trained ImageNet weights: " + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Downloading data from https://github.com/fchollet/deep-learning-models/releases/download/v0.1/vgg16_weights_tf_dim_ordering_tf_kernels.h5\n", + "553467904/553467096 [==============================] - 7s 0us/step\n", + "Model: \"vgg16\"\n", + "_________________________________________________________________\n", + "Layer (type) Output Shape Param # \n", + "=================================================================\n", + "input_1 (InputLayer) (None, 224, 224, 3) 0 \n", + "_________________________________________________________________\n", + "block1_conv1 (Conv2D) (None, 224, 224, 64) 1792 \n", + "_________________________________________________________________\n", + "block1_conv2 (Conv2D) (None, 224, 224, 64) 36928 \n", + "_________________________________________________________________\n", + "block1_pool (MaxPooling2D) (None, 112, 112, 64) 0 \n", + "_________________________________________________________________\n", + "block2_conv1 (Conv2D) (None, 112, 112, 128) 73856 \n", + "_________________________________________________________________\n", + "block2_conv2 (Conv2D) (None, 112, 112, 128) 147584 \n", + "_________________________________________________________________\n", + "block2_pool (MaxPooling2D) (None, 56, 56, 128) 0 \n", + "_________________________________________________________________\n", + "block3_conv1 (Conv2D) (None, 56, 56, 256) 295168 \n", + "_________________________________________________________________\n", + "block3_conv2 (Conv2D) (None, 56, 56, 256) 590080 \n", + "_________________________________________________________________\n", + "block3_conv3 (Conv2D) (None, 56, 56, 256) 590080 \n", + "_________________________________________________________________\n", + "block3_pool (MaxPooling2D) (None, 28, 28, 256) 0 \n", + "_________________________________________________________________\n", + "block4_conv1 (Conv2D) (None, 28, 28, 512) 1180160 \n", + "_________________________________________________________________\n", + "block4_conv2 (Conv2D) (None, 28, 28, 512) 2359808 \n", + "_________________________________________________________________\n", + "block4_conv3 (Conv2D) (None, 28, 28, 512) 2359808 \n", + "_________________________________________________________________\n", + "block4_pool (MaxPooling2D) (None, 14, 14, 512) 0 \n", + "_________________________________________________________________\n", + "block5_conv1 (Conv2D) (None, 14, 14, 512) 2359808 \n", + "_________________________________________________________________\n", + "block5_conv2 (Conv2D) (None, 14, 14, 512) 2359808 \n", + "_________________________________________________________________\n", + "block5_conv3 (Conv2D) (None, 14, 14, 512) 2359808 \n", + "_________________________________________________________________\n", + "block5_pool (MaxPooling2D) (None, 7, 7, 512) 0 \n", + "_________________________________________________________________\n", + "flatten (Flatten) (None, 25088) 0 \n", + "_________________________________________________________________\n", + "fc1 (Dense) (None, 4096) 102764544 \n", + "_________________________________________________________________\n", + "fc2 (Dense) (None, 4096) 16781312 \n", + "_________________________________________________________________\n", + "predictions (Dense) (None, 1000) 4097000 \n", + "=================================================================\n", + "Total params: 138,357,544\n", + "Trainable params: 138,357,544\n", + "Non-trainable params: 0\n", + "_________________________________________________________________\n" + ] + } + ], + "source": [ + "model = VGG16(include_top=True, weights='imagenet')\n", + "model.summary()" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "transfer_layer = model.get_layer('block5_pool')\n", + "vgg_model = Model(inputs=model.input,\n", + " outputs=transfer_layer.output)" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "## Now, choose which layers of VGG16 we actually want to fine-tune (if any)\n", + "## Here, we'll freeze all but the last convolutional layer\n", + "for layer in vgg_model.layers[0:17]:\n", + " layer.trainable = False" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "input_1 False\n", + "block1_conv1 False\n", + "block1_conv2 False\n", + "block1_pool False\n", + "block2_conv1 False\n", + "block2_conv2 False\n", + "block2_pool False\n", + "block3_conv1 False\n", + "block3_conv2 False\n", + "block3_conv3 False\n", + "block3_pool False\n", + "block4_conv1 False\n", + "block4_conv2 False\n", + "block4_conv3 False\n", + "block4_pool False\n", + "block5_conv1 False\n", + "block5_conv2 False\n", + "block5_conv3 True\n", + "block5_pool True\n" + ] + } + ], + "source": [ + "for layer in vgg_model.layers:\n", + " print(layer.name, layer.trainable)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "new_model = Sequential()\n", + "\n", + "# Add the convolutional part of the VGG16 model from above.\n", + "new_model.add(vgg_model)\n", + "\n", + "# Flatten the output of the VGG16 model because it is from a\n", + "# convolutional layer.\n", + "new_model.add(Flatten())\n", + "\n", + "# Add a dense (aka. fully-connected) layer.\n", + "# This is for combining features that the VGG16 model has\n", + "# recognized in the image.\n", + "new_model.add(Dense(1, activation='sigmoid'))" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "## Set our optimizer, loss function, and learning rate\n", + "optimizer = Adam(lr=1e-4)\n", + "loss = 'binary_crossentropy'\n", + "metrics = ['binary_accuracy']" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [], + "source": [ + "new_model.compile(optimizer=optimizer, loss=loss, metrics=metrics)" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 1/5\n", + "3/3 [==============================] - 8s 3s/step - loss: 0.7176 - binary_accuracy: 0.5500 - val_loss: 0.7353 - val_binary_accuracy: 0.5000\n", + "Epoch 2/5\n", + "3/3 [==============================] - 0s 142ms/step - loss: 0.6999 - binary_accuracy: 0.4500 - val_loss: 0.6940 - val_binary_accuracy: 0.5000\n", + "Epoch 3/5\n", + "3/3 [==============================] - 1s 175ms/step - loss: 0.6610 - binary_accuracy: 0.6500 - val_loss: 0.6592 - val_binary_accuracy: 0.5000\n", + "Epoch 4/5\n", + "3/3 [==============================] - 1s 183ms/step - loss: 0.5746 - binary_accuracy: 0.7500 - val_loss: 0.6607 - val_binary_accuracy: 0.5000\n", + "Epoch 5/5\n", + "3/3 [==============================] - 1s 172ms/step - loss: 0.5632 - binary_accuracy: 0.6000 - val_loss: 0.6222 - val_binary_accuracy: 0.5000\n" + ] + }, + { + "data": { + "text/plain": [ + "<keras.callbacks.callbacks.History at 0x7f6181c55ed0>" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "## Just run a single epoch to see how it does:\n", + "new_model.fit_generator(train_gen, \n", + " validation_data = (testX, testY), \n", + " epochs = 5)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Let's try another experiment where we add a few more dense layers:" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [], + "source": [ + "new_model = Sequential()\n", + "\n", + "# Add the convolutional part of the VGG16 model from above.\n", + "new_model.add(vgg_model)\n", + "\n", + "# Flatten the output of the VGG16 model because it is from a\n", + "# convolutional layer.\n", + "new_model.add(Flatten())\n", + "\n", + "# Add a dense (aka. fully-connected) layer.\n", + "# This is for combining features that the VGG16 model has\n", + "# recognized in the image.\n", + "new_model.add(Dense(1024, activation='relu'))\n", + "\n", + "# Add a dense (aka. fully-connected) layer.\n", + "# This is for combining features that the VGG16 model has\n", + "# recognized in the image.\n", + "new_model.add(Dense(512, activation='relu'))\n", + "\n", + "# Add a dense (aka. fully-connected) layer.\n", + "# Change the activation function to sigmoid \n", + "# so output of the last layer is in the range of [0,1] \n", + "new_model.add(Dense(1, activation='sigmoid'))" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [], + "source": [ + "new_model.compile(optimizer=optimizer, loss=loss, metrics=metrics)" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 1/5\n", + "3/3 [==============================] - 1s 365ms/step - loss: 0.7317 - binary_accuracy: 0.6000 - val_loss: 0.7886 - val_binary_accuracy: 0.5000\n", + "Epoch 2/5\n", + "3/3 [==============================] - 0s 166ms/step - loss: 0.7462 - binary_accuracy: 0.5500 - val_loss: 0.6289 - val_binary_accuracy: 0.5000\n", + "Epoch 3/5\n", + "3/3 [==============================] - 1s 183ms/step - loss: 0.4993 - binary_accuracy: 0.7500 - val_loss: 1.0301 - val_binary_accuracy: 0.5000\n", + "Epoch 4/5\n", + "3/3 [==============================] - 0s 165ms/step - loss: 0.6508 - binary_accuracy: 0.8000 - val_loss: 0.7457 - val_binary_accuracy: 0.6667\n", + "Epoch 5/5\n", + "3/3 [==============================] - 1s 190ms/step - loss: 0.3542 - binary_accuracy: 0.7500 - val_loss: 0.6752 - val_binary_accuracy: 0.6667\n" + ] + }, + { + "data": { + "text/plain": [ + "<keras.callbacks.callbacks.History at 0x7f6181ef4cd0>" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "## Just run a single epoch to see how it does:\n", + "new_model.fit_generator(train_gen, \n", + " validation_data = (testX, testY), \n", + " epochs = 5)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Now let's add dropout and another fully connected layer:" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [], + "source": [ + "new_model = Sequential()\n", + "\n", + "# Add the convolutional part of the VGG16 model from above.\n", + "new_model.add(vgg_model)\n", + "\n", + "# Flatten the output of the VGG16 model because it is from a\n", + "# convolutional layer.\n", + "new_model.add(Flatten())\n", + "\n", + "# Add a dropout-layer which may prevent overfitting and\n", + "# improve generalization ability to unseen data e.g. the test-set.\n", + "new_model.add(Dropout(0.5))\n", + "\n", + "# Add a dense (aka. fully-connected) layer.\n", + "# This is for combining features that the VGG16 model has\n", + "# recognized in the image.\n", + "new_model.add(Dense(1024, activation='relu'))\n", + "\n", + "# Add a dropout-layer which may prevent overfitting and\n", + "# improve generalization ability to unseen data e.g. the test-set.\n", + "new_model.add(Dropout(0.5))\n", + "\n", + "# Add a dense (aka. fully-connected) layer.\n", + "# This is for combining features that the VGG16 model has\n", + "# recognized in the image.\n", + "new_model.add(Dense(512, activation='relu'))\n", + "\n", + "# Add a dropout-layer which may prevent overfitting and\n", + "# improve generalization ability to unseen data e.g. the test-set.\n", + "new_model.add(Dropout(0.5))\n", + "\n", + "# Add a dense (aka. fully-connected) layer.\n", + "# This is for combining features that the VGG16 model has\n", + "# recognized in the image.\n", + "new_model.add(Dense(256, activation='relu'))\n", + "\n", + "# Add a dense (aka. fully-connected) layer.\n", + "# Change the activation function to sigmoid \n", + "# so output of the last layer is in the range of [0,1] \n", + "new_model.add(Dense(1, activation='sigmoid'))" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [], + "source": [ + "new_model.compile(optimizer=optimizer, loss=loss, metrics=metrics)" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 1/5\n", + "3/3 [==============================] - 1s 407ms/step - loss: 0.9829 - binary_accuracy: 0.5000 - val_loss: 0.7160 - val_binary_accuracy: 0.5000\n", + "Epoch 2/5\n", + "3/3 [==============================] - 1s 203ms/step - loss: 0.8588 - binary_accuracy: 0.4000 - val_loss: 0.8992 - val_binary_accuracy: 0.5000\n", + "Epoch 3/5\n", + "3/3 [==============================] - 1s 199ms/step - loss: 0.8550 - binary_accuracy: 0.4500 - val_loss: 1.0988 - val_binary_accuracy: 0.5000\n", + "Epoch 4/5\n", + "3/3 [==============================] - 1s 172ms/step - loss: 1.1873 - binary_accuracy: 0.4500 - val_loss: 1.1166 - val_binary_accuracy: 0.5000\n", + "Epoch 5/5\n", + "3/3 [==============================] - 1s 168ms/step - loss: 0.9431 - binary_accuracy: 0.4000 - val_loss: 0.8109 - val_binary_accuracy: 0.5000\n" + ] + }, + { + "data": { + "text/plain": [ + "<keras.callbacks.callbacks.History at 0x7f612c060f50>" + ] + }, + "execution_count": 19, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "## Just run a single epoch to see how it does:\n", + "new_model.fit_generator(train_gen, \n", + " validation_data = (testX, testY), \n", + " epochs = 5)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "What's interesting about the small number of epochs we ran on the three different architectures above is that the simplest archiecture seemed to show the fastest learning. Why might that be? \n", + "\n", + "Answer: there were the fewest parameters to train because we didn't add any fully-connected layers, and were only fine-tuning the last layer of VGG16. \n", + "\n", + "The last architecture we tried seemed to show more stable and promise than the second, and this is likely due to the fact that we added Dropout. This helps our model from overfitting and usually using Dropout, we see better learning on the validation set (val_loss going down over epochs as opposed to only the training loss). " + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.6" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +}