{

 "cells": [

  {

   "cell_type": "markdown",

   "metadata": {

    "id": "VW1lne1tWskl",

    "colab_type": "text"

   },

   "source": [

    "# Random Survival Forest model\n",

    "\n",

    "Feature selection with RSF in Owkin dataset."

   ]

  },

  {

   "cell_type": "markdown",

   "metadata": {

    "id": "BBI8oFpGcMUg",

    "colab_type": "text"

   },

   "source": [

    "External packages."

   ]

  },

  {

   "cell_type": "code",

   "execution_count": 0,

   "metadata": {

    "id": "pn1cU1CocQ_G",

    "colab_type": "code",

    "outputId": "a5e9d172-c200-4daf-8a68-a611a41c5766",

    "colab": {

     "base_uri": "https://localhost:8080/",

     "height": 595.0

    }

   },

   "outputs": [

    {

     "name": "stdout",

     "output_type": "stream",

     "text": [

      "Requirement already satisfied: scikit-survival in /usr/local/lib/python3.6/dist-packages (0.11)\n",

      "Requirement already satisfied: scipy!=1.3.0,>=1.0 in /usr/local/lib/python3.6/dist-packages (from scikit-survival) (1.4.1)\n",

      "Requirement already satisfied: cvxpy>=1.0 in /usr/local/lib/python3.6/dist-packages (from scikit-survival) (1.0.25)\n",

      "Requirement already satisfied: cython>=0.29 in /usr/local/lib/python3.6/dist-packages (from scikit-survival) (0.29.15)\n",

      "Requirement already satisfied: scikit-learn<0.22,>=0.21.0 in /usr/local/lib/python3.6/dist-packages (from scikit-survival) (0.21.3)\n",

      "Requirement already satisfied: numexpr in /usr/local/lib/python3.6/dist-packages (from scikit-survival) (2.7.1)\n",

      "Requirement already satisfied: osqp!=0.6.0,!=0.6.1 in /usr/local/lib/python3.6/dist-packages (from scikit-survival) (0.5.0)\n",

      "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from scikit-survival) (1.17.5)\n",

      "Requirement already satisfied: cvxopt in /usr/local/lib/python3.6/dist-packages (from scikit-survival) (1.2.4)\n",

      "Requirement already satisfied: pandas<0.26,>=0.21 in /usr/local/lib/python3.6/dist-packages (from scikit-survival) (0.25.3)\n",

      "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from cvxpy>=1.0->scikit-survival) (1.12.0)\n",

      "Requirement already satisfied: multiprocess in /usr/local/lib/python3.6/dist-packages (from cvxpy>=1.0->scikit-survival) (0.70.9)\n",

      "Requirement already satisfied: ecos>=2 in /usr/local/lib/python3.6/dist-packages (from cvxpy>=1.0->scikit-survival) (2.0.7.post1)\n",

      "Requirement already satisfied: scs>=1.1.3 in /usr/local/lib/python3.6/dist-packages (from cvxpy>=1.0->scikit-survival) (2.1.1.post2)\n",

      "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.6/dist-packages (from scikit-learn<0.22,>=0.21.0->scikit-survival) (0.14.1)\n",

      "Requirement already satisfied: future in /usr/local/lib/python3.6/dist-packages (from osqp!=0.6.0,!=0.6.1->scikit-survival) (0.16.0)\n",

      "Requirement already satisfied: pytz>=2017.2 in /usr/local/lib/python3.6/dist-packages (from pandas<0.26,>=0.21->scikit-survival) (2018.9)\n",

      "Requirement already satisfied: python-dateutil>=2.6.1 in /usr/local/lib/python3.6/dist-packages (from pandas<0.26,>=0.21->scikit-survival) (2.6.1)\n",

      "Requirement already satisfied: dill>=0.3.1 in /usr/local/lib/python3.6/dist-packages (from multiprocess->cvxpy>=1.0->scikit-survival) (0.3.1.1)\n",

      "Collecting eli5\n",

      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/97/2f/c85c7d8f8548e460829971785347e14e45fa5c6617da374711dec8cb38cc/eli5-0.10.1-py2.py3-none-any.whl (105kB)\n",

      "\u001b[K     |████████████████████████████████| 112kB 2.8MB/s \n",

      "\u001b[?25hRequirement already satisfied: attrs>16.0.0 in /usr/local/lib/python3.6/dist-packages (from eli5) (19.3.0)\n",

      "Requirement already satisfied: scikit-learn>=0.18 in /usr/local/lib/python3.6/dist-packages (from eli5) (0.21.3)\n",

      "Requirement already satisfied: tabulate>=0.7.7 in /usr/local/lib/python3.6/dist-packages (from eli5) (0.8.6)\n",

      "Requirement already satisfied: scipy in /usr/local/lib/python3.6/dist-packages (from eli5) (1.4.1)\n",

      "Requirement already satisfied: graphviz in /usr/local/lib/python3.6/dist-packages (from eli5) (0.10.1)\n",

      "Requirement already satisfied: jinja2 in /usr/local/lib/python3.6/dist-packages (from eli5) (2.11.1)\n",

      "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from eli5) (1.12.0)\n",

      "Requirement already satisfied: numpy>=1.9.0 in /usr/local/lib/python3.6/dist-packages (from eli5) (1.17.5)\n",

      "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.6/dist-packages (from scikit-learn>=0.18->eli5) (0.14.1)\n",

      "Requirement already satisfied: MarkupSafe>=0.23 in /usr/local/lib/python3.6/dist-packages (from jinja2->eli5) (1.1.1)\n",

      "Installing collected packages: eli5\n",

      "Successfully installed eli5-0.10.1\n"

     ]

    }

   ],

   "source": [

    "!pip install scikit-survival\n",

    "!pip install eli5"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": 0,

   "metadata": {

    "id": "wvGuplLr3lW1",

    "colab_type": "code",

    "colab": {}

   },

   "outputs": [],

   "source": [

    "import pandas as pd\n",

    "import matplotlib.pyplot as plt\n",

    "import numpy as np\n",

    "import read_data"

   ]

  },

  {

   "cell_type": "markdown",

   "metadata": {

    "id": "s0zye152Zilo",

    "colab_type": "text"

   },

   "source": [

    "## Load data"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": 0,

   "metadata": {

    "id": "RSMkfEPyE9s6",

    "colab_type": "code",

    "colab": {}

   },

   "outputs": [],

   "source": [

    "features = ['original_shape_Compactness2',\n",

    " 'original_shape_SphericalDisproportion',\n",

    " 'original_shape_SurfaceVolumeRatio',\n",

    " 'original_firstorder_Kurtosis',\n",

    " 'original_firstorder_MeanAbsoluteDeviation',\n",

    " 'original_firstorder_Minimum',\n",

    " 'original_glcm_ClusterProminence',\n",

    " 'original_glcm_Contrast',\n",

    " 'original_glcm_DifferenceEntropy',\n",

    " 'original_glcm_DifferenceAverage',\n",

    " 'original_glcm_JointEnergy',\n",

    " 'original_glcm_Id',\n",

    " 'original_glcm_Idm',\n",

    " 'original_glcm_Imc1',\n",

    " 'original_glcm_Imc2',\n",

    " 'original_glcm_Idmn',\n",

    " 'original_glcm_Idn',\n",

    " 'original_glrlm_ShortRunEmphasis',\n",

    " 'original_glrlm_LongRunEmphasis',\n",

    " 'original_glrlm_GrayLevelNonUniformity',\n",

    " 'original_glrlm_RunPercentage',\n",

    " 'original_glrlm_ShortRunLowGrayLevelEmphasis',\n",

    " 'original_glrlm_LongRunLowGrayLevelEmphasis',\n",

    " 'original_glrlm_LongRunHighGrayLevelEmphasis',\n",

    " 'Nstage',\n",

    " 'age',\n",

    " 'SourceDataset_l1',\n",

    " 'SourceDataset_l2']"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": 0,

   "metadata": {

    "id": "EB3sdksn3lW8",

    "colab_type": "code",

    "outputId": "c0b0a729-baeb-4284-a640-589169d38762",

    "colab": {

     "base_uri": "https://localhost:8080/",

     "height": 470.0

    }

   },

   "outputs": [

    {

     "data": {

      "text/html": [

       "<div>\n",

       "<style scoped>\n",

       "    .dataframe tbody tr th:only-of-type {\n",

       "        vertical-align: middle;\n",

       "    }\n",

       "\n",

       "    .dataframe tbody tr th {\n",

       "        vertical-align: top;\n",

       "    }\n",

       "\n",

       "    .dataframe thead th {\n",

       "        text-align: right;\n",

       "    }\n",

       "</style>\n",

       "<table border=\"1\" class=\"dataframe\">\n",

       "  <thead>\n",

       "    <tr style=\"text-align: right;\">\n",

       "      <th></th>\n",

       "      <th>original_shape_Compactness2</th>\n",

       "      <th>original_shape_SphericalDisproportion</th>\n",

       "      <th>original_shape_SurfaceVolumeRatio</th>\n",

       "      <th>original_firstorder_Kurtosis</th>\n",

       "      <th>original_firstorder_MeanAbsoluteDeviation</th>\n",

       "      <th>original_firstorder_Minimum</th>\n",

       "      <th>original_glcm_ClusterProminence</th>\n",

       "      <th>original_glcm_Contrast</th>\n",

       "      <th>original_glcm_DifferenceEntropy</th>\n",

       "      <th>original_glcm_DifferenceAverage</th>\n",

       "      <th>original_glcm_JointEnergy</th>\n",

       "      <th>original_glcm_Id</th>\n",

       "      <th>original_glcm_Idm</th>\n",

       "      <th>original_glcm_Imc1</th>\n",

       "      <th>original_glcm_Imc2</th>\n",

       "      <th>original_glcm_Idmn</th>\n",

       "      <th>original_glcm_Idn</th>\n",

       "      <th>original_glrlm_ShortRunEmphasis</th>\n",

       "      <th>original_glrlm_LongRunEmphasis</th>\n",

       "      <th>original_glrlm_GrayLevelNonUniformity</th>\n",

       "      <th>original_glrlm_RunPercentage</th>\n",

       "      <th>original_glrlm_ShortRunLowGrayLevelEmphasis</th>\n",

       "      <th>original_glrlm_LongRunLowGrayLevelEmphasis</th>\n",

       "      <th>original_glrlm_LongRunHighGrayLevelEmphasis</th>\n",

       "      <th>Nstage</th>\n",

       "      <th>age</th>\n",

       "      <th>SourceDataset_l1</th>\n",

       "      <th>SourceDataset_l2</th>\n",

       "    </tr>\n",

       "    <tr>\n",

       "      <th>PatientID</th>\n",

       "      <th></th>\n",

       "      <th></th>\n",

       "      <th></th>\n",

       "      <th></th>\n",

       "      <th></th>\n",

       "      <th></th>\n",

       "      <th></th>\n",

       "      <th></th>\n",

       "      <th></th>\n",

       "      <th></th>\n",

       "      <th></th>\n",

       "      <th></th>\n",

       "      <th></th>\n",

       "      <th></th>\n",

       "      <th></th>\n",

       "      <th></th>\n",

       "      <th></th>\n",

       "      <th></th>\n",

       "      <th></th>\n",

       "      <th></th>\n",

       "      <th></th>\n",

       "      <th></th>\n",

       "      <th></th>\n",

       "      <th></th>\n",

       "      <th></th>\n",

       "      <th></th>\n",

       "      <th></th>\n",

       "      <th></th>\n",

       "    </tr>\n",

       "  </thead>\n",

       "  <tbody>\n",

       "    <tr>\n",

       "      <th>202</th>\n",

       "      <td>0.274892</td>\n",

       "      <td>1.537964</td>\n",

       "      <td>0.275228</td>\n",

       "      <td>1.592466</td>\n",

       "      <td>288.311105</td>\n",

       "      <td>-1006.0</td>\n",

       "      <td>546253.499762</td>\n",

       "      <td>54.336816</td>\n",

       "      <td>3.845383</td>\n",

       "      <td>5.036775</td>\n",

       "      <td>0.005920</td>\n",

       "      <td>0.338672</td>\n",

       "      <td>0.262780</td>\n",

       "      <td>-0.175185</td>\n",

       "      <td>0.901525</td>\n",

       "      <td>0.981053</td>\n",

       "      <td>0.917509</td>\n",

       "      <td>0.926391</td>\n",

       "      <td>1.457980</td>\n",

       "      <td>555.378594</td>\n",

       "      <td>0.891330</td>\n",

       "      <td>0.003384</td>\n",

       "      <td>0.004444</td>\n",

       "      <td>1651.710761</td>\n",

       "      <td>0</td>\n",

       "      <td>66.0000</td>\n",

       "      <td>0</td>\n",

       "      <td>1</td>\n",

       "    </tr>\n",

       "    <tr>\n",

       "      <th>371</th>\n",

       "      <td>0.188210</td>\n",

       "      <td>1.744961</td>\n",

       "      <td>0.240727</td>\n",

       "      <td>3.371013</td>\n",

       "      <td>191.395154</td>\n",

       "      <td>-945.0</td>\n",

       "      <td>331155.230396</td>\n",

       "      <td>16.868781</td>\n",

       "      <td>2.984034</td>\n",

       "      <td>2.570017</td>\n",

       "      <td>0.025532</td>\n",

       "      <td>0.495719</td>\n",

       "      <td>0.439879</td>\n",

       "      <td>-0.250370</td>\n",

       "      <td>0.932847</td>\n",

       "      <td>0.993282</td>\n",

       "      <td>0.953464</td>\n",

       "      <td>0.855154</td>\n",

       "      <td>2.512125</td>\n",

       "      <td>1766.619373</td>\n",

       "      <td>0.766032</td>\n",

       "      <td>0.002069</td>\n",

       "      <td>0.003633</td>\n",

       "      <td>3386.995589</td>\n",

       "      <td>2</td>\n",

       "      <td>64.5722</td>\n",

       "      <td>1</td>\n",

       "      <td>0</td>\n",

       "    </tr>\n",

       "    <tr>\n",

       "      <th>246</th>\n",

       "      <td>0.265740</td>\n",

       "      <td>1.555420</td>\n",

       "      <td>0.200766</td>\n",

       "      <td>7.585494</td>\n",

       "      <td>94.132136</td>\n",

       "      <td>-785.0</td>\n",

       "      <td>54523.086867</td>\n",

       "      <td>7.040623</td>\n",

       "      <td>2.333939</td>\n",

       "      <td>1.496020</td>\n",

       "      <td>0.066569</td>\n",

       "      <td>0.616607</td>\n",

       "      <td>0.581458</td>\n",

       "      <td>-0.249969</td>\n",

       "      <td>0.884911</td>\n",

       "      <td>0.995095</td>\n",

       "      <td>0.963887</td>\n",

       "      <td>0.769265</td>\n",

       "      <td>4.096838</td>\n",

       "      <td>3462.816344</td>\n",

       "      <td>0.644554</td>\n",

       "      <td>0.001635</td>\n",

       "      <td>0.004869</td>\n",

       "      <td>4232.418080</td>\n",

       "      <td>3</td>\n",

       "      <td>66.0452</td>\n",

       "      <td>1</td>\n",

       "      <td>0</td>\n",

       "    </tr>\n",

       "    <tr>\n",

       "      <th>240</th>\n",

       "      <td>0.255406</td>\n",

       "      <td>1.576120</td>\n",

       "      <td>0.323878</td>\n",

       "      <td>4.281865</td>\n",

       "      <td>149.072998</td>\n",

       "      <td>-850.0</td>\n",

       "      <td>166735.779994</td>\n",

       "      <td>13.656782</td>\n",

       "      <td>2.926791</td>\n",

       "      <td>2.492079</td>\n",

       "      <td>0.015048</td>\n",

       "      <td>0.457545</td>\n",

       "      <td>0.391681</td>\n",

       "      <td>-0.214233</td>\n",

       "      <td>0.902811</td>\n",

       "      <td>0.993509</td>\n",

       "      <td>0.950374</td>\n",

       "      <td>0.880123</td>\n",

       "      <td>1.824328</td>\n",

       "      <td>683.869792</td>\n",

       "      <td>0.828138</td>\n",

       "      <td>0.002680</td>\n",

       "      <td>0.003901</td>\n",

       "      <td>1844.746046</td>\n",

       "      <td>2</td>\n",

       "      <td>59.3566</td>\n",

       "      <td>1</td>\n",

       "      <td>0</td>\n",

       "    </tr>\n",

       "    <tr>\n",

       "      <th>284</th>\n",

       "      <td>0.199424</td>\n",

       "      <td>1.711620</td>\n",

       "      <td>0.327241</td>\n",

       "      <td>8.999616</td>\n",

       "      <td>143.210231</td>\n",

       "      <td>-1023.0</td>\n",

       "      <td>658805.553990</td>\n",

       "      <td>20.121147</td>\n",

       "      <td>2.787526</td>\n",

       "      <td>2.372269</td>\n",

       "      <td>0.029317</td>\n",

       "      <td>0.544607</td>\n",

       "      <td>0.501892</td>\n",

       "      <td>-0.238174</td>\n",

       "      <td>0.911356</td>\n",

       "      <td>0.992658</td>\n",

       "      <td>0.958908</td>\n",

       "      <td>0.819049</td>\n",

       "      <td>2.652580</td>\n",

       "      <td>962.094760</td>\n",

       "      <td>0.734898</td>\n",

       "      <td>0.004704</td>\n",

       "      <td>0.023273</td>\n",

       "      <td>4444.272812</td>\n",

       "      <td>3</td>\n",

       "      <td>71.0554</td>\n",

       "      <td>1</td>\n",

       "      <td>0</td>\n",

       "    </tr>\n",

       "    <tr>\n",

       "      <th>...</th>\n",

       "      <td>...</td>\n",

       "      <td>...</td>\n",

       "      <td>...</td>\n",

       "      <td>...</td>\n",

       "      <td>...</td>\n",

       "      <td>...</td>\n",

       "      <td>...</td>\n",

       "      <td>...</td>\n",

       "      <td>...</td>\n",

       "      <td>...</td>\n",

       "      <td>...</td>\n",

       "      <td>...</td>\n",

       "      <td>...</td>\n",

       "      <td>...</td>\n",

       "      <td>...</td>\n",

       "      <td>...</td>\n",

       "      <td>...</td>\n",

       "      <td>...</td>\n",

       "      <td>...</td>\n",

       "      <td>...</td>\n",

       "      <td>...</td>\n",

       "      <td>...</td>\n",

       "      <td>...</td>\n",

       "      <td>...</td>\n",

       "      <td>...</td>\n",

       "      <td>...</td>\n",

       "      <td>...</td>\n",

       "      <td>...</td>\n",

       "    </tr>\n",

       "    <tr>\n",

       "      <th>89</th>\n",

       "      <td>0.250187</td>\n",

       "      <td>1.587005</td>\n",

       "      <td>0.286576</td>\n",

       "      <td>1.927350</td>\n",

       "      <td>233.308909</td>\n",

       "      <td>-1015.0</td>\n",

       "      <td>367468.618766</td>\n",

       "      <td>28.824354</td>\n",

       "      <td>3.484871</td>\n",

       "      <td>3.972802</td>\n",

       "      <td>0.001746</td>\n",

       "      <td>0.336821</td>\n",

       "      <td>0.249734</td>\n",

       "      <td>-0.214544</td>\n",

       "      <td>0.939555</td>\n",

       "      <td>0.988997</td>\n",

       "      <td>0.930211</td>\n",

       "      <td>0.934151</td>\n",

       "      <td>1.351356</td>\n",

       "      <td>454.228333</td>\n",

       "      <td>0.907916</td>\n",

       "      <td>0.003590</td>\n",

       "      <td>0.005985</td>\n",

       "      <td>1145.996232</td>\n",

       "      <td>2</td>\n",

       "      <td>69.1828</td>\n",

       "      <td>1</td>\n",

       "      <td>0</td>\n",

       "    </tr>\n",

       "    <tr>\n",

       "      <th>261</th>\n",

       "      <td>0.454151</td>\n",

       "      <td>1.300968</td>\n",

       "      <td>0.281441</td>\n",

       "      <td>1.925799</td>\n",

       "      <td>214.212310</td>\n",

       "      <td>-901.0</td>\n",

       "      <td>227251.294990</td>\n",

       "      <td>17.384511</td>\n",

       "      <td>3.111318</td>\n",

       "      <td>2.831998</td>\n",

       "      <td>0.016200</td>\n",

       "      <td>0.451176</td>\n",

       "      <td>0.383987</td>\n",

       "      <td>-0.271559</td>\n",

       "      <td>0.953994</td>\n",

       "      <td>0.990157</td>\n",

       "      <td>0.939545</td>\n",

       "      <td>0.885250</td>\n",

       "      <td>2.126745</td>\n",

       "      <td>336.573622</td>\n",

       "      <td>0.807157</td>\n",

       "      <td>0.002829</td>\n",

       "      <td>0.004400</td>\n",

       "      <td>2243.689318</td>\n",

       "      <td>0</td>\n",

       "      <td>87.1266</td>\n",

       "      <td>1</td>\n",

       "      <td>0</td>\n",

       "    </tr>\n",

       "    <tr>\n",

       "      <th>129</th>\n",

       "      <td>0.234096</td>\n",

       "      <td>1.622565</td>\n",

       "      <td>0.224665</td>\n",

       "      <td>51.791276</td>\n",

       "      <td>27.675471</td>\n",

       "      <td>-751.0</td>\n",

       "      <td>6889.281619</td>\n",

       "      <td>1.439822</td>\n",

       "      <td>1.440679</td>\n",

       "      <td>0.633557</td>\n",

       "      <td>0.130721</td>\n",

       "      <td>0.746369</td>\n",

       "      <td>0.735862</td>\n",

       "      <td>-0.264988</td>\n",

       "      <td>0.824795</td>\n",

       "      <td>0.999412</td>\n",

       "      <td>0.987616</td>\n",

       "      <td>0.631183</td>\n",

       "      <td>7.254939</td>\n",

       "      <td>4513.100699</td>\n",

       "      <td>0.511173</td>\n",

       "      <td>0.000784</td>\n",

       "      <td>0.006983</td>\n",

       "      <td>7752.599509</td>\n",

       "      <td>1</td>\n",

       "      <td>59.8439</td>\n",

       "      <td>1</td>\n",

       "      <td>0</td>\n",

       "    </tr>\n",

       "    <tr>\n",

       "      <th>273</th>\n",

       "      <td>0.375695</td>\n",

       "      <td>1.385866</td>\n",

       "      <td>0.605798</td>\n",

       "      <td>1.585915</td>\n",

       "      <td>273.935074</td>\n",

       "      <td>-948.0</td>\n",

       "      <td>407492.165534</td>\n",

       "      <td>78.081431</td>\n",

       "      <td>4.126944</td>\n",

       "      <td>6.580868</td>\n",

       "      <td>0.002750</td>\n",

       "      <td>0.272701</td>\n",

       "      <td>0.192854</td>\n",

       "      <td>-0.212124</td>\n",

       "      <td>0.939127</td>\n",

       "      <td>0.959449</td>\n",

       "      <td>0.873813</td>\n",

       "      <td>0.952612</td>\n",

       "      <td>1.231149</td>\n",

       "      <td>40.140124</td>\n",

       "      <td>0.934473</td>\n",

       "      <td>0.012984</td>\n",

       "      <td>0.016086</td>\n",

       "      <td>726.945564</td>\n",

       "      <td>0</td>\n",

       "      <td>70.0000</td>\n",

       "      <td>0</td>\n",

       "      <td>1</td>\n",

       "    </tr>\n",

       "    <tr>\n",

       "      <th>366</th>\n",

       "      <td>0.347965</td>\n",

       "      <td>1.421744</td>\n",

       "      <td>0.183806</td>\n",

       "      <td>3.987643</td>\n",

       "      <td>163.293001</td>\n",

       "      <td>-951.0</td>\n",

       "      <td>231987.589044</td>\n",

       "      <td>12.072458</td>\n",

       "      <td>2.801756</td>\n",

       "      <td>2.259543</td>\n",

       "      <td>0.022109</td>\n",

       "      <td>0.484819</td>\n",

       "      <td>0.425104</td>\n",

       "      <td>-0.234808</td>\n",

       "      <td>0.914060</td>\n",

       "      <td>0.994511</td>\n",

       "      <td>0.955753</td>\n",

       "      <td>0.858965</td>\n",

       "      <td>2.085791</td>\n",

       "      <td>2982.109496</td>\n",

       "      <td>0.797732</td>\n",

       "      <td>0.001148</td>\n",

       "      <td>0.002128</td>\n",

       "      <td>2947.807968</td>\n",

       "      <td>3</td>\n",

       "      <td>54.6940</td>\n",

       "      <td>1</td>\n",

       "      <td>0</td>\n",

       "    </tr>\n",

       "  </tbody>\n",

       "</table>\n",

       "<p>284 rows × 28 columns</p>\n",

       "</div>"

      ],

      "text/plain": [

       "           original_shape_Compactness2  ...  SourceDataset_l2\n",

       "PatientID                               ...                  \n",

       "202                           0.274892  ...                 1\n",

       "371                           0.188210  ...                 0\n",

       "246                           0.265740  ...                 0\n",

       "240                           0.255406  ...                 0\n",

       "284                           0.199424  ...                 0\n",

       "...                                ...  ...               ...\n",

       "89                            0.250187  ...                 0\n",

       "261                           0.454151  ...                 0\n",

       "129                           0.234096  ...                 0\n",

       "273                           0.375695  ...                 1\n",

       "366                           0.347965  ...                 0\n",

       "\n",

       "[284 rows x 28 columns]"

      ]

     },

     "execution_count": 77,

     "metadata": {

      "tags": []

     },

     "output_type": "execute_result"

    }

   ],

   "source": [

    "radiomics_path = 'radiomics.csv'\n",

    "clinical_path = 'clinical_data_cleaned.csv'\n",

    "label_path = 'y_train.csv'\n",

    "\n",

    "def load_data(features=None):\n",

    "    X_df, y_df = read_data.load_owkin_data(radiomics_path, clinical_path, label_path)\n",

    "    if features != None:\n",

    "        X_df = X_df[features]\n",

    "    X = X_df.to_numpy()\n",

    "    y = read_data.y_dataframe_to_rsf_input(y_df)\n",

    "    return X_df, y_df, X, y\n",

    "\n",

    "X_df, y_df, X, y = load_data(features)\n",

    "feature_name = list(X_df.columns.values)\n",

    "X_df"

   ]

  },

  {

   "cell_type": "markdown",

   "metadata": {

    "id": "iCQyaFW1Zn-B",

    "colab_type": "text"

   },

   "source": [

    "## Selection via permutation importance\n",

    "See https://eli5.readthedocs.io/en/latest/blackbox/permutation_importance.html"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": 0,

   "metadata": {

    "id": "E7sOnhFNirxk",

    "colab_type": "code",

    "colab": {}

   },

   "outputs": [],

   "source": [

    "import eli5\n",

    "from eli5.sklearn import PermutationImportance\n",

    "from sklearn.feature_selection import SelectFromModel\n",

    "from sksurv.ensemble import RandomSurvivalForest\n",

    "from sklearn.model_selection import RandomizedSearchCV"

   ]

  },

  {

   "cell_type": "markdown",

   "metadata": {

    "id": "27we-i40p8ag",

    "colab_type": "text"

   },

   "source": [

    "Random search for parameters of the random forest, with all features."

   ]

  },

  {

   "cell_type": "code",

   "execution_count": 0,

   "metadata": {

    "id": "zMZKrq3T3lYP",

    "colab_type": "code",

    "colab": {}

   },

   "outputs": [],

   "source": [

    "tuned_params = {\"n_estimators\": [10, 50, 100, 200, 500],\n",

    "                \"min_samples_split\": [2, 5, 10, 20, 40],\n",

    "                \"min_samples_leaf\": [1, 2, 4, 8, 16]\n",

    "                }"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": 0,

   "metadata": {

    "id": "VgdhrodkqAyf",

    "colab_type": "code",

    "outputId": "d582b5a0-18a1-4001-fcd6-96ef790d3fc3",

    "colab": {

     "base_uri": "https://localhost:8080/",

     "height": 513.0

    }

   },

   "outputs": [

    {

     "name": "stdout",

     "output_type": "stream",

     "text": [

      "Fitting 5 folds for each of 50 candidates, totalling 250 fits\n"

     ]

    },

    {

     "name": "stderr",

     "output_type": "stream",

     "text": [

      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 2 concurrent workers.\n",

      "[Parallel(n_jobs=-1)]: Done  37 tasks      | elapsed:  1.5min\n",

      "/usr/local/lib/python3.6/dist-packages/joblib/externals/loky/process_executor.py:706: UserWarning: A worker stopped while some jobs were given to the executor. This can be caused by a too short worker timeout or by a memory leak.\n",

      "  \"timeout or by a memory leak.\", UserWarning\n",

      "[Parallel(n_jobs=-1)]: Done 158 tasks      | elapsed:  5.0min\n",

      "[Parallel(n_jobs=-1)]: Done 250 out of 250 | elapsed:  7.8min finished\n",

      "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:814: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",

      "  DeprecationWarning)\n"

     ]

    },

    {

     "data": {

      "text/plain": [

       "RandomizedSearchCV(cv=5, error_score='raise-deprecating',\n",

       "                   estimator=RandomSurvivalForest(bootstrap=True,\n",

       "                                                  max_depth=None,\n",

       "                                                  max_features='auto',\n",

       "                                                  max_leaf_nodes=None,\n",

       "                                                  min_samples_leaf=3,\n",

       "                                                  min_samples_split=6,\n",

       "                                                  min_weight_fraction_leaf=0.0,\n",

       "                                                  n_estimators=100, n_jobs=None,\n",

       "                                                  oob_score=False,\n",

       "                                                  random_state=None, verbose=0,\n",

       "                                                  warm_start=False),\n",

       "                   iid='warn', n_iter=50, n_jobs=-1,\n",

       "                   param_distributions={'min_samples_leaf': [1, 2, 4, 8, 16],\n",

       "                                        'min_samples_split': [2, 5, 10, 20, 40],\n",

       "                                        'n_estimators': [10, 50, 100, 200,\n",

       "                                                         500]},\n",

       "                   pre_dispatch='2*n_jobs', random_state=None, refit=True,\n",

       "                   return_train_score=False, scoring=None, verbose=2)"

      ]

     },

     "execution_count": 80,

     "metadata": {

      "tags": []

     },

     "output_type": "execute_result"

    }

   ],

   "source": [

    "search = RandomizedSearchCV(RandomSurvivalForest(), tuned_params, cv=5, verbose=2, n_jobs=-1, n_iter=50)\n",

    "search.fit(X, y)"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": 0,

   "metadata": {

    "id": "XauGW-tBuEEV",

    "colab_type": "code",

    "outputId": "35bb5481-1447-4e37-8257-eedccd02d36c",

    "colab": {

     "base_uri": "https://localhost:8080/",

     "height": 34.0

    }

   },

   "outputs": [

    {

     "data": {

      "text/plain": [

       "{'min_samples_leaf': 1, 'min_samples_split': 10, 'n_estimators': 10}"

      ]

     },

     "execution_count": 81,

     "metadata": {

      "tags": []

     },

     "output_type": "execute_result"

    }

   ],

   "source": [

    "import json\n",

    "with open('best_params.json', 'w') as fp:\n",

    "    json.dump(search.best_params_, fp)\n",

    "\n",

    "search.best_params_"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": 0,

   "metadata": {

    "id": "YuROoGezuHiu",

    "colab_type": "code",

    "outputId": "ea3237a5-8dd7-4580-cece-78fea3c0ddc9",

    "colab": {

     "base_uri": "https://localhost:8080/",

     "height": 34.0

    }

   },

   "outputs": [

    {

     "data": {

      "text/plain": [

       "0.681492222449054"

      ]

     },

     "execution_count": 82,

     "metadata": {

      "tags": []

     },

     "output_type": "execute_result"

    }

   ],

   "source": [

    "search.best_score_"

   ]

  },

  {

   "cell_type": "markdown",

   "metadata": {

    "id": "hZMB39pax0Ch",

    "colab_type": "text"

   },

   "source": [

    "Creating best model from tuning."

   ]

  },

  {

   "cell_type": "code",

   "execution_count": 0,

   "metadata": {

    "id": "FPSs3b6GxzmD",

    "colab_type": "code",

    "outputId": "e6916639-6c74-46b7-89b9-88b5ed3a2c74",

    "colab": {

     "base_uri": "https://localhost:8080/",

     "height": 68.0

    }

   },

   "outputs": [

    {

     "data": {

      "text/plain": [

       "{'fit_time': array([0.22310948, 0.20801306, 0.21998215, 0.22359776, 0.21602607]),\n",

       " 'score_time': array([0.10423088, 0.10348821, 0.10338593, 0.10330081, 0.11683774]),\n",

       " 'test_score': array([0.71041667, 0.65082109, 0.58180367, 0.6196319 , 0.67475728])}"

      ]

     },

     "execution_count": 83,

     "metadata": {

      "tags": []

     },

     "output_type": "execute_result"

    }

   ],

   "source": [

    "params = search.best_params_\n",

    "rsf = RandomSurvivalForest(n_estimators = params['n_estimators'],\n",

    "                           min_samples_split = params['min_samples_split'],\n",

    "                           min_samples_leaf = params['min_samples_leaf'],\n",

    "                           max_features=\"sqrt\",\n",

    "                           n_jobs=-1\n",

    "                           )\n",

    "cross_validate(rsf, X, y, cv=5)"

   ]

  },

  {

   "cell_type": "markdown",

   "metadata": {

    "id": "omkSLicTjVLX",

    "colab_type": "text"

   },

   "source": [

    "Feature importances are computed for a sequence of estimators trained and evaluated on train/test splits according to ``cv``, and then averaged."

   ]

  },

  {

   "cell_type": "code",

   "execution_count": 0,

   "metadata": {

    "id": "uMMDbQVzioLu",

    "colab_type": "code",

    "colab": {}

   },

   "outputs": [],

   "source": [

    "perm = PermutationImportance(rsf, cv=5)\n",

    "perm.fit(X, y)\n",

    "sel = SelectFromModel(perm, prefit=True)"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": 0,

   "metadata": {

    "id": "rzgEjt0ql5sU",

    "colab_type": "code",

    "outputId": "483500b0-7623-4f34-a237-35a6b2f5b39c",

    "colab": {

     "base_uri": "https://localhost:8080/",

     "height": 238.0

    }

   },

   "outputs": [

    {

     "data": {

      "text/plain": [

       "['original_shape_SurfaceVolumeRatio',\n",

       " 'original_firstorder_Minimum',\n",

       " 'original_glcm_ClusterProminence',\n",

       " 'original_glcm_JointEnergy',\n",

       " 'original_glcm_Idm',\n",

       " 'original_glcm_Imc1',\n",

       " 'original_glrlm_ShortRunEmphasis',\n",

       " 'original_glrlm_LongRunEmphasis',\n",

       " 'original_glrlm_GrayLevelNonUniformity',\n",

       " 'original_glrlm_ShortRunLowGrayLevelEmphasis',\n",

       " 'original_glrlm_LongRunHighGrayLevelEmphasis',\n",

       " 'Nstage',\n",

       " 'SourceDataset_l2']"

      ]

     },

     "execution_count": 85,

     "metadata": {

      "tags": []

     },

     "output_type": "execute_result"

    }

   ],

   "source": [

    "X_best = sel.transform(X)\n",

    "id_features = sel.get_support(indices=True)\n",

    "best_features = [feature_name[id] for id in id_features]\n",

    "best_features"

   ]

  },

  {

   "cell_type": "markdown",

   "metadata": {

    "id": "CFmH3cz8SUXT",

    "colab_type": "text"

   },

   "source": [

    "Problem of the feature selection by permutation importance: the errors (std) on the feature importance weights are too high, and the computed feature importance weights don't make sense."

   ]

  },

  {

   "cell_type": "code",

   "execution_count": 0,

   "metadata": {

    "id": "M0y9U5vWJgoE",

    "colab_type": "code",

    "outputId": "27dffd13-5dfd-495c-fa9e-0d2905c74947",

    "colab": {

     "base_uri": "https://localhost:8080/",

     "height": 238.0

    }

   },

   "outputs": [

    {

     "name": "stdout",

     "output_type": "stream",

     "text": [

      "original_shape_SurfaceVolumeRatio: 0.020968142322508116 +/- 0.01987413782583521\n",

      "original_firstorder_Minimum: 0.004782756396357914 +/- 0.012517019893694161\n",

      "original_glcm_ClusterProminence: 0.0028753696565334552 +/- 0.014174580908375901\n",

      "original_glcm_JointEnergy: 0.00710193285525678 +/- 0.009908880621070757\n",

      "original_glcm_Idm: 0.005363079617221596 +/- 0.01372982140990025\n",

      "original_glcm_Imc1: 0.009299617695135192 +/- 0.024793343027533932\n",

      "original_glrlm_ShortRunEmphasis: 0.013550379256367803 +/- 0.02382858097433002\n",

      "original_glrlm_LongRunEmphasis: 0.020862170600605516 +/- 0.012444436857730468\n",

      "original_glrlm_GrayLevelNonUniformity: 0.016629158199686443 +/- 0.020176409780966232\n",

      "original_glrlm_ShortRunLowGrayLevelEmphasis: 0.0037956133982889415 +/- 0.01722860897105692\n",

      "original_glrlm_LongRunHighGrayLevelEmphasis: 0.00821198522155985 +/- 0.01777512119568821\n",

      "Nstage: 0.003868734861434109 +/- 0.011216352542439795\n",

      "SourceDataset_l2: 0.00734324786519267 +/- 0.010657670355138135\n"

     ]

    }

   ],

   "source": [

    "for id in id_features:\n",

    "    print(f'{feature_name[id]}: {perm.feature_importances_[id]} +/- {perm.feature_importances_std_[id]}')"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": 0,

   "metadata": {

    "id": "DG1PCy-I0p5a",

    "colab_type": "code",

    "outputId": "cbf3893b-4b6b-4776-c070-1e4aed03a297",

    "colab": {

     "base_uri": "https://localhost:8080/",

     "height": 238.0

    }

   },

   "outputs": [

    {

     "data": {

      "text/plain": [

       "['original_shape_Compactness2',\n",

       " 'original_glcm_DifferenceAverage',\n",

       " 'original_glcm_Idm',\n",

       " 'original_glcm_Contrast',\n",

       " 'original_glcm_DifferenceEntropy',\n",

       " 'original_firstorder_MeanAbsoluteDeviation',\n",

       " 'original_firstorder_Kurtosis',\n",

       " 'original_shape_SphericalDisproportion',\n",

       " 'original_shape_SurfaceVolumeRatio',\n",

       " 'original_glcm_Id',\n",

       " 'original_firstorder_Minimum',\n",

       " 'original_glcm_JointEnergy',\n",

       " 'original_glcm_ClusterProminence']"

      ]

     },

     "execution_count": 86,

     "metadata": {

      "tags": []

     },

     "output_type": "execute_result"

    }

   ],

   "source": [

    "best_indices = np.flip(np.argsort(perm.feature_importances_))[:len(best_features)]\n",

    "[feature_name[i] for i in np.argsort(best_indices)] "

   ]

  },

  {

   "cell_type": "markdown",

   "metadata": {

    "id": "j3CPwSYgnVFL",

    "colab_type": "text"

   },

   "source": [

    "Cross validation."

   ]

  },

  {

   "cell_type": "code",

   "execution_count": 0,

   "metadata": {

    "id": "ZZy_tXRinL4t",

    "colab_type": "code",

    "outputId": "0e9b7a17-c8ad-4812-ba70-4294fa869d5c",

    "colab": {

     "base_uri": "https://localhost:8080/",

     "height": 68.0

    }

   },

   "outputs": [

    {

     "data": {

      "text/plain": [

       "{'fit_time': array([0.12452412, 0.13066578, 0.12156296, 0.12446737, 0.1260767 ]),\n",

       " 'score_time': array([0.10334563, 0.10343194, 0.10331035, 0.10328126, 0.1031146 ]),\n",

       " 'test_score': array([0.69333333, 0.66378565, 0.57222666, 0.67382413, 0.65695793])}"

      ]

     },

     "execution_count": 87,

     "metadata": {

      "tags": []

     },

     "output_type": "execute_result"

    }

   ],

   "source": [

    "from sklearn.model_selection import cross_validate\n",

    "X_best = sel.transform(X)\n",

    "cross_validate(rsf, X_best, y, cv=5)"

   ]

  },

  {

   "cell_type": "markdown",

   "metadata": {

    "id": "aExvM8zMKD4W",

    "colab_type": "text"

   },

   "source": [

    "## Growing strategy for feature selection"

   ]

  },

  {

   "cell_type": "markdown",

   "metadata": {

    "id": "S-n_aC2jKpor",

    "colab_type": "text"

   },

   "source": [

    "Choose a set of features, fine tune, cross validate."

   ]

  },

  {

   "cell_type": "code",

   "execution_count": 0,

   "metadata": {

    "id": "IPKZA7f7LDt5",

    "colab_type": "code",

    "colab": {}

   },

   "outputs": [],

   "source": [

    "class RSFModel():\n",

    "\n",

    "    def __init__(self, features):\n",

    "        self.X_df, self.y_df, self.X, self.y = load_data(features)\n",

    "\n",

    "    def fine_tune(self, hyperparams):\n",

    "        search = RandomizedSearchCV(RandomSurvivalForest(), tuned_params, cv=5, verbose=1, n_jobs=-1, n_iter=10)\n",

    "        search.fit(self.X, self.y)\n",

    "        return search.best_params_\n",

    "\n",

    "    def cross_validate(self, params):\n",

    "        params = search.best_params_\n",

    "        rsf = RandomSurvivalForest(n_estimators = params['n_estimators'],\n",

    "                                min_samples_split = params['min_samples_split'],\n",

    "                                min_samples_leaf = params['min_samples_leaf'],\n",

    "                                max_features=\"sqrt\",\n",

    "                                n_jobs=-1\n",

    "                                )\n",