Datasets
Models
Downloads: 1
[973ab6]:
/
TCARER_summaryReports.ipynb
2481 lines (2480 with data), 123.6 kB
{
"cells": [
{
"cell_type": "markdown",
"metadata": {
"deletable": true,
"editable": true
},
"source": [
"# Temporal-Comorbidity Adjusted Risk of Emergency Readmission (TCARER)\n",
"## <font style=\"font-weight:bold;color:gray\">Summary Reports</font>"
]
},
{
"cell_type": "markdown",
"metadata": {
"deletable": true,
"editable": true
},
"source": [
"## 1. Initialise"
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {
"collapsed": false,
"deletable": true,
"editable": true,
"scrolled": true
},
"outputs": [],
"source": [
"# reload modules\n",
"# Reload all modules (except those excluded by %aimport) every time before executing the Python code typed.\n",
"%load_ext autoreload \n",
"%autoreload 2"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {
"collapsed": false,
"deletable": true,
"editable": true
},
"outputs": [],
"source": [
"# import libraries\n",
"import logging\n",
"import os\n",
"import sys\n",
"import gc\n",
"import pandas as pd\n",
"import numpy as np\n",
"import random\n",
"import statistics\n",
"from datetime import datetime\n",
"from collections import OrderedDict\n",
"from sklearn import preprocessing\n",
"from scipy.stats import stats\n",
"from IPython.display import display, HTML\n",
"from pprint import pprint\n",
"from pivottablejs import pivot_ui\n",
"from IPython.display import clear_output\n",
"import imblearn.over_sampling as oversampling\n",
"import matplotlib.pyplot as plt"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {
"collapsed": false,
"deletable": true,
"editable": true
},
"outputs": [],
"source": [
"# import local classes\n",
"from Configs.CONSTANTS import CONSTANTS\n",
"from Configs.Logger import Logger\n",
"from Features.Variables import Variables\n",
"from ReadersWriters.ReadersWriters import ReadersWriters\n",
"from Stats.PreProcess import PreProcess\n",
"from Stats.FeatureSelection import FeatureSelection\n",
"from Stats.TrainingMethod import TrainingMethod\n",
"from Stats.Plots import Plots\n",
"from Stats.Stats import Stats"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {
"collapsed": false,
"deletable": true,
"editable": true,
"scrolled": true
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"Make sure the correct Python interpreter is used!\n",
"3.5.3 (v3.5.3:1880cb95a742, Jan 16 2017, 16:02:32) [MSC v.1900 64 bit (AMD64)]\n",
"\n",
"Make sure sys.path of the Python interpreter is correct!\n",
"C:\\Users\\eagle\\Documents\\GitHub\\Analytics_UoW\\TCARER\n"
]
}
],
"source": [
"# Check the interpreter\n",
"print(\"\\nMake sure the correct Python interpreter is used!\")\n",
"print(sys.version)\n",
"print(\"\\nMake sure sys.path of the Python interpreter is correct!\")\n",
"print(os.getcwd())"
]
},
{
"cell_type": "markdown",
"metadata": {
"deletable": true,
"editable": true
},
"source": [
"### 1.1. Initialise General Settings"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {
"collapsed": false,
"deletable": true,
"editable": true
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Output path: C:\\Users\\eagle\\Documents\\GitHub\\tmp\\TCARER\\Basic_prototype\n"
]
}
],
"source": [
"# init paths & directories\n",
"config_path = os.path.abspath(\"ConfigInputs/CONFIGURATIONS.ini\")\n",
"io_path = os.path.abspath(\"../../tmp/TCARER/Basic_prototype\")\n",
"schema = \"parr_sample_prototype\" \n",
"app_name = \"T-CARER\"\n",
"\n",
"print(\"Output path:\", io_path)"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {
"collapsed": false,
"deletable": true,
"editable": true
},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"2017-10-29 13:03:26,935 - T-CARER - INFO - Creating 'C:\\Users\\eagle\\Documents\\GitHub\\tmp\\TCARER\\Basic_prototype\\T-CARER.log' File.\n"
]
}
],
"source": [
"# init logs\n",
"if not os.path.exists(io_path):\n",
" os.makedirs(io_path, exist_ok=True)\n",
"\n",
"logger = Logger(path=io_path, app_name=app_name, ext=\"log\")\n",
"logger = logging.getLogger(app_name)"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {
"collapsed": false,
"deletable": true,
"editable": true
},
"outputs": [],
"source": [
"# init constants \n",
"CONSTANTS.set(io_path, app_name)"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {
"collapsed": false,
"deletable": true,
"editable": true
},
"outputs": [],
"source": [
"# initialise other classes\n",
"readers_writers = ReadersWriters()\n",
"plots = Plots()"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {
"collapsed": true,
"deletable": true,
"editable": true
},
"outputs": [],
"source": [
"# other Constant variables\n",
"submodel_name = \"hesIp\"\n",
"submodel_input_name = \"tcarer_model_features_ip\""
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {
"collapsed": true,
"deletable": true,
"editable": true
},
"outputs": [],
"source": [
"# set print settings\n",
"pd.set_option('display.width', 1600, 'display.max_colwidth', 800)"
]
},
{
"cell_type": "markdown",
"metadata": {
"deletable": true,
"editable": true
},
"source": [
" Common variables:\n",
"* Readmission\n",
" * 'label30', 'label365'\n",
"* Admissions Methods:\n",
" * 'admimeth\\_0t30d\\_prevalence\\_1\\_cnt', ...\n",
"* Prior Spells: \n",
" * 'prior\\_spells'\n",
"* Male:\n",
" * 'gender\\_1'\n",
"* LoS:\n",
" * 'trigger\\_los'\n",
"* Age:\n",
" * 'trigger\\_age'\n",
"* Charlson Score:\n",
" * 'trigger\\_charlsonFoster'\n",
"* predictions score\n",
" * score\n",
"* Most prevalent diagnoses groups (0-30-day, 0-730-day):\n",
" * 0-30-day: 'diagCCS\\_0t30d\\_prevalence\\_1\\_cnt', ...\n",
" * 0-730-day: 'diagCCS\\_0t30d\\_prevalence\\_1\\_cnt' + 'diagCCS\\_30t90d\\_prevalence\\_1\\_cnt' + \n",
" 'diagCCS\\_90t180d\\_prevalence\\_1\\_cnt' + 'diagCCS\\_180t365d\\_prevalence\\_1\\_cnt' + \n",
" 'diagCCS\\_365t730d\\_prevalence\\_1\\_cnt', ...\n",
"* Comorbidity diagnoses groups (0-730-day):\n",
" * 'prior\\_admiOther', 'prior\\_admiAcute', 'prior\\_spells', 'prior\\_asthma', 'prior\\_copd', 'prior\\_depression', 'prior\\_diabetes', 'prior\\_hypertension', 'prior\\_cancer', 'prior\\_chd', 'prior\\_chf'\n",
"* Charlson diagnoses groups (trigger):\n",
" * 'diagCci\\_01\\_myocardial\\_freq\\_\\_trigger',..."
]
},
{
"cell_type": "markdown",
"metadata": {
"deletable": true,
"editable": true
},
"source": [
"<br/><br/>"
]
},
{
"cell_type": "markdown",
"metadata": {
"deletable": true,
"editable": true
},
"source": [
"## 2. Load the Saved Model Outputs"
]
},
{
"cell_type": "markdown",
"metadata": {
"deletable": true,
"editable": true
},
"source": [
"<font style=\"font-weight:bold;color:red\">Note: Make sure the following files are located at the input path</font>\n",
"* Step\\_05\\_Features.bz2\n",
"* Step\\_07\\_Top\\_Features\\_...\n",
"* Step\\_07\\_Model\\_Train\\_model\\_rank\\_summaries\\_...\n",
"* Step\\_09\\_Model\\_..."
]
},
{
"cell_type": "markdown",
"metadata": {
"deletable": true,
"editable": true
},
"source": [
"<font style=\"font-weight:bold;color:red\">Note: Create features extra (Run only once)</font>"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true,
"deletable": true,
"editable": true
},
"outputs": [],
"source": [
"# settings\n",
"feature_table = 'tcarer_features'\n",
"featureExtra_table = 'tcarer_featuresExtra'"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": false,
"deletable": true,
"editable": true,
"scrolled": true
},
"outputs": [],
"source": [
"result = readers_writers.load_mysql_procedure(\"tcarer_set_featuresExtra\", [feature_table, featureExtra_table], schema)"
]
},
{
"cell_type": "markdown",
"metadata": {
"deletable": true,
"editable": true
},
"source": [
"<br/><br/>"
]
},
{
"cell_type": "markdown",
"metadata": {
"deletable": true,
"editable": true
},
"source": [
"### 2.1. Initialise"
]
},
{
"cell_type": "code",
"execution_count": 16,
"metadata": {
"collapsed": true,
"deletable": true,
"editable": true
},
"outputs": [],
"source": [
"# select the target variable\n",
"target_feature = \"label365\" # \"label365\", \"label30\" \n",
"method_name = \"rfc\" # \"rfc\", \"gbrt\", \"randLogit\", \"wdnn\"\n",
"rank_models = [\"rfc\"] # [\"rfc\", \"gbrt\", \"randLogit\"]"
]
},
{
"cell_type": "markdown",
"metadata": {
"deletable": true,
"editable": true
},
"source": [
"<br/><br/>"
]
},
{
"cell_type": "markdown",
"metadata": {
"collapsed": true,
"deletable": true,
"editable": true
},
"source": [
"### 2.2. Load Features"
]
},
{
"cell_type": "markdown",
"metadata": {
"deletable": true,
"editable": true
},
"source": [
"Load pre-processed features"
]
},
{
"cell_type": "code",
"execution_count": 17,
"metadata": {
"collapsed": false,
"deletable": true,
"editable": true
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"File size: 97692\n",
"Number of columns: 458\n",
"features: {train: 2500 , test: 2499 }\n"
]
}
],
"source": [
"file_name = \"Step_07_Features\"\n",
"features = readers_writers.load_serialised_compressed(path=CONSTANTS.io_path, title=file_name)\n",
"\n",
"# print \n",
"print(\"File size: \", os.stat(os.path.join(CONSTANTS.io_path, file_name + \".bz2\")).st_size)\n",
"print(\"Number of columns: \", len(features[\"train_indep\"].columns)) \n",
"print(\"features: {train: \", len(features[\"train_indep\"]), \", test: \", len(features[\"test_indep\"]), \"}\")"
]
},
{
"cell_type": "markdown",
"metadata": {
"deletable": true,
"editable": true
},
"source": [
"### 2.3. Load Features Names"
]
},
{
"cell_type": "code",
"execution_count": 18,
"metadata": {
"collapsed": false,
"deletable": true,
"editable": true
},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>0</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>epidur_0t30d_avg</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>epidur_365t730d_avg</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>preopdur_0t30d_avg</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>gapDays_0t30d_others_cnt</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>epidur_365t730d_others_cnt</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5</th>\n",
" <td>preopdur_30t90d_others_cnt</td>\n",
" </tr>\n",
" <tr>\n",
" <th>6</th>\n",
" <td>epidur_0t30d_others_cnt</td>\n",
" </tr>\n",
" <tr>\n",
" <th>7</th>\n",
" <td>preopdur_0t30d_others_cnt</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8</th>\n",
" <td>epidur_30t90d_others_cnt</td>\n",
" </tr>\n",
" <tr>\n",
" <th>9</th>\n",
" <td>preopdur_90t180d_others_cnt</td>\n",
" </tr>\n",
" <tr>\n",
" <th>10</th>\n",
" <td>epidur_180t365d_avg</td>\n",
" </tr>\n",
" <tr>\n",
" <th>11</th>\n",
" <td>preopdur_30t90d_avg</td>\n",
" </tr>\n",
" <tr>\n",
" <th>12</th>\n",
" <td>preopdur_90t180d_avg</td>\n",
" </tr>\n",
" <tr>\n",
" <th>13</th>\n",
" <td>gapDays_365t730d_avg</td>\n",
" </tr>\n",
" <tr>\n",
" <th>14</th>\n",
" <td>operOPCSL1_0t30d_prevalence_1_cnt</td>\n",
" </tr>\n",
" <tr>\n",
" <th>15</th>\n",
" <td>operOPCSL1_0t30d_others_cnt</td>\n",
" </tr>\n",
" <tr>\n",
" <th>16</th>\n",
" <td>posopdur_30t90d_others_cnt</td>\n",
" </tr>\n",
" <tr>\n",
" <th>17</th>\n",
" <td>epidur_180t365d_others_cnt</td>\n",
" </tr>\n",
" <tr>\n",
" <th>18</th>\n",
" <td>posopdur_0t30d_avg</td>\n",
" </tr>\n",
" <tr>\n",
" <th>19</th>\n",
" <td>preopdur_180t365d_others_cnt</td>\n",
" </tr>\n",
" <tr>\n",
" <th>20</th>\n",
" <td>operOPCSL1_0t30d_prevalence_3_cnt</td>\n",
" </tr>\n",
" <tr>\n",
" <th>21</th>\n",
" <td>operOPCSL1_0t30d_prevalence_4_cnt</td>\n",
" </tr>\n",
" <tr>\n",
" <th>22</th>\n",
" <td>posopdur_365t730d_avg</td>\n",
" </tr>\n",
" <tr>\n",
" <th>23</th>\n",
" <td>operOPCSL1_0t30d_prevalence_2_cnt</td>\n",
" </tr>\n",
" <tr>\n",
" <th>24</th>\n",
" <td>epidur_30t90d_avg</td>\n",
" </tr>\n",
" <tr>\n",
" <th>25</th>\n",
" <td>posopdur_30t90d_avg</td>\n",
" </tr>\n",
" <tr>\n",
" <th>26</th>\n",
" <td>operOPCSL1_0t30d_prevalence_5_cnt</td>\n",
" </tr>\n",
" <tr>\n",
" <th>27</th>\n",
" <td>preopdur_180t365d_avg</td>\n",
" </tr>\n",
" <tr>\n",
" <th>28</th>\n",
" <td>preopdur_365t730d_others_cnt</td>\n",
" </tr>\n",
" <tr>\n",
" <th>29</th>\n",
" <td>preopdur_365t730d_avg</td>\n",
" </tr>\n",
" <tr>\n",
" <th>...</th>\n",
" <td>...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>370</th>\n",
" <td>operOPCSL1_180t365d_prevalence_30_cnt</td>\n",
" </tr>\n",
" <tr>\n",
" <th>371</th>\n",
" <td>diagCCS_0t30d_prevalence_25_cnt</td>\n",
" </tr>\n",
" <tr>\n",
" <th>372</th>\n",
" <td>diagCCS_30t90d_prevalence_1_cnt</td>\n",
" </tr>\n",
" <tr>\n",
" <th>373</th>\n",
" <td>diagCCS_0t30d_prevalence_23_cnt</td>\n",
" </tr>\n",
" <tr>\n",
" <th>374</th>\n",
" <td>diagCCS_90t180d_prevalence_5_cnt</td>\n",
" </tr>\n",
" <tr>\n",
" <th>375</th>\n",
" <td>diagCCS_90t180d_prevalence_4_cnt</td>\n",
" </tr>\n",
" <tr>\n",
" <th>376</th>\n",
" <td>diagCCS_0t30d_prevalence_19_cnt</td>\n",
" </tr>\n",
" <tr>\n",
" <th>377</th>\n",
" <td>operOPCSL1_365t730d_prevalence_4_cnt</td>\n",
" </tr>\n",
" <tr>\n",
" <th>378</th>\n",
" <td>diagCCS_90t180d_prevalence_2_cnt</td>\n",
" </tr>\n",
" <tr>\n",
" <th>379</th>\n",
" <td>diagCCS_90t180d_prevalence_3_cnt</td>\n",
" </tr>\n",
" <tr>\n",
" <th>380</th>\n",
" <td>diagCCS_0t30d_prevalence_21_cnt</td>\n",
" </tr>\n",
" <tr>\n",
" <th>381</th>\n",
" <td>diagCCS_30t90d_prevalence_3_cnt</td>\n",
" </tr>\n",
" <tr>\n",
" <th>382</th>\n",
" <td>diagCCS_30t90d_prevalence_4_cnt</td>\n",
" </tr>\n",
" <tr>\n",
" <th>383</th>\n",
" <td>diagCCS_30t90d_prevalence_2_cnt</td>\n",
" </tr>\n",
" <tr>\n",
" <th>384</th>\n",
" <td>operOPCSL1_365t730d_prevalence_5_cnt</td>\n",
" </tr>\n",
" <tr>\n",
" <th>385</th>\n",
" <td>diagCCS_30t90d_prevalence_5_cnt</td>\n",
" </tr>\n",
" <tr>\n",
" <th>386</th>\n",
" <td>diagCCS_0t30d_prevalence_20_cnt</td>\n",
" </tr>\n",
" <tr>\n",
" <th>387</th>\n",
" <td>diagCCS_30t90d_prevalence_6_cnt</td>\n",
" </tr>\n",
" <tr>\n",
" <th>388</th>\n",
" <td>diagCCS_30t90d_prevalence_7_cnt</td>\n",
" </tr>\n",
" <tr>\n",
" <th>389</th>\n",
" <td>diagCCS_30t90d_prevalence_8_cnt</td>\n",
" </tr>\n",
" <tr>\n",
" <th>390</th>\n",
" <td>operOPCSL1_365t730d_prevalence_6_cnt</td>\n",
" </tr>\n",
" <tr>\n",
" <th>391</th>\n",
" <td>diagCCS_90t180d_prevalence_1_cnt</td>\n",
" </tr>\n",
" <tr>\n",
" <th>392</th>\n",
" <td>diagCCS_90t180d_others_cnt</td>\n",
" </tr>\n",
" <tr>\n",
" <th>393</th>\n",
" <td>operOPCSL1_365t730d_prevalence_7_cnt</td>\n",
" </tr>\n",
" <tr>\n",
" <th>394</th>\n",
" <td>diagCCS_0t30d_prevalence_17_cnt</td>\n",
" </tr>\n",
" <tr>\n",
" <th>395</th>\n",
" <td>diagCCS_0t30d_prevalence_18_cnt</td>\n",
" </tr>\n",
" <tr>\n",
" <th>396</th>\n",
" <td>diagCCS_0t30d_prevalence_16_cnt</td>\n",
" </tr>\n",
" <tr>\n",
" <th>397</th>\n",
" <td>operOPCSL1_365t730d_prevalence_8_cnt</td>\n",
" </tr>\n",
" <tr>\n",
" <th>398</th>\n",
" <td>diagCCS_30t90d_prevalence_30_cnt</td>\n",
" </tr>\n",
" <tr>\n",
" <th>399</th>\n",
" <td>diagCCS_30t90d_prevalence_29_cnt</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"<p>400 rows × 1 columns</p>\n",
"</div>"
],
"text/plain": [
" 0\n",
"0 epidur_0t30d_avg\n",
"1 epidur_365t730d_avg\n",
"2 preopdur_0t30d_avg\n",
"3 gapDays_0t30d_others_cnt\n",
"4 epidur_365t730d_others_cnt\n",
"5 preopdur_30t90d_others_cnt\n",
"6 epidur_0t30d_others_cnt\n",
"7 preopdur_0t30d_others_cnt\n",
"8 epidur_30t90d_others_cnt\n",
"9 preopdur_90t180d_others_cnt\n",
"10 epidur_180t365d_avg\n",
"11 preopdur_30t90d_avg\n",
"12 preopdur_90t180d_avg\n",
"13 gapDays_365t730d_avg\n",
"14 operOPCSL1_0t30d_prevalence_1_cnt\n",
"15 operOPCSL1_0t30d_others_cnt\n",
"16 posopdur_30t90d_others_cnt\n",
"17 epidur_180t365d_others_cnt\n",
"18 posopdur_0t30d_avg\n",
"19 preopdur_180t365d_others_cnt\n",
"20 operOPCSL1_0t30d_prevalence_3_cnt\n",
"21 operOPCSL1_0t30d_prevalence_4_cnt\n",
"22 posopdur_365t730d_avg\n",
"23 operOPCSL1_0t30d_prevalence_2_cnt\n",
"24 epidur_30t90d_avg\n",
"25 posopdur_30t90d_avg\n",
"26 operOPCSL1_0t30d_prevalence_5_cnt\n",
"27 preopdur_180t365d_avg\n",
"28 preopdur_365t730d_others_cnt\n",
"29 preopdur_365t730d_avg\n",
".. ...\n",
"370 operOPCSL1_180t365d_prevalence_30_cnt\n",
"371 diagCCS_0t30d_prevalence_25_cnt\n",
"372 diagCCS_30t90d_prevalence_1_cnt\n",
"373 diagCCS_0t30d_prevalence_23_cnt\n",
"374 diagCCS_90t180d_prevalence_5_cnt\n",
"375 diagCCS_90t180d_prevalence_4_cnt\n",
"376 diagCCS_0t30d_prevalence_19_cnt\n",
"377 operOPCSL1_365t730d_prevalence_4_cnt\n",
"378 diagCCS_90t180d_prevalence_2_cnt\n",
"379 diagCCS_90t180d_prevalence_3_cnt\n",
"380 diagCCS_0t30d_prevalence_21_cnt\n",
"381 diagCCS_30t90d_prevalence_3_cnt\n",
"382 diagCCS_30t90d_prevalence_4_cnt\n",
"383 diagCCS_30t90d_prevalence_2_cnt\n",
"384 operOPCSL1_365t730d_prevalence_5_cnt\n",
"385 diagCCS_30t90d_prevalence_5_cnt\n",
"386 diagCCS_0t30d_prevalence_20_cnt\n",
"387 diagCCS_30t90d_prevalence_6_cnt\n",
"388 diagCCS_30t90d_prevalence_7_cnt\n",
"389 diagCCS_30t90d_prevalence_8_cnt\n",
"390 operOPCSL1_365t730d_prevalence_6_cnt\n",
"391 diagCCS_90t180d_prevalence_1_cnt\n",
"392 diagCCS_90t180d_others_cnt\n",
"393 operOPCSL1_365t730d_prevalence_7_cnt\n",
"394 diagCCS_0t30d_prevalence_17_cnt\n",
"395 diagCCS_0t30d_prevalence_18_cnt\n",
"396 diagCCS_0t30d_prevalence_16_cnt\n",
"397 operOPCSL1_365t730d_prevalence_8_cnt\n",
"398 diagCCS_30t90d_prevalence_30_cnt\n",
"399 diagCCS_30t90d_prevalence_29_cnt\n",
"\n",
"[400 rows x 1 columns]"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"file_name = \"Step_07_Top_Features_rfc_adhoc\" \n",
"\n",
"features_names_selected = readers_writers.load_csv(path=CONSTANTS.io_path, title=file_name, dataframing=False)[0]\n",
"features_names_selected = [f.replace(\"\\n\", \"\") for f in features_names_selected]\n",
"display(pd.DataFrame(features_names_selected))"
]
},
{
"cell_type": "markdown",
"metadata": {
"deletable": true,
"editable": true
},
"source": [
"### 2.4. Load the fitted model"
]
},
{
"cell_type": "markdown",
"metadata": {
"deletable": true,
"editable": true
},
"source": [
"#### <font style=\"font-weight:bold;color:blue\">2.4.1. Basic Models</font>"
]
},
{
"cell_type": "markdown",
"metadata": {
"deletable": true,
"editable": true
},
"source": [
"Initialise"
]
},
{
"cell_type": "code",
"execution_count": 19,
"metadata": {
"collapsed": false,
"deletable": true,
"editable": true
},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"2017-10-29 13:04:19,778 - T-CARER - INFO - Running Random Forest Classifier\n"
]
}
],
"source": [
"training_method = TrainingMethod(method_name)\n",
"\n",
"# file name\n",
"file_name = \"Step_09_Model_\" + method_name + \"_\" + target_feature"
]
},
{
"cell_type": "markdown",
"metadata": {
"deletable": true,
"editable": true
},
"source": [
"Load the model"
]
},
{
"cell_type": "code",
"execution_count": 20,
"metadata": {
"collapsed": false,
"deletable": true,
"editable": true
},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"2017-10-29 13:04:19,866 - T-CARER - INFO - Running Random Forest Classifier\n"
]
}
],
"source": [
"training_method.load(path=CONSTANTS.io_path, title=file_name)"
]
},
{
"cell_type": "markdown",
"metadata": {
"deletable": true,
"editable": true
},
"source": [
"#### <font style=\"font-weight:bold;color:blue\">2.4.2. TensorFlow Models</font>"
]
},
{
"cell_type": "code",
"execution_count": 21,
"metadata": {
"collapsed": false,
"deletable": true,
"editable": true
},
"outputs": [],
"source": [
"class TrainingMethodTensorflow: \n",
" def __init__(self, summaries, features_names, num_features, cut_off, train_size, test_size):\n",
" self.model_predict = {\"train\": {'score': [], 'model_labels': []}, \n",
" \"test\": {'score': [], 'model_labels': []}}\n",
" self.__stats = Stats()\n",
" # summaries[\"fit\"][\"get_variable_names\"]\n",
" # summaries[\"fit\"][\"get_variable_value\"]\n",
" # summaries[\"fit\"][\"get_params\"]\n",
" # summaries[\"fit\"][\"export\"]\n",
" # summaries[\"fit\"][\"get_variable_names()\"]\n",
" # summaries[\"fit\"][\"params\"]\n",
" # summaries[\"fit\"][\"dnn_bias_\"]\n",
" # summaries[\"fit\"][\"dnn_weights_\"] \n",
" # summaries[\"train\"][\"results\"]\n",
" # summaries[\"test\"][\"results\"]\n",
" \n",
" self.model_predict[\"train\"]['pred'] = np.asarray([1 if i[1] >= 0.5 else 0 for i in summaries[\"train\"][\"predict_proba\"]][0:train_size])\n",
" self.model_predict[\"test\"]['pred'] = np.asarray([1 if i[1] >= 0.5 else 0 for i in summaries[\"test\"][\"predict_proba\"]][0:test_size])\n",
" \n",
" self.model_predict[\"train\"]['score'] = np.asarray([i[1] for i in summaries[\"train\"][\"predict_proba\"]][0:train_size])\n",
" self.model_predict[\"test\"]['score'] = np.asarray([i[1] for i in summaries[\"test\"][\"predict_proba\"]][0:test_size])\n",
" \n",
" self.model_predict[\"train\"]['score_0'] = np.asarray([i[0] for i in summaries[\"train\"][\"predict_proba\"]][0:train_size])\n",
" self.model_predict[\"test\"]['score_0'] = np.asarray([i[0] for i in summaries[\"test\"][\"predict_proba\"]][0:test_size])\n",
" \n",
" def train_summaries(self):\n",
" return {\"feature_importances_\": self.__weights}\n",
" \n",
" def predict_summaries(self, feature_target, sample_name):\n",
" return self.__stats.predict_summaries(self.model_predict[sample_name], feature_target)"
]
},
{
"cell_type": "code",
"execution_count": 22,
"metadata": {
"collapsed": false,
"deletable": true,
"editable": true
},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"2017-10-29 13:04:20,044 - T-CARER - ERROR - ReadersWriters._PickleSerialised - Can not open the file: \n",
"C:\\Users\\eagle\\Documents\\GitHub\\tmp\\TCARER\\Basic_prototype\\model_tensorflow_summaries_label365.bz2\n",
"\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"[Errno 2] No such file or directory: 'C:\\\\Users\\\\eagle\\\\Documents\\\\GitHub\\\\tmp\\\\TCARER\\\\Basic_prototype\\\\model_tensorflow_summaries_label365.bz2'\n"
]
},
{
"ename": "SystemExit",
"evalue": "",
"output_type": "error",
"traceback": [
"An exception has occurred, use %tb to see the full traceback.\n",
"\u001b[0;31mSystemExit\u001b[0m\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"c:\\users\\eagle\\appdata\\local\\programs\\python\\python35\\lib\\site-packages\\IPython\\core\\interactiveshell.py:2889: UserWarning: To exit: use 'exit', 'quit', or Ctrl-D.\n",
" warn(\"To exit: use 'exit', 'quit', or Ctrl-D.\", stacklevel=1)\n"
]
}
],
"source": [
"file_name = \"model_tensorflow_summaries_\" + target_feature\n",
"summaries = readers_writers.load_serialised_compressed(path=CONSTANTS.io_path, title=file_name)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": false,
"deletable": true,
"editable": true
},
"outputs": [],
"source": [
"num_features = 300\n",
"cut_off = 0.5\n",
"\n",
"training_method = TrainingMethodTensorflow(summaries, features_names_selected, num_features, cut_off,\n",
" len(features[\"train_indep\"].index), len(features[\"test_indep\"].index))"
]
},
{
"cell_type": "markdown",
"metadata": {
"deletable": true,
"editable": true
},
"source": [
"<br/><br/>"
]
},
{
"cell_type": "markdown",
"metadata": {
"deletable": true,
"editable": true
},
"source": [
"Performance"
]
},
{
"cell_type": "code",
"execution_count": 23,
"metadata": {
"collapsed": false,
"deletable": true,
"editable": true
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"accuracy_score 0.6592\n",
"average_precision_score 0.47970482114\n",
"brier_score_loss 0.218148003816\n",
"classification_report precision recall f1-score support\n",
"\n",
" 0 0.88 0.63 0.74 1885\n",
" 1 0.40 0.74 0.52 615\n",
"\n",
"avg / total 0.76 0.66 0.68 2500\n",
"\n",
"confusion_matrix [[1194 691]\n",
" [ 161 454]]\n",
"f1_score 0.515909090909\n",
"fbeta_score 0.436958614052\n",
"hamming_loss 0.3408\n",
"jaccard_similarity_score 0.6592\n",
"log_loss 11.7710360041\n",
"matthews_corrcoef 0.32124418171\n",
"precision_recall_fscore_support (array([ 0.88118081, 0.39650655]), array([ 0.63342175, 0.73821138]), array([ 0.73703704, 0.51590909]), array([1885, 615], dtype=int64))\n",
"precision_score 0.396506550218\n",
"recall_score 0.738211382114\n",
"roc_auc_score 0.742965646633\n",
"zero_one_loss 0.3408\n",
"\n",
"\n",
"accuracy_score 0.654661864746\n",
"average_precision_score 0.423030738094\n",
"brier_score_loss 0.22022287805\n",
"classification_report precision recall f1-score support\n",
"\n",
" 0 0.90 0.63 0.74 1959\n",
" 1 0.36 0.75 0.48 540\n",
"\n",
"avg / total 0.78 0.65 0.69 2499\n",
"\n",
"confusion_matrix [[1230 729]\n",
" [ 134 406]]\n",
"f1_score 0.484776119403\n",
"fbeta_score 0.399606299213\n",
"hamming_loss 0.345338135254\n",
"jaccard_similarity_score 0.654661864746\n",
"log_loss 11.9277898901\n",
"matthews_corrcoef 0.313889024973\n",
"precision_recall_fscore_support (array([ 0.90175953, 0.35770925]), array([ 0.62787136, 0.75185185]), array([ 0.74029491, 0.48477612]), array([1959, 540], dtype=int64))\n",
"precision_score 0.357709251101\n",
"recall_score 0.751851851852\n",
"roc_auc_score 0.73645898323\n",
"zero_one_loss 0.345338135254\n"
]
}
],
"source": [
"# train\n",
"o_summaries = training_method.predict_summaries(features[\"train_target\"][target_feature], \"train\")\n",
"for k in o_summaries.keys():\n",
" print(k, o_summaries[k])\n",
" \n",
"print(\"\\n\")\n",
"# test\n",
"o_summaries = training_method.predict_summaries(features[\"test_target\"][target_feature], \"test\")\n",
"for k in o_summaries.keys():\n",
" print(k, o_summaries[k])"
]
},
{
"cell_type": "markdown",
"metadata": {
"deletable": true,
"editable": true
},
"source": [
"<br/><br/>"
]
},
{
"cell_type": "markdown",
"metadata": {
"deletable": true,
"editable": true
},
"source": [
"### 2.5. Load the Extra Features for Benchmarking"
]
},
{
"cell_type": "markdown",
"metadata": {
"deletable": true,
"editable": true
},
"source": [
"Read the extra features"
]
},
{
"cell_type": "code",
"execution_count": 24,
"metadata": {
"collapsed": false,
"deletable": true,
"editable": true
},
"outputs": [],
"source": [
"table = 'tcarer_featuresExtra'\n",
"features_extra_dtypes = {'patientID': 'U32', 'trigger_charlsonFoster': 'i4', 'trigger_los': 'i4', 'trigger_age': 'i4', 'prior_admiOther': 'i4', 'prior_admiAcute': 'i4', \n",
" 'prior_spells': 'i4', 'prior_asthma': 'i4', 'prior_copd': 'i4', 'prior_depression': 'i4', 'prior_diabetes': 'i4', 'prior_hypertension': 'i4', 'prior_cancer': 'i4', 'prior_chd': 'i4', 'prior_chf': 'i4', \n",
" 'diagCci_01_myocardial_freq': 'i4', 'diagCci_02_chf_freq': 'i4', 'diagCci_03_pvd_freq': 'i4', 'diagCci_04_cerebrovascular_freq': 'i4', 'diagCci_05_dementia_freq': 'i4', 'diagCci_06_cpd_freq': 'i4', 'diagCci_07_rheumatic_freq': 'i4', 'diagCci_08_ulcer_freq': 'i4', 'diagCci_09_liverMild_freq': 'i4', 'diagCci_10_diabetesNotChronic_freq': 'i4', 'diagCci_11_diabetesChronic_freq': 'i4', 'diagCci_12_hemiplegia_freq': 'i4', 'diagCci_13_renal_freq': 'i4', 'diagCci_14_malignancy_freq': 'i4', 'diagCci_15_liverSevere_freq': 'i4', 'diagCci_16_tumorSec_freq': 'i4', 'diagCci_17_aids_freq': 'i4', 'diagCci_18_depression_freq': 'i4', 'diagCci_19_cardiac_freq': 'i4', 'diagCci_20_valvular_freq': 'i4', 'diagCci_21_pulmonary_freq': 'i4', 'diagCci_22_vascular_freq': 'i4', 'diagCci_23_hypertensionNotComplicated_freq': 'i4', 'diagCci_24_hypertensionComplicated_freq': 'i4', 'diagCci_25_paralysis_freq': 'i4', 'diagCci_26_neuroOther_freq': 'i4', 'diagCci_27_pulmonaryChronic_freq': 'i4', 'diagCci_28_diabetesNotComplicated_freq': 'i4', 'diagCci_29_diabetesComplicated_freq': 'i4', 'diagCci_30_hypothyroidism_freq': 'i4', 'diagCci_31_renal_freq': 'i4', 'diagCci_32_liver_freq': 'i4', 'diagCci_33_ulcerNotBleeding_freq': 'i4', 'diagCci_34_psychoses_freq': 'i4', 'diagCci_35_lymphoma_freq': 'i4', 'diagCci_36_cancerSec_freq': 'i4', 'diagCci_37_tumorNotSec_freq': 'i4', 'diagCci_38_rheumatoid_freq': 'i4', 'diagCci_39_coagulopathy_freq': 'i4', 'diagCci_40_obesity_freq': 'i4', 'diagCci_41_weightLoss_freq': 'i4', 'diagCci_42_fluidDisorder_freq': 'i4', 'diagCci_43_bloodLoss_freq': 'i4', 'diagCci_44_anemia_freq': 'i4', 'diagCci_45_alcohol_freq': 'i4', 'diagCci_46_drug_freq': 'i4'}\n",
"features_extra_name = features_extra_dtypes.keys()"
]
},
{
"cell_type": "code",
"execution_count": 25,
"metadata": {
"collapsed": false,
"deletable": true,
"editable": true
},
"outputs": [
{
"ename": "ProgrammingError",
"evalue": "(_mysql_exceptions.ProgrammingError) (1146, \"Table 'parr_sample_prototype.tcarer_featuresextra' doesn't exist\") [SQL: 'SELECT * FROM tcarer_featuresExtra']",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mProgrammingError\u001b[0m Traceback (most recent call last)",
"\u001b[0;32mc:\\users\\eagle\\appdata\\local\\programs\\python\\python35\\lib\\site-packages\\sqlalchemy\\engine\\base.py\u001b[0m in \u001b[0;36m_execute_context\u001b[0;34m(self, dialect, constructor, statement, parameters, *args)\u001b[0m\n\u001b[1;32m 1181\u001b[0m \u001b[0mparameters\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1182\u001b[0;31m context)\n\u001b[0m\u001b[1;32m 1183\u001b[0m \u001b[1;32mexcept\u001b[0m \u001b[0mBaseException\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
"\u001b[0;32mc:\\users\\eagle\\appdata\\local\\programs\\python\\python35\\lib\\site-packages\\sqlalchemy\\engine\\default.py\u001b[0m in \u001b[0;36mdo_execute\u001b[0;34m(self, cursor, statement, parameters, context)\u001b[0m\n\u001b[1;32m 469\u001b[0m \u001b[1;32mdef\u001b[0m \u001b[0mdo_execute\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcursor\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstatement\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mparameters\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcontext\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m--> 470\u001b[0;31m \u001b[0mcursor\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mexecute\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mstatement\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mparameters\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 471\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
"\u001b[0;32mc:\\users\\eagle\\appdata\\local\\programs\\python\\python35\\lib\\site-packages\\MySQLdb\\cursors.py\u001b[0m in \u001b[0;36mexecute\u001b[0;34m(self, query, args)\u001b[0m\n\u001b[1;32m 249\u001b[0m \u001b[0mexc\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mvalue\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0msys\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mexc_info\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;36m2\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m--> 250\u001b[0;31m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0merrorhandler\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mexc\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 251\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_executed\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mquery\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
"\u001b[0;32mc:\\users\\eagle\\appdata\\local\\programs\\python\\python35\\lib\\site-packages\\MySQLdb\\connections.py\u001b[0m in \u001b[0;36mdefaulterrorhandler\u001b[0;34m(***failed resolving arguments***)\u001b[0m\n\u001b[1;32m 41\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0merrorvalue\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mBaseException\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m---> 42\u001b[0;31m \u001b[1;32mraise\u001b[0m \u001b[0merrorvalue\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 43\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0merrorclass\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
"\u001b[0;32mc:\\users\\eagle\\appdata\\local\\programs\\python\\python35\\lib\\site-packages\\MySQLdb\\cursors.py\u001b[0m in \u001b[0;36mexecute\u001b[0;34m(self, query, args)\u001b[0m\n\u001b[1;32m 246\u001b[0m \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m--> 247\u001b[0;31m \u001b[0mres\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_query\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mquery\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 248\u001b[0m \u001b[1;32mexcept\u001b[0m \u001b[0mException\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
"\u001b[0;32mc:\\users\\eagle\\appdata\\local\\programs\\python\\python35\\lib\\site-packages\\MySQLdb\\cursors.py\u001b[0m in \u001b[0;36m_query\u001b[0;34m(self, q)\u001b[0m\n\u001b[1;32m 410\u001b[0m \u001b[1;32mdef\u001b[0m \u001b[0m_query\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mq\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m--> 411\u001b[0;31m \u001b[0mrowcount\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_do_query\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mq\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 412\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_post_get_result\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
"\u001b[0;32mc:\\users\\eagle\\appdata\\local\\programs\\python\\python35\\lib\\site-packages\\MySQLdb\\cursors.py\u001b[0m in \u001b[0;36m_do_query\u001b[0;34m(self, q)\u001b[0m\n\u001b[1;32m 373\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_last_executed\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mq\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m--> 374\u001b[0;31m \u001b[0mdb\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mquery\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mq\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 375\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_do_get_result\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
"\u001b[0;32mc:\\users\\eagle\\appdata\\local\\programs\\python\\python35\\lib\\site-packages\\MySQLdb\\connections.py\u001b[0m in \u001b[0;36mquery\u001b[0;34m(self, query)\u001b[0m\n\u001b[1;32m 269\u001b[0m \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m--> 270\u001b[0;31m \u001b[0m_mysql\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mconnection\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mquery\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mquery\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 271\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
"\u001b[0;31mProgrammingError\u001b[0m: (1146, \"Table 'parr_sample_prototype.tcarer_featuresextra' doesn't exist\")",
"\nThe above exception was the direct cause of the following exception:\n",
"\u001b[0;31mProgrammingError\u001b[0m Traceback (most recent call last)",
"\u001b[0;32m<ipython-input-25-086a730ff34b>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[1;31m# Read features from the MySQL\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m 2\u001b[0m \u001b[0mfeatures_extra\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mdict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0mfeatures_extra\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'train'\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mreaders_writers\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mload_mysql_table\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mschema\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtable\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdataframing\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mTrue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 4\u001b[0m \u001b[0mfeatures_extra\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'train'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mastype\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdtype\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mfeatures_extra_dtypes\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m 5\u001b[0m \u001b[0mfeatures_extra\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'test'\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mfeatures_extra\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'train'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
"\u001b[0;32mC:\\Users\\eagle\\Documents\\GitHub\\Analytics_UoW\\TCARER\\ReadersWriters\\ReadersWriters.py\u001b[0m in \u001b[0;36mload_mysql_table\u001b[0;34m(db_schema, db_table, dataframing)\u001b[0m\n\u001b[1;32m 323\u001b[0m \"\"\"\n\u001b[1;32m 324\u001b[0m \u001b[0mquery\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;34m\"SELECT * FROM \"\u001b[0m \u001b[1;33m+\u001b[0m \u001b[0mdb_table\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m--> 325\u001b[0;31m \u001b[1;32mreturn\u001b[0m \u001b[0mReadersWriters\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mload_mysql_query\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mquery\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdb_schema\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdataframing\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 326\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m 327\u001b[0m \u001b[1;33m@\u001b[0m\u001b[0mstaticmethod\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
"\u001b[0;32mC:\\Users\\eagle\\Documents\\GitHub\\Analytics_UoW\\TCARER\\ReadersWriters\\ReadersWriters.py\u001b[0m in \u001b[0;36mload_mysql_query\u001b[0;34m(query, db_schema, dataframing, batch, float_round_vars, float_round)\u001b[0m\n\u001b[1;32m 345\u001b[0m \u001b[0mengine\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mdb\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mopen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m 346\u001b[0m \u001b[0mdbc\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mMysqlCommand\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mengine\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdb\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdb_session_vars\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m--> 347\u001b[0;31m \u001b[0moutput\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mdbc\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mread\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mquery\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdataframing\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mbatch\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfloat_round_vars\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfloat_round\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 348\u001b[0m \u001b[0mdb\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mclose\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m 349\u001b[0m \u001b[1;32mreturn\u001b[0m \u001b[0moutput\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
"\u001b[0;32mC:\\Users\\eagle\\Documents\\GitHub\\Analytics_UoW\\TCARER\\ReadersWriters\\_MysqlCommand.py\u001b[0m in \u001b[0;36mread\u001b[0;34m(self, query, dataframing, batch, float_round_vars, float_round)\u001b[0m\n\u001b[1;32m 71\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m__logger\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdebug\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"Reading from MySQL database.\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m 72\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0mdataframing\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m---> 73\u001b[0;31m \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m__read_df\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mquery\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mbatch\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfloat_round_vars\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfloat_round\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 74\u001b[0m \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m 75\u001b[0m \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m__read_arr\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mquery\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
"\u001b[0;32mC:\\Users\\eagle\\Documents\\GitHub\\Analytics_UoW\\TCARER\\ReadersWriters\\_MysqlCommand.py\u001b[0m in \u001b[0;36m__read_df\u001b[0;34m(self, query, batch, float_round_vars, float_round)\u001b[0m\n\u001b[1;32m 104\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m 105\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0mbatch\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m--> 106\u001b[0;31m \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mpds\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mread_sql\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msql\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mquery\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcon\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mconn\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcoerce_float\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mFalse\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mchunksize\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mbatch\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 107\u001b[0m \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m 108\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mdf\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mpds\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mread_sql\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msql\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mquery\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcon\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mconn\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcoerce_float\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mFalse\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mchunksize\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mbatch\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
"\u001b[0;32mc:\\users\\eagle\\appdata\\local\\programs\\python\\python35\\lib\\site-packages\\pandas\\io\\sql.py\u001b[0m in \u001b[0;36mread_sql\u001b[0;34m(sql, con, index_col, coerce_float, params, parse_dates, columns, chunksize)\u001b[0m\n\u001b[1;32m 413\u001b[0m \u001b[0msql\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mindex_col\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mindex_col\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mparams\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mparams\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m 414\u001b[0m \u001b[0mcoerce_float\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mcoerce_float\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mparse_dates\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mparse_dates\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m--> 415\u001b[0;31m chunksize=chunksize)\n\u001b[0m\u001b[1;32m 416\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m 417\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
"\u001b[0;32mc:\\users\\eagle\\appdata\\local\\programs\\python\\python35\\lib\\site-packages\\pandas\\io\\sql.py\u001b[0m in \u001b[0;36mread_query\u001b[0;34m(self, sql, index_col, coerce_float, parse_dates, params, chunksize)\u001b[0m\n\u001b[1;32m 1082\u001b[0m \u001b[0margs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0m_convert_params\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msql\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mparams\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m 1083\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1084\u001b[0;31m \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mexecute\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1085\u001b[0m \u001b[0mcolumns\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mresult\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mkeys\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m 1086\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
"\u001b[0;32mc:\\users\\eagle\\appdata\\local\\programs\\python\\python35\\lib\\site-packages\\pandas\\io\\sql.py\u001b[0m in \u001b[0;36mexecute\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 973\u001b[0m \u001b[1;32mdef\u001b[0m \u001b[0mexecute\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m 974\u001b[0m \u001b[1;34m\"\"\"Simple passthrough to SQLAlchemy connectable\"\"\"\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m--> 975\u001b[0;31m \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mconnectable\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mexecute\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 976\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m 977\u001b[0m def read_table(self, table_name, index_col=None, coerce_float=True,\n",
"\u001b[0;32mc:\\users\\eagle\\appdata\\local\\programs\\python\\python35\\lib\\site-packages\\sqlalchemy\\engine\\base.py\u001b[0m in \u001b[0;36mexecute\u001b[0;34m(self, object, *multiparams, **params)\u001b[0m\n\u001b[1;32m 937\u001b[0m \"\"\"\n\u001b[1;32m 938\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mobject\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mutil\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstring_types\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m--> 939\u001b[0;31m \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_execute_text\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mobject\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmultiparams\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mparams\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 940\u001b[0m \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m 941\u001b[0m \u001b[0mmeth\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mobject\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_execute_on_connection\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
"\u001b[0;32mc:\\users\\eagle\\appdata\\local\\programs\\python\\python35\\lib\\site-packages\\sqlalchemy\\engine\\base.py\u001b[0m in \u001b[0;36m_execute_text\u001b[0;34m(self, statement, multiparams, params)\u001b[0m\n\u001b[1;32m 1095\u001b[0m \u001b[0mstatement\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m 1096\u001b[0m \u001b[0mparameters\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1097\u001b[0;31m \u001b[0mstatement\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mparameters\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1098\u001b[0m )\n\u001b[1;32m 1099\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_has_events\u001b[0m \u001b[1;32mor\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mengine\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_has_events\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
"\u001b[0;32mc:\\users\\eagle\\appdata\\local\\programs\\python\\python35\\lib\\site-packages\\sqlalchemy\\engine\\base.py\u001b[0m in \u001b[0;36m_execute_context\u001b[0;34m(self, dialect, constructor, statement, parameters, *args)\u001b[0m\n\u001b[1;32m 1187\u001b[0m \u001b[0mparameters\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m 1188\u001b[0m \u001b[0mcursor\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1189\u001b[0;31m context)\n\u001b[0m\u001b[1;32m 1190\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m 1191\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_has_events\u001b[0m \u001b[1;32mor\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mengine\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_has_events\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
"\u001b[0;32mc:\\users\\eagle\\appdata\\local\\programs\\python\\python35\\lib\\site-packages\\sqlalchemy\\engine\\base.py\u001b[0m in \u001b[0;36m_handle_dbapi_exception\u001b[0;34m(self, e, statement, parameters, cursor, context)\u001b[0m\n\u001b[1;32m 1391\u001b[0m util.raise_from_cause(\n\u001b[1;32m 1392\u001b[0m \u001b[0msqlalchemy_exception\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1393\u001b[0;31m \u001b[0mexc_info\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1394\u001b[0m )\n\u001b[1;32m 1395\u001b[0m \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
"\u001b[0;32mc:\\users\\eagle\\appdata\\local\\programs\\python\\python35\\lib\\site-packages\\sqlalchemy\\util\\compat.py\u001b[0m in \u001b[0;36mraise_from_cause\u001b[0;34m(exception, exc_info)\u001b[0m\n\u001b[1;32m 201\u001b[0m \u001b[0mexc_type\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mexc_value\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mexc_tb\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mexc_info\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m 202\u001b[0m \u001b[0mcause\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mexc_value\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0mexc_value\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mexception\u001b[0m \u001b[1;32melse\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m--> 203\u001b[0;31m \u001b[0mreraise\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtype\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mexception\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mexception\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtb\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mexc_tb\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcause\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mcause\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 204\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m 205\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0mpy3k\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
"\u001b[0;32mc:\\users\\eagle\\appdata\\local\\programs\\python\\python35\\lib\\site-packages\\sqlalchemy\\util\\compat.py\u001b[0m in \u001b[0;36mreraise\u001b[0;34m(tp, value, tb, cause)\u001b[0m\n\u001b[1;32m 184\u001b[0m \u001b[0mvalue\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m__cause__\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcause\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m 185\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0mvalue\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m__traceback__\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mtb\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m--> 186\u001b[0;31m \u001b[1;32mraise\u001b[0m \u001b[0mvalue\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mwith_traceback\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtb\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 187\u001b[0m \u001b[1;32mraise\u001b[0m \u001b[0mvalue\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m 188\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
"\u001b[0;32mc:\\users\\eagle\\appdata\\local\\programs\\python\\python35\\lib\\site-packages\\sqlalchemy\\engine\\base.py\u001b[0m in \u001b[0;36m_execute_context\u001b[0;34m(self, dialect, constructor, statement, parameters, *args)\u001b[0m\n\u001b[1;32m 1180\u001b[0m \u001b[0mstatement\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m 1181\u001b[0m \u001b[0mparameters\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1182\u001b[0;31m context)\n\u001b[0m\u001b[1;32m 1183\u001b[0m \u001b[1;32mexcept\u001b[0m \u001b[0mBaseException\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m 1184\u001b[0m self._handle_dbapi_exception(\n",
"\u001b[0;32mc:\\users\\eagle\\appdata\\local\\programs\\python\\python35\\lib\\site-packages\\sqlalchemy\\engine\\default.py\u001b[0m in \u001b[0;36mdo_execute\u001b[0;34m(self, cursor, statement, parameters, context)\u001b[0m\n\u001b[1;32m 468\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m 469\u001b[0m \u001b[1;32mdef\u001b[0m \u001b[0mdo_execute\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcursor\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstatement\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mparameters\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcontext\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m--> 470\u001b[0;31m \u001b[0mcursor\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mexecute\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mstatement\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mparameters\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 471\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m 472\u001b[0m \u001b[1;32mdef\u001b[0m \u001b[0mdo_execute_no_params\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcursor\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstatement\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcontext\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
"\u001b[0;32mc:\\users\\eagle\\appdata\\local\\programs\\python\\python35\\lib\\site-packages\\MySQLdb\\cursors.py\u001b[0m in \u001b[0;36mexecute\u001b[0;34m(self, query, args)\u001b[0m\n\u001b[1;32m 248\u001b[0m \u001b[1;32mexcept\u001b[0m \u001b[0mException\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m 249\u001b[0m \u001b[0mexc\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mvalue\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0msys\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mexc_info\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;36m2\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m--> 250\u001b[0;31m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0merrorhandler\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mexc\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 251\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_executed\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mquery\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m 252\u001b[0m \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_defer_warnings\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
"\u001b[0;32mc:\\users\\eagle\\appdata\\local\\programs\\python\\python35\\lib\\site-packages\\MySQLdb\\connections.py\u001b[0m in \u001b[0;36mdefaulterrorhandler\u001b[0;34m(***failed resolving arguments***)\u001b[0m\n\u001b[1;32m 40\u001b[0m \u001b[1;32mdel\u001b[0m \u001b[0mconnection\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m 41\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0merrorvalue\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mBaseException\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m---> 42\u001b[0;31m \u001b[1;32mraise\u001b[0m \u001b[0merrorvalue\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 43\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0merrorclass\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m 44\u001b[0m \u001b[1;32mraise\u001b[0m \u001b[0merrorclass\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0merrorvalue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
"\u001b[0;32mc:\\users\\eagle\\appdata\\local\\programs\\python\\python35\\lib\\site-packages\\MySQLdb\\cursors.py\u001b[0m in \u001b[0;36mexecute\u001b[0;34m(self, query, args)\u001b[0m\n\u001b[1;32m 245\u001b[0m \u001b[0mres\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m 246\u001b[0m \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m--> 247\u001b[0;31m \u001b[0mres\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_query\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mquery\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 248\u001b[0m \u001b[1;32mexcept\u001b[0m \u001b[0mException\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m 249\u001b[0m \u001b[0mexc\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mvalue\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0msys\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mexc_info\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;36m2\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
"\u001b[0;32mc:\\users\\eagle\\appdata\\local\\programs\\python\\python35\\lib\\site-packages\\MySQLdb\\cursors.py\u001b[0m in \u001b[0;36m_query\u001b[0;34m(self, q)\u001b[0m\n\u001b[1;32m 409\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m 410\u001b[0m \u001b[1;32mdef\u001b[0m \u001b[0m_query\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mq\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m--> 411\u001b[0;31m \u001b[0mrowcount\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_do_query\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mq\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 412\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_post_get_result\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m 413\u001b[0m \u001b[1;32mreturn\u001b[0m \u001b[0mrowcount\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
"\u001b[0;32mc:\\users\\eagle\\appdata\\local\\programs\\python\\python35\\lib\\site-packages\\MySQLdb\\cursors.py\u001b[0m in \u001b[0;36m_do_query\u001b[0;34m(self, q)\u001b[0m\n\u001b[1;32m 372\u001b[0m \u001b[0mdb\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_get_db\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m 373\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_last_executed\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mq\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m--> 374\u001b[0;31m \u001b[0mdb\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mquery\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mq\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 375\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_do_get_result\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m 376\u001b[0m \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrowcount\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
"\u001b[0;32mc:\\users\\eagle\\appdata\\local\\programs\\python\\python35\\lib\\site-packages\\MySQLdb\\connections.py\u001b[0m in \u001b[0;36mquery\u001b[0;34m(self, query)\u001b[0m\n\u001b[1;32m 268\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mread_query_result\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m 269\u001b[0m \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m--> 270\u001b[0;31m \u001b[0m_mysql\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mconnection\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mquery\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mquery\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 271\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m 272\u001b[0m \u001b[1;32mdef\u001b[0m \u001b[0m__enter__\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
"\u001b[0;31mProgrammingError\u001b[0m: (_mysql_exceptions.ProgrammingError) (1146, \"Table 'parr_sample_prototype.tcarer_featuresextra' doesn't exist\") [SQL: 'SELECT * FROM tcarer_featuresExtra']"
]
}
],
"source": [
"# Read features from the MySQL\n",
"features_extra = dict()\n",
"features_extra['train'] = readers_writers.load_mysql_table(schema, table, dataframing=True)\n",
"features_extra['train'].astype(dtype=features_extra_dtypes)\n",
"features_extra['test'] = features_extra['train']\n",
"\n",
"print(\"Number of columns: \", len(features_extra['train'].columns), \"; Total records: \", len(features_extra['train'].index))"
]
},
{
"cell_type": "markdown",
"metadata": {
"deletable": true,
"editable": true
},
"source": [
"Replace NaN appears in the Charlson-Index feature "
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true,
"deletable": true,
"editable": true
},
"outputs": [],
"source": [
"features_extra['train'].loc[:, \"trigger_charlsonFoster\"] = np.nan_to_num(features_extra['train'][\"trigger_charlsonFoster\"])\n",
"features_extra['test'].loc[:, \"trigger_charlsonFoster\"] = np.nan_to_num(features_extra['test'][\"trigger_charlsonFoster\"])"
]
},
{
"cell_type": "markdown",
"metadata": {
"deletable": true,
"editable": true
},
"source": [
"Combine (join by PatientID)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": false,
"deletable": true,
"editable": true
},
"outputs": [],
"source": [
"features_extra['train'] = features_extra['train'].merge(\n",
" pd.concat([features['train_id'], features['train_target'], \n",
" pd.DataFrame({'score': training_method.model_predict[\"train\"]['score']}), features['train_indep']], axis=1), \n",
" how=\"inner\", on=\"patientID\")\n",
"features_extra['test'] = features_extra['test'].merge(\n",
" pd.concat([features['test_id'], features['test_target'], \n",
" pd.DataFrame({'score': training_method.model_predict[\"test\"]['score']}), features['test_indep']], axis=1), \n",
" how=\"inner\", on=\"patientID\")"
]
},
{
"cell_type": "markdown",
"metadata": {
"deletable": true,
"editable": true
},
"source": [
"<font style=\"font-weight:bold;color:red\">Clean-up</font>"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": false,
"deletable": true,
"editable": true,
"scrolled": true
},
"outputs": [],
"source": [
"features = None\n",
"gc.collect()"
]
},
{
"cell_type": "markdown",
"metadata": {
"deletable": true,
"editable": true
},
"source": [
"<br/><br/>"
]
},
{
"cell_type": "markdown",
"metadata": {
"deletable": true,
"editable": true
},
"source": [
"## 3. Charlson Index Model"
]
},
{
"cell_type": "markdown",
"metadata": {
"deletable": true,
"editable": true
},
"source": [
"### 3.1. Algorithm"
]
},
{
"cell_type": "markdown",
"metadata": {
"deletable": true,
"editable": true
},
"source": [
"<font style=\"font-weight:bold;color:brown\">Algorithm 1</font>: Random Forest"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": false,
"deletable": true,
"editable": true
},
"outputs": [],
"source": [
"charlson_method_name = \"rfc\"\n",
"kwargs = {\"n_estimators\": 20, \"criterion\": 'gini', \"max_depth\": None, \"min_samples_split\": 100,\n",
" \"min_samples_leaf\": 50, \"min_weight_fraction_leaf\": 0.0, \"max_features\": 'auto',\n",
" \"max_leaf_nodes\": None, \"bootstrap\": True, \"oob_score\": False, \"n_jobs\": -1, \"random_state\": None,\n",
" \"verbose\": 0, \"warm_start\": False, \"class_weight\": \"balanced_subsample\"}"
]
},
{
"cell_type": "markdown",
"metadata": {
"deletable": true,
"editable": true
},
"source": [
"<font style=\"font-weight:bold;color:brown\">Algorithm 2</font>: Logistic Regression"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": false,
"deletable": true,
"editable": true
},
"outputs": [],
"source": [
"charlson_method_name = \"lr\"\n",
"kwargs = {\"penalty\": 'l2', \"dual\": False, \"tol\": 0.0001, \"C\": 1, \"fit_intercept\": True, \"intercept_scaling\": 1,\n",
" \"class_weight\": None, \"random_state\": None, \"solver\": 'liblinear', \"max_iter\": 100, \"multi_class\": 'ovr',\n",
" \"verbose\": 0, \"warm_start\": False, \"n_jobs\": -1}"
]
},
{
"cell_type": "markdown",
"metadata": {
"deletable": true,
"editable": true
},
"source": [
"<br/><br/>"
]
},
{
"cell_type": "markdown",
"metadata": {
"deletable": true,
"editable": true
},
"source": [
"### 3.2. Initialise"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": false,
"deletable": true,
"editable": true
},
"outputs": [],
"source": [
"# set features\n",
"charlson_features_names = ['trigger_charlsonFoster']"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": false,
"deletable": true,
"editable": true
},
"outputs": [],
"source": [
"# select the target variable\n",
"charlson_target_feature = \"label30\" # \"label30\", \"label365\" \n",
"\n",
"# file name\n",
"file_name = \"report_Model_Charlson_\" + charlson_method_name + \"_\" + charlson_target_feature\n",
"\n",
"# initialise\n",
"charlson_training_method = TrainingMethod(charlson_method_name)"
]
},
{
"cell_type": "markdown",
"metadata": {
"deletable": true,
"editable": true
},
"source": [
"### 3.3. Fit"
]
},
{
"cell_type": "markdown",
"metadata": {
"deletable": true,
"editable": true
},
"source": [
"Fit Model"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": false,
"deletable": true,
"editable": true
},
"outputs": [],
"source": [
"o_summaries = dict()\n",
"# Fit\n",
"model = charlson_training_method.train(features_extra[\"train\"][charlson_features_names], features_extra[\"train\"][target_feature], **kwargs)\n",
"charlson_training_method.save_model(path=CONSTANTS.io_path, title=file_name)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true,
"deletable": true,
"editable": true
},
"outputs": [],
"source": [
"# load model\n",
"# charlson_training_method.load(path=CONSTANTS.io_path, title=file_name)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": false,
"deletable": true,
"editable": true
},
"outputs": [],
"source": [
"# short summary\n",
"o_summaries = charlson_training_method.train_summaries()"
]
},
{
"cell_type": "markdown",
"metadata": {
"deletable": true,
"editable": true
},
"source": [
"Fit Performance"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": false,
"deletable": true,
"editable": true
},
"outputs": [],
"source": [
"o_summaries = dict()\n",
"model = charlson_training_method.predict(features_extra[\"train\"][charlson_features_names], \"train\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": false,
"deletable": true,
"editable": true
},
"outputs": [],
"source": [
"# short summary\n",
"o_summaries = charlson_training_method.predict_summaries(pd.Series(features_extra[\"train\"][target_feature]), \"train\")\n",
"print(\"ROC AUC:\", o_summaries['roc_auc_score_1'], \"\\n\", o_summaries['classification_report'])\n",
"for k in o_summaries.keys():\n",
" print(k, o_summaries[k])"
]
},
{
"cell_type": "markdown",
"metadata": {
"deletable": true,
"editable": true
},
"source": [
"### 3.4. Predict"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": false,
"deletable": true,
"editable": true
},
"outputs": [],
"source": [
"o_summaries = dict()\n",
"model = charlson_training_method.predict(features_extra[\"test\"][charlson_features_names], \"test\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": false,
"deletable": true,
"editable": true
},
"outputs": [],
"source": [
"# short summary\n",
"o_summaries = charlson_training_method.predict_summaries(pd.Series(features_extra[\"test\"][target_feature]), \"test\")\n",
"print(\"ROC AUC:\", o_summaries['roc_auc_score_1'], \"\\n\", o_summaries['classification_report'])\n",
"for k in o_summaries.keys():\n",
" print(k, o_summaries[k])"
]
},
{
"cell_type": "markdown",
"metadata": {
"deletable": true,
"editable": true
},
"source": [
"### 3.5. Cross-Validate"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": false,
"deletable": true,
"editable": true
},
"outputs": [],
"source": [
"o_summaries = dict()\n",
"score = charlson_training_method.cross_validate(features_extra[\"test\"][charlson_features_names], features_extra[\"test\"][target_feature], \n",
" scoring=\"neg_mean_squared_error\", cv=10)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": false,
"deletable": true,
"editable": true
},
"outputs": [],
"source": [
"# short summary\n",
"o_summaries = charlson_training_method.cross_validate_summaries()\n",
"print(\"Scores: \", o_summaries)"
]
},
{
"cell_type": "markdown",
"metadata": {
"deletable": true,
"editable": true
},
"source": [
"### 3.6. Save"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": false,
"deletable": true,
"editable": true,
"scrolled": true
},
"outputs": [],
"source": [
"charlson_training_method.save_model(path=CONSTANTS.io_path, title=file_name)"
]
},
{
"cell_type": "markdown",
"metadata": {
"deletable": true,
"editable": true
},
"source": [
"<br/><br/>"
]
},
{
"cell_type": "markdown",
"metadata": {
"deletable": true,
"editable": true
},
"source": [
"## 4. Features Statistics"
]
},
{
"cell_type": "markdown",
"metadata": {
"collapsed": true,
"deletable": true,
"editable": true
},
"source": [
"### 4.1. Features Rank"
]
},
{
"cell_type": "markdown",
"metadata": {
"collapsed": false,
"deletable": true,
"editable": true
},
"source": [
"<i>It is produced during modelling</i>"
]
},
{
"cell_type": "markdown",
"metadata": {
"deletable": true,
"editable": true
},
"source": [
"### 4.2. Descriptive Statistics"
]
},
{
"cell_type": "markdown",
"metadata": {
"collapsed": false,
"deletable": true,
"editable": true
},
"source": [
"<i>It is produced during modelling</i>"
]
},
{
"cell_type": "markdown",
"metadata": {
"deletable": true,
"editable": true
},
"source": [
"### 4.3. Features Weigths"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": false,
"deletable": true,
"editable": true
},
"outputs": [],
"source": [
"def features_importance_rank(fitting_method, ranking_file_name=None, rank_models=[\"rfc\", \"gbrt\", \"randLogit\"]):\n",
" # Fitting weight\n",
" o_summaries = pd.DataFrame({\"Name\": fitting_method.model_labels,\n",
" \"Fitting Weight\": fitting_method.train_summaries()[\"feature_importances_\"]},\n",
" index = fitting_method.model_labels)\n",
" o_summaries = o_summaries.sort_values(\"Fitting Weight\", ascending=False)\n",
" o_summaries = o_summaries.reset_index(drop=True)\n",
" \n",
" # Ranking scores\n",
" if ranking_file_name is not None:\n",
" for rank_model in rank_models:\n",
" o_summaries_ranks = readers_writers.load_serialised_compressed(\n",
" path=CONSTANTS.io_path, title=ranking_file_name + rank_model)\n",
" for trial in range(len(o_summaries_ranks)):\n",
" o_summaries_rank = pd.DataFrame(o_summaries_ranks[trial])\n",
" o_summaries_rank.columns = [\"Name\", \"Importance - \" + rank_model + \" - Trial_\" + str(trial),\n",
" \"Order - \" + rank_model + \" - Trial_\" + str(trial)]\n",
" o_summaries = o_summaries.merge(o_summaries_rank, how=\"outer\", on=\"Name\")\n",
" \n",
" return o_summaries"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": false,
"deletable": true,
"editable": true
},
"outputs": [],
"source": [
"file_name = \"Step_07_Model_Train_model_rank_summaries_\"\n",
"\n",
"o_summaries = features_importance_rank(fitting_method=training_method, ranking_file_name=file_name, rank_models=rank_models)\n",
"\n",
"file_name = \"report_weights_ranks\"\n",
"readers_writers.save_csv(path=CONSTANTS.io_path, title=file_name, data=o_summaries, append=False, extension=\"csv\", header=o_summaries.columns)\n",
"\n",
"display(o_summaries.head())"
]
},
{
"cell_type": "markdown",
"metadata": {
"deletable": true,
"editable": true
},
"source": [
"<br/><br/>"
]
},
{
"cell_type": "markdown",
"metadata": {
"deletable": true,
"editable": true
},
"source": [
"## 5. Model Performance"
]
},
{
"cell_type": "markdown",
"metadata": {
"deletable": true,
"editable": true
},
"source": [
"### 5.1. Performance Indicators"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true,
"deletable": true,
"editable": true
},
"outputs": [],
"source": [
"measures = [\"accuracy_score\", \"precision_score\", \"recall_score\",\n",
" \"roc_auc_score_1\", \"f1_score\", \"fbeta_score\", \"average_precision_score\", \n",
" \"log_loss\", \"zero_one_loss\", \"hamming_loss\", \"jaccard_similarity_score\", \"matthews_corrcoef\"]"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": false,
"deletable": true,
"editable": true
},
"outputs": [],
"source": [
"# train\n",
"o_summaries = training_method.predict_summaries(features_extra[\"train\"][target_feature], \"train\")\n",
"o_summaries = np.array([(m, o_summaries[m]) for m in measures])\n",
"report_performance = pd.DataFrame({\"Measure\": o_summaries[:, 0], \n",
" \"Sample Train\": o_summaries[:, 1], \n",
" \"Sample Test\": [None] * len(measures)})\n",
"\n",
"# test\n",
"o_summaries = training_method.predict_summaries(features_extra[\"test\"][target_feature], \"test\")\n",
"o_summaries = np.array([(m, o_summaries[m]) for m in measures])\n",
"report_performance[\"Sample Test\"] = o_summaries[:, 1]"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": false,
"deletable": true,
"editable": true,
"scrolled": true
},
"outputs": [],
"source": [
"# print\n",
"file_name = \"report_performance_\" + method_name + \"_\" + target_feature\n",
"display(report_performance)\n",
"readers_writers.save_csv(path=CONSTANTS.io_path, title=file_name, data=report_performance, append=False)"
]
},
{
"cell_type": "markdown",
"metadata": {
"deletable": true,
"editable": true
},
"source": [
"### 5.2. Population Statistics"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": false,
"deletable": true,
"editable": true
},
"outputs": [],
"source": [
"def population_statistics(df, diagnoses, cutpoints=[0.50, 0.60, 0.70, 0.80, 0.90]):\n",
" o_summaries = pd.DataFrame(columns=['Name'], index=diagnoses)\n",
" o_summaries['Name'] = diagnoses\n",
" \n",
" for diagnose in diagnoses:\n",
" o_summaries.loc[diagnose, 'Total'] = len(df.index)\n",
" if diagnose not in df:\n",
" continue\n",
" \n",
" o_summaries.loc[diagnose, 'Total - diagnose'] = len(df.loc[(df[diagnose] > 0)].index)\n",
" o_summaries.loc[diagnose, 'Total - diagnose - label_1'] = len(df.loc[(df[diagnose] > 0) & (df[target_feature] > 0)].index)\n",
" o_summaries.loc[diagnose, 'Emergency Readmission Rate - cnt 1'] = len(df.loc[(df[diagnose] > 0) & (df['admimeth_0t30d_prevalence_1_cnt'] > 0) & (df[target_feature] > 0)].index)\n",
" o_summaries.loc[diagnose, 'Emergency Readmission Rate - cnt 2'] = len(df.loc[(df[diagnose] > 0) & (df['admimeth_0t30d_prevalence_2_cnt'] > 0) & (df[target_feature] > 0)].index)\n",
" o_summaries.loc[diagnose, 'Emergency Readmission Rate - cnt 3'] = len(df.loc[(df[diagnose] > 0) & (df['admimeth_0t30d_prevalence_3_cnt'] > 0) & (df[target_feature] > 0)].index)\n",
" o_summaries.loc[diagnose, 'Prior Spells'] = len(df.loc[(df[diagnose] > 0) & (df['prior_spells'] > 0) & (df[target_feature] > 0)].index)\n",
" o_summaries.loc[diagnose, 'Male - perc'] = len(df.loc[(df[diagnose] > 0) & (df['gender_1'] > 0) & (df[target_feature] > 0)].index)\n",
" age = df.loc[(df[diagnose] > 0) & (df[target_feature] > 0)]['trigger_age'].describe(percentiles=[.25, .5, .75])\n",
" o_summaries.loc[diagnose, 'Age - IQR_min'] = age['min']\n",
" o_summaries.loc[diagnose, 'Age - IQR_25'] = age['25%']\n",
" o_summaries.loc[diagnose, 'Age - IQR_50'] = age['50%']\n",
" o_summaries.loc[diagnose, 'Age - IQR_75'] = age['75%']\n",
" o_summaries.loc[diagnose, 'Age - IQR_max'] = age['max']\n",
" los = df.loc[(df[diagnose] > 0) & (df[target_feature] > 0)]['trigger_los'].describe(percentiles=[.25, .5, .75])\n",
" o_summaries.loc[diagnose, 'LoS - IQR_min'] = los['min']\n",
" o_summaries.loc[diagnose, 'LoS - IQR_25'] = los['25%']\n",
" o_summaries.loc[diagnose, 'LoS - IQR_50'] = los['50%']\n",
" o_summaries.loc[diagnose, 'LoS - IQR_75'] = los['75%']\n",
" o_summaries.loc[diagnose, 'LoS - IQR_max'] = los['max']\n",
" for cutpoint in cutpoints:\n",
" o_summaries.loc[diagnose, 'score - ' + str(cutpoint)] = len(df.loc[(df[diagnose] > 0) & (df['score'] > cutpoint)].index)\n",
" o_summaries.loc[diagnose, 'TP - ' + str(cutpoint)] = len(df.loc[(df[diagnose] > 0) & (df[target_feature] > 0) & (df['score'] > cutpoint)].index)\n",
" o_summaries.loc[diagnose, 'FP - ' + str(cutpoint)] = len(df.loc[(df[diagnose] > 0) & (df[target_feature] == 0) & (df['score'] > cutpoint)].index)\n",
" o_summaries.loc[diagnose, 'FN - ' + str(cutpoint)] = len(df.loc[(df[diagnose] > 0) & (df[target_feature] > 0) & (df['score'] <= cutpoint)].index)\n",
" o_summaries.loc[diagnose, 'TN - ' + str(cutpoint)] = len(df.loc[(df[diagnose] > 0) & (df[target_feature] == 0) & (df['score'] <= cutpoint)].index)\n",
" \n",
" \n",
" o_summaries.loc[diagnose, 'Charlson - 0'] = len(df.loc[(df[diagnose] > 0) & (df[\"trigger_charlsonFoster\"] == 0)].index)\n",
" o_summaries.loc[diagnose, 'Charlson - 0 - label_1'] = len(df.loc[(df[diagnose] > 0) & (df[\"trigger_charlsonFoster\"] == 0) & (df[target_feature] > 0)].index)\n",
" o_summaries.loc[diagnose, 'Charlson - 1'] = len(df.loc[(df[diagnose] > 0) & (df[\"trigger_charlsonFoster\"] == 1)].index)\n",
" o_summaries.loc[diagnose, 'Charlson - 1 - label_1'] = len(df.loc[(df[diagnose] > 0) & (df[\"trigger_charlsonFoster\"] == 1) & (df[target_feature] > 0)].index)\n",
" o_summaries.loc[diagnose, 'Charlson - 2'] = len(df.loc[(df[diagnose] > 0) & (df[\"trigger_charlsonFoster\"] == 2)].index)\n",
" o_summaries.loc[diagnose, 'Charlson - 2 - label_1'] = len(df.loc[(df[diagnose] > 0) & (df[\"trigger_charlsonFoster\"] == 2) & (df[target_feature] > 0)].index)\n",
" o_summaries.loc[diagnose, 'Charlson - 3'] = len(df.loc[(df[diagnose] > 0) & (df[\"trigger_charlsonFoster\"] == 3)].index)\n",
" o_summaries.loc[diagnose, 'Charlson - 3 - label_1'] = len(df.loc[(df[diagnose] > 0) & (df[\"trigger_charlsonFoster\"] == 3) & (df[target_feature] > 0)].index)\n",
" o_summaries.loc[diagnose, 'Charlson - 4+'] = len(df.loc[(df[diagnose] > 0) & (df[\"trigger_charlsonFoster\"] >= 4)].index)\n",
" o_summaries.loc[diagnose, 'Charlson - 4+ - label_1'] = len(df.loc[(df[diagnose] > 0) & (df[\"trigger_charlsonFoster\"] >= 4) & (df[target_feature] > 0)].index)\n",
" \n",
" for cutpoint in cutpoints:\n",
" o_summaries.loc[diagnose, 'Charlson - 0 - label_1 - TP - ' + str(cutpoint)] = \\\n",
" len(df.loc[(df[diagnose] > 0) & (df[\"trigger_charlsonFoster\"] == 0) & (df[target_feature] > 0) & (df['score'] > cutpoint)].index)\n",
" \n",
" return o_summaries"
]
},
{
"cell_type": "markdown",
"metadata": {
"deletable": true,
"editable": true
},
"source": [
"#### 5.2.1. Most Prevalent Diagnoses Groups\n",
"Most prevalent diagnoses groups (30-day, 1-year readmission): \n",
"* Total, Admissions, Emergency Readmission Rate, Prior Spells, Male (%), Age (IQR), LoS (IQR), TP, FP, FN, TN"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true,
"deletable": true,
"editable": true
},
"outputs": [],
"source": [
"diagnoses = ['diagCCS_0t30d_others_cnt', 'diagCCS_0t30d_prevalence_1_cnt', 'diagCCS_0t30d_prevalence_2_cnt', 'diagCCS_0t30d_prevalence_3_cnt', 'diagCCS_0t30d_prevalence_4_cnt', 'diagCCS_0t30d_prevalence_5_cnt', 'diagCCS_0t30d_prevalence_6_cnt', 'diagCCS_0t30d_prevalence_7_cnt', 'diagCCS_0t30d_prevalence_8_cnt', 'diagCCS_0t30d_prevalence_9_cnt', 'diagCCS_0t30d_prevalence_10_cnt', 'diagCCS_0t30d_prevalence_11_cnt', 'diagCCS_0t30d_prevalence_12_cnt', 'diagCCS_0t30d_prevalence_13_cnt', 'diagCCS_0t30d_prevalence_14_cnt', 'diagCCS_0t30d_prevalence_15_cnt', 'diagCCS_0t30d_prevalence_16_cnt', 'diagCCS_0t30d_prevalence_17_cnt', 'diagCCS_0t30d_prevalence_18_cnt', 'diagCCS_0t30d_prevalence_19_cnt', 'diagCCS_0t30d_prevalence_20_cnt', 'diagCCS_0t30d_prevalence_21_cnt', 'diagCCS_0t30d_prevalence_22_cnt', 'diagCCS_0t30d_prevalence_23_cnt', 'diagCCS_0t30d_prevalence_24_cnt', 'diagCCS_0t30d_prevalence_25_cnt', 'diagCCS_0t30d_prevalence_26_cnt', 'diagCCS_0t30d_prevalence_27_cnt', 'diagCCS_0t30d_prevalence_28_cnt', 'diagCCS_0t30d_prevalence_29_cnt', 'diagCCS_0t30d_prevalence_30_cnt'\n",
" , 'diagCCS_30t90d_others_cnt', 'diagCCS_30t90d_prevalence_1_cnt', 'diagCCS_30t90d_prevalence_2_cnt', 'diagCCS_30t90d_prevalence_3_cnt', 'diagCCS_30t90d_prevalence_4_cnt', 'diagCCS_30t90d_prevalence_5_cnt', 'diagCCS_30t90d_prevalence_6_cnt', 'diagCCS_30t90d_prevalence_7_cnt', 'diagCCS_30t90d_prevalence_8_cnt', 'diagCCS_30t90d_prevalence_9_cnt', 'diagCCS_30t90d_prevalence_10_cnt', 'diagCCS_30t90d_prevalence_11_cnt', 'diagCCS_30t90d_prevalence_12_cnt', 'diagCCS_30t90d_prevalence_13_cnt', 'diagCCS_30t90d_prevalence_14_cnt', 'diagCCS_30t90d_prevalence_15_cnt', 'diagCCS_30t90d_prevalence_16_cnt', 'diagCCS_30t90d_prevalence_17_cnt', 'diagCCS_30t90d_prevalence_18_cnt', 'diagCCS_30t90d_prevalence_19_cnt', 'diagCCS_30t90d_prevalence_20_cnt', 'diagCCS_30t90d_prevalence_21_cnt', 'diagCCS_30t90d_prevalence_22_cnt', 'diagCCS_30t90d_prevalence_23_cnt', 'diagCCS_30t90d_prevalence_24_cnt', 'diagCCS_30t90d_prevalence_25_cnt', 'diagCCS_30t90d_prevalence_26_cnt', 'diagCCS_30t90d_prevalence_27_cnt', 'diagCCS_30t90d_prevalence_28_cnt', 'diagCCS_30t90d_prevalence_29_cnt', 'diagCCS_30t90d_prevalence_30_cnt'\n",
" , 'diagCCS_90t180d_others_cnt', 'diagCCS_90t180d_prevalence_1_cnt', 'diagCCS_90t180d_prevalence_2_cnt', 'diagCCS_90t180d_prevalence_3_cnt', 'diagCCS_90t180d_prevalence_4_cnt', 'diagCCS_90t180d_prevalence_5_cnt', 'diagCCS_90t180d_prevalence_6_cnt', 'diagCCS_90t180d_prevalence_7_cnt', 'diagCCS_90t180d_prevalence_8_cnt', 'diagCCS_90t180d_prevalence_9_cnt', 'diagCCS_90t180d_prevalence_10_cnt', 'diagCCS_90t180d_prevalence_11_cnt', 'diagCCS_90t180d_prevalence_12_cnt', 'diagCCS_90t180d_prevalence_13_cnt', 'diagCCS_90t180d_prevalence_14_cnt', 'diagCCS_90t180d_prevalence_15_cnt', 'diagCCS_90t180d_prevalence_16_cnt', 'diagCCS_90t180d_prevalence_17_cnt', 'diagCCS_90t180d_prevalence_18_cnt', 'diagCCS_90t180d_prevalence_19_cnt', 'diagCCS_90t180d_prevalence_20_cnt', 'diagCCS_90t180d_prevalence_21_cnt', 'diagCCS_90t180d_prevalence_22_cnt', 'diagCCS_90t180d_prevalence_23_cnt', 'diagCCS_90t180d_prevalence_24_cnt', 'diagCCS_90t180d_prevalence_25_cnt', 'diagCCS_90t180d_prevalence_26_cnt', 'diagCCS_90t180d_prevalence_27_cnt', 'diagCCS_90t180d_prevalence_28_cnt', 'diagCCS_90t180d_prevalence_29_cnt', 'diagCCS_90t180d_prevalence_30_cnt'\n",
" , 'diagCCS_180t365d_others_cnt', 'diagCCS_180t365d_prevalence_1_cnt', 'diagCCS_180t365d_prevalence_2_cnt', 'diagCCS_180t365d_prevalence_3_cnt', 'diagCCS_180t365d_prevalence_4_cnt', 'diagCCS_180t365d_prevalence_5_cnt', 'diagCCS_180t365d_prevalence_6_cnt', 'diagCCS_180t365d_prevalence_7_cnt', 'diagCCS_180t365d_prevalence_8_cnt', 'diagCCS_180t365d_prevalence_9_cnt', 'diagCCS_180t365d_prevalence_10_cnt', 'diagCCS_180t365d_prevalence_11_cnt', 'diagCCS_180t365d_prevalence_12_cnt', 'diagCCS_180t365d_prevalence_13_cnt', 'diagCCS_180t365d_prevalence_14_cnt', 'diagCCS_180t365d_prevalence_15_cnt', 'diagCCS_180t365d_prevalence_16_cnt', 'diagCCS_180t365d_prevalence_17_cnt', 'diagCCS_180t365d_prevalence_18_cnt', 'diagCCS_180t365d_prevalence_19_cnt', 'diagCCS_180t365d_prevalence_20_cnt', 'diagCCS_180t365d_prevalence_21_cnt', 'diagCCS_180t365d_prevalence_22_cnt', 'diagCCS_180t365d_prevalence_23_cnt', 'diagCCS_180t365d_prevalence_24_cnt', 'diagCCS_180t365d_prevalence_25_cnt', 'diagCCS_180t365d_prevalence_26_cnt', 'diagCCS_180t365d_prevalence_27_cnt', 'diagCCS_180t365d_prevalence_28_cnt', 'diagCCS_180t365d_prevalence_29_cnt', 'diagCCS_180t365d_prevalence_30_cnt'\n",
" , 'diagCCS_365t730d_others_cnt', 'diagCCS_365t730d_prevalence_1_cnt', 'diagCCS_365t730d_prevalence_2_cnt', 'diagCCS_365t730d_prevalence_3_cnt', 'diagCCS_365t730d_prevalence_4_cnt', 'diagCCS_365t730d_prevalence_5_cnt', 'diagCCS_365t730d_prevalence_6_cnt', 'diagCCS_365t730d_prevalence_7_cnt', 'diagCCS_365t730d_prevalence_8_cnt', 'diagCCS_365t730d_prevalence_9_cnt', 'diagCCS_365t730d_prevalence_10_cnt', 'diagCCS_365t730d_prevalence_11_cnt', 'diagCCS_365t730d_prevalence_12_cnt', 'diagCCS_365t730d_prevalence_13_cnt', 'diagCCS_365t730d_prevalence_14_cnt', 'diagCCS_365t730d_prevalence_15_cnt', 'diagCCS_365t730d_prevalence_16_cnt', 'diagCCS_365t730d_prevalence_17_cnt', 'diagCCS_365t730d_prevalence_18_cnt', 'diagCCS_365t730d_prevalence_19_cnt', 'diagCCS_365t730d_prevalence_20_cnt', 'diagCCS_365t730d_prevalence_21_cnt', 'diagCCS_365t730d_prevalence_22_cnt', 'diagCCS_365t730d_prevalence_23_cnt', 'diagCCS_365t730d_prevalence_24_cnt', 'diagCCS_365t730d_prevalence_25_cnt', 'diagCCS_365t730d_prevalence_26_cnt', 'diagCCS_365t730d_prevalence_27_cnt', 'diagCCS_365t730d_prevalence_28_cnt', 'diagCCS_365t730d_prevalence_29_cnt', 'diagCCS_365t730d_prevalence_30_cnt']\n",
"file_name = \"report_population_prevalent_diagnoses_\" + method_name + \"_\" + target_feature + \"_\""
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": false,
"deletable": true,
"editable": true
},
"outputs": [],
"source": [
"o_summaries = population_statistics(features_extra['train'], diagnoses)\n",
"readers_writers.save_csv(path=CONSTANTS.io_path, title=file_name + \"train\", data=o_summaries, append=False, extension=\"csv\", header=o_summaries.columns)\n",
"\n",
"o_summaries = population_statistics(features_extra['test'], diagnoses)\n",
"readers_writers.save_csv(path=CONSTANTS.io_path, title=file_name + \"test\", data=o_summaries, append=False, extension=\"csv\", header=o_summaries.columns)"
]
},
{
"cell_type": "markdown",
"metadata": {
"deletable": true,
"editable": true
},
"source": [
"#### 5.2.2. Major Comorbidity Groups\n",
"Comorbidity diagnoses groups (30-day, 1-year readmission): \n",
"* Total, Admissions, Emergency Readmission Rate, Prior Spells, Male (%), Age (IQR), LoS (IQR), TP, FP, FN, TN"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": false,
"deletable": true,
"editable": true
},
"outputs": [],
"source": [
"diagnoses = ['prior_admiOther', 'prior_admiAcute', 'prior_spells', 'prior_asthma', 'prior_copd', 'prior_depression', 'prior_diabetes', 'prior_hypertension', 'prior_cancer', 'prior_chd', 'prior_chf']\n",
"file_name = \"report_population_comorbidity_diagnoses_\" + method_name + \"_\" + target_feature + \"_\""
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": false,
"deletable": true,
"editable": true
},
"outputs": [],
"source": [
"o_summaries = population_statistics(features_extra['train'], diagnoses)\n",
"readers_writers.save_csv(path=CONSTANTS.io_path, title=file_name + \"train\", data=o_summaries, append=False, extension=\"csv\", header=o_summaries.columns)\n",
"\n",
"o_summaries = population_statistics(features_extra['test'], diagnoses)\n",
"readers_writers.save_csv(path=CONSTANTS.io_path, title=file_name + \"test\", data=o_summaries, append=False, extension=\"csv\", header=o_summaries.columns)"
]
},
{
"cell_type": "markdown",
"metadata": {
"deletable": true,
"editable": true
},
"source": [
"#### 5.2.3. Charlson Comorbidity Groups\n",
"Charlson diagnoses groups (30-day, 1-year readmission): \n",
"* Total, Admissions, Emergency Readmission Rate, Prior Spells, Male (%), Age (IQR), LoS (IQR), TP, FP, FN, TN"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true,
"deletable": true,
"editable": true
},
"outputs": [],
"source": [
"diagnoses = ['diagCci_01_myocardial_freq', 'diagCci_02_chf_freq', 'diagCci_03_pvd_freq', 'diagCci_04_cerebrovascular_freq', 'diagCci_05_dementia_freq', 'diagCci_06_cpd_freq', 'diagCci_07_rheumatic_freq', 'diagCci_08_ulcer_freq', 'diagCci_09_liverMild_freq', 'diagCci_10_diabetesNotChronic_freq', 'diagCci_11_diabetesChronic_freq', 'diagCci_12_hemiplegia_freq', 'diagCci_13_renal_freq', 'diagCci_14_malignancy_freq', 'diagCci_15_liverSevere_freq', 'diagCci_16_tumorSec_freq', 'diagCci_17_aids_freq', 'diagCci_18_depression_freq', 'diagCci_19_cardiac_freq', 'diagCci_20_valvular_freq', 'diagCci_21_pulmonary_freq', 'diagCci_22_vascular_freq', 'diagCci_23_hypertensionNotComplicated_freq', 'diagCci_24_hypertensionComplicated_freq', 'diagCci_25_paralysis_freq', 'diagCci_26_neuroOther_freq', 'diagCci_27_pulmonaryChronic_freq', 'diagCci_28_diabetesNotComplicated_freq', 'diagCci_29_diabetesComplicated_freq', 'diagCci_30_hypothyroidism_freq', 'diagCci_31_renal_freq', 'diagCci_32_liver_freq', 'diagCci_33_ulcerNotBleeding_freq', 'diagCci_34_psychoses_freq', 'diagCci_35_lymphoma_freq', 'diagCci_36_cancerSec_freq', 'diagCci_37_tumorNotSec_freq', 'diagCci_38_rheumatoid_freq', 'diagCci_39_coagulopathy_freq', 'diagCci_40_obesity_freq', 'diagCci_41_weightLoss_freq', 'diagCci_42_fluidDisorder_freq', 'diagCci_43_bloodLoss_freq', 'diagCci_44_anemia_freq', 'diagCci_45_alcohol_freq', 'diagCci_46_drug_freq']\n",
"file_name = \"report_population_charlson_diagnoses_\" + method_name + \"_\" + target_feature + \"_\""
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": false,
"deletable": true,
"editable": true
},
"outputs": [],
"source": [
"o_summaries = population_statistics(features_extra['train'], diagnoses)\n",
"readers_writers.save_csv(path=CONSTANTS.io_path, title=file_name + \"train\", data=o_summaries, append=False, extension=\"csv\", header=o_summaries.columns)\n",
"\n",
"o_summaries = population_statistics(features_extra['test'], diagnoses)\n",
"readers_writers.save_csv(path=CONSTANTS.io_path, title=file_name + \"test\", data=o_summaries, append=False, extension=\"csv\", header=o_summaries.columns)"
]
},
{
"cell_type": "markdown",
"metadata": {
"deletable": true,
"editable": true
},
"source": [
"#### 5.2.4. Most Prevalent Operatons\n",
"Most prevalent operations variables (30-day, 1-year readmission): \n",
"* Total, Admissions, Emergency Readmission Rate, Prior Spells, Male (%), Age (IQR), LoS (IQR), TP, FP, FN, TN"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true,
"deletable": true,
"editable": true
},
"outputs": [],
"source": [
"diagnoses = ['operOPCSL1_0t30d_others_cnt', 'operOPCSL1_0t30d_prevalence_1_cnt', 'operOPCSL1_0t30d_prevalence_2_cnt', 'operOPCSL1_0t30d_prevalence_3_cnt', 'operOPCSL1_0t30d_prevalence_4_cnt', 'operOPCSL1_0t30d_prevalence_5_cnt', 'operOPCSL1_0t30d_prevalence_6_cnt', 'operOPCSL1_0t30d_prevalence_7_cnt', 'operOPCSL1_0t30d_prevalence_8_cnt', 'operOPCSL1_0t30d_prevalence_9_cnt', 'operOPCSL1_0t30d_prevalence_10_cnt', 'operOPCSL1_0t30d_prevalence_11_cnt', 'operOPCSL1_0t30d_prevalence_12_cnt', 'operOPCSL1_0t30d_prevalence_13_cnt', 'operOPCSL1_0t30d_prevalence_14_cnt', 'operOPCSL1_0t30d_prevalence_15_cnt', 'operOPCSL1_0t30d_prevalence_16_cnt', 'operOPCSL1_0t30d_prevalence_17_cnt', 'operOPCSL1_0t30d_prevalence_18_cnt', 'operOPCSL1_0t30d_prevalence_19_cnt', 'operOPCSL1_0t30d_prevalence_20_cnt', 'operOPCSL1_0t30d_prevalence_21_cnt', 'operOPCSL1_0t30d_prevalence_22_cnt', 'operOPCSL1_0t30d_prevalence_23_cnt', 'operOPCSL1_0t30d_prevalence_24_cnt', 'operOPCSL1_0t30d_prevalence_25_cnt', 'operOPCSL1_0t30d_prevalence_26_cnt', 'operOPCSL1_0t30d_prevalence_27_cnt', 'operOPCSL1_0t30d_prevalence_28_cnt', 'operOPCSL1_0t30d_prevalence_29_cnt', 'operOPCSL1_0t30d_prevalence_30_cnt'\n",
" , 'operOPCSL1_30t90d_others_cnt', 'operOPCSL1_30t90d_prevalence_1_cnt', 'operOPCSL1_30t90d_prevalence_2_cnt', 'operOPCSL1_30t90d_prevalence_3_cnt', 'operOPCSL1_30t90d_prevalence_4_cnt', 'operOPCSL1_30t90d_prevalence_5_cnt', 'operOPCSL1_30t90d_prevalence_6_cnt', 'operOPCSL1_30t90d_prevalence_7_cnt', 'operOPCSL1_30t90d_prevalence_8_cnt', 'operOPCSL1_30t90d_prevalence_9_cnt', 'operOPCSL1_30t90d_prevalence_10_cnt', 'operOPCSL1_30t90d_prevalence_11_cnt', 'operOPCSL1_30t90d_prevalence_12_cnt', 'operOPCSL1_30t90d_prevalence_13_cnt', 'operOPCSL1_30t90d_prevalence_14_cnt', 'operOPCSL1_30t90d_prevalence_15_cnt', 'operOPCSL1_30t90d_prevalence_16_cnt', 'operOPCSL1_30t90d_prevalence_17_cnt', 'operOPCSL1_30t90d_prevalence_18_cnt', 'operOPCSL1_30t90d_prevalence_19_cnt', 'operOPCSL1_30t90d_prevalence_20_cnt', 'operOPCSL1_30t90d_prevalence_21_cnt', 'operOPCSL1_30t90d_prevalence_22_cnt', 'operOPCSL1_30t90d_prevalence_23_cnt', 'operOPCSL1_30t90d_prevalence_24_cnt', 'operOPCSL1_30t90d_prevalence_25_cnt', 'operOPCSL1_30t90d_prevalence_26_cnt', 'operOPCSL1_30t90d_prevalence_27_cnt', 'operOPCSL1_30t90d_prevalence_28_cnt', 'operOPCSL1_30t90d_prevalence_29_cnt', 'operOPCSL1_30t90d_prevalence_30_cnt'\n",
" , 'operOPCSL1_90t180d_others_cnt', 'operOPCSL1_90t180d_prevalence_1_cnt', 'operOPCSL1_90t180d_prevalence_2_cnt', 'operOPCSL1_90t180d_prevalence_3_cnt', 'operOPCSL1_90t180d_prevalence_4_cnt', 'operOPCSL1_90t180d_prevalence_5_cnt', 'operOPCSL1_90t180d_prevalence_6_cnt', 'operOPCSL1_90t180d_prevalence_7_cnt', 'operOPCSL1_90t180d_prevalence_8_cnt', 'operOPCSL1_90t180d_prevalence_9_cnt', 'operOPCSL1_90t180d_prevalence_10_cnt', 'operOPCSL1_90t180d_prevalence_11_cnt', 'operOPCSL1_90t180d_prevalence_12_cnt', 'operOPCSL1_90t180d_prevalence_13_cnt', 'operOPCSL1_90t180d_prevalence_14_cnt', 'operOPCSL1_90t180d_prevalence_15_cnt', 'operOPCSL1_90t180d_prevalence_16_cnt', 'operOPCSL1_90t180d_prevalence_17_cnt', 'operOPCSL1_90t180d_prevalence_18_cnt', 'operOPCSL1_90t180d_prevalence_19_cnt', 'operOPCSL1_90t180d_prevalence_20_cnt', 'operOPCSL1_90t180d_prevalence_21_cnt', 'operOPCSL1_90t180d_prevalence_22_cnt', 'operOPCSL1_90t180d_prevalence_23_cnt', 'operOPCSL1_90t180d_prevalence_24_cnt', 'operOPCSL1_90t180d_prevalence_25_cnt', 'operOPCSL1_90t180d_prevalence_26_cnt', 'operOPCSL1_90t180d_prevalence_27_cnt', 'operOPCSL1_90t180d_prevalence_28_cnt', 'operOPCSL1_90t180d_prevalence_29_cnt', 'operOPCSL1_90t180d_prevalence_30_cnt'\n",
" , 'operOPCSL1_180t365d_others_cnt', 'operOPCSL1_180t365d_prevalence_1_cnt', 'operOPCSL1_180t365d_prevalence_2_cnt', 'operOPCSL1_180t365d_prevalence_3_cnt', 'operOPCSL1_180t365d_prevalence_4_cnt', 'operOPCSL1_180t365d_prevalence_5_cnt', 'operOPCSL1_180t365d_prevalence_6_cnt', 'operOPCSL1_180t365d_prevalence_7_cnt', 'operOPCSL1_180t365d_prevalence_8_cnt', 'operOPCSL1_180t365d_prevalence_9_cnt', 'operOPCSL1_180t365d_prevalence_10_cnt', 'operOPCSL1_180t365d_prevalence_11_cnt', 'operOPCSL1_180t365d_prevalence_12_cnt', 'operOPCSL1_180t365d_prevalence_13_cnt', 'operOPCSL1_180t365d_prevalence_14_cnt', 'operOPCSL1_180t365d_prevalence_15_cnt', 'operOPCSL1_180t365d_prevalence_16_cnt', 'operOPCSL1_180t365d_prevalence_17_cnt', 'operOPCSL1_180t365d_prevalence_18_cnt', 'operOPCSL1_180t365d_prevalence_19_cnt', 'operOPCSL1_180t365d_prevalence_20_cnt', 'operOPCSL1_180t365d_prevalence_21_cnt', 'operOPCSL1_180t365d_prevalence_22_cnt', 'operOPCSL1_180t365d_prevalence_23_cnt', 'operOPCSL1_180t365d_prevalence_24_cnt', 'operOPCSL1_180t365d_prevalence_25_cnt', 'operOPCSL1_180t365d_prevalence_26_cnt', 'operOPCSL1_180t365d_prevalence_27_cnt', 'operOPCSL1_180t365d_prevalence_28_cnt', 'operOPCSL1_180t365d_prevalence_29_cnt', 'operOPCSL1_180t365d_prevalence_30_cnt'\n",
" , 'operOPCSL1_365t730d_others_cnt', 'operOPCSL1_365t730d_prevalence_1_cnt', 'operOPCSL1_365t730d_prevalence_2_cnt', 'operOPCSL1_365t730d_prevalence_3_cnt', 'operOPCSL1_365t730d_prevalence_4_cnt', 'operOPCSL1_365t730d_prevalence_5_cnt', 'operOPCSL1_365t730d_prevalence_6_cnt', 'operOPCSL1_365t730d_prevalence_7_cnt', 'operOPCSL1_365t730d_prevalence_8_cnt', 'operOPCSL1_365t730d_prevalence_9_cnt', 'operOPCSL1_365t730d_prevalence_10_cnt', 'operOPCSL1_365t730d_prevalence_11_cnt', 'operOPCSL1_365t730d_prevalence_12_cnt', 'operOPCSL1_365t730d_prevalence_13_cnt', 'operOPCSL1_365t730d_prevalence_14_cnt', 'operOPCSL1_365t730d_prevalence_15_cnt', 'operOPCSL1_365t730d_prevalence_16_cnt', 'operOPCSL1_365t730d_prevalence_17_cnt', 'operOPCSL1_365t730d_prevalence_18_cnt', 'operOPCSL1_365t730d_prevalence_19_cnt', 'operOPCSL1_365t730d_prevalence_20_cnt', 'operOPCSL1_365t730d_prevalence_21_cnt', 'operOPCSL1_365t730d_prevalence_22_cnt', 'operOPCSL1_365t730d_prevalence_23_cnt', 'operOPCSL1_365t730d_prevalence_24_cnt', 'operOPCSL1_365t730d_prevalence_25_cnt', 'operOPCSL1_365t730d_prevalence_26_cnt', 'operOPCSL1_365t730d_prevalence_27_cnt', 'operOPCSL1_365t730d_prevalence_28_cnt', 'operOPCSL1_365t730d_prevalence_29_cnt', 'operOPCSL1_365t730d_prevalence_30_cnt']\n",
"file_name = \"report_population_operations_\" + method_name + \"_\" + target_feature + \"_\""
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true,
"deletable": true,
"editable": true
},
"outputs": [],
"source": [
"o_summaries = population_statistics(features_extra['train'], diagnoses)\n",
"readers_writers.save_csv(path=CONSTANTS.io_path, title=file_name + \"train\", data=o_summaries, append=False, extension=\"csv\", header=o_summaries.columns)\n",
"\n",
"o_summaries = population_statistics(features_extra['test'], diagnoses)\n",
"readers_writers.save_csv(path=CONSTANTS.io_path, title=file_name + \"test\", data=o_summaries, append=False, extension=\"csv\", header=o_summaries.columns)"
]
},
{
"cell_type": "markdown",
"metadata": {
"deletable": true,
"editable": true
},
"source": [
"#### 5.2.4. Most Prevalent Main Speciality\n",
"Most prevalent operations variables (30-day, 1-year readmission): \n",
"* Total, Admissions, Emergency Readmission Rate, Prior Spells, Male (%), Age (IQR), LoS (IQR), TP, FP, FN, TN"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true,
"deletable": true,
"editable": true
},
"outputs": [],
"source": [
"diagnoses = ['mainspef_0t30d_others_cnt', 'mainspef_0t30d_prevalence_1_cnt', 'mainspef_0t30d_prevalence_2_cnt', 'mainspef_0t30d_prevalence_3_cnt', 'mainspef_0t30d_prevalence_4_cnt', 'mainspef_0t30d_prevalence_5_cnt', 'mainspef_0t30d_prevalence_6_cnt', 'mainspef_0t30d_prevalence_7_cnt', 'mainspef_0t30d_prevalence_8_cnt', 'mainspef_0t30d_prevalence_9_cnt', 'mainspef_0t30d_prevalence_10_cnt'\n",
" , 'mainspef_30t90d_others_cnt', 'mainspef_30t90d_prevalence_1_cnt', 'mainspef_30t90d_prevalence_2_cnt', 'mainspef_30t90d_prevalence_3_cnt', 'mainspef_30t90d_prevalence_4_cnt', 'mainspef_30t90d_prevalence_5_cnt', 'mainspef_30t90d_prevalence_6_cnt', 'mainspef_30t90d_prevalence_7_cnt', 'mainspef_30t90d_prevalence_8_cnt', 'mainspef_30t90d_prevalence_9_cnt', 'mainspef_30t90d_prevalence_10_cnt'\n",
" , 'mainspef_90t180d_others_cnt', 'mainspef_90t180d_prevalence_1_cnt', 'mainspef_90t180d_prevalence_2_cnt', 'mainspef_90t180d_prevalence_3_cnt', 'mainspef_90t180d_prevalence_4_cnt', 'mainspef_90t180d_prevalence_5_cnt', 'mainspef_90t180d_prevalence_6_cnt', 'mainspef_90t180d_prevalence_7_cnt', 'mainspef_90t180d_prevalence_8_cnt', 'mainspef_90t180d_prevalence_9_cnt', 'mainspef_90t180d_prevalence_10_cnt'\n",
" , 'mainspef_180t365d_others_cnt', 'mainspef_180t365d_prevalence_1_cnt', 'mainspef_180t365d_prevalence_2_cnt', 'mainspef_180t365d_prevalence_3_cnt', 'mainspef_180t365d_prevalence_4_cnt', 'mainspef_180t365d_prevalence_5_cnt', 'mainspef_180t365d_prevalence_6_cnt', 'mainspef_180t365d_prevalence_7_cnt', 'mainspef_180t365d_prevalence_8_cnt', 'mainspef_180t365d_prevalence_9_cnt', 'mainspef_180t365d_prevalence_10_cnt'\n",
" , 'mainspef_365t730d_others_cnt', 'mainspef_365t730d_prevalence_1_cnt', 'mainspef_365t730d_prevalence_2_cnt', 'mainspef_365t730d_prevalence_3_cnt', 'mainspef_365t730d_prevalence_4_cnt', 'mainspef_365t730d_prevalence_5_cnt', 'mainspef_365t730d_prevalence_6_cnt', 'mainspef_365t730d_prevalence_7_cnt', 'mainspef_365t730d_prevalence_8_cnt', 'mainspef_365t730d_prevalence_9_cnt', 'mainspef_365t730d_prevalence_10_cnt']\n",
"file_name = \"report_population_operations_\" + method_name + \"_\" + target_feature + \"_\""
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": false,
"deletable": true,
"editable": true
},
"outputs": [],
"source": [
"o_summaries = population_statistics(features_extra['train'], diagnoses)\n",
"readers_writers.save_csv(path=CONSTANTS.io_path, title=file_name + \"train\", data=o_summaries, append=False, extension=\"csv\", header=o_summaries.columns)\n",
"\n",
"o_summaries = population_statistics(features_extra['test'], diagnoses)\n",
"readers_writers.save_csv(path=CONSTANTS.io_path, title=file_name + \"test\", data=o_summaries, append=False, extension=\"csv\", header=o_summaries.columns)"
]
},
{
"cell_type": "markdown",
"metadata": {
"deletable": true,
"editable": true
},
"source": [
"#### 5.2.5. Other Variables\n",
"Other variables (30-day, 1-year readmission): \n",
"* Total, Admissions, Emergency Readmission Rate, Prior Spells, Male (%), Age (IQR), LoS (IQR), TP, FP, FN, TN"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true,
"deletable": true,
"editable": true
},
"outputs": [],
"source": [
"diagnoses = ['gapDays_0t30d_avg', 'gapDays_30t90d_avg', 'gapDays_90t180d_avg', 'gapDays_180t365d_avg', 'gapDays_365t730d_avg', \n",
" 'epidur_0t30d_avg', 'epidur_30t90d_avg', 'epidur_90t180d_avg', 'epidur_180t365d_avg', 'epidur_365t730d_avg', \n",
" 'preopdur_0t30d_avg', 'preopdur_30t90d_avg', 'preopdur_90t180d_avg', 'preopdur_180t365d_avg', 'preopdur_365t730d_avg', \n",
" 'posopdur_0t30d_avg', 'posopdur_30t90d_avg', 'posopdur_90t180d_avg', 'posopdur_180t365d_avg', 'posopdur_365t730d_avg']\n",
"file_name = \"report_population_other_variables_\" + method_name + \"_\" + target_feature + \"_\""
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true,
"deletable": true,
"editable": true
},
"outputs": [],
"source": [
"o_summaries = population_statistics(features_extra['train'], diagnoses)\n",
"readers_writers.save_csv(path=CONSTANTS.io_path, title=file_name + \"train\", data=o_summaries, append=False, extension=\"csv\", header=o_summaries.columns)\n",
"\n",
"o_summaries = population_statistics(features_extra['test'], diagnoses)\n",
"readers_writers.save_csv(path=CONSTANTS.io_path, title=file_name + \"test\", data=o_summaries, append=False, extension=\"csv\", header=o_summaries.columns)"
]
},
{
"cell_type": "markdown",
"metadata": {
"deletable": true,
"editable": true
},
"source": [
"<br/><br/>"
]
},
{
"cell_type": "markdown",
"metadata": {
"deletable": true,
"editable": true
},
"source": [
"### 5.3. Plots"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true,
"deletable": true,
"editable": true
},
"outputs": [],
"source": [
"file_name = \"report_population_\" + method_name + \"_\" + target_feature + \"_\""
]
},
{
"cell_type": "markdown",
"metadata": {
"deletable": true,
"editable": true
},
"source": [
"#### 5.3.1. ROC"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": false,
"deletable": true,
"editable": true
},
"outputs": [],
"source": [
"fig, summaries = plots.roc(training_method.model_predict[\"test\"], features_extra[\"test\"][target_feature], \n",
" title=\"ROC Curve\", lw=2)\n",
"display(fig)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": false,
"deletable": true,
"editable": true
},
"outputs": [],
"source": [
"# save\n",
"plt.savefig(os.path.join(CONSTANTS.io_path, file_name + \"_roc\" + \".pdf\"), \n",
" dpi=300, facecolor='w', edgecolor='w', orientation='portrait', papertype=None, format=\"pdf\",\n",
" transparent=False, bbox_inches=None, pad_inches=0.1, frameon=None)"
]
},
{
"cell_type": "markdown",
"metadata": {
"deletable": true,
"editable": true
},
"source": [
"#### 5.3.2. Precision Recall"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": false,
"deletable": true,
"editable": true,
"scrolled": true
},
"outputs": [],
"source": [
"fig, summaries = plots.precision_recall(training_method.model_predict[\"test\"], \n",
" features_extra[\"test\"][target_feature], \n",
" title=\"Precision-Recall Curve\", lw=2)\n",
"display(fig)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true,
"deletable": true,
"editable": true
},
"outputs": [],
"source": [
"# save\n",
"plt.savefig(os.path.join(CONSTANTS.io_path, file_name + \"_precision_recall\" + \".pdf\"), \n",
" dpi=300, facecolor='w', edgecolor='w', orientation='portrait', papertype=None, format=\"pdf\",\n",
" transparent=False, bbox_inches=None, pad_inches=0.1, frameon=None)"
]
},
{
"cell_type": "markdown",
"metadata": {
"deletable": true,
"editable": true
},
"source": [
"#### 5.3.3. Learning Curve"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": false,
"deletable": true,
"editable": true
},
"outputs": [],
"source": [
"fig, summaries = plots.learning_curve(training_method.model_train, \n",
" features_extra[\"test\"][features_names_selected], \n",
" features_extra[\"test\"][target_feature],\n",
" title=\"Learning Curve\", ylim=None, cv=None, n_jobs=-1, train_sizes=np.linspace(.1, 1.0, 5))\n",
"display(fig)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true,
"deletable": true,
"editable": true
},
"outputs": [],
"source": [
"# save\n",
"plt.savefig(os.path.join(CONSTANTS.io_path, file_name + \"_learning_curve\" + \".pdf\"), \n",
" dpi=300, facecolor='w', edgecolor='w', orientation='portrait', papertype=None, format=\"pdf\",\n",
" transparent=False, bbox_inches=None, pad_inches=0.1, frameon=None)"
]
},
{
"cell_type": "markdown",
"metadata": {
"deletable": true,
"editable": true
},
"source": [
"#### 5.3.4. Validation Curve"
]
},
{
"cell_type": "markdown",
"metadata": {
"deletable": true,
"editable": true
},
"source": [
"Set the model's metadata"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": false,
"deletable": true,
"editable": true
},
"outputs": [],
"source": [
"# method metadata\n",
"if method_name == \"lr\":\n",
" param_name = \"clf__C\"\n",
" param_range = [0.001, 0.01, 0.1, 1.0, 10.0, 100.0]\n",
"elif method_name == \"rfc\":\n",
" param_name = \"max_features\"\n",
" param_range = range(1, 4, 1) # range(1, 20, 1)\n",
"elif method_name == \"nn\":\n",
" param_name = \"alpha\"\n",
" param_range = range(1e4, 1e6, 9e4)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": false,
"deletable": true,
"editable": true
},
"outputs": [],
"source": [
"fig, summaries = plots.validation_curve(training_method.model_train, \n",
" features_extra[\"test\"][features_names_selected], \n",
" features_extra[\"test\"][target_feature],\n",
" param_name, param_range, \n",
" title=\"Learning Curve\", ylim=None, cv=None, lw=2, n_jobs=-1)\n",
"display(fig)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true,
"deletable": true,
"editable": true
},
"outputs": [],
"source": [
"# save\n",
"plt.savefig(os.path.join(CONSTANTS.io_path, file_name + \"_validation_curve\" + \".pdf\"), \n",
" dpi=300, facecolor='w', edgecolor='w', orientation='portrait', papertype=None, format=\"pdf\",\n",
" transparent=False, bbox_inches=None, pad_inches=0.1, frameon=None)"
]
},
{
"cell_type": "markdown",
"metadata": {
"deletable": true,
"editable": true
},
"source": [
"<br/><br/>"
]
},
{
"cell_type": "markdown",
"metadata": {
"collapsed": true,
"deletable": true,
"editable": true
},
"source": [
"Fin!"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.5.3"
}
},
"nbformat": 4,
"nbformat_minor": 2
}