282 lines (281 with data), 8.6 kB

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "#from pyimzml.ImzMLParser import ImzMLParser\n",
    "from tqdm import tqdm\n",
    "import os\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.neural_network import MLPClassifier\n",
    "from sklearn.model_selection import train_test_split \n",
    "from sklearn.ensemble import RandomForestClassifier, VotingClassifier\n",
    "from sklearn import preprocessing\n",
    "from sklearn.metrics import roc_curve, auc,classification_report\n",
    "from utils import print_confusion_matrix, assemble_dataset_supervised_learning\n",
    "from sklearn.utils import shuffle\n",
    "from sklearn.svm import SVC\n",
    "from itertools import product\n",
    "import xgboost as xgb\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from itertools import cycle\n",
    "from scipy import interp\n",
    "from sklearn.calibration import calibration_curve"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## load data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "peaklist = np.array(pd.read_csv(r'.\\\\regions_peaklist_from_marta.txt', sep = \" \"))\n",
    "\n",
    "\n",
    "\n",
    "path_data = r'.\\msi_tables_filtered'\n",
    "list_dataset = os.listdir(path_data)\n",
    "\n",
    "##classification per tiles _ supervised \n",
    "\n",
    "labels = pd.read_csv('.\\labels_frozen.txt',sep = ';' ) #table with slide;label;unified_label;image_name\n",
    "\n",
    "full_dataset, y_labels = assemble_dataset_supervised_learning(labels,list_dataset,path_data, \"grade\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## pre-process data per patient with box cox and 10**5 factor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dict_X_gauss = {}\n",
    "\n",
    "pt = preprocessing.PowerTransformer(method='box-cox', standardize=False)\n",
    "name_images = full_dataset[full_dataset[\"dataset_name\"]==\"SlideA1\"][\"image_name\"]\n",
    "temp_patient_data = full_dataset[full_dataset[\"dataset_name\"]==\"SlideA1\"].drop(columns = ['dataset_name','image_name'])*10**5\n",
    "X_gaus = pt.fit_transform(temp_patient_data)\n",
    "\n",
    "columns = np.unique(full_dataset[\"dataset_name\"])\n",
    "\n",
    "for col in tqdm( columns[1:]):\n",
    "    name_images = full_dataset[full_dataset[\"dataset_name\"]==col][\"image_name\"]\n",
    "    temp_patient_data = full_dataset[full_dataset[\"dataset_name\"]==col].drop(columns = ['dataset_name','image_name'])*10**5\n",
    "    array_trans = pt.fit_transform(temp_patient_data)\n",
    "    X_gaus=np.concatenate((X_gaus,array_trans),axis =0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test_and_valid, y_train, y_test_and_valid, data_train, data_test_and_valid = train_test_split(X_gaus,y_labels , full_dataset[[\"dataset_name\",'image_name']],test_size = 0.30, random_state=10) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#create test dataset\n",
    "len_half = len(y_test_and_valid)//2\n",
    "X_test = X_test_and_valid[:len_half]\n",
    "data_test = data_test_and_valid[:len_half]\n",
    "y_test = y_test_and_valid[:len_half]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#create validation dataset\n",
    "X_valid = X_test_and_valid[len_half:]\n",
    "data_valid = data_test_and_valid[len_half:]\n",
    "y_valid = y_test_and_valid[len_half:]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## balancing training data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "max_len = X_train[y_train == 'high grade'].shape[0]\n",
    "len_h = X_train[y_train == 'non-dysplasia'].shape[0]\n",
    "len_lg = X_train[y_train == 'low grade'].shape[0]\n",
    "\n",
    "balanced_X_train = np.concatenate((X_train[y_train == 'non-dysplasia'][np.random.randint(0,len_h,max_len)], X_train[y_train == 'low grade'][np.random.randint(0,len_lg,max_len)],X_train[y_train == 'high grade']))\n",
    "balanced_y_train = np.array(['non-dysplasia']*max_len + ['low grade']*max_len  + ['high grade']*X_train[y_train == 'highgrade'].shape[0])\n",
    "balanced_X_train,balanced_y_train = shuffle(balanced_X_train,balanced_y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## grid search for MLP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "parameters = { 'batch_size':[32,64,128,356], 'alpha': 10.0 ** -np.arange(1, 10), 'hidden_layer_sizes':list(product(np.arange(10,21,10),np.arange(10,21,10)))}\n",
    "mlp_model = GridSearchCV(MLPClassifier(solver='adam',max_iter = 1100), parameters, n_jobs=20, , cv= 5, verbose = 2)\n",
    "\n",
    "mlp_model.fit(balanced_X_train,balanced_y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## gridsearchCV for random forest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "param_grid = { \n",
    "   'n_estimators': [100,200,500],\n",
    "   'max_depth' : [4,8,16],\n",
    "   'criterion' :['gini', 'entropy']\n",
    "}\n",
    "\n",
    "rf_model = GridSearchCV(estimator=rfc, param_grid=param_grid, cv= 5, verbose=1,n_jobs=20)\n",
    "rf_model.fit(balanced_X_train,balanced_y_train)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## gridsearchCV for XGBoost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "params = {\n",
    "   \"min_child_weight\":range(1,6,2),\n",
    "   \"gamma\": uniform(0, 0.5),\n",
    "   \"learning_rate\": uniform(0.03, 0.3),\n",
    "   \"max_depth\": range(3,10,2), \n",
    "   \"n_estimators\": randint(100, 150),\n",
    "   \"subsample\": uniform(0.6, 0.4)\n",
    "}\n",
    "\n",
    "xgb_model = GridSearchCV(estimator = xgb.XGBClassifier(colsample_bytree=0.8,\n",
    " objective= 'binary:logistic', nthread=4, scale_pos_weight=1, seed=27), \n",
    " param_grid = params, scoring='roc_auc',n_jobs=12,iid=False, cv=5,verbose=1)\n",
    "\n",
    "xgb_model.fit(balanced_X_train,balanced_y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## save feature importance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "results=pd.DataFrame()\n",
    "results['columns']=list(full_dataset.columns)[2:]\n",
    "results['importances_rf'] = CV_rfc.feature_importances_\n",
    "results['importances_xgboost'] = xgb1.feature_importances_\n",
    "results['importances_mean'] = np.mean([xgb1.feature_importances_,CV_rfc.feature_importances_],axis=0)\n",
    "results.sort_values(by='importances_mean',ascending=False,inplace=True)\n",
    "results.to_excel(r\".\\features_rf_xgboost_msi_grade.xlsx\",index=None)\n",
    "other_results= pd.read_excel(r\".\\features_rf_xgboost_msi_gland_vs_tissue.xlsx\")\n",
    "other_results.sort_values(by='importances_mean',ascending=False,inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## ensemble all best model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#ensemble all best model\n",
    "\n",
    "vc = VotingClassifier(estimators=[\n",
    "     ('mlp', mlp_model.best_estimator_), ('rf', rf_model.best_estimator_), ('xgb', xgb_model.best_estimator_)],\n",
    "     voting='soft',n_jobs=12)\n",
    "vc = vc.fit(balanced_X_train,balanced_y_train)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}