--- a
+++ b/full pipeline for classification grade with MSI data.ipynb
@@ -0,0 +1,281 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import numpy as np\n",
+    "#from pyimzml.ImzMLParser import ImzMLParser\n",
+    "from tqdm import tqdm\n",
+    "import os\n",
+    "import pandas as pd\n",
+    "import matplotlib.pyplot as plt\n",
+    "from sklearn.neural_network import MLPClassifier\n",
+    "from sklearn.model_selection import train_test_split \n",
+    "from sklearn.ensemble import RandomForestClassifier, VotingClassifier\n",
+    "from sklearn import preprocessing\n",
+    "from sklearn.metrics import roc_curve, auc,classification_report\n",
+    "from utils import print_confusion_matrix, assemble_dataset_supervised_learning\n",
+    "from sklearn.utils import shuffle\n",
+    "from sklearn.svm import SVC\n",
+    "from itertools import product\n",
+    "import xgboost as xgb\n",
+    "from sklearn.model_selection import GridSearchCV\n",
+    "from itertools import cycle\n",
+    "from scipy import interp\n",
+    "from sklearn.calibration import calibration_curve"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## load data"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "peaklist = np.array(pd.read_csv(r'.\\\\regions_peaklist_from_marta.txt', sep = \" \"))\n",
+    "\n",
+    "\n",
+    "\n",
+    "path_data = r'.\\msi_tables_filtered'\n",
+    "list_dataset = os.listdir(path_data)\n",
+    "\n",
+    "##classification per tiles _ supervised \n",
+    "\n",
+    "labels = pd.read_csv('.\\labels_frozen.txt',sep = ';' ) #table with slide;label;unified_label;image_name\n",
+    "\n",
+    "full_dataset, y_labels = assemble_dataset_supervised_learning(labels,list_dataset,path_data, \"grade\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## pre-process data per patient with box cox and 10**5 factor"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "dict_X_gauss = {}\n",
+    "\n",
+    "pt = preprocessing.PowerTransformer(method='box-cox', standardize=False)\n",
+    "name_images = full_dataset[full_dataset[\"dataset_name\"]==\"SlideA1\"][\"image_name\"]\n",
+    "temp_patient_data = full_dataset[full_dataset[\"dataset_name\"]==\"SlideA1\"].drop(columns = ['dataset_name','image_name'])*10**5\n",
+    "X_gaus = pt.fit_transform(temp_patient_data)\n",
+    "\n",
+    "columns = np.unique(full_dataset[\"dataset_name\"])\n",
+    "\n",
+    "for col in tqdm( columns[1:]):\n",
+    "    name_images = full_dataset[full_dataset[\"dataset_name\"]==col][\"image_name\"]\n",
+    "    temp_patient_data = full_dataset[full_dataset[\"dataset_name\"]==col].drop(columns = ['dataset_name','image_name'])*10**5\n",
+    "    array_trans = pt.fit_transform(temp_patient_data)\n",
+    "    X_gaus=np.concatenate((X_gaus,array_trans),axis =0)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "X_train, X_test_and_valid, y_train, y_test_and_valid, data_train, data_test_and_valid = train_test_split(X_gaus,y_labels , full_dataset[[\"dataset_name\",'image_name']],test_size = 0.30, random_state=10) "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#create test dataset\n",
+    "len_half = len(y_test_and_valid)//2\n",
+    "X_test = X_test_and_valid[:len_half]\n",
+    "data_test = data_test_and_valid[:len_half]\n",
+    "y_test = y_test_and_valid[:len_half]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#create validation dataset\n",
+    "X_valid = X_test_and_valid[len_half:]\n",
+    "data_valid = data_test_and_valid[len_half:]\n",
+    "y_valid = y_test_and_valid[len_half:]"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## balancing training data"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "max_len = X_train[y_train == 'high grade'].shape[0]\n",
+    "len_h = X_train[y_train == 'non-dysplasia'].shape[0]\n",
+    "len_lg = X_train[y_train == 'low grade'].shape[0]\n",
+    "\n",
+    "balanced_X_train = np.concatenate((X_train[y_train == 'non-dysplasia'][np.random.randint(0,len_h,max_len)], X_train[y_train == 'low grade'][np.random.randint(0,len_lg,max_len)],X_train[y_train == 'high grade']))\n",
+    "balanced_y_train = np.array(['non-dysplasia']*max_len + ['low grade']*max_len  + ['high grade']*X_train[y_train == 'highgrade'].shape[0])\n",
+    "balanced_X_train,balanced_y_train = shuffle(balanced_X_train,balanced_y_train)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## grid search for MLP"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "parameters = { 'batch_size':[32,64,128,356], 'alpha': 10.0 ** -np.arange(1, 10), 'hidden_layer_sizes':list(product(np.arange(10,21,10),np.arange(10,21,10)))}\n",
+    "mlp_model = GridSearchCV(MLPClassifier(solver='adam',max_iter = 1100), parameters, n_jobs=20, , cv= 5, verbose = 2)\n",
+    "\n",
+    "mlp_model.fit(balanced_X_train,balanced_y_train)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## gridsearchCV for random forest"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "param_grid = { \n",
+    "   'n_estimators': [100,200,500],\n",
+    "   'max_depth' : [4,8,16],\n",
+    "   'criterion' :['gini', 'entropy']\n",
+    "}\n",
+    "\n",
+    "rf_model = GridSearchCV(estimator=rfc, param_grid=param_grid, cv= 5, verbose=1,n_jobs=20)\n",
+    "rf_model.fit(balanced_X_train,balanced_y_train)\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## gridsearchCV for XGBoost"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "params = {\n",
+    "   \"min_child_weight\":range(1,6,2),\n",
+    "   \"gamma\": uniform(0, 0.5),\n",
+    "   \"learning_rate\": uniform(0.03, 0.3),\n",
+    "   \"max_depth\": range(3,10,2), \n",
+    "   \"n_estimators\": randint(100, 150),\n",
+    "   \"subsample\": uniform(0.6, 0.4)\n",
+    "}\n",
+    "\n",
+    "xgb_model = GridSearchCV(estimator = xgb.XGBClassifier(colsample_bytree=0.8,\n",
+    " objective= 'binary:logistic', nthread=4, scale_pos_weight=1, seed=27), \n",
+    " param_grid = params, scoring='roc_auc',n_jobs=12,iid=False, cv=5,verbose=1)\n",
+    "\n",
+    "xgb_model.fit(balanced_X_train,balanced_y_train)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## save feature importance"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "results=pd.DataFrame()\n",
+    "results['columns']=list(full_dataset.columns)[2:]\n",
+    "results['importances_rf'] = CV_rfc.feature_importances_\n",
+    "results['importances_xgboost'] = xgb1.feature_importances_\n",
+    "results['importances_mean'] = np.mean([xgb1.feature_importances_,CV_rfc.feature_importances_],axis=0)\n",
+    "results.sort_values(by='importances_mean',ascending=False,inplace=True)\n",
+    "results.to_excel(r\".\\features_rf_xgboost_msi_grade.xlsx\",index=None)\n",
+    "other_results= pd.read_excel(r\".\\features_rf_xgboost_msi_gland_vs_tissue.xlsx\")\n",
+    "other_results.sort_values(by='importances_mean',ascending=False,inplace=True)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## ensemble all best model"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#ensemble all best model\n",
+    "\n",
+    "vc = VotingClassifier(estimators=[\n",
+    "     ('mlp', mlp_model.best_estimator_), ('rf', rf_model.best_estimator_), ('xgb', xgb_model.best_estimator_)],\n",
+    "     voting='soft',n_jobs=12)\n",
+    "vc = vc.fit(balanced_X_train,balanced_y_train)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.7.3"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}