--- a
+++ b/full pipeline for classification gland tissue with MSI data.ipynb
@@ -0,0 +1,377 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import numpy as np\n",
+    "from tqdm import tqdm\n",
+    "import os\n",
+    "import pandas as pd\n",
+    "import matplotlib.pyplot as plt\n",
+    "from sklearn.neural_network import MLPClassifier\n",
+    "from sklearn.ensemble import RandomForestClassifier\n",
+    "from sklearn import preprocessing\n",
+    "from sklearn.metrics import roc_curve, auc, classification_report\n",
+    "from sklearn.preprocessing import MultiLabelBinarizer\n",
+    "import xgboost as xgb\n",
+    "from sklearn.model_selection import GridSearchCV, train_test_split\n",
+    "from utils import print_confusion_matrix, assemble_dataset_supervised_learning\n",
+    "from sklearn.ensemble import VotingClassifier"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## load data"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "main_path = r\"./\" #path were the data is stored\n",
+    "\n",
+    "peaklist = np.array(pd.read_csv(r'.\\\\regions_peaklist.txt', sep = \" \")) #load the preselected of peaklist\n",
+    "\n",
+    "\n",
+    "\n",
+    "path_data = r'.\\msi_tables_filtered'\n",
+    "list_dataset = os.listdir(path_data)\n",
+    "\n",
+    "\n",
+    "\n",
+    "\n",
+    "labels = pd.read_csv(os.path.join(main_path,'labels_frozen.txt'),sep = ';' )\n",
+    "\n",
+    "full_dataset, y_labels = assemble_dataset_supervised_learning(labels,list_dataset,path_data, data_type = \"stroma\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## pre-process data per patient with box cox and 10**5 factor"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "dict_X_gauss = {}\n",
+    "\n",
+    "pt = preprocessing.PowerTransformer(method='box-cox', standardize=False)\n",
+    "name_images = full_dataset[full_dataset[\"dataset_name\"]==\"SlideA1\"][\"image_name\"]\n",
+    "temp_patient_data = full_dataset[full_dataset[\"dataset_name\"]==\"SlideA1\"].drop(columns = ['dataset_name','image_name'])*10**5\n",
+    "X_gaus = pt.fit_transform(temp_patient_data)\n",
+    "\n",
+    "columns = np.unique(full_dataset[\"dataset_name\"])\n",
+    "\n",
+    "for col in tqdm( columns[1:]):\n",
+    "    name_images = full_dataset[full_dataset[\"dataset_name\"]==col][\"image_name\"]\n",
+    "    temp_patient_data = full_dataset[full_dataset[\"dataset_name\"]==col].drop(columns = ['dataset_name','image_name'])*10**5\n",
+    "    array_trans = pt.fit_transform(temp_patient_data)\n",
+    "    X_gaus=np.concatenate((X_gaus,array_trans),axis =0)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "X_train, X_test_and_valid, y_train, y_test_and_valid, data_train, data_test_and_valid = train_test_split(X_gaus,y_labels , full_dataset[[\"dataset_name\",'image_name']],test_size = 0.30, random_state=10) "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#create test dataset\n",
+    "len_half = len(y_test_and_valid)//2\n",
+    "X_test = X_test_and_valid[:len_half]\n",
+    "data_test = data_test_and_valid[:len_half]\n",
+    "y_test = y_test_and_valid[:len_half]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#create validation dataset\n",
+    "X_valid = X_test_and_valid[len_half:]\n",
+    "data_valid = data_test_and_valid[len_half:]\n",
+    "y_valid = y_test_and_valid[len_half:]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "##saving data train, test and valid to reuse in H&E pipeline\n",
+    "\n",
+    "#data_train.insert(loc=1, column='labels', value=y_train)\n",
+    "#data_test.insert(loc=1, column='labels', value=y_test)\n",
+    "#data_valid.insert(loc=1, column='labels', value=y_valid)\n",
+    "\n",
+    "#\n",
+    "#data_train.to_csv('data_train_stroma_vs_epithelial_tissue.csv',index=False)\n",
+    "#data_test.to_csv('data_test_stroma_vs_epithelial_tissue.csv',index=False)\n",
+    "#data_valid.to_csv('data_valid_stroma_vs_epithelial_tissue.csv',index=False)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "data = pd.read_csv(os.path.join(main_path,'data_train_stroma_vs_epithelial_tissue.csv'),sep = ',' )\n",
+    "\n",
+    "X_train = []\n",
+    "y_train = []\n",
+    "train_paths = []\n",
+    "for slide in tqdm(os.listdir(os.path.join(main_path, 'Slides'))):\n",
+    "    tile_path = os.path.join(main_path, 'Slides',slide,'tiles')\n",
+    "    gland = data[(data['labels']=='stroma') & (data['dataset_name']==slide)]['image_name']\n",
+    "    tissue = data[(data['labels']== \"epithelial tissue\") & (data['dataset_name']==slide)]['image_name']\n",
+    "    gland = list(gland)\n",
+    "    tissue = list(tissue)\n",
+    "    for image_path in gland:\n",
+    "        if os.path.isfile(os.path.join(tile_path, image_path)):\n",
+    "            X_train.append(dict_X_gauss[slide+image_path][0])\n",
+    "            y_train.append(\"stroma\")\n",
+    "            train_paths.append(os.path.join(tile_path, image_path))\n",
+    "        else:\n",
+    "            print(\"error for gland\")\n",
+    "    for image_path in tissue:\n",
+    "        if os.path.isfile(os.path.join(tile_path, image_path)):\n",
+    "            X_train.append(dict_X_gauss[slide+image_path][0])\n",
+    "            y_train.append(\"epithelial tissue\")\n",
+    "            train_paths.append(os.path.join(tile_path, image_path))\n",
+    "\n",
+    "y_train = np.ravel(np.array(y_train))\n",
+    "            \n",
+    "df_features_train = pd.DataFrame(X_train,index= train_paths,columns=list(full_dataset.columns)[2:])\n",
+    "df_features_train[\"labels\"]=y_train\n",
+    "#df_features_train.to_csv(r\".\\train_features_tissue_type_msi.csv\")\n",
+    "            "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "data = pd.read_csv(os.path.join(main_path,'data_test_stroma_vs_epithelial_tissue.csv'),sep = ',' )\n",
+    "\n",
+    "X_test = []\n",
+    "y_test = []\n",
+    "test_paths = []\n",
+    "        \n",
+    "for slide in tqdm(os.listdir(os.path.join(main_path, 'Slides'))):\n",
+    "    tile_path = os.path.join(main_path, 'Slides',slide,'tiles')\n",
+    "    gland = data[(data['labels']=='stroma') & (data['dataset_name']==slide)]['image_name']\n",
+    "    tissue = data[(data['labels']=='epithelial tissue') & (data['dataset_name']==slide)]['image_name']\n",
+    "    gland = list(gland)\n",
+    "    tissue = list(tissue)\n",
+    "    for image_path in gland:\n",
+    "        if os.path.isfile(os.path.join(tile_path, image_path)):\n",
+    "            X_test.append(dict_X_gauss[slide+image_path][0])\n",
+    "            y_test.append(\"stroma\")\n",
+    "            test_paths.append(os.path.join(tile_path, image_path))\n",
+    "    for image_path in tissue:\n",
+    "        if os.path.isfile(os.path.join(tile_path, image_path)):\n",
+    "            X_test.append(dict_X_gauss[slide+image_path][0])\n",
+    "            y_test.append(\"epithelial tissue\")\n",
+    "            test_paths.append(os.path.join(tile_path, image_path))\n",
+    "            \n",
+    "y_test = np.ravel(np.array(y_test))\n",
+    "\n",
+    "df_features_test = pd.DataFrame(X_test[:len(y_test)//2],index= test_paths[:len(y_test)//2],columns=list(full_dataset.columns)[2:])\n",
+    "df_features_test['labels']=y_test[:len(y_test)//2]\n",
+    "#df_features_test.to_csv(r\".\\test_features_tissue_type_msi.csv\")\n",
+    "\n",
+    "df_features_valid = pd.DataFrame(X_test[len(y_test)//2:],index= test_paths[len(y_test)//2:],columns=list(full_dataset.columns)[2:])\n",
+    "df_features_valid['labels']=y_test[len(y_test)//2:]\n",
+    "#df_features_valid.to_csv(r\".\\valid_features_tissue_type_msi.csv\")\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## gridsearchCV for MLPclassifier"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "parameters = { 'batch_size':[32,64,128,356], 'alpha': 10.0 ** -np.arange(1, 10), 'hidden_layer_sizes':list(product(np.arange(10,21,10),np.arange(10,21,10)))}\n",
+    "mlp_model = GridSearchCV(MLPClassifier(solver='adam',max_iter = 1100), parameters, n_jobs=20, , cv= 5, verbose = 2)\n",
+    "\n",
+    "mlp_model.fit(X_train,y_train)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## gridsearchCV for random forest"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "param_grid = { \n",
+    "   'n_estimators': [100,200,500],\n",
+    "   'max_depth' : [4,8,16],\n",
+    "   'criterion' :['gini', 'entropy']\n",
+    "}\n",
+    "\n",
+    "rf_model = GridSearchCV(estimator=rfc, param_grid=param_grid, cv= 5, verbose=1,n_jobs=20)\n",
+    "rf_model.fit(X_train, y_train)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## gridsearchCV for XGBoost"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "params = {\n",
+    "   \"min_child_weight\":range(1,6,2),\n",
+    "   \"gamma\": uniform(0, 0.5),\n",
+    "   \"learning_rate\": uniform(0.03, 0.3),\n",
+    "   \"max_depth\": range(3,10,2), \n",
+    "   \"n_estimators\": randint(100, 150),\n",
+    "   \"subsample\": uniform(0.6, 0.4)\n",
+    "}\n",
+    "\n",
+    "xgb_model = GridSearchCV(estimator = xgb.XGBClassifier(colsample_bytree=0.8,\n",
+    " objective= 'binary:logistic', nthread=4, scale_pos_weight=1, seed=27), \n",
+    " param_grid = params, scoring='roc_auc',n_jobs=12,iid=False, cv=5,verbose=1)\n",
+    "\n",
+    "xgb_model.fit(X_train, y_train)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## create feature importance file"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "results=pd.DataFrame()\n",
+    "results['columns']=list(full_dataset.columns)[2:]\n",
+    "results['importances_rf'] = CV_rfc.feature_importances_\n",
+    "results['importances_xgboost'] = xgb1.feature_importances_\n",
+    "results.sort_values(by='importances_xgboost',ascending=False,inplace=True)\n",
+    "results.to_excel(r\".\\features_rf_xgboost_msi_gland_vs_tissue.xlsx\",index=None)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## ensemble all best model"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "vc = VotingClassifier(estimators=[\n",
+    "     ('mlp', mlp_model.best_estimator_), ('rf', rf_model.best_estimator_), ('xgb', xgb_model.best_estimator_)],\n",
+    "     voting='soft',n_jobs=12)\n",
+    "vc = vc.fit(np.array(X_train),np.array(y_train))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "predictions_test =vc.predict(np.array(X_test))\n",
+    "predictions_valid =vc.predict(np.array(X_valid))\n",
+    "predictions_train =vc.predict(np.array(X_train))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## classification report"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "print(classification_report(y_train,predictions_train))  \n",
+    "print(classification_report(y_valid,predictions_valid))  \n",
+    "print(classification_report(y_test,predictions_test))  "
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.7.3"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}