Datasets
Models
Diff of
/reproduce_table3.ipynb
[000000]
..
[0f2bcf]
Switch to side-by-side view
--- a +++ b/reproduce_table3.ipynb @@ -0,0 +1,419 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Tutorial - TCGA\n", + "\n", + "### Generating Results in Table 3 (TCGA Dataset)" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "import warnings\n", + "warnings.filterwarnings('ignore')\n", + "\n", + "import numpy as np\n", + "import tensorflow as tf\n", + "\n", + "import random\n", + "import sys, os" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "from sklearn.model_selection import train_test_split" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "import import_data as impt\n", + "from helper import f_get_minibatch_set, evaluate\n", + "from class_DeepIMV_AISTATS import DeepIMV_AISTATS" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "year = 1\n", + "DATASET_PATH = 'TCGA_{}YR'.format(int(year))\n", + "DATASET = 'TCGA'\n", + "\n", + "X_set_comp, Y_onehot_comp, Mask_comp, X_set_incomp, Y_onehot_incomp, Mask_incomp = impt.import_dataset_TCGA(year)\n", + "\n", + "MODE = 'incomplete'\n", + "model_name = 'DeepIMV_AISTATS'\n", + "\n", + "M = len(X_set_comp)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "SEED = 1234\n", + "OUTITERATION = 5\n", + "\n", + "RESULTS_AUROC_RAND = np.zeros([4, OUTITERATION+2])\n", + "RESULTS_AUPRC_RAND = np.zeros([4, OUTITERATION+2])" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "out_itr = 1\n", + "\n", + "tr_X_set, te_X_set, va_X_set = {}, {}, {}\n", + "for m in range(M):\n", + " tr_X_set[m],te_X_set[m] = train_test_split(X_set_comp[m], test_size=0.2, random_state=SEED + out_itr)\n", + " tr_X_set[m],va_X_set[m] = train_test_split(tr_X_set[m], test_size=0.2, random_state=SEED + out_itr)\n", + " \n", + "tr_Y_onehot,te_Y_onehot, tr_M,te_M = train_test_split(Y_onehot_comp, Mask_comp, test_size=0.2, random_state=SEED + out_itr)\n", + "tr_Y_onehot,va_Y_onehot, tr_M,va_M = train_test_split(tr_Y_onehot, tr_M, test_size=0.2, random_state=SEED + out_itr)" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "(5850, 4)\n" + ] + } + ], + "source": [ + "if MODE == 'incomplete':\n", + " for m in range(M):\n", + " tr_X_set[m] = np.concatenate([tr_X_set[m], X_set_incomp[m]], axis=0)\n", + "\n", + " tr_Y_onehot = np.concatenate([tr_Y_onehot, Y_onehot_incomp], axis=0)\n", + " tr_M = np.concatenate([tr_M, Mask_incomp], axis=0)\n", + " \n", + " print(tr_M.shape)\n", + "elif MODE == 'complete':\n", + " print(tr_M.shape)\n", + "else:\n", + " raise ValueError('WRONG MODE!!!')\n", + " \n", + "\n", + "save_path = '{}/M{}_{}/{}/'.format(DATASET_PATH, M, MODE, model_name)\n", + " \n", + " \n", + "if not os.path.exists(save_path + 'itr{}/'.format(out_itr)):\n", + " os.makedirs(save_path + 'itr{}/'.format(out_itr))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Hyper-parameters" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [], + "source": [ + "### training coefficients\n", + "alpha = 1.0\n", + "beta = 0.01 # IB coefficient\n", + "lr_rate = 1e-4\n", + "k_prob = 0.7\n", + "\n", + "\n", + "### network parameters\n", + "mb_size = 32 \n", + "steps_per_batch = int(np.shape(tr_M)[0]/mb_size)\n", + "steps_per_batch = 500\n", + "\n", + "x_dim_set = [tr_X_set[m].shape[1] for m in range(len(tr_X_set))]\n", + "y_dim = np.shape(tr_Y_onehot)[1]\n", + "y_type = 'binary'\n", + "z_dim = 100\n", + "\n", + "h_dim_p = 100\n", + "num_layers_p = 2\n", + "\n", + "h_dim_e = 300\n", + "num_layers_e = 3\n", + "\n", + "input_dims = {\n", + " 'x_dim_set': x_dim_set,\n", + " 'y_dim': y_dim,\n", + " 'y_type': y_type,\n", + " 'z_dim': z_dim,\n", + " \n", + " 'steps_per_batch': steps_per_batch\n", + "}\n", + "\n", + "network_settings = {\n", + " 'h_dim_p1': h_dim_p,\n", + " 'num_layers_p1': num_layers_p, #view-specific\n", + " 'h_dim_p2': h_dim_p,\n", + " 'num_layers_p2': num_layers_p, #multi-view\n", + " 'h_dim_e': h_dim_e,\n", + " 'num_layers_e': num_layers_e,\n", + " 'fc_activate_fn': tf.nn.relu,\n", + " 'reg_scale': 0., #1e-4,\n", + "}\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Training" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [], + "source": [ + "tf.reset_default_graph()\n", + "\n", + "# gpu_options = tf.GPUOptions()\n", + "gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.22)\n", + "sess = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options))\n", + "\n", + "model = DeepIMV_AISTATS(sess, \"DeepIMV_AISTATS\", input_dims, network_settings)\n", + "\n", + "saver = tf.train.Saver()\n", + "sess.run(tf.global_variables_initializer())" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "00500: TRAIN| Lt=6.454 Lp=0.994 Lkl=42.950 Lps=4.288 Lkls=74.204 Lc=49.079 | VALID| Lt=5.554 Lp=0.929 Lkl=45.010 Lps=3.849 Lkls=32.576 Lc=46.531 score=(0.5993706489643726, 0.20887111832184874)\n", + "saved...\n", + "01000: TRAIN| Lt=5.237 Lp=0.856 Lkl=39.923 Lps=3.558 Lkls=42.400 Lc=43.212 | VALID| Lt=4.698 Lp=0.793 Lkl=43.676 Lps=3.274 Lkls=19.390 Lc=43.746 score=(0.7675711791710671, 0.2945589414475165)\n", + "saved...\n", + "01500: TRAIN| Lt=4.808 Lp=0.744 Lkl=38.987 Lps=3.332 Lkls=34.243 Lc=41.684 | VALID| Lt=4.419 Lp=0.717 Lkl=42.991 Lps=3.106 Lkls=16.574 Lc=43.175 score=(0.7953315947151756, 0.32247570838825707)\n", + "saved...\n", + "02000: TRAIN| Lt=4.523 Lp=0.687 Lkl=38.207 Lps=3.151 Lkls=30.336 Lc=40.726 | VALID| Lt=4.272 Lp=0.679 Lkl=42.528 Lps=3.014 Lkls=15.384 Lc=42.915 score=(0.7921632864193805, 0.36678881900359384)\n", + "saved...\n", + "02500: TRAIN| Lt=4.286 Lp=0.633 Lkl=37.788 Lps=2.994 Lkls=28.058 Lc=40.309 | VALID| Lt=4.211 Lp=0.677 Lkl=42.190 Lps=2.970 Lkls=14.196 Lc=42.683 score=(0.7951160635385909, 0.36298993132414803)\n", + "saved...\n", + "03000: TRAIN| Lt=4.229 Lp=0.621 Lkl=37.510 Lps=2.975 Lkls=25.871 Lc=39.875 | VALID| Lt=4.188 Lp=0.676 Lkl=41.952 Lps=2.961 Lkls=13.201 Lc=42.500 score=(0.8097075241933746, 0.41376471395985764)\n", + "saved...\n", + "03500: TRAIN| Lt=4.001 Lp=0.570 Lkl=37.365 Lps=2.802 Lkls=25.548 Lc=39.800 | VALID| Lt=4.182 Lp=0.675 Lkl=41.812 Lps=2.959 Lkls=12.999 Lc=42.445 score=(0.7994482401879431, 0.38641092091905127)\n", + "saved...\n", + "04000: TRAIN| Lt=3.809 Lp=0.524 Lkl=37.191 Lps=2.659 Lkls=25.413 Lc=39.659 | VALID| Lt=4.171 Lp=0.673 Lkl=41.692 Lps=2.951 Lkls=12.952 Lc=42.425 score=(0.8013664676595469, 0.40857793712669693)\n", + "saved...\n", + "04500: TRAIN| Lt=3.721 Lp=0.500 Lkl=37.069 Lps=2.603 Lkls=24.795 Lc=39.503 | VALID| Lt=4.199 Lp=0.691 Lkl=41.590 Lps=2.970 Lkls=12.149 Lc=42.290 score=(0.8095350992521069, 0.3908763759078532)\n", + "05000: TRAIN| Lt=3.584 Lp=0.468 Lkl=36.996 Lps=2.499 Lkls=24.732 Lc=39.387 | VALID| Lt=4.222 Lp=0.683 Lkl=41.568 Lps=3.004 Lkls=11.931 Lc=42.247 score=(0.8059680582796301, 0.3793825246248715)\n", + "05500: TRAIN| Lt=3.491 Lp=0.415 Lkl=37.158 Lps=2.448 Lkls=25.743 Lc=39.493 | VALID| Lt=4.229 Lp=0.691 Lkl=41.672 Lps=2.999 Lkls=12.135 Lc=42.268 score=(0.8071858094273336, 0.3889992215812622)\n", + "06000: TRAIN| Lt=3.203 Lp=0.369 Lkl=37.194 Lps=2.200 Lkls=26.217 Lc=39.626 | VALID| Lt=4.317 Lp=0.713 Lkl=41.652 Lps=3.064 Lkls=12.409 Lc=42.303 score=(0.8036295450136862, 0.37662407670056136)\n", + "06500: TRAIN| Lt=3.164 Lp=0.335 Lkl=37.223 Lps=2.193 Lkls=26.342 Lc=39.580 | VALID| Lt=4.451 Lp=0.739 Lkl=41.631 Lps=3.180 Lkls=11.647 Lc=42.185 score=(0.8091255900165959, 0.3946424315886283)\n", + "07000: TRAIN| Lt=2.999 Lp=0.304 Lkl=37.251 Lps=2.057 Lkls=26.575 Lc=39.603 | VALID| Lt=4.551 Lp=0.769 Lkl=41.639 Lps=3.248 Lkls=11.771 Lc=42.199 score=(0.8006552147768174, 0.370863714209815)\n", + "07500: TRAIN| Lt=2.906 Lp=0.282 Lkl=37.167 Lps=1.988 Lkls=26.441 Lc=39.570 | VALID| Lt=4.596 Lp=0.778 Lkl=41.552 Lps=3.290 Lkls=11.228 Lc=42.119 score=(0.8097290773110329, 0.39510639377535883)\n", + "08000: TRAIN| Lt=2.769 Lp=0.247 Lkl=37.136 Lps=1.886 Lkls=26.457 Lc=39.533 | VALID| Lt=4.599 Lp=0.801 Lkl=41.528 Lps=3.270 Lkls=11.238 Lc=42.125 score=(0.8128542793715111, 0.41143309089456515)\n", + "08500: TRAIN| Lt=2.716 Lp=0.232 Lkl=37.163 Lps=1.841 Lkls=27.146 Lc=39.555 | VALID| Lt=4.615 Lp=0.807 Lkl=41.499 Lps=3.282 Lkls=11.072 Lc=42.104 score=(0.8085221027221589, 0.38547322403694323)\n", + "09000: TRAIN| Lt=2.657 Lp=0.220 Lkl=37.092 Lps=1.803 Lkls=26.330 Lc=39.509 | VALID| Lt=4.735 Lp=0.817 Lkl=41.439 Lps=3.395 Lkls=10.821 Lc=42.076 score=(0.8157639502554044, 0.39705578680902437)\n", + "09500: TRAIN| Lt=2.580 Lp=0.201 Lkl=37.165 Lps=1.741 Lkls=26.648 Lc=39.570 | VALID| Lt=4.852 Lp=0.872 Lkl=41.456 Lps=3.461 Lkls=10.475 Lc=42.012 score=(0.8036079918960277, 0.3835206157275273)\n", + "10000: TRAIN| Lt=2.514 Lp=0.188 Lkl=37.163 Lps=1.687 Lkls=26.721 Lc=39.563 | VALID| Lt=4.793 Lp=0.864 Lkl=41.406 Lps=3.411 Lkls=10.398 Lc=42.004 score=(0.8004612367178912, 0.384517869168757)\n", + "10500: TRAIN| Lt=2.440 Lp=0.169 Lkl=37.154 Lps=1.631 Lkls=26.868 Lc=39.589 | VALID| Lt=4.870 Lp=0.884 Lkl=41.363 Lps=3.468 Lkls=10.429 Lc=42.012 score=(0.7988663060111645, 0.3839728905168826)\n", + "11000: TRAIN| Lt=2.398 Lp=0.161 Lkl=37.106 Lps=1.602 Lkls=26.449 Lc=39.480 | VALID| Lt=4.816 Lp=0.854 Lkl=41.332 Lps=3.449 Lkls=9.962 Lc=41.958 score=(0.7941677263616183, 0.3776828074521237)\n", + "11500: TRAIN| Lt=2.306 Lp=0.141 Lkl=37.107 Lps=1.524 Lkls=27.002 Lc=39.504 | VALID| Lt=5.125 Lp=0.943 Lkl=41.346 Lps=3.669 Lkls=10.021 Lc=41.967 score=(0.8140612539603854, 0.3916890388476985)\n", + "12000: TRAIN| Lt=2.222 Lp=0.129 Lkl=37.141 Lps=1.452 Lkls=26.963 Lc=39.599 | VALID| Lt=5.303 Lp=0.986 Lkl=41.324 Lps=3.803 Lkls=10.081 Lc=41.972 score=(0.795999741362588, 0.38850082170618294)\n", + "12500: TRAIN| Lt=2.237 Lp=0.127 Lkl=37.083 Lps=1.468 Lkls=27.086 Lc=39.497 | VALID| Lt=5.258 Lp=0.980 Lkl=41.319 Lps=3.767 Lkls=9.768 Lc=41.938 score=(0.8033062482488093, 0.3854664963559042)\n", + "13000: TRAIN| Lt=2.173 Lp=0.116 Lkl=37.029 Lps=1.416 Lkls=27.096 Lc=39.430 | VALID| Lt=5.308 Lp=1.004 Lkl=41.298 Lps=3.793 Lkls=9.795 Lc=41.938 score=(0.7964954630687329, 0.37666072443224835)\n", + "13500: TRAIN| Lt=2.111 Lp=0.113 Lkl=36.941 Lps=1.363 Lkls=26.571 Lc=39.395 | VALID| Lt=5.416 Lp=1.027 Lkl=41.269 Lps=3.882 Lkls=9.464 Lc=41.890 score=(0.8053969006616808, 0.38825824120686886)\n", + "14000: TRAIN| Lt=2.074 Lp=0.105 Lkl=37.012 Lps=1.334 Lkls=26.482 Lc=39.457 | VALID| Lt=5.368 Lp=1.013 Lkl=41.223 Lps=3.849 Lkls=9.415 Lc=41.889 score=(0.7898139965946074, 0.3696094392582078)\n", + "FINISHED...\n" + ] + } + ], + "source": [ + "saver = tf.train.Saver()\n", + "sess.run(tf.global_variables_initializer())\n", + "\n", + "ITERATION = 500000\n", + "STEPSIZE = 500\n", + "\n", + "min_loss = 1e+8 \n", + "max_acc = 0.0\n", + "max_flag = 20\n", + "\n", + "tr_avg_Lt, tr_avg_Lp, tr_avg_Lkl, tr_avg_Lps, tr_avg_Lkls, tr_avg_Lc = 0, 0, 0, 0, 0, 0\n", + "va_avg_Lt, va_avg_Lp, va_avg_Lkl, va_avg_Lps, va_avg_Lkls, va_avg_Lc = 0, 0, 0, 0, 0, 0\n", + " \n", + "stop_flag = 0\n", + "for itr in range(ITERATION):\n", + " x_mb_set, y_mb, m_mb = f_get_minibatch_set(mb_size, tr_X_set, tr_Y_onehot, tr_M) \n", + " \n", + " _, Lt, Lp, Lkl, Lps, Lkls, Lc = model.train(x_mb_set, y_mb, m_mb, alpha, beta, lr_rate, k_prob)\n", + "\n", + " tr_avg_Lt += Lt/STEPSIZE\n", + " tr_avg_Lp += Lp/STEPSIZE\n", + " tr_avg_Lkl += Lkl/STEPSIZE\n", + " tr_avg_Lps += Lps/STEPSIZE\n", + " tr_avg_Lkls += Lkls/STEPSIZE\n", + " tr_avg_Lc += Lc/STEPSIZE\n", + "\n", + " \n", + " x_mb_set, y_mb, m_mb = f_get_minibatch_set(min(np.shape(va_M)[0], mb_size), va_X_set, va_Y_onehot, va_M) \n", + " Lt, Lp, Lkl, Lps, Lkls, Lc, _, _ = model.get_loss(x_mb_set, y_mb, m_mb, alpha, beta)\n", + " \n", + " va_avg_Lt += Lt/STEPSIZE\n", + " va_avg_Lp += Lp/STEPSIZE\n", + " va_avg_Lkl += Lkl/STEPSIZE\n", + " va_avg_Lps += Lps/STEPSIZE\n", + " va_avg_Lkls += Lkls/STEPSIZE\n", + " va_avg_Lc += Lc/STEPSIZE\n", + " \n", + " if (itr+1)%STEPSIZE == 0:\n", + " y_pred, y_preds = model.predict_ys(va_X_set, va_M)\n", + " \n", + "# score = \n", + "\n", + " print( \"{:05d}: TRAIN| Lt={:.3f} Lp={:.3f} Lkl={:.3f} Lps={:.3f} Lkls={:.3f} Lc={:.3f} | VALID| Lt={:.3f} Lp={:.3f} Lkl={:.3f} Lps={:.3f} Lkls={:.3f} Lc={:.3f} score={}\".format(\n", + " itr+1, tr_avg_Lt, tr_avg_Lp, tr_avg_Lkl, tr_avg_Lps, tr_avg_Lkls, tr_avg_Lc, \n", + " va_avg_Lt, va_avg_Lp, va_avg_Lkl, va_avg_Lps, va_avg_Lkls, va_avg_Lc, evaluate(va_Y_onehot, np.mean(y_preds, axis=0), y_type))\n", + " )\n", + " \n", + " if min_loss > va_avg_Lt:\n", + " min_loss = va_avg_Lt\n", + " stop_flag = 0\n", + " saver.save(sess,save_path + 'itr{}/best_model'.format(out_itr))\n", + " print('saved...')\n", + " else:\n", + " stop_flag += 1\n", + " \n", + " tr_avg_Lt, tr_avg_Lp, tr_avg_Lkl, tr_avg_Lps, tr_avg_Lkls, tr_avg_Lc = 0, 0, 0, 0, 0, 0\n", + " va_avg_Lt, va_avg_Lp, va_avg_Lkl, va_avg_Lps, va_avg_Lkls, va_avg_Lc = 0, 0, 0, 0, 0, 0\n", + " \n", + " if stop_flag >= max_flag:\n", + " break\n", + " \n", + "print('FINISHED...')" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "INFO:tensorflow:Restoring parameters from TCGA_1YR/M4_incomplete/DeepIMV_AISTATS/itr1/best_model\n" + ] + } + ], + "source": [ + "saver.restore(sess, save_path + 'itr{}/best_model'.format(out_itr))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Evaluation -- (Results in Table 3)" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "TEST - INCOMPLETE: auroc=0.7433 auprc=0.3541\n", + "TEST - INCOMPLETE: auroc=0.7747 auprc=0.4077\n", + "TEST - INCOMPLETE: auroc=0.7955 auprc=0.4004\n", + "TEST - INCOMPLETE: auroc=0.8011 auprc=0.4138\n" + ] + } + ], + "source": [ + "for m_available in [1,2,3,4]:\n", + "\n", + " tmp_M_mis = np.zeros_like(te_M)#np.copy(te_M)\n", + "\n", + "\n", + " for i in range(len(tmp_M_mis)):\n", + " np.random.seed(SEED+out_itr+i)\n", + " idx = np.random.choice(4, m_available, replace=False)\n", + " tmp_M_mis[i, idx] = 1\n", + "\n", + "\n", + " #for stablity of reducing randomness..\n", + " for i in range(100):\n", + " _, tmp_preds_all = model.predict_ys(te_X_set, tmp_M_mis)\n", + " if i == 0:\n", + " y_preds_all = tmp_preds_all\n", + " else:\n", + " y_preds_all = np.concatenate([y_preds_all, tmp_preds_all], axis=0)\n", + "\n", + " auc1, apc1 = evaluate(te_Y_onehot, y_preds_all.mean(axis=0), y_type)\n", + "\n", + " RESULTS_AUROC_RAND[m_available-1, out_itr] = auc1\n", + " RESULTS_AUPRC_RAND[m_available-1, out_itr] = apc1\n", + "\n", + " print(\"TEST - {} - #VIEW {}: auroc={:.4f} auprc={:.4f}\".format(MODE.upper(), m_available, auc1, apc1))" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.9" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +}