{

 "cells": [

  {

   "cell_type": "markdown",

   "metadata": {},

   "source": [

    "* [Baseline models](#Baseline-models)\n",

    "* [Load and prepare data](#Load-and-prepare-data)\n",

    "    * [Load and prepare the text](#Load-and-prepare-the-text)\n",

    "    * [Compute LACE features](#Compute-LACE-features)\n",

    "* [Train or load Word2Vec](#Train-or-load-Word2Vec)\n",

    "* [Model](#Model)\n",

    "    * [Neural network with LACE features](#Neural-network-with-LACE-features)\n",

    "    * [Random forest with TF-IDF matrix](#Random-forest-with-TF-IDF-matrix)\n",

    "    * [2-layer feed forward neural network](#2-layer-feed-forward-neural-network)\n",

    "    * [Logistic regression](#Logistic-regression)"

   ]

  },

  {

   "cell_type": "markdown",

   "metadata": {},

   "source": [

    "# Baseline models"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": null,

   "metadata": {

    "collapsed": true

   },

   "outputs": [],

   "source": [

    "# Data prep\n",

    "import numpy as np\n",

    "import pandas as pd\n",

    "from   sklearn.model_selection import train_test_split\n",

    "\n",

    "# Word2Vec\n",

    "import os\n",

    "import logging\n",

    "import string\n",

    "from   gensim.models import word2vec\n",

    "import gensim\n",

    "logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO)\n",

    "\n",

    "# Neural networks \n",

    "import keras\n",

    "from   keras.models import Model\n",

    "from   keras.preprocessing.text import Tokenizer\n",

    "from   keras.preprocessing.sequence import pad_sequences\n",

    "from   keras.layers import Embedding, Input, Conv1D, Dense, GlobalMaxPooling1D\n",

    "from   keras.optimizers import RMSprop\n",

    "import keras.backend as K\n",

    "\n",

    "# Random forest\n",

    "from sklearn.feature_extraction.text import TfidfVectorizer\n",

    "from sklearn.ensemble import RandomForestClassifier\n",

    "\n",

    "# Logistic regression\n",

    "import statsmodels.api as sm"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": null,

   "metadata": {

    "collapsed": true

   },

   "outputs": [],

   "source": [

    "# Data frame created by TextSections/TextPrep\n",

    "TRAIN_TEXT_LOC = \"\"\n",

    "TEST_TEXT_LOC  = \"\"\n",

    "\n",

    "# Data frame containing LACE features.\n",

    "# Assumes presence of:\n",

    "# - LengthOfStay\n",

    "# - Charlson\n",

    "# - PrevERVisits\n",

    "# - AdmittedViaER\n",

    "TRAIN_AUX_LOC  = \"\"\n",

    "TEST_AUX_LOC   = \"\"\n",

    "\n",

    "# Unique visit identifier to merge the train/test text with LACE data\n",

    "MERGE_ON       = \"\"\n",

    "\n",

    "# Other column names\n",

    "VISITID        = \"\"\n",

    "OUTCOME        = \"\" # e.g. ReadmissionInLessThan30Days"

   ]

  },

  {

   "cell_type": "markdown",

   "metadata": {},

   "source": [

    "# Load and prepare data"

   ]

  },

  {

   "cell_type": "markdown",

   "metadata": {},

   "source": [

    "## Load and prepare the text"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": null,

   "metadata": {

    "collapsed": true

   },

   "outputs": [],

   "source": [

    "# Read train and test text data.\n",

    "trainTXT = pd.read_csv(TRAIN_TEXT_LOC)\n",

    "testTXT  = pd.read_csv(TEST_TEXT_LOC)\n",

    "\n",

    "# Read train and test LACE data.\n",

    "trainLACE = pd.read_csv(TRAIN_AUX_LOC)\n",

    "testLACE  = pd.read_csv(TEST_AUX_LOC)\n",

    "\n",

    "# Combine data\n",

    "train = pd.merge(trainTXT, trainLACE, on = MERGE_ON)\n",

    "test  = pd.merge(testTXT,  testLACE,  on = MERGE_ON)\n",

    "\n",

    "# Split the train data into a train and validation set.\n",

    "train, valid = train_test_split(train, \n",

    "                                stratify     = train[OUTCOME], \n",

    "                                train_size   = .9, \n",

    "                                random_state = 1234)\n",

    "\n",

    "# Prepare the sections.\n",

    "# If `sectiontext` is present, then include \"SECTIONNAME sectiontext\".\n",

    "# If not present, include only \"SECTIONNAME\".\n",

    "SECTIONNAMES = [x for x in trainTXT.columns if VISITID not in x and OUTCOME not in x]\n",

    "for x in SECTIONNAMES:\n",

    "    rep      = x.replace(\" \", \"_\").upper()\n",

    "    train[x] = [\" \".join([rep, t]) if not pd.isnull(t) else rep for t in train[x]]\n",

    "    valid[x] = [\" \".join([rep, t]) if not pd.isnull(t) else rep for t in valid[x]]\n",

    "    test[x]  = [\" \".join([rep, t]) if not pd.isnull(t) else rep for t in test[x]]"

   ]

  },

  {

   "cell_type": "markdown",

   "metadata": {},

   "source": [

    "## Compute LACE features"

   ]

  },

  {

   "cell_type": "markdown",

   "metadata": {},

   "source": [

    "This code assumes that, for each hospital visit, you have computed:\n",

    " * the Charlson index\n",

    " * the number of ER visits in the last 6 months\n",

    " * whether the patient was admitted through the ER\n",

    " * the length of stay, in days\n",

    "\n",

    "We then using these data to compute LACE."

   ]

  },

  {

   "cell_type": "code",

   "execution_count": null,

   "metadata": {

    "collapsed": true

   },

   "outputs": [],

   "source": [

    "def LOS(los):\n",

    "    if los <= 3:\n",

    "        return(los)\n",

    "    elif los <= 6:\n",

    "        return(4)\n",

    "    elif los <= 13:\n",

    "        return(5)\n",

    "    else:\n",

    "        return(7)\n",

    "    \n",

    "def ACUITY(erboolean):\n",

    "    if erboolean:\n",

    "        return(3)\n",

    "    else:\n",

    "        return(0)\n",

    "    \n",

    "def LACE(data):\n",

    "    return(LOS(data.LengthOfStay) + ACUITY(data.AdmittedViaER) + data.Charlson + data.PrevERVisits)\n",

    "\n",

    "train[\"LACE\"] = train.apply(LACE, axis=1)\n",

    "valid[\"LACE\"] = valid.apply(LACE, axis=1)\n",

    "test[\"LACE\"]  = test.apply(LACE,  axis=1)"

   ]

  },

  {

   "cell_type": "markdown",

   "metadata": {},

   "source": [

    "For their use in modeling, we also transform the LACE variables by subtracting the mean of the train data:"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": null,

   "metadata": {

    "collapsed": true

   },

   "outputs": [],

   "source": [

    "# We transform \"length of stay\" following the precedent set by LACE.\n",

    "train[\"LOS_Quantized\"]            = train.LengthOfStay.apply(LOS)\n",

    "test[\"LOS_Quantized\"]             = test.LengthOfStay.apply(LOS)\n",

    "valid[\"LOS_Quantized\"]            = valid.LengthOfStay.apply(LOS)\n",

    "\n",

    "train[\"Charlson_Transformed\"]     = train.Charlson - train.Charlson.mean()\n",

    "train[\"LOS_Transformed\"]          = train.LOS_Quantized - train.LOS_Quantized.mean()\n",

    "train[\"PrevERVisits_Transformed\"] = train.PrevERVisits - train.PrevERVisits.mean()\n",

    "\n",

    "test[\"Charlson_Transformed\"]      = test.Charlson - train.Charlson.mean()\n",

    "test[\"LOS_Transformed\"]           = test.LOS_Quantized - train.LOS_Quantized.mean()\n",

    "test[\"PrevERVisits_Transformed\"]  = test.PrevERVisits - train.PrevERVisits.mean()\n",

    "\n",

    "valid[\"Charlson_Transformed\"]     = valid.Charlson - train.Charlson.mean()\n",

    "valid[\"LOS_Transformed\"]          = valid.LOS_Quantized - train.LOS_Quantized.mean()\n",

    "valid[\"PrevERVisits_Transformed\"] = valid.PrevERVisits - train.PrevERVisits.mean()"

   ]

  },

  {

   "cell_type": "markdown",

   "metadata": {},

   "source": [

    "# Train or load Word2Vec"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": null,

   "metadata": {

    "collapsed": true

   },

   "outputs": [],

   "source": [

    "# Word2Vec hyperparameters\n",

    "window    = 2\n",

    "dimension = 1000\n",

    "min_count = 5\n",

    "sg        = 1  \n",

    "hs        = 0  \n",

    "\n",

    "# Where to save the model:\n",

    "modelFile = './word2vec/w2v_dims_' + str(dimension) + \"_window_\" + str(window) + '.bin'\n",

    "\n",

    "# We will remove digits and punctuation:\n",

    "remove_digits_punc = str.maketrans('', '', string.digits + ''.join([x for x in string.punctuation if '_' not in x]))\n",

    "remove_digits_punc = {a:\" \" for a in remove_digits_punc.keys()}\n",

    "\n",

    "# (If the model already exists, don't recompute.)\n",

    "if not os.path.isfile(modelFile):\n",

    "    # Use only training data to train word2vec:\n",

    "    notes = train[SECTIONNAMES].apply(lambda x: \" \".join(x), axis=1).values  \n",

    "    stop  = set([x for x in string.ascii_lowercase]) \n",

    "    for i in range(len(notes)):\n",

    "        notes[i] = [w for w in notes[i].translate(remove_digits_punc).split() if (w not in stop)]\n",

    "    \n",

    "    w2v = word2vec.Word2Vec(notes, \n",

    "                            size=dimension, \n",

    "                            window=window, \n",

    "                            sg=sg, \n",

    "                            hs=hs, \n",

    "                            min_count=min_count, \n",

    "                            workers=50)\n",

    "    w2v.wv.save_word2vec_format(modelFile, binary=True)\n",

    "else:\n",

    "    w2v = gensim.models.KeyedVectors.load_word2vec_format(modelFile, binary=True)"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": null,

   "metadata": {

    "collapsed": true

   },

   "outputs": [],

   "source": [

    "# Make the embedding matrix.\n",

    "# We include one extra word, `PADDING`. This is the word that will right-pad short notes.\n",

    "# For `PADDING`'s vector representation, we choose the zero vector.\n",

    "vocab = [\"PADDING\"] + sorted(list(w2v.wv.vocab.keys()))\n",

    "vset  = set(vocab)\n",

    "\n",

    "embeddings_index = {}\n",

    "for i in range(len(vocab)):\n",

    "    embeddings_index[vocab[i]] = i\n",

    "\n",

    "reverse_embeddings_index = {b:a for a,b in embeddings_index.items()}\n",

    "embeddings_matrix        = np.matrix(np.concatenate(([[0.]*1000], [w2v[x] for x in vocab[1:]])))"

   ]

  },

  {

   "cell_type": "markdown",

   "metadata": {},

   "source": [

    "# Model"

   ]

  },

  {

   "cell_type": "markdown",

   "metadata": {},

   "source": [

    "## Neural network with LACE features"

   ]

  },

  {

   "cell_type": "markdown",

   "metadata": {},

   "source": [

    "Prepare text using our embeddings index:"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": null,

   "metadata": {

    "collapsed": true

   },

   "outputs": [],

   "source": [

    "train_x = train[SECTIONNAMES].apply(lambda x: (\" \".join(x)).translate(remove_digits_punc), axis=1).values  \n",

    "test_x  = test[ SECTIONNAMES].apply(lambda x: (\" \".join(x)).translate(remove_digits_punc), axis=1).values  \n",

    "valid_x = valid[SECTIONNAMES].apply(lambda x: (\" \".join(x)).translate(remove_digits_punc), axis=1).values  \n",

    "\n",

    "train_x = [[embeddings_index[x] for x in note.split() if x in vset] for note in train_x]\n",

    "valid_x = [[embeddings_index[x] for x in note.split() if x in vset] for note in valid_x]\n",

    "test_x  = [[embeddings_index[x] for x in note.split() if x in vset] for note in test_x]\n",

    "\n",

    "train_y = train[OUTCOME]\n",

    "valid_y = valid[OUTCOME]\n",

    "test_y  = test[OUTCOME]"

   ]

  },

  {

   "cell_type": "markdown",

   "metadata": {},

   "source": [

    "And model:"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": null,

   "metadata": {

    "collapsed": true

   },

   "outputs": [],

   "source": [

    "UNITS      = 500\n",

    "FILTERSIZE = 3\n",

    "embedding_layer = Embedding(embeddings_matrix.shape[0],\n",

    "                            embeddings_matrix.shape[1],\n",

    "                            weights=[embeddings_matrix],\n",

    "                            input_length=maxlen,\n",

    "                            trainable=True)\n",

    "\n",

    "sequence_input     = Input(shape=(maxlen,), dtype='int32')\n",

    "embedded_sequences = embedding_layer(sequence_input)\n",

    "\n",

    "lace_in            = Input(shape=(4,))\n",

    "lace               = keras.layers.Reshape((1,4,))(lace_in)\n",

    "lace               = keras.layers.UpSampling1D(700)(lace)\n",

    "\n",

    "combined           = keras.layers.concatenate([embedded_sequences, lace])\n",

    "\n",

    "conv               = Conv1D(UNITS, FILTERSIZE, activation=\"tanh\", use_bias=True)(combined)\n",

    "pool               = GlobalMaxPooling1D()(conv)\n",

    "\n",

    "\n",

    "out                = Dense(1, \n",

    "                           activation='sigmoid', \n",

    "                           activity_regularizer=keras.regularizers.l1(l=.05)\n",

    "                        )(pool)\n",

    "\n",

    "optimizer = keras.optimizers.RMSprop(lr = .0001)\n",

    "model=Model(inputs=[sequence_input, lace_in], outputs=out)\n",

    "model.compile(loss='binary_crossentropy', optimizer=optimizer)\n",

    "\n",

    "model.fit(train_x, train_y, batch_size=100, epochs=4, validation_data=(valid_x, valid_y), verbose=1)"

   ]

  },

  {

   "cell_type": "markdown",

   "metadata": {},

   "source": [

    "## Random forest with TF-IDF matrix"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": null,

   "metadata": {

    "collapsed": true

   },

   "outputs": [],

   "source": [

    "# Prepare the text for sklearn's tfidf vectorizer:\n",

    "train_x = train[SECTIONNAMES].apply(lambda x: (\" \".join(x)).translate(remove_digits_punc), axis=1).values  \n",

    "test_x  = test[ SECTIONNAMES].apply(lambda x: (\" \".join(x)).translate(remove_digits_punc), axis=1).values  \n",

    "valid_x = valid[SECTIONNAMES].apply(lambda x: (\" \".join(x)).translate(remove_digits_punc), axis=1).values  \n",

    "\n",

    "train_y = train[OUTCOME]\n",

    "valid_y = valid[OUTCOME]\n",

    "test_y  = test[OUTCOME]\n",

    "\n",

    "tfidf = TfidfVectorizer()\n",

    "tr_x  = tfidf.fit_transform(train_x)\n",

    "te_x  = tfidf.transform(test_x)\n",

    "va_x  = tfidf.transform(valid_x)"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": null,

   "metadata": {

    "collapsed": true

   },

   "outputs": [],

   "source": [

    "# Model:\n",

    "rfc = RandomForestClassifier(n_estimators=1000, max_depth=100, n_jobs=-1)\n",

    "rfc.fit(tr_x, train_y)"

   ]

  },

  {

   "cell_type": "markdown",

   "metadata": {},

   "source": [

    "## 2-layer feed forward neural network "

   ]

  },

  {

   "cell_type": "markdown",

   "metadata": {},

   "source": [

    "This model uses only the components of LACE together with the LACE score:"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": null,

   "metadata": {

    "collapsed": true

   },

   "outputs": [],

   "source": [

    "lace  = Input(shape=(5,))\n",

    "dense = Dense(50, activation='tanh')(lace)\n",

    "out   = Dense(1, activation='sigmoid')(dense)\n",

    "\n",

    "model = Model(inputs=lace, outputs=out)\n",

    "model.compile(loss='binary_crossentropy', optimizer=\"nadam\")"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": null,

   "metadata": {

    "collapsed": true

   },

   "outputs": [],

   "source": [

    "model.fit(train[[\"LOS_Transformed\", \"AdmittedViaER\", \"Charlson_Transformed\", \"PrevERVisits_Transformed\", \"LACE\"]].values, \n",

    "           train_y,\n",

    "           class_weight={0:1, 1:10}, \n",

    "           epochs=1)"

   ]

  },

  {

   "cell_type": "markdown",

   "metadata": {},

   "source": [

    "## Logistic regression"

   ]

  },

  {

   "cell_type": "code",

   "execution_count": null,

   "metadata": {

    "collapsed": true

   },

   "outputs": [],

   "source": [

    "model = logit(formula = OUTCOME + \" ~ (LOS_Transformed + AdmittedViaER + Charlson_Transformed + PrevERVisits_Transformed + LACE)\", \n",

    "              data = train\n",

    "        ).fit(maxiter = 1000, method = 'lbfgs')"

   ]

  }

 ],

 "metadata": {

  "kernelspec": {

   "display_name": "Python 3",

   "language": "python",

   "name": "python3"

  },

  "language_info": {

   "codemirror_mode": {

    "name": "ipython",

    "version": 3

   },

   "file_extension": ".py",

   "mimetype": "text/x-python",

   "name": "python",

   "nbconvert_exporter": "python",

   "pygments_lexer": "ipython3",

   "version": "3.5.2"

  }

 },

 "nbformat": 4,

 "nbformat_minor": 2

}