520 lines (519 with data), 16.2 kB

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* [Baseline models](#Baseline-models)\n",
    "* [Load and prepare data](#Load-and-prepare-data)\n",
    "    * [Load and prepare the text](#Load-and-prepare-the-text)\n",
    "    * [Compute LACE features](#Compute-LACE-features)\n",
    "* [Train or load Word2Vec](#Train-or-load-Word2Vec)\n",
    "* [Model](#Model)\n",
    "    * [Neural network with LACE features](#Neural-network-with-LACE-features)\n",
    "    * [Random forest with TF-IDF matrix](#Random-forest-with-TF-IDF-matrix)\n",
    "    * [2-layer feed forward neural network](#2-layer-feed-forward-neural-network)\n",
    "    * [Logistic regression](#Logistic-regression)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Baseline models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Data prep\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from   sklearn.model_selection import train_test_split\n",
    "\n",
    "# Word2Vec\n",
    "import os\n",
    "import logging\n",
    "import string\n",
    "from   gensim.models import word2vec\n",
    "import gensim\n",
    "logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO)\n",
    "\n",
    "# Neural networks \n",
    "import keras\n",
    "from   keras.models import Model\n",
    "from   keras.preprocessing.text import Tokenizer\n",
    "from   keras.preprocessing.sequence import pad_sequences\n",
    "from   keras.layers import Embedding, Input, Conv1D, Dense, GlobalMaxPooling1D\n",
    "from   keras.optimizers import RMSprop\n",
    "import keras.backend as K\n",
    "\n",
    "# Random forest\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "# Logistic regression\n",
    "import statsmodels.api as sm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Data frame created by TextSections/TextPrep\n",
    "TRAIN_TEXT_LOC = \"\"\n",
    "TEST_TEXT_LOC  = \"\"\n",
    "\n",
    "# Data frame containing LACE features.\n",
    "# Assumes presence of:\n",
    "# - LengthOfStay\n",
    "# - Charlson\n",
    "# - PrevERVisits\n",
    "# - AdmittedViaER\n",
    "TRAIN_AUX_LOC  = \"\"\n",
    "TEST_AUX_LOC   = \"\"\n",
    "\n",
    "# Unique visit identifier to merge the train/test text with LACE data\n",
    "MERGE_ON       = \"\"\n",
    "\n",
    "# Other column names\n",
    "VISITID        = \"\"\n",
    "OUTCOME        = \"\" # e.g. ReadmissionInLessThan30Days"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load and prepare data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load and prepare the text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Read train and test text data.\n",
    "trainTXT = pd.read_csv(TRAIN_TEXT_LOC)\n",
    "testTXT  = pd.read_csv(TEST_TEXT_LOC)\n",
    "\n",
    "# Read train and test LACE data.\n",
    "trainLACE = pd.read_csv(TRAIN_AUX_LOC)\n",
    "testLACE  = pd.read_csv(TEST_AUX_LOC)\n",
    "\n",
    "# Combine data\n",
    "train = pd.merge(trainTXT, trainLACE, on = MERGE_ON)\n",
    "test  = pd.merge(testTXT,  testLACE,  on = MERGE_ON)\n",
    "\n",
    "# Split the train data into a train and validation set.\n",
    "train, valid = train_test_split(train, \n",
    "                                stratify     = train[OUTCOME], \n",
    "                                train_size   = .9, \n",
    "                                random_state = 1234)\n",
    "\n",
    "# Prepare the sections.\n",
    "# If `sectiontext` is present, then include \"SECTIONNAME sectiontext\".\n",
    "# If not present, include only \"SECTIONNAME\".\n",
    "SECTIONNAMES = [x for x in trainTXT.columns if VISITID not in x and OUTCOME not in x]\n",
    "for x in SECTIONNAMES:\n",
    "    rep      = x.replace(\" \", \"_\").upper()\n",
    "    train[x] = [\" \".join([rep, t]) if not pd.isnull(t) else rep for t in train[x]]\n",
    "    valid[x] = [\" \".join([rep, t]) if not pd.isnull(t) else rep for t in valid[x]]\n",
    "    test[x]  = [\" \".join([rep, t]) if not pd.isnull(t) else rep for t in test[x]]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Compute LACE features"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This code assumes that, for each hospital visit, you have computed:\n",
    " * the Charlson index\n",
    " * the number of ER visits in the last 6 months\n",
    " * whether the patient was admitted through the ER\n",
    " * the length of stay, in days\n",
    "\n",
    "We then using these data to compute LACE."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def LOS(los):\n",
    "    if los <= 3:\n",
    "        return(los)\n",
    "    elif los <= 6:\n",
    "        return(4)\n",
    "    elif los <= 13:\n",
    "        return(5)\n",
    "    else:\n",
    "        return(7)\n",
    "    \n",
    "def ACUITY(erboolean):\n",
    "    if erboolean:\n",
    "        return(3)\n",
    "    else:\n",
    "        return(0)\n",
    "    \n",
    "def LACE(data):\n",
    "    return(LOS(data.LengthOfStay) + ACUITY(data.AdmittedViaER) + data.Charlson + data.PrevERVisits)\n",
    "\n",
    "train[\"LACE\"] = train.apply(LACE, axis=1)\n",
    "valid[\"LACE\"] = valid.apply(LACE, axis=1)\n",
    "test[\"LACE\"]  = test.apply(LACE,  axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For their use in modeling, we also transform the LACE variables by subtracting the mean of the train data:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# We transform \"length of stay\" following the precedent set by LACE.\n",
    "train[\"LOS_Quantized\"]            = train.LengthOfStay.apply(LOS)\n",
    "test[\"LOS_Quantized\"]             = test.LengthOfStay.apply(LOS)\n",
    "valid[\"LOS_Quantized\"]            = valid.LengthOfStay.apply(LOS)\n",
    "\n",
    "train[\"Charlson_Transformed\"]     = train.Charlson - train.Charlson.mean()\n",
    "train[\"LOS_Transformed\"]          = train.LOS_Quantized - train.LOS_Quantized.mean()\n",
    "train[\"PrevERVisits_Transformed\"] = train.PrevERVisits - train.PrevERVisits.mean()\n",
    "\n",
    "test[\"Charlson_Transformed\"]      = test.Charlson - train.Charlson.mean()\n",
    "test[\"LOS_Transformed\"]           = test.LOS_Quantized - train.LOS_Quantized.mean()\n",
    "test[\"PrevERVisits_Transformed\"]  = test.PrevERVisits - train.PrevERVisits.mean()\n",
    "\n",
    "valid[\"Charlson_Transformed\"]     = valid.Charlson - train.Charlson.mean()\n",
    "valid[\"LOS_Transformed\"]          = valid.LOS_Quantized - train.LOS_Quantized.mean()\n",
    "valid[\"PrevERVisits_Transformed\"] = valid.PrevERVisits - train.PrevERVisits.mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train or load Word2Vec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Word2Vec hyperparameters\n",
    "window    = 2\n",
    "dimension = 1000\n",
    "min_count = 5\n",
    "sg        = 1  \n",
    "hs        = 0  \n",
    "\n",
    "# Where to save the model:\n",
    "modelFile = './word2vec/w2v_dims_' + str(dimension) + \"_window_\" + str(window) + '.bin'\n",
    "\n",
    "# We will remove digits and punctuation:\n",
    "remove_digits_punc = str.maketrans('', '', string.digits + ''.join([x for x in string.punctuation if '_' not in x]))\n",
    "remove_digits_punc = {a:\" \" for a in remove_digits_punc.keys()}\n",
    "\n",
    "# (If the model already exists, don't recompute.)\n",
    "if not os.path.isfile(modelFile):\n",
    "    # Use only training data to train word2vec:\n",
    "    notes = train[SECTIONNAMES].apply(lambda x: \" \".join(x), axis=1).values  \n",
    "    stop  = set([x for x in string.ascii_lowercase]) \n",
    "    for i in range(len(notes)):\n",
    "        notes[i] = [w for w in notes[i].translate(remove_digits_punc).split() if (w not in stop)]\n",
    "    \n",
    "    w2v = word2vec.Word2Vec(notes, \n",
    "                            size=dimension, \n",
    "                            window=window, \n",
    "                            sg=sg, \n",
    "                            hs=hs, \n",
    "                            min_count=min_count, \n",
    "                            workers=50)\n",
    "    w2v.wv.save_word2vec_format(modelFile, binary=True)\n",
    "else:\n",
    "    w2v = gensim.models.KeyedVectors.load_word2vec_format(modelFile, binary=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Make the embedding matrix.\n",
    "# We include one extra word, `PADDING`. This is the word that will right-pad short notes.\n",
    "# For `PADDING`'s vector representation, we choose the zero vector.\n",
    "vocab = [\"PADDING\"] + sorted(list(w2v.wv.vocab.keys()))\n",
    "vset  = set(vocab)\n",
    "\n",
    "embeddings_index = {}\n",
    "for i in range(len(vocab)):\n",
    "    embeddings_index[vocab[i]] = i\n",
    "\n",
    "reverse_embeddings_index = {b:a for a,b in embeddings_index.items()}\n",
    "embeddings_matrix        = np.matrix(np.concatenate(([[0.]*1000], [w2v[x] for x in vocab[1:]])))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Neural network with LACE features"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Prepare text using our embeddings index:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_x = train[SECTIONNAMES].apply(lambda x: (\" \".join(x)).translate(remove_digits_punc), axis=1).values  \n",
    "test_x  = test[ SECTIONNAMES].apply(lambda x: (\" \".join(x)).translate(remove_digits_punc), axis=1).values  \n",
    "valid_x = valid[SECTIONNAMES].apply(lambda x: (\" \".join(x)).translate(remove_digits_punc), axis=1).values  \n",
    "\n",
    "train_x = [[embeddings_index[x] for x in note.split() if x in vset] for note in train_x]\n",
    "valid_x = [[embeddings_index[x] for x in note.split() if x in vset] for note in valid_x]\n",
    "test_x  = [[embeddings_index[x] for x in note.split() if x in vset] for note in test_x]\n",
    "\n",
    "train_y = train[OUTCOME]\n",
    "valid_y = valid[OUTCOME]\n",
    "test_y  = test[OUTCOME]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "And model:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "UNITS      = 500\n",
    "FILTERSIZE = 3\n",
    "embedding_layer = Embedding(embeddings_matrix.shape[0],\n",
    "                            embeddings_matrix.shape[1],\n",
    "                            weights=[embeddings_matrix],\n",
    "                            input_length=maxlen,\n",
    "                            trainable=True)\n",
    "\n",
    "sequence_input     = Input(shape=(maxlen,), dtype='int32')\n",
    "embedded_sequences = embedding_layer(sequence_input)\n",
    "\n",
    "lace_in            = Input(shape=(4,))\n",
    "lace               = keras.layers.Reshape((1,4,))(lace_in)\n",
    "lace               = keras.layers.UpSampling1D(700)(lace)\n",
    "\n",
    "combined           = keras.layers.concatenate([embedded_sequences, lace])\n",
    "\n",
    "conv               = Conv1D(UNITS, FILTERSIZE, activation=\"tanh\", use_bias=True)(combined)\n",
    "pool               = GlobalMaxPooling1D()(conv)\n",
    "\n",
    "\n",
    "out                = Dense(1, \n",
    "                           activation='sigmoid', \n",
    "                           activity_regularizer=keras.regularizers.l1(l=.05)\n",
    "                        )(pool)\n",
    "\n",
    "optimizer = keras.optimizers.RMSprop(lr = .0001)\n",
    "model=Model(inputs=[sequence_input, lace_in], outputs=out)\n",
    "model.compile(loss='binary_crossentropy', optimizer=optimizer)\n",
    "\n",
    "model.fit(train_x, train_y, batch_size=100, epochs=4, validation_data=(valid_x, valid_y), verbose=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Random forest with TF-IDF matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Prepare the text for sklearn's tfidf vectorizer:\n",
    "train_x = train[SECTIONNAMES].apply(lambda x: (\" \".join(x)).translate(remove_digits_punc), axis=1).values  \n",
    "test_x  = test[ SECTIONNAMES].apply(lambda x: (\" \".join(x)).translate(remove_digits_punc), axis=1).values  \n",
    "valid_x = valid[SECTIONNAMES].apply(lambda x: (\" \".join(x)).translate(remove_digits_punc), axis=1).values  \n",
    "\n",
    "train_y = train[OUTCOME]\n",
    "valid_y = valid[OUTCOME]\n",
    "test_y  = test[OUTCOME]\n",
    "\n",
    "tfidf = TfidfVectorizer()\n",
    "tr_x  = tfidf.fit_transform(train_x)\n",
    "te_x  = tfidf.transform(test_x)\n",
    "va_x  = tfidf.transform(valid_x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Model:\n",
    "rfc = RandomForestClassifier(n_estimators=1000, max_depth=100, n_jobs=-1)\n",
    "rfc.fit(tr_x, train_y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2-layer feed forward neural network "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This model uses only the components of LACE together with the LACE score:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "lace  = Input(shape=(5,))\n",
    "dense = Dense(50, activation='tanh')(lace)\n",
    "out   = Dense(1, activation='sigmoid')(dense)\n",
    "\n",
    "model = Model(inputs=lace, outputs=out)\n",
    "model.compile(loss='binary_crossentropy', optimizer=\"nadam\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model.fit(train[[\"LOS_Transformed\", \"AdmittedViaER\", \"Charlson_Transformed\", \"PrevERVisits_Transformed\", \"LACE\"]].values, \n",
    "           train_y,\n",
    "           class_weight={0:1, 1:10}, \n",
    "           epochs=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Logistic regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model = logit(formula = OUTCOME + \" ~ (LOS_Transformed + AdmittedViaER + Charlson_Transformed + PrevERVisits_Transformed + LACE)\", \n",
    "              data = train\n",
    "        ).fit(maxiter = 1000, method = 'lbfgs')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}