# coding: utf-8
# In[59]:
import pandas as pd
from sklearn.externals import joblib
import re
from nltk.stem.snowball import SnowballStemmer
from collections import defaultdict
import operator
import numpy as np
import sklearn.feature_extraction.text as text
from sklearn import decomposition
from nltk.stem import PorterStemmer, WordNetLemmatizer
from sklearn.decomposition import PCA
from numpy.linalg import norm
# In[60]:
def load_data():
## Intitializing data paths
base_path = r'D:\ORGANIZATION\UCSD_Life\Work\4. Quarter-3\Subjects\MED 277\Project\DATA\\'
data_file = base_path+"NOTEEVENTS.csv.gz"
## Loading data frames from CSV file
#df = pd.read_csv(data_file, compression='gzip')
#df = df[:10000]
#joblib.dump(df,base_path+'data10.pkl')
## loading data frames from PKL memory
df1 = joblib.load(base_path+'data10.pkl')
df = df1[:50]
## Filtering dataframe for "Discharge summaries" and "TEXT"
df = df.loc[df['CATEGORY'] == 'Discharge summary'] #Extracting only discharge summaries
df_text = df['TEXT']
return df_text
# ## EXTRACT ALL THE TOPICS
# In[61]:
'''Method that processes the entire document string'''
def process_text(txt):
txt1 = re.sub('[\n]'," ",txt)
txt1 = re.sub('[^A-Za-z \.]+', '', txt1)
return txt1
# In[62]:
'''Method that processes the document string not considering separate lines'''
def process(txt):
txt1 = re.sub('[\n]'," ",txt)
txt1 = re.sub('[^A-Za-z ]+', '', txt1)
_wrds = txt1.split()
stemmer = SnowballStemmer("english") ## May use porter stemmer
wrds = [stemmer.stem(wrd) for wrd in _wrds]
return wrds
# In[63]:
'''Method that processes raw string and gets a processes list containing lines'''
def get_processed_sentences(snt_txt):
snt_list = []
for line in snt_txt.split('.'):
line = line.strip()
if len(line.split()) >= 5:
snt_list.append(line)
return snt_list
# In[64]:
'''This method extracts topic from sentence'''
def extract_topic(str_arg, num_topics = 1, num_top_words = 3):
vectorizer = text.CountVectorizer(input='content', analyzer='word', lowercase=True, stop_words='english')
try:
dtm = vectorizer.fit_transform(str_arg.split())
vocab = np.array(vectorizer.get_feature_names())
#clf = decomposition.NMF(n_components=num_topics, random_state=1) ## topic extraction
clf = decomposition.LatentDirichletAllocation(n_components=num_topics, learning_method='online')
clf.fit_transform(dtm)
topic_words = []
for topic in clf.components_:
word_idx = np.argsort(topic)[::-1][0:num_top_words] ##[::-1] reverses the list
topic_words.append([vocab[i] for i in word_idx])
return topic_words
except:
return None
# In[65]:
'''This method extracts topics of each sentence and returns a list'''
def extract_topics_all(doc_string):
#One entry per sentence in list
doc_str = process_text(doc_string)
doc_str = get_processed_sentences(doc_str)
res = []
for i in range (0, len(doc_str)):
snd_str = doc_str[i].lower()
#print("Sending ----------------------------",snd_str,"==========",len(snd_str))
tmp_topic = extract_topic(snd_str, num_topics = 2, num_top_words = 1)
for top in tmp_topic:
for wrd in top:
res.append(wrd)
return res
# In[66]:
'''This function takes a dataframe and returns all the topics in the entire corpus'''
def extract_corpus_topics(arg_df):
all_topics = set()
cnt = 1
for txt in arg_df:
all_topics = all_topics.union(extract_topics_all(txt))
print("Processed ",cnt," records")
cnt += 1
all_topics = list(all_topics)
return all_topics
# ## GET A VECTORIZED REPRESENTATION OF ALL THE TOPICS
# In[67]:
'''data_set = words list per document.
vocabulary = list of all the words present
_vocab = dict of word counts for words in vocabulary'''
def get_vocab_wrd_map(df_text):
data_set = []
vocabulary = []
_vocab = defaultdict(int)
for i in range(0,df_text.size):
txt = process(df_text[i])
data_set.append(txt)
for wrd in txt:
_vocab[wrd] += 1
vocabulary = vocabulary + txt
vocabulary = list(set(vocabulary))
if(i%100 == 0):
print("%5d records processed"%(i))
return data_set, vocabulary, _vocab
# In[68]:
'''vocab = return sorted list of most common words in vocabulary'''
def get_common_vocab(num_arg, vocab):
vocab = sorted(vocab.items(), key=operator.itemgetter(1), reverse=True)
vocab = vocab[:num_arg]
return vocab
# In[69]:
'''Convert vocabulary and most common words to map for faster access'''
def get_vocab_map(vocabulary, vocab):
vocab_map = {}
for i in range(0,len(vocab)):
vocab_map[vocab[i][0]] = i
vocabulary_map = {}
for i in range(0,len(vocabulary)):
vocabulary_map[vocabulary[i]] = i
return vocabulary_map, vocab_map
# In[70]:
def get_embedding(word, data_set, vocab_map, wdw_size):
embedding = [0]*len(vocab_map)
for docs in data_set:
for i in range(wdw_size, len(docs)-wdw_size):
if docs[i] == word:
for j in range(i-wdw_size, i-1):
if docs[j] in vocab_map:
embedding[vocab_map[docs[j]]] += 1
for j in range(i+1, i+wdw_size):
if docs[j] in vocab_map:
embedding[vocab_map[docs[j]]] += 1
total_words = sum(embedding)
if total_words != 0:
embedding[:] = [e/total_words for e in embedding]
return embedding
# In[71]:
def get_embedding_all(all_topics, data_set, vocab_map, wdw_size):
embeddings = []
for i in range(0, len(all_topics)):
embeddings.append(get_embedding(all_topics[i], data_set, vocab_map, wdw_size))
return embeddings
# ## Get similarity function
# In[72]:
def cos_matrix_multiplication(matrix, vector):
"""
Calculating pairwise cosine distance using matrix vector multiplication.
"""
dotted = matrix.dot(vector)
matrix_norms = np.linalg.norm(matrix, axis=1)
vector_norm = np.linalg.norm(vector)
matrix_vector_norms = np.multiply(matrix_norms, vector_norm)
neighbors = np.divide(dotted, matrix_vector_norms)
return neighbors
# In[73]:
def get_most_similar_topics(embd, embeddings, all_topics, num_wrd=10):
sim_top = []
cos_sim = cos_matrix_multiplication(np.array(embeddings), embd)
#closest_match = cos_sim.argsort()[-num_wrd:][::-1] ## This sorts all matches in order
## This just takes 80% and above similar matches
idx = list(np.where(cos_sim > 0.9)[0])
val = list(cos_sim[np.where(cos_sim > 0.9)])
closest_match, list2 = (list(t) for t in zip(*sorted(zip(idx, val), reverse=True)))
closest_match = np.array(closest_match)
for i in range(0, closest_match.shape[0]):
sim_top.append(all_topics[closest_match[i]])
return sim_top
# ## Topic Modelling
# In[74]:
def get_regex_match(regex, str_arg):
srch = re.search(regex,str_arg)
if srch is not None:
return srch.group(0).strip()
else:
return "Not found"
# In[75]:
def extract(key,str_arg):
if key == 'dob':
return get_regex_match('Date of Birth:(.*)] ', str_arg)
elif key == 'a_date':
return get_regex_match('Admission Date:(.*)] ', str_arg)
elif key == 'd_date':
return get_regex_match('Discharge Date:(.*)]\n', str_arg)
elif key == 'sex':
return get_regex_match('Sex:(.*)\n', str_arg)
elif key == 'service':
return get_regex_match('Service:(.*)\n', str_arg)
elif key == 'allergy':
return get_regex_match('Allergies:(.*)\n(.*)\n', str_arg)
elif key == 'attdng':
return get_regex_match('Attending:(.*)]\n', str_arg)
else:
return "I Don't know"
# In[76]:
'''This method extracts topic from sentence'''
def extract_topic(str_arg, num_topics = 1, num_top_words = 3):
vectorizer = text.CountVectorizer(input='content', analyzer='word', lowercase=True, stop_words='english')
dtm = vectorizer.fit_transform(str_arg.split())
vocab = np.array(vectorizer.get_feature_names())
#clf = decomposition.NMF(n_components=num_topics, random_state=1) ## topic extraction
clf = decomposition.LatentDirichletAllocation(n_components=num_topics, learning_method='online')
clf.fit_transform(dtm)
topic_words = []
for topic in clf.components_:
word_idx = np.argsort(topic)[::-1][0:num_top_words] ##[::-1] reverses the list
topic_words.append([vocab[i] for i in word_idx])
return topic_words
# In[77]:
'''This method extracts topics in a question'''
def extract_Q_topic(str_arg):
try:
return extract_topic(str_arg)
except:
return None
## TODO fix later for more comprehensive results
# In[78]:
def get_extract_map(key_wrd):
## A Stemmed mapping for simple extractions
extract_map = {'birth':'dob', 'dob':'dob',
'admiss':'a_date', 'discharg':'d_date',
'sex':'sex', 'gender':'sex', 'servic':'service',
'allergi':'allergy', 'attend':'attdng'}
if key_wrd in extract_map.keys():
return extract_map[key_wrd]
else:
return None
# In[79]:
'''Method that generates the answer for text extraction questions'''
def get_extracted_answer(topic_str, text):
port = PorterStemmer()
for i in range(0, len(topic_str)):
rel_wrd = topic_str[i]
for wrd in rel_wrd:
key = get_extract_map(port.stem(wrd))
if key is not None:
return extract(key, text)
return None
# In[80]:
'''This method extracts topics of each sentence and returns a list'''
def get_topic_mapping(doc_string):
#One entry per sentence in list
doc_str = process_text(doc_string)
doc_str = get_processed_sentences(doc_str)
res = defaultdict(list)
for i in range (0, len(doc_str)):
snd_str = doc_str[i].lower()
#print("Sending ----------------------------",snd_str,"==========",len(snd_str))
tmp_topic = extract_topic(snd_str, num_topics = 2, num_top_words = 1)
for top in tmp_topic:
for wrd in top:
res[wrd].append(doc_str[i])
return res
# In[81]:
def get_direct_answer(topic_str, topic_map):
## Maybe apply lemmatizer here
for i in range(0, len(topic_str)):
rel_wrd = topic_str[i]
for wrd in rel_wrd:
if wrd in topic_map.keys():
return topic_map[wrd]
return None
# In[82]:
def get_answer(topic, topic_map, embedding_short, all_topics, data_set, vocab_map, pca, wdw_size=5):
## Get most similar topics
tpc_embedding = get_embedding(topic, data_set, vocab_map, wdw_size)
tpc_embedding = pca.transform([tpc_embedding])
sim_topics = get_most_similar_topics(tpc_embedding[0], embedding_short, all_topics, num_wrd = len(all_topics))
for topic in sim_topics:
if topic in topic_map.keys():
return topic_map[topic]
return None
# In[83]:
'''This function checks if the user input text is an instruction allowed in chatbot or not'''
def is_instruction_option(str_arg):
if str_arg == "exit" or str_arg == "summary" or str_arg == "reveal":
return True
else:
return False
def print_bot():
print(r" _ _ _")
print(r" | o o |")
print(r" \| = |/")
print(r" -------")
print(r" |||||||")
print(r" // \\")
def print_caption():
print(r" ||\\ || || ||= =||")
print(r" || \\ || || ||= =||")
print(r" || \\ || || ||")
print(r" || \\|| ||_ _ _ ||")
# In[ ]:
if __name__ == "__main__":
print("Loading data ...","\n")
df_text = load_data()
print("Getting Vocabulary ...")
data_set, vocabulary, _vocab = get_vocab_wrd_map(df_text)
print("Creating context ...")
vocab = get_common_vocab(1000, _vocab)
vocabulary_map, vocab_map = get_vocab_map(vocabulary, vocab)
print("Learning topics ...")
all_topics = extract_corpus_topics(df_text)
print("Getting Embeddings")
embeddings = get_embedding_all(all_topics, data_set, vocab_map, 5)
pca = PCA(n_components=10)
embedding_short = pca.fit_transform(embeddings)
print_caption()
print_bot()
print("Bot:> I am online!")
print("Bot:> Type \"exit\" to switch to end a patient's session")
print("Bot:> Type \"summary\" to view patient's discharge summary")
while(True):
while(True):
try:
pid = int(input("Bot:> What is your Patient Id [0 to "+str(df_text.shape[0]-1)+"?]"))
except:
continue
if pid < 0 or pid > df_text.shape[0]-1:
print("Bot:> Patient Id out or range!")
continue
else:
print("Bot:> Reading Discharge Summary for Patient Id: ",pid)
break
personal_topics = extract_topics_all(df_text[pid])
topic_mapping = get_topic_mapping(df_text[pid])
ques = "random starter"
while(ques != "exit"):
## Read Question
ques = input("Bot:> How can I help ?\nPerson:>")
## Check if it is an instructional question
if is_instruction_option(ques):
if ques == "summary":
print("Bot:> ================= Discharge Summary for Patient Id ",pid,"\n")
print(df_text[pid])
elif ques == "reveal":
print(topic_mapping, topic_mapping.keys())
continue
## Extract Question topic
topic_q = extract_Q_topic(ques)
if topic_q is None:
print("Bot:> I am a specialized NLP bot, please as a more specific question for me!")
continue
ans = get_extracted_answer(topic_q, df_text[pid])
if ans is not None:
print("Bot:> ",ans)
else:
ans = get_direct_answer(topic_q, topic_mapping)
if ans is not None:
print("Bot:> ",ans)
else:
ans = get_answer(topic_q, topic_mapping, embedding_short, all_topics, data_set, vocab_map, pca, 5)
if ans is not None:
print("Bot:> ",ans)
else:
print("Bot:> Sorry but, I have no information on this topic!")
