#!/usr/bin/python !/usr/bin/env python
# -*- coding: utf-8 -*
# Function to extract knowledge from medical text
import json
import os
import py2neo
import csv
import subprocess
import urllib2
import requests
import unicodecsv as csv2
import pandas as pd
from nltk.tokenize import sent_tokenize
from config import settings
def mmap_extract(text):
"""
Function-wrapper for metamap binary. Extracts concepts
found in text.
!!!! REMEMBER TO START THE METAMAP TAGGER AND
WordSense DISAMBIGUATION SERVER !!!!
Input:
- text: str,
a piece of text or sentence
Output:
- concepts: list,
list of metamap concepts extracted
"""
# Tokenize into sentences
sents = sent_tokenize(text)
mm = MetaMap.get_instance(settings['load']['path']['metamap'])
concepts, errors = mm.extract_concepts(sents, range(len(sents)),
word_sense_disambiguation=True)
if errors:
print 'Errors with extracting concepts!'
print errors
return concepts
def runProcess(exe, working_dir):
"""
Function that opens a command line and runs a command.
Captures the output and returns.
Input:
- exe: str,
string of the command to be run. ! REMEMBER TO ESCAPE CHARS!
- working_dir: str,
directory where the cmd should be executed
Output:
- lines: list,
list of strings generated from the command
"""
p = subprocess.Popen(exe, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, cwd=working_dir, shell=True)
lines = p.stdout.readlines()
return lines
def stopw_removal(inp, stop):
"""
Stopwords removal in line of text.
Input:
- inp: str,
string of the text input
- stop: list,
list of stop-words to be removed
"""
# Final string to be returned
final = ''
for w in inp.lower().split():
if w not in stop:
final += w + ' '
# Remove last whitespace that was added ' '
final = final[:-1]
return final
def reverb_wrapper(text, stop=None):
"""
Function-wrapper for ReVerb binary. Extracts relations
found in text.
Input:
- text: str,
a piece of text or sentence
- stop: list,
list of stopwords to remove from the relations
Output:
- total: list,
list of lists. Each inner list contains one relation in the form
[subject, predicate, object]
"""
total = []
for sent in sent_tokenize(text):
cmd = 'echo "' + sent + '"' "| ./reverb -q | tr '\t' '\n' | cat -n"
reverb_dir = settings['load']['path']['reverb']
result = runProcess(cmd, reverb_dir)
# Extract relations from reverb output
result = result[-3:]
result = [row.split('\t')[1].strip('\n') for row in result]
# Remove common stopwords from relations
if stop:
result = [stopw_removal(res, stop) for res in result]
total.append(result)
# Remove empty relations
total = [t for t in total if t]
return total
def cui_to_uri(api_key, cui):
"""
Function to map from cui to uri if possible. Uses biontology portal
Input:
- api_key: str,
api usage key change it in setting.yaml
- cui: str,
cui of the entity we wish to map the uri
Output:
- the uri found in string format or None
"""
REST_URL = "http://data.bioontology.org"
annotations = get_json_with_api(api_key, REST_URL + "/search?include_properties=true&q=" + urllib2.quote(cui))
try:
return annotations['collection'][0]['@id']
except Exception,e:
print Exception
print e
return None
def get_json_with_api(api_key, url):
"""
Helper funtion to retrieve a json from a url through urlib2
Input:
- api_key: str,
api usage key change it in setting.yaml
- url: str,
url to curl
Output:
- json-style dictionary with the curl results
"""
opener = urllib2.build_opener()
opener.addheaders = [('Authorization', 'apikey token=' + api_key)]
return json.loads(opener.open(url).read())
def threshold_concepts(concepts, hard_num=3, score=None):
"""
Thresholding concepts from metamap to keep only the most probable ones.
Currently supporting thresholding on the first-N (hard_num) or based on
the concept score.
Input:
- concepts: list,
list of Metamap Class concepts
- hard_num: int,
the first-N concepts to keep, if this thresholidng is selected
- score: float,
lowest accepted concept score, if this thresholidng is selected
"""
if hard_num:
if hard_num >= len(concepts):
return concepts
elif hard_num < len(concepts):
return concepts[:hard_num]
elif score:
return [c for c in concepts if c.score > score]
else:
return concepts
def get_name_concept(concept):
"""
Get name from the metamap concept. Tries different variations and
returns the name found.
Input:
- concept: Metamap class concept, as generated from mmap_extract
for example
Output:
- name: str,
the name found for this concept
"""
name = ''
if hasattr(concept, 'preferred_name'):
name = concept.preferred_name
elif hasattr(concept, 'long_form') and hasattr(concept, 'short_form'):
name = concept.long_form + '|' + concept.short_form
elif hasattr(concept, 'long_form'):
name = concept.long_form
elif hasattr(concept, 'short_form'):
name = concept.short_form
else:
name = 'NO NAME IN CONCEPT'
return name
def metamap_ents(x):
"""
Function to get entities in usable form.
Exctracts metamap concepts first, thresholds them and
tries to extract names and uris for the concepts to be
more usable.
Input:
- x: str,
sentence to extract entities
Output:
- ents: list,
list of entities found. Each entity is a dictionary with
fields id (no. found in sentence), name if retrieved, cui if
available and uri if found
"""
# API KEY to biontology mapping from cui to uri
API_KEY = settings['apis']['biont']
concepts = mmap_extract(x)
concepts = threshold_concepts(concepts)
ents = []
for i, concept in enumerate(concepts):
ent = {}
ent['ent_id'] = i
ent['name'] = get_name_concept(concept)
if hasattr(concept, 'cui'):
ent['cui'] = concept.cui
ent['uri'] = cui_to_uri(API_KEY, ent['cui'])
else:
ent['cui'] = None
ent['uri'] = None
ents.append(ent)
return ents
def extract_entities(text, json_={}):
"""
Extract entities from a given text using metamap and
generate a json, preserving infro regarding the sentence
of each entity that was found. For the time being, we preserve
both concepts and the entities related to them
Input:
- text: str,
a piece of text or sentence
- json_: dic,
sometimes the json to be returned is given to us to be enriched
Defaults to an empty json_
Output:
- json_: dic,
json with fields text, sents, concepts and entities
containg the final results
"""
json_['text'] = text
# Tokenize the text
sents = sent_tokenize(text)
json_['sents'] = [{'sent_id': i, 'sent_text': sent} for i, sent in enumerate(sents)]
json_['concepts'], _ = mmap_extract(text)
json_['entities'] = {}
for i, sent in enumerate(json_['sents']):
ents = metamap_ents(sent)
json_['entities'][sent['sent_id']] = ents
return json_
def enrich_with_triples(results, subject, pred='MENTIONED_IN'):
"""
Enrich with rdf triples a json dictionary in the form of:
entity-URI -- MENTIONED_IN -- 'Text 'Title'. Only entities with
uri's are considered.
Input:
- results: dic,
json-style dictionary genereated from the extract_entities function
- subject: str,
the name of the text document in which the entities are mentioned
- pred: str,
the predicate to be used as a link between the uri and the title
Output:
- results: dic,
the same dictionary with one more
"""
triples = []
for sent_key, ents in results['entities'].iteritems():
for ent in ents:
if ent['uri']:
triples.append({'subj': ent['uri'], 'pred': pred, 'obj': subject})
results['triples'] = triples
return results
def semrep_wrapper(text):
"""
Function wrapper for SemRep binary. It is called with flags
-F only and changing this will cause this parsing to fail, cause
the resulting lines won't have the same structure.
Input:
- text: str,
a piece of text or sentence
Output:
- results: dic,
jston-style dictionary with fields text and sents. Each
sentence has entities and relations found in it. Each entity and
each relation has attributes denoted in the corresponding
mappings dictionary.
"""
# Exec the binary
cmd = "echo " + text + " | ./semrep.v1.7 -L 2015 -Z 2015AA -F"
semrep_dir = settings['load']['path']['semrep']
lines = runProcess(cmd, semrep_dir)
# mapping of line elements to fields
mappings = {
"text": {
"sent_id": 4,
"sent_text": 6
},
"entity": {
'cuid': 6,
'label': 7,
'sem_types': 8,
'score': 15
},
"relation": {
'subject__cui': 8,
'subject__label': 9,
'subject__sem_types': 10,
'subject__sem_type': 11,
'subject__score': 18,
'predicate__type': 21,
'predicate': 22,
'negation': 23,
'object__cui': 28,
'object__label': 29,
'object__sem_types': 30,
'object__sem_type': 31,
'object__score': 38,
}
}
results = {'sents': [], 'text': text}
for line in lines:
# If Sentence
if line.startswith('SE'):
elements = line.split('|')
# New sentence that was processed
if elements[5] == 'text':
tmp = {"entities": [], "relations": []}
for key, ind in mappings['text'].iteritems():
tmp[key] = elements[ind]
results['sents'].append(tmp)
# A line containing entity info
if elements[5] == 'entity':
tmp = {}
for key, ind in mappings['entity'].iteritems():
if key == 'sem_types':
tmp[key] = elements[ind].split(',')
tmp[key] = elements[ind]
results['sents'][-1]['entities'].append(tmp)
# A line containing relation info
if elements[5] == 'relation':
tmp = {}
for key, ind in mappings['relation'].iteritems():
if 'sem_types' in key:
tmp[key] = elements[ind].split(',')
else:
tmp[key] = elements[ind]
results['sents'][-1]['relations'].append(tmp)
return results
results = extract_entities(text)
results = enrich_with_triples(results, subject='Text Title')
