#!/usr/bin/python !/usr/bin/env python
# -*- coding: utf-8 -*
# Functions to extract knowledge from medical text. Everything related to
# extraction needed for the knowledge base. Also, some wrappers for SemRep,
# MetaMap and Reverb. Contains some enrichment routines for utilizing UTS
# services.
import json
import subprocess
import urllib2
import pymongo
import numpy as np
from nltk.tokenize import sent_tokenize
from config import settings
from pymetamap import MetaMap
from utilities import time_log, get_concept_from_cui, get_concept_from_source
from itertools import product
from multiprocessing import cpu_count, Pool
from unidecode import unidecode
def metamap_wrapper(text):
"""
Function-wrapper for metamap binary. Extracts concepts
found in text.
!!!! REMEMBER TO START THE METAMAP TAGGER AND
WordSense DISAMBIGUATION SERVER !!!!
Input:
- text: str,
a piece of text or sentence
Output:
- a dictionary with key sents and values
a list of the concepts found
"""
# Tokenize into sentences
sents = sent_tokenize(text)
# Load Metamap Instance
mm = MetaMap.get_instance(settings['load']['path']['metamap'])
concepts, errors = mm.extract_concepts(sents, range(len(sents)))
# Keep the sentence ids
ids = np.array([int(concept[0]) for concept in concepts])
sentences = []
for i in xrange(len(sents)):
tmp = {'sent_id': i+1, 'entities': [], 'relations': []}
# Wanted concepts according to sentence
wanted = np.where(ids == i)[0].tolist()
for w_ind in wanted:
w_conc = concepts[w_ind]
if hasattr(w_conc, 'cui'):
tmp_conc = {'label': w_conc.preferred_name, 'cui': w_conc.cui,
'sem_types': w_conc.semtypes, 'score': w_conc.score}
tmp['entities'].append(tmp_conc)
sentences.append(tmp)
if errors:
time_log('Errors with extracting concepts!')
time_log(errors)
return {'sents': sentences, 'sent_text':text}
def runProcess(exe, working_dir):
"""
Function that opens a command line and runs a command.
Captures the output and returns.
Input:
- exe: str,
string of the command to be run. ! REMEMBER TO ESCAPE CHARS!
- working_dir: str,
directory where the cmd should be executed
Output:
- lines: list,
list of strings generated from the command
"""
p = subprocess.Popen(exe, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, cwd=working_dir, shell=True)
lines = p.stdout.readlines()
return lines
def stopw_removal(inp, stop):
"""
Stopwords removal in line of text.
Input:
- inp: str,
string of the text input
- stop: list,
list of stop-words to be removed
"""
# Final string to be returned
final = ''
for w in inp.lower().split():
if w not in stop:
final += w + ' '
# Remove last whitespace that was added ' '
final = final[:-1]
return final
def create_text_batches(text, N=5000, buffer_ = 100):
"""
Function that takes a long string and split it into
batches of approximately length N. The actual length
of each batch differs, as each batch end in the next
dot found in the string after the N chars.
Input:
- text: str,
piece of text to clean
- N: int,
split into strings of 5000 characters each
Output:
- chunks: list,
list containing the string parts
"""
M = len(text)
chunks_num = M // N
if M % N != 0:
chunks_num += 1
chunks = []
end_ind = 0
start_ind = 0
i = 0
while i < chunks_num:
start_ind = end_ind
prob_text = text[start_ind + N: start_ind + N + buffer_]
if '.' in prob_text:
end_ind = start_ind + N + prob_text.index('.')+1
else:
end_ind = start_ind + N
chunks.append(text[start_ind:end_ind])
i += 1
chunks = [ch for ch in chunks if ch]
return chunks
def reverb_wrapper(text, stop=None):
"""
Function-wrapper for ReVerb binary. Extracts relations
found in text.
Input:
- text: str,
a piece of text or sentence
- stop: list,
list of stopwords to remove from the relations
Output:
- total: list,
list of lists. Each inner list contains one relation in the form
[subject, predicate, object]
"""
total = []
for sent in sent_tokenize(text):
cmd = 'echo "' + sent + '"' "| ./reverb -q | tr '\t' '\n' | cat -n"
reverb_dir = settings['load']['path']['reverb']
result = runProcess(cmd, reverb_dir)
# Extract relations from reverb output
result = result[-3:]
result = [row.split('\t')[1].strip('\n') for row in result]
# Remove common stopwords from relations
if stop:
result = [stopw_removal(res, stop) for res in result]
total.append(result)
# Remove empty relations
total = [t for t in total if t]
return total
def cui_to_uri(api_key, cui):
"""
Function to map from cui to uri if possible. Uses biontology portal
Input:
- api_key: str,
api usage key change it in setting.yaml
- cui: str,
cui of the entity we wish to map the uri
Output:
- the uri found in string format or None
"""
REST_URL = "http://data.bioontology.org"
annotations = get_json_with_api(api_key, REST_URL + "/search?include_properties=true&q=" + urllib2.quote(cui))
try:
return annotations['collection'][0]['@id']
except Exception, e:
time_log(Exception)
time_log(e)
return None
def get_json_with_api(api_key, url):
"""
Helper funtion to retrieve a json from a url through urlib2
Input:
- api_key: str,
api usage key change it in setting.yaml
- url: str,
url to curl
Output:
- json-style dictionary with the curl results
"""
opener = urllib2.build_opener()
opener.addheaders = [('Authorization', 'apikey token=' + api_key)]
return json.loads(opener.open(url).read())
def threshold_concepts(concepts, hard_num=3, score=None):
"""
Thresholding concepts from metamap to keep only the most probable ones.
Currently supporting thresholding on the first-N (hard_num) or based on
the concept score.
Input:
- concepts: list,
list of Metamap Class concepts
- hard_num: int,
the first-N concepts to keep, if this thresholidng is selected
- score: float,
lowest accepted concept score, if this thresholidng is selected
"""
if hard_num:
if hard_num >= len(concepts):
return concepts
elif hard_num < len(concepts):
return concepts[:hard_num]
elif score:
return [c for c in concepts if c.score > score]
else:
return concepts
def get_name_concept(concept):
"""
Get name from the metamap concept. Tries different variations and
returns the name found.
Input:
- concept: Metamap class concept, as generated from mmap_extract
for example
Output:
- name: str,
the name found for this concept
"""
name = ''
if hasattr(concept, 'preferred_name'):
name = concept.preferred_name
elif hasattr(concept, 'long_form') and hasattr(concept, 'short_form'):
name = concept.long_form + '|' + concept.short_form
elif hasattr(concept, 'long_form'):
name = concept.long_form
elif hasattr(concept, 'short_form'):
name = concept.short_form
else:
name = 'NO NAME IN CONCEPT'
return name
def metamap_ents(x):
"""
Function to get entities in usable form.
Exctracts metamap concepts first, thresholds them and
tries to extract names and uris for the concepts to be
more usable.
Input:
- x: str,
sentence to extract entities
Output:
- ents: list,
list of entities found. Each entity is a dictionary with
fields id (no. found in sentence), name if retrieved, cui if
available and uri if found
"""
# API KEY to biontology mapping from cui to uri
API_KEY = settings['apis']['biont']
concepts = mmap_extract(x)
concepts = threshold_concepts(concepts)
ents = []
for i, concept in enumerate(concepts):
ent = {}
ent['ent_id'] = i
ent['name'] = get_name_concept(concept)
if hasattr(concept, 'cui'):
ent['cui'] = concept.cui
ent['uri'] = cui_to_uri(API_KEY, ent['cui'])
else:
ent['cui'] = None
ent['uri'] = None
ents.append(ent)
return ents
def extract_entities(text, json_={}):
"""
Extract entities from a given text using metamap and
generate a json, preserving infro regarding the sentence
of each entity that was found. For the time being, we preserve
both concepts and the entities related to them
Input:
- text: str,
a piece of text or sentence
- json_: dic,
sometimes the json to be returned is given to us to be enriched
Defaults to an empty json_
Output:
- json_: dic,
json with fields text, sents, concepts and entities
containg the final results
"""
json_['text'] = text
# Tokenize the text
sents = sent_tokenize(text)
json_['sents'] = [{'sent_id': i, 'sent_text': sent} for i, sent in enumerate(sents)]
json_['concepts'], _ = mmap_extract(text)
json_['entities'] = {}
for i, sent in enumerate(json_['sents']):
ents = metamap_ents(sent)
json_['entities'][sent['sent_id']] = ents
return json_
def extract_metamap(json_, key):
"""
Task function to parse and extract concepts from json_ style dic, using
the MetaMap binary.
Input:
- json_ : dic,
json-style dictionary generated from the Parse object related
to the specific type of input
- key : str,
string denoting the type of medical text to read from. Used to
find the correct paragraph in the settings.yaml file.
Output:
- json_ : dic,
the previous json-style dictionary enriched with medical concepts
"""
# outerfield for the documents in json
docfield = settings['out']['json']['itemfield']
# textfield to read text from
textfield = settings['out']['json']['json_text_field']
N = len(json_[docfield])
for i, doc in enumerate(json_[docfield]):
text = clean_text(doc[textfield])
if len(text) > 5000:
chunks = create_text_batches(text)
results = {'text': text, 'sents': []}
sent_id = 0
for chunk in chunks:
tmp = metamap_wrapper(chunk)
for sent in tmp['sents']:
sent['sent_id'] = sent_id
sent_id += 1
results['sents'].append(sent)
else:
results = metamap_wrapper(text)
json_[docfield][i].update(results)
proc = int(i/float(N)*100)
if proc % 10 == 0 and proc > 0:
time_log('We are at %d/%d documents -- %0.2f %%' % (i, N, proc))
return json_
def enrich_with_triples(results, subject, pred='MENTIONED_IN'):
"""
Enrich with rdf triples a json dictionary in the form of:
entity-URI -- MENTIONED_IN -- 'Text 'Title'. Only entities with
uri's are considered.
Input:
- results: dic,
json-style dictionary genereated from the extract_entities function
- subject: str,
the name of the text document in which the entities are mentioned
- pred: str,
the predicate to be used as a link between the uri and the title
Output:
- results: dic,
the same dictionary with one more
"""
triples = []
for sent_key, ents in results['entities'].iteritems():
for ent in ents:
if ent['uri']:
triples.append({'subj': ent['uri'], 'pred': pred, 'obj': subject})
results['triples'] = triples
return results
def force_to_unicode(text):
"If text is unicode, it is returned as is. If it's str, convert it to Unicode using UTF-8 encoding"
return text if isinstance(text, unicode) else text.decode('utf8', 'ignore')
def toAscii_wrapper(text):
"""
Function wrapper for Lexical Tool toAscii:
https://lexsrv3.nlm.nih.gov/LexSysGroup/Projects/lvg/current/docs/userDoc/tools/toAscii.html
Converts input to ascii ready for SemRep
Input:
- text: str,
a piece of text or sentence'
Output:
- text: str,
the same text with changes
"""
text = clean_text(text)
#text = repr(text)
cmd = 'echo "' + text + '" | ./toAscii'
toAscii_dir = settings['load']['path']['toAscii']
lines = runProcess(cmd, toAscii_dir)
return lines[0]
def semrep_wrapper(text):
"""
Function wrapper for SemRep binary. It is called with flags
-F only and changing this will cause this parsing to fail, cause
the resulting lines won't have the same structure.
Input:
- text: str,
a piece of text or sentence
Output:
- results: dic,
jston-style dictionary with fields text and sents. Each
sentence has entities and relations found in it. Each entity and
each relation has attributes denoted in the corresponding
mappings dictionary.
"""
# Exec the binary
# THIS SHOULD FIX ENCODING PROBLEMS???
text = clean_text(text)
utf8 = force_to_unicode(text)
text = unidecode(utf8)
# text = toAscii_wrapper(text)
# THIS IS NEEDED FOR ANY ARTIFACTS!
text = repr(text)
cmd = "echo " + text + " | ./semrep.v1.7 -L 2015 -Z 2015AA -F"
#print cmd
semrep_dir = settings['load']['path']['semrep']
lines = runProcess(cmd, semrep_dir)
#print(lines)
# mapping of line elements to fields
mappings = {
"text": {
"sent_id": 4,
"sent_text": 6
},
"entity": {
'cuid': 6,
'label': 7,
'sem_types': 8,
'score': 15
},
"relation": {
'subject__cui': 8,
'subject__label': 9,
'subject__sem_types': 10,
'subject__sem_type': 11,
'subject__score': 18,
'predicate__type': 21,
'predicate': 22,
'negation': 23,
'object__cui': 28,
'object__label': 29,
'object__sem_types': 30,
'object__sem_type': 31,
'object__score': 38,
}
}
results = {'sents': [], 'text': text}
for line in lines:
# If Sentence
if line.startswith('SE'):
##### DEPRECATED AS IN CLEAN TEXT WE REMOVE TABS FROM TEXT #######
# Temporary workaround to remove read |-delimited semrep output
# Without mixing up tabs contained in the text
# line = line.replace('\|', '!@#$')
# elements = line.split('|')
# elements = [el.replace('!@#$', '\|') for el in elements]
######################### DEPRECATED ###########################
elements = line.split('|')
# New sentence that was processed
if elements[5] == 'text':
tmp = {"entities": [], "relations": []}
for key, ind in mappings['text'].iteritems():
tmp[key] = elements[ind]
results['sents'].append(tmp)
# A line containing entity info
if elements[5] == 'entity':
tmp = {}
for key, ind in mappings['entity'].iteritems():
if key == 'sem_types':
tmp[key] = elements[ind].split(',')
tmp[key] = elements[ind]
results['sents'][-1]['entities'].append(tmp)
# A line containing relation info
if elements[5] == 'relation':
tmp = {}
for key, ind in mappings['relation'].iteritems():
if 'sem_types' in key:
tmp[key] = elements[ind].split(',')
else:
tmp[key] = elements[ind]
results['sents'][-1]['relations'].append(tmp)
return results
def clean_text(text):
"""
Escape specific characters for command line call of SemRep. This
could be updated in the future to more sophisticated transformations.
Input:
- text: str,
piece of text to clean
Output:
- text: str,
the same text with cmd escaped parenthesis and removing '
"""
replace_chars = [('(', ' '), (')', ' '), ("'", ' '), ('\n', " "), ('\t', ' '), (';', " "),
("}", " "), ("{", " "), ("|", " "), ("&", " "), ("/", ' ')]
for unw_pair in replace_chars:
text = text.replace(unw_pair[0], unw_pair[1])
text = ' '.join(text.split())
return text
def extract_semrep(json_, key):
"""
Task function to parse and extract concepts from json_ style dic, using
the SemRep binary.
Input:
- json_ : dic,
json-style dictionary generated from the Parse object related
to the specific type of input
- key : str,
string denoting the type of medical text to read from. Used to
find the correct paragraph in the settings.yaml file.
Output:
- json_ : dic,
the previous json-style dictionary enriched with medical concepts
"""
# outerfield for the documents in json
if key == 'mongo':
key = 'json'
docfield = settings['out']['json']['itemfield']
# textfield to read text from
textfield = settings['out']['json']['json_text_field']
N = len(json_[docfield])
for i, doc in enumerate(json_[docfield]):
print doc['id']
text = doc[textfield]
if len(text) > 5000:
chunks = create_text_batches(text)
results = {'text': text, 'sents': []}
sent_id = 0
c = 0
for chunk in chunks:
c += 1
tmp = semrep_wrapper(chunk)
for sent in tmp['sents']:
sent['sent_id'] = sent_id
sent_id += 1
results['sents'].append(sent)
else:
results = semrep_wrapper(text)
json_[docfield][i].update(results)
proc = int(i/float(N)*100)
if proc % 10 == 0 and proc > 0:
time_log('We are at %d/%d documents -- %0.2f %%' % (i, N, proc))
return json_
def extract_semrep_parallel(json_, key):
"""
Task function to parse and extract concepts from json_ style dic, using
the SemRep binary. It uses multiprocessing for efficiency.
Input:
- json_ : dic,
json-style dictionary generated from the Parse object related
to the specific type of input
- key : str,
string denoting the type of medical text to read from. Used to
find the correct paragraph in the settings.yaml file.
Output:
- json_ : dic,
the previous json-style dictionary enriched with medical concepts
"""
# outerfield for the documents in json
docfield = settings['out']['json']['itemfield']
N = len(json_[docfield])
try:
N_THREADS = int(settings['num_cores'])
except:
N_THREADS = cpu_count()
batches = chunk_document_collection(json_[docfield], N_THREADS)
len_col = " | ".join([str(len(b)) for b in batches])
time_log('Will break the collection into batches of: %s documents!' % len_col)
batches = [{docfield: batch} for batch in batches]
data = zip(batches, [key for batch in batches])
pool = Pool(N_THREADS, maxtasksperchild=1)
res = pool.map(semrep_parallel_worker, data)
pool.close()
pool.join()
del pool
tmp = {docfield: []}
for batch_res in res:
tmp[docfield].extend(batch_res[docfield])
for i, sub_doc in enumerate(json_[docfield]):
for sub_doc_new in tmp[docfield]:
if sub_doc_new['id'] == sub_doc['id']:
json_[docfield][i].update(sub_doc_new)
break
time_log('Completed multiprocessing extraction!')
return json_
def chunk_document_collection(seq, num):
"""
Helper function to break a collection of N = len(seq) documents
to num batches.
Input:
- seq: list,
a list of documents
- num: int,
number of batches to be broken into. This will usually be
equal to the number of cores available
Output:
- out: list,
a list of lists. Each sublist contains the batch-collection
of documents to be used.
"""
avg = len(seq) / float(num)
out = []
last = 0.0
while last < len(seq):
out.append(seq[int(last):int(last + avg)])
last += avg
return out
def semrep_parallel_worker((json_, key)):
"""
Just a worker interface for the different SemRep
executions.
Input:
- json_ : dic,
json-style dictionary generated from the Parse object related
to the specific type of input
- key : str,
string denoting the type of medical text to read from. Used to
find the correct paragraph in the settings.yaml file.
Output:
- res : dic,
the previous json-style dictionary enriched with medical concepts
"""
res = extract_semrep(json_, key)
return res
def get_concepts_from_edges_parallel(json_, key):
"""
Same work as the get_concepts_from_edges_paralle. It uses multiprocessing
for efficiency.
Input:
- json: dict,
json-style dictionary with a field containing
relations
- key : str,
string denoting the type of medical text to read from. Used to
find the correct paragraph in the settings.yaml file.
Output:
- json: dict,
the updated json-style dictionary where the relations
in the list have been updated and each subject-object has been
mapped to the according
"""
outfield = settings['load'][key]['itemfield']
N = len(json_[outfield])
try:
N_THREADS = int(settings['num_cores'])
except:
N_THREADS = cpu_count()
batches = chunk_document_collection(json_[outfield], N_THREADS)
len_col = " | ".join([str(len(b)) for b in batches])
time_log('Will break the edges into batches of: %s documents!' % len_col)
batches = [{outfield: batch} for batch in batches]
data = zip(batches, [key for batch in batches])
pool = Pool(N_THREADS, maxtasksperchild=1)
res = pool.map(edges_parallel_worker, data)
pool.close()
pool.join()
del pool
json_ = {outfield: []}
for batch_res in res:
json_[outfield].extend(batch_res[outfield])
time_log('Completed multiprocessing extraction!')
return json_
def edges_parallel_worker((json_, key)):
"""
Just a worker interface for the parallel enrichment
executions.
Input:
- json_ : dic,
json-style dictionary generated from the Parse object related
to the specific type of input
- key : str,
string denoting the type of medical text to read from. Used to
find the correct paragraph in the settings.yaml file.
Output:
- res : dic,
expected outcome of get_concepts_from_edges
"""
res = get_concepts_from_edges(json_, key)
return res
def get_concepts_from_edges(json_, key):
"""
Get concept-specific info related to an entity from a list
containing relations. Each subject-object in the relations
list is expressed in a another data source(MESH, DRUGBANK etc)
and their unique identifier is provided. Also, articles and new
kinde of sub-obj are handled.
Input:
- json: dict,
json-style dictionary with a field containing
relations
- key : str,
string denoting the type of medical text to read from. Used to
find the correct paragraph in the settings.yaml file.
Output:
- json: dict,
the updated json-style dictionary where the relations
in the list have been updated and each subject-object has been
mapped to the according
"""
# docfield containing list of elements containing the relations
outfield = settings['load'][key]['itemfield']
# field containing the type of the node for the subject
sub_type = settings['load'][key]['sub_type']
# field containing the source of the node for the subject
sub_source = settings['load'][key]['sub_source']
# field containing the type of the node for the object
obj_type = settings['load'][key]['obj_type']
# field containing the source of the node for the object
obj_source = settings['load'][key]['obj_source']
new_relations = []
uri = settings['load']['mongo']['uri']
db_name = settings['load']['mongo']['db']
collection_name = settings['load']['mongo']['cache_collection']
client = pymongo.MongoClient(uri)
db = client[db_name]
collection = db[collection_name]
cur = collection.find({})
cache = {}
for item in cur:
cache[item['key']] = item['value']
N = len(json_[outfield])
for ii, triple in enumerate(json_[outfield]):
print triple
try:
if sub_source == 'UMLS':
if not(triple['s'] in cache):
ent = get_concept_from_cui(triple['s'])
cache[triple['s']] = ent
collection.insert_one({'key':triple['s'],'value':ent})
print 'INSERTED in UMLS %s' % triple['s']
else:
ent = cache[triple['s']]
if (type(ent['sem_types']) == list and len(ent['sem_types']) > 1):
sem_types = ';'.join(ent['sem_types'])
elif (',' in ent['sem_types']):
sem_types = ';'.join(ent['sem_types'].split(','))
else:
sem_types = ent['sem_types']
triple_subj = [{'id:ID': ent['cuid'],
'label': ent['label'],
'sem_types:string[]': sem_types}]
elif (sub_source == 'PMC') or (sub_source == 'TEXT') or (sub_source == 'None'):
triple_subj = [{'id:ID': triple['s']}]
else:
if not(triple['s'] in cache):
ents = get_concept_from_source(triple['s'], sub_source)
cache[triple['s']] = ents
collection.insert_one({'key':triple['s'],'value':ents})
print 'INSERTED in other %s' % triple['s']
else:
ents = cache[triple['s']]
triple_subj = []
for ent in ents:
if (type(ent['sem_types']) == list and len(ent['sem_types']) > 1):
sem_types = ';'.join(ent['sem_types'])
elif (',' in ent['sem_types']):
sem_types = ';'.join(ent['sem_types'].split(','))
else:
sem_types = ent['sem_types']
triple_subj.append({'id:ID': ent['cuid'],
'label': ent['label'],
'sem_types:string[]': sem_types})
if obj_source == 'UMLS':
if not(triple['o'] in cache):
ent = get_concept_from_cui(triple['o'])
cache[triple['o']] = ent
collection.insert_one({'key':triple['o'],'value':ent})
print 'INSERTED in UMLS %s' % triple['o']
else:
ent = cache[triple['o']]
if (type(ent['sem_types']) == list and len(ent['sem_types']) > 1):
sem_types = ';'.join(ent['sem_types'])
elif (',' in ent['sem_types']):
sem_types = ';'.join(ent['sem_types'].split(','))
else:
sem_types = ent['sem_types']
triple_obj = [{'id:ID': ent['cuid'],
'label': ent['label'],
'sem_types:string[]': sem_types}]
elif (obj_source == 'PMC') or (obj_source == 'TEXT') or (obj_source == 'None'):
triple_obj = [{'id:ID': triple['o']}]
else:
if not(triple['o'] in cache):
ents = get_concept_from_source(triple['o'], obj_source)
cache[triple['o']] = ents
collection.insert_one({'key':triple['o'],'value':ents})
print 'INSERTED in other %s' % triple['o']
else:
ents = cache[triple['o']]
triple_obj = []
for ent in ents:
if (type(ent['sem_types']) == list and len(ent['sem_types']) > 1):
sem_types = ';'.join(ent['sem_types'])
elif (',' in ent['sem_types']):
sem_types = ';'.join(ent['sem_types'].split(','))
else:
sem_types = ent['sem_types']
triple_obj.append({'id:ID': ent['cuid'],
'label': ent['label'],
'sem_types:string[]': sem_types})
combs = product(triple_subj, triple_obj)
for comb in combs:
new_relations.append({'s':comb[0], 'p':triple['p'], 'o':comb[1]})
except Exception, e:
time_log(e)
time_log('S: %s | P: %s | O: %s' % (triple['s'],triple['p'],triple['o']))
time_log('Skipped the above edge! Probably due to concept-fetching errors!')
proc = int(ii/float(N)*100)
if proc % 10 == 0 and proc > 0:
time_log('We are at %d/%d edges transformed -- %0.2f %%' % (ii, N, proc))
# if ii % 100 == 0 and ii > 9:
# time_log("Edges Transformation Process: %d -- %0.2f %%" % (ii, 100*ii/float(len(json_[outfield]))))
json_[outfield] = new_relations
return json_
