'''
input: 370K data
1. ClinicalTrialGov/NCTxxxx/xxxxxx.xml & all_xml
1. data/diseases.csv
2. data/drug2smiles.pkl
output: data/raw_data.csv
processing:
0.1 Interventional: 273k data (348k total, e.g., observatorial, surgery, )
0.2 intervention_type == Drug (drug not empty)
0.3 drop_set 96k data (273k), (we don't use drop_set to filter out)
0.4 -1 -> 0 based on "why_stop"
0.5 filter out -1(invalid)
1. disease -> icd
2. drug -> smiles
3. inclusive / exclusive criteria ---- to do
requires ~10 minutes.
'''
##### standard library
import os, csv, pickle
from xml.dom import minidom
from xml.etree import ElementTree as ET
from collections import defaultdict
from time import time
import re
from tqdm import tqdm
from utils import get_path_of_all_xml_file, walkData
drop_set = ['Active, not recruiting', 'Enrolling by invitation', 'No longer available',
'Not yet recruiting', 'Recruiting', 'Temporarily not available', 'Unknown status']
'''
14 overall_status
Active, not recruiting
Approved for marketing
Available
Completed
Enrolling by invitation
No longer available
Not yet recruiting
Recruiting
Suspended
Temporarily not available
Terminated
Unknown status
Withdrawn
Withheld
'''
def load_disease2icd():
disease2icd = dict()
with open('data/diseases.csv', 'r') as csvfile:
rows = list(csv.reader(csvfile, delimiter = ','))[1:]
for row in rows:
disease = row[0]
icd = row[1]
disease2icd[disease] = icd
return disease2icd
def nctid2label_dict():
nctid2outcome = dict()
nctid2label = dict()
with open("IQVIA/outcome2label.txt", 'r') as fin:
lines = fin.readlines()
outcome2label = {line.split('\t')[0]:int(line.strip().split('\t')[1]) for line in lines}
with open("IQVIA/trial_outcomes_v1.csv", 'r') as csvfile:
csvreader = list(csv.reader(csvfile))[1:]
nctid2outcome = {row[0]:row[1] for row in csvreader}
for nctid,outcome in nctid2outcome.items():
nctid2label[nctid] = outcome2label[outcome]
return nctid2label
nctid2label = nctid2label_dict()
def root2outcome(root):
result_list = []
walkData(root, prefix = '', result_list = result_list)
filter_func = lambda x:'p_value' in x[0]
outcome_list = list(filter(filter_func, result_list))
if len(outcome_list)==0:
return None
outcome = outcome_list[0][1]
if outcome[0]=='<':
return 1
if outcome[0]=='>':
return 0
if outcome[0]=='=':
outcome = outcome[1:]
try:
label = float(outcome)
if label < 0.05:
return 1
else:
return 0
except:
return None
def xml_file_2_tuple(xml_file):
tree = ET.parse(xml_file)
root = tree.getroot()
nctid = root.find('id_info').find('nct_id').text ### nctid: 'NCT00000102'
study_type = root.find('study_type').text
if study_type != 'Interventional':
return ("non-Interventional",)
interventions = [i for i in root.findall('intervention')]
drug_interventions = [i.find('intervention_name').text for i in interventions \
if i.find('intervention_type').text=='Drug']
# or i.find('intervention_type').text=='Biological']
if len(drug_interventions)==0:
return ("Biological",)
try:
status = root.find('overall_status').text
except:
status = ''
# if status in drop_set:
# return (None,) ### invalid
try:
why_stop = root.find('why_stopped').text
except:
why_stop = ''
##### p-value
# label = root2outcome(root) ######## p-value
# label = -1 if label is None else label
##### IQVIA internal data
if nctid not in nctid2label:
label = -1
else:
label = nctid2label[nctid]
# if nctid == "NCT00924001":
# print(nctid, label)
# exit()
try:
phase = root.find('phase').text
# print("phase\n\t\t", phase)
except:
phase = ''
conditions = [i.text for i in root.findall('condition')]
try:
criteria = root.find('eligibility').find('criteria').find('textblock').text
# print("criteria\n\t\t", criteria)
except:
criteria = ''
#if criteria != '':
# assert "Inclusion Criteria:" in criteria
# assert "Exclusion Criteria:" in criteria
# title = root.find('brief_title').text
# try:
# summary = root.find('brief_summary').text
# # print("summary\n\t\t", summary)
# except:
# summary = ''
conditions = [i.lower() for i in conditions]
drugs = [i.lower() for i in drug_interventions]
return nctid, status.lower(), why_stop.lower(), label, phase.lower(), conditions, drugs, criteria
# return nctid, status.lower(), why_stop.lower(), label, phase.lower(), conditions, drugs, title, criteria, summary
def process_all():
from raw_data_to_feature import load_drug2smiles_pkl, drug_hit_smiles
### input
drug2smiles = load_drug2smiles_pkl()
disease2icd = load_disease2icd()
input_file_lst = get_path_of_all_xml_file()
### output
output_file = 'data/raw_data.csv'
# iqvia_disease2icd, public_disease2icd = load_disease2icd_pkl()
# iqvia_disease2diseaseset = disease_dict_reorganize(iqvia_disease2icd)
# disease2icd = public_disease2icd
# disease2diseaseset = disease_dict_reorganize(public_disease2icd)
t1 = time()
disease_hit, disease_all, drug_hit, drug_all = 0,0,0,0 ### disease hit icd && drug hit smiles
# fieldname = ['nctid', 'status', 'why_stop', 'label', 'phase',
# 'diseases', 'icdcodes', 'drugs', 'smiless',
# 'title', 'criteria', 'summary']
fieldname = ['nctid', 'status', 'why_stop', 'label', 'phase',
'diseases', 'icdcodes', 'drugs', 'smiless',
'criteria']
num_noninterventional, num_biologics = 0, 0,
num_nodrug = 0
num_nolabel = 0
num_nodisease = 0
with open(output_file, 'w') as csvfile:
writer = csv.DictWriter(csvfile, fieldnames=fieldname)
writer.writeheader()
data_count = 0
for file in tqdm(input_file_lst[:]):
result = xml_file_2_tuple(file)
## 0.1 & 0.2
if len(result)==1 and result[0] == 'non-Interventional':
num_noninterventional += 1
continue
elif len(result)==1 and result[0]== 'Biological':
num_biologics += 1
continue
nctid, status, why_stop, label, phase, conditions, drugs, criteria = result
# nctid, status, why_stop, label, phase, conditions, drugs, title, criteria, summary = result
## 0.4
if (label == -1) and ('lack of efficacy' in why_stop or 'efficacy concern' in why_stop or \
'accrual' in why_stop):
label = 0
## 0.5
if label == -1:
num_nolabel += 1
continue
## 1. disease -> icd
icdcode_lst = []
for disease in conditions:
icdcode = disease2icd[disease] if disease in disease2icd else None
icdcode_lst.append(icdcode)
## 2. drug -> smiles
smiles_lst = []
print("drugs ", drugs)
for drug in drugs:
# drug_all += 1
smiles = drug_hit_smiles(drug, drug2smiles)
if smiles is not None:
# drug_hit += 1
smiles_lst.append(smiles)
else:
print("unfounded drug: ", drug)
if smiles_lst == []:
num_nodrug += 1
continue
icdcode_lst = list(filter(lambda x:x!='None' and x!=None, icdcode_lst))
if icdcode_lst == []:
num_nodisease += 1
continue
data_count += 1
writer.writerow({'nctid':nctid, \
'status': status, \
'why_stop': why_stop, \
'label':label, \
'phase':phase, \
'diseases':conditions, \
'icdcodes': icdcode_lst, \
'drugs':drugs, \
'smiless': smiles_lst, \
'criteria':criteria, })
t2 = time()
# print("disease hit icdcode", disease_hit, "disease all", disease_all, "\n drug hit smiles", drug_hit, "drug all", drug_all)
print(str(int((t2-t1)/60)) + " minutes. " + str(data_count) + " data samples. ")
print("number of non-Interventional:", num_noninterventional)
print("number of Biological:", num_biologics)
print("number of non-label:", num_nolabel)
print("number of non-drug", num_nodrug)
print("number of non-disease", num_nodisease)
return
## write csv file
if __name__ == "__main__":
process_all()
