#AUTHOR: RAHUL VERMA and SPIRO RAZIS
import sys
import re
import pprint
import numpy
from sklearn import svm
from sklearn import linear_model
import time
from random import shuffle
start_time = time.time()
numpy.set_printoptions(threshold=numpy.nan)
def parseTextViaPMCID(textFile, pmcidFeatureList, uniqueWordsDictionary,lim):
if textFile.startswith("beneficial"):
#print("beneficial")
fileType = "beneficial".encode('utf-8')
elif textFile.startswith("harmful"):
#print("harmful")
fileType = "harmful".encode('utf-8')
else:
#print("invalid file name")
sys.exit(2)
limit = 0
entryCount = 0
disease = ""
causeOrTreatment = ""
relation = ""
newEntry = False
with open(textFile, "r") as openedTextFile:
for line in openedTextFile:
if limit < lim:
if line.startswith("pmcid : "): #it's the idNumber
entryCount += 1
newEntry = True
elif line.startswith("sentence: "): #it's a sentence
pass
elif line.startswith("entities: "): #it's the two in a relationship
disease = line[11:line.index(",")].lower().encode('utf-8')
causeOrTreatment = line[(line.index(",")+2):-2].lower().encode('utf-8')
#add disease and cause/treatment to dictionary of unique words/phrases
if disease not in uniqueWordsDictionary:
uniqueWordsDictionary[disease] = {}
if causeOrTreatment not in uniqueWordsDictionary:
uniqueWordsDictionary[causeOrTreatment] = {}
elif line.startswith("offsets : "): #the position of the entities
pass
elif line.startswith("relation: "): #the actual relationship
relation = line[10:-1].lower().encode('utf-8')
else:
if line.startswith("\n") and (newEntry == True):
pmcidFeatureList.append([disease, causeOrTreatment, relation, fileType])
disease = ""
causeOrTreatment = ""
relation = ""
newEntry = False
limit += 1
else:
print("invalid line: %s" %(line))
sys.exit(2)
else: break
return (pmcidFeatureList, entryCount, uniqueWordsDictionary)
def printFeatureWithCellValue(numpyRow, featureRow):
for index, feature in enumerate(featureRow):
print("%s: %d" %(feature, numpyRow[index]))
print("harmfulOrBeneficial: %d" %(numpyRow[-1]))
return
def printFeaturesWithValuesEqualOne(numpyRow, featureRow):
for index, feature in enumerate(featureRow):
if numpyRow[index] == 1:
print("%s: %d" %(feature, numpyRow[index]))
print("harmfulOrBeneficial: %d" %(numpyRow[-1]))
return
def parseEntitiesIntoUnigrams(beneficialFile, harmfulFile, beneficialLimit, harmfulLimit):
beneficialEntry = 0
harmfulEntry = 0
entitiesTrainingDictionary= {}
entityUnigramList = []
beneficialFullEntitiesList = []
harmfulFullEntitiesList = []
sentenceUnigramList = []
beneficialSplitSentences = []
harmfulSplitSentences = []
entityUnigrams = {}
harmfulUnigrams = {}
beneficialUnigrams = {}
sentenceFeatureUnigrams = {}
testArrayForWritingEntries = numpy.empty(shape = (1, 1), dtype = "S128")
#WORKING ON THE UNIGRAMS OF THE TRAINING BENEFICIAL ENTITIES HERE
with open(beneficialFile, "r") as openedBeneficialFile:
for line in openedBeneficialFile:
if beneficialEntry < beneficialLimit:
if line.startswith("entities: "):
#individual entities
disease = line[11:line.index(",")].lower().encode('utf-8')
causeOrTreatment = line[(line.index(",")+2):-2].lower().encode('utf-8')
if disease not in entitiesTrainingDictionary:
entitiesTrainingDictionary[disease] = {}
if causeOrTreatment not in entitiesTrainingDictionary:
entitiesTrainingDictionary[causeOrTreatment] = {}
#unigrams composing the entities
entityUnigramList = re.split("-|, |\. |\/| ", line[11:-2].lower())
for entry in entityUnigramList:
if (entry != "") and (entry not in entityUnigrams):
entityUnigrams[entry] = {}
beneficialEntry += 1
else: break
#WORKING ON THE TRAINING HARMFUL ENTITIES HERE
with open(harmfulFile, "r") as openedHarmfulFile:
for line in openedHarmfulFile:
if harmfulEntry < harmfulLimit:
if line.startswith("entities: "):
#individual entities
disease = line[11:line.index(",")].lower().encode('utf-8')
causeOrTreatment = line[(line.index(",")+2):-2].lower().encode('utf-8')
if disease not in entitiesTrainingDictionary:
entitiesTrainingDictionary[disease] = {}
if causeOrTreatment not in entitiesTrainingDictionary:
entitiesTrainingDictionary[causeOrTreatment] = {}
entityUnigramList = re.split("-|, |\. |\/| ", line[11:-2].lower())
for entry in entityUnigramList:
if (entry != "") and (entry not in entityUnigrams):
entityUnigrams[entry] = {}
harmfulEntry += 1
else: break
beneficialEntry = 0
mostRecentPMCID = ""
with open(beneficialFile, "r") as openedBeneficialFile:
for line in openedBeneficialFile:
if line.startswith("pmcid : "): #it's the pmcid line
mostRecentPMCID = line[11:-1]
elif line.startswith("sentence: "):
sentenceUnigramList = re.split("\—|\-|\, |\.|\/|\(|\)|\'|\"|\[|\]|\ |\“|\”|\,|\d|\<|\>|\:|\$|\%|\*|\′", line[10:-2].lower())
beneficialSplitSentences.append(sentenceUnigramList)
if beneficialEntry < beneficialLimit:
for word in sentenceUnigramList:
if (word != "") and (word not in entityUnigrams):
if word not in sentenceFeatureUnigrams:
try:
testArrayForWritingEntries[0,0] = word
sentenceFeatureUnigrams[word] = {}
sentenceFeatureUnigrams[word]["beneficial"] = {}
sentenceFeatureUnigrams[word]["beneficial"]["pmcid"] = {}
sentenceFeatureUnigrams[word]["beneficial"]["pmcid"][mostRecentPMCID] = {}
sentenceFeatureUnigrams[word]["beneficial"]["count"] = 0
sentenceFeatureUnigrams[word]["harmful"] = {}
sentenceFeatureUnigrams[word]["harmful"]["pmcid"] = {}
sentenceFeatureUnigrams[word]["harmful"]["count"] = 0
except UnicodeEncodeError: pass
else: #it is in the feature unigrams already, so add the
if mostRecentPMCID not in sentenceFeatureUnigrams[word]["beneficial"]["pmcid"]: #and the same pmcid isn't already there
sentenceFeatureUnigrams[word]["beneficial"]["pmcid"][mostRecentPMCID] = {}
beneficialEntry += 1
elif line.startswith("entities: "):
#individual entities
disease = line[11:line.index(",")].lower().encode('utf-8')
causeOrTreatment = line[(line.index(",")+2):-2].lower().encode('utf-8')
beneficialFullEntitiesList.append([disease, causeOrTreatment])
else: pass
harmfulEntry = 0
mostRecentPMCID = ""
with open(harmfulFile, "r") as openedHarmfulFile:
for line in openedHarmfulFile:
if line.startswith("pmcid : "): #it's the pmcid line
mostRecentPMCID = line[11:-1]
elif line.startswith("sentence: "):
sentenceUnigramList = re.split("\—|\-|\, |\.|\/|\(|\)|\'|\"|\[|\]|\ |\“|\”|\,|\d|\<|\>|\:|\$|\%|\*|\′", line[10:-2].lower())
harmfulSplitSentences.append(sentenceUnigramList)
if harmfulEntry < harmfulLimit:
for word in sentenceUnigramList:
if (word != "") and (word not in entityUnigrams):
if word not in sentenceFeatureUnigrams:
try:
testArrayForWritingEntries[0,0] = word
sentenceFeatureUnigrams[word] = {}
sentenceFeatureUnigrams[word]["beneficial"] = {}
sentenceFeatureUnigrams[word]["beneficial"]["pmcid"] = {}
sentenceFeatureUnigrams[word]["beneficial"]["count"] = 0
sentenceFeatureUnigrams[word]["harmful"] = {}
sentenceFeatureUnigrams[word]["harmful"]["pmcid"] = {}
sentenceFeatureUnigrams[word]["harmful"]["pmcid"][mostRecentPMCID] = {}
sentenceFeatureUnigrams[word]["harmful"]["count"] = 0
except UnicodeEncodeError: pass
else:
if mostRecentPMCID not in sentenceFeatureUnigrams[word]["harmful"]["pmcid"]: #and the same pmcid isn't already there
sentenceFeatureUnigrams[word]["harmful"]["pmcid"][mostRecentPMCID] = {}
harmfulEntry += 1
elif line.startswith("entities: "):
disease = line[11:line.index(",")].lower().encode('utf-8')
causeOrTreatment = line[(line.index(",")+2):-2].lower().encode('utf-8')
harmfulFullEntitiesList.append([disease, causeOrTreatment])
else: pass
for word in sentenceFeatureUnigrams:
for benefitHarmfulOrEntity in sentenceFeatureUnigrams[word]:
#start counting!
for pmcid in sentenceFeatureUnigrams[word][benefitHarmfulOrEntity]["pmcid"]:
sentenceFeatureUnigrams[word][benefitHarmfulOrEntity]["count"] += 1
if (sentenceFeatureUnigrams[word]["beneficial"]["count"] > 1) or (sentenceFeatureUnigrams[word]["harmful"]["count"] > 1):
if sentenceFeatureUnigrams[word]["beneficial"]["count"] > (2*sentenceFeatureUnigrams[word]["harmful"]["count"]):
beneficialUnigrams[word] = {}
elif sentenceFeatureUnigrams[word]["harmful"]["count"] > (2*sentenceFeatureUnigrams[word]["beneficial"]["count"]):
harmfulUnigrams[word] = {}
else: pass #the words can't be categorized one way or the other
return (entitiesTrainingDictionary,
beneficialUnigrams, harmfulUnigrams,
beneficialEntry, harmfulEntry,
beneficialSplitSentences, harmfulSplitSentences,
beneficialFullEntitiesList, harmfulFullEntitiesList)
def main(argv):
#Python3 training.py beneficial.txt harmful.txt
if len(argv) != 3:
print("invalid number of arguments")
sys.exit(2)
#two separate lists because don't know how many entries in each, so dividing one list will be difficult
(entitiesTrainingDictionary, beneficialUnigrams, harmfulUnigrams,
beneficialCount, harmfulCount, pmcidBeneficialSentences, pmcidHarmfulSentences,
beneficialFullEntitiesList, harmfulFullEntitiesList) = parseEntitiesIntoUnigrams(argv[1], argv[2], 10356, 9797)
benprec = 10356/beneficialCount
harmprec = 9797/harmfulCount
numFeatures = len(entitiesTrainingDictionary) + len(beneficialUnigrams) + len(harmfulUnigrams) + 1 #plus 1 for harmful or beneficial
uniqueFeaturesArray = numpy.empty(shape = (1, numFeatures), dtype="S128")
#place the dictionary words into the array
for index, feature in enumerate(entitiesTrainingDictionary):
uniqueFeaturesArray[0, index] = feature
finalColumn = len(entitiesTrainingDictionary)
for index, feature in enumerate(beneficialUnigrams):
currentColumn = index + finalColumn
uniqueFeaturesArray[0, currentColumn] = feature
finalColumn += len(beneficialUnigrams)
for index, feature in enumerate(harmfulUnigrams):
currentColumn = index + finalColumn
uniqueFeaturesArray[0, currentColumn] = feature
uniqueFeaturesArray[0][:-1].sort()
beneficial80Percent = int(beneficialCount * benprec)-1
beneficial20Percent = int(beneficialCount - beneficial80Percent)
harmful80Percent = int(harmfulCount * harmprec)-1
harmful20Percent = int(harmfulCount - harmful80Percent)
trainArray = numpy.empty(shape=((beneficial80Percent + harmful80Percent), numFeatures), dtype=numpy.int8) #Default is numpy.float64
testArray = numpy.empty(shape=((beneficial20Percent + harmful20Percent), numFeatures), dtype=numpy.int8)
#training data
for entry in range(0, beneficial80Percent):
#for each entry, find the index of the given feature
for word in pmcidBeneficialSentences[entry]:
#get the index of the given feature
featureColumn = numpy.searchsorted(uniqueFeaturesArray[0][:-1], word.encode("utf-8"))
if uniqueFeaturesArray[0][featureColumn] == word.encode("utf-8"):
trainArray[entry, featureColumn] = 1
for entity in beneficialFullEntitiesList[entry]:
featureColumn = numpy.searchsorted(uniqueFeaturesArray[0][:-1], entity)
if uniqueFeaturesArray[0][featureColumn] == entity:
trainArray[entry, featureColumn] = 1
trainArray[entry, -1] = 1
for entry in range(0, harmful80Percent):
trainingEntry = entry + beneficial80Percent
for word in pmcidHarmfulSentences[entry]:
featureColumn = numpy.searchsorted(uniqueFeaturesArray[0][:-1], word.encode("utf-8"))
if uniqueFeaturesArray[0][featureColumn] == word.encode("utf-8"):
trainArray[trainingEntry, featureColumn] = 1
for entity in harmfulFullEntitiesList[entry]:
#get the index of the given feature
featureColumn = numpy.searchsorted(uniqueFeaturesArray[0][:-1], entity)
if uniqueFeaturesArray[0][featureColumn] == entity:
trainArray[trainingEntry, featureColumn] = 1
#test data
for entry in range(0, beneficial20Percent):
dataEntry = entry + beneficial80Percent #finding next beneficial entry, starting from 60% until 80%
for word in pmcidBeneficialSentences[dataEntry]:
#get the index of the given feature
featureColumn = numpy.searchsorted(uniqueFeaturesArray[0][:-1], word.encode("utf-8"))
if uniqueFeaturesArray[0][featureColumn] == word.encode("utf-8"):
testArray[entry, featureColumn] = 1
for entity in beneficialFullEntitiesList[dataEntry]:
featureColumn = numpy.searchsorted(uniqueFeaturesArray[0][:-1], entity)
if uniqueFeaturesArray[0][featureColumn] == entity:
testArray[entry, featureColumn] = 1
testArray[entry, -1] = 1
for entry in range(0, harmful20Percent):
dataEntry = entry + harmful80Percent # finding the next harmful entry starting from 60% until 80%
testEntry = entry + beneficial20Percent #because the prior data entered ended with beneficial20Percent
for word in pmcidHarmfulSentences[dataEntry]:
featureColumn = numpy.searchsorted(uniqueFeaturesArray[0][:-1], feature.encode("utf-8"))
if uniqueFeaturesArray[0][featureColumn] == word.encode("utf-8"):
testArray[testEntry, featureColumn] = 1
for entity in harmfulFullEntitiesList[dataEntry]:
featureColumn = numpy.searchsorted(uniqueFeaturesArray[0][:-1], entity)
if uniqueFeaturesArray[0][featureColumn] == entity:
testArray[testEntry, featureColumn] = 1
###################################################CLASSIFICATION SECTION################################################################
#Here we set up our list for support vectors and our list for classes.
#We will setup lists to hold our support vectors our classes.
supportVectorsL = []
classesListL = []
for row in trainArray:
y1 = row[len(row)-1]
supportVectorsL.append(row[:-1])
classesListL.append(y1)
#Here we initialize our Linear classifier
supportVectors = numpy.asarray(supportVectorsL)
classesList = numpy.asarray(classesListL)
#Here we try out the linear regresion stuff
classifier = linear_model.LogisticRegression()
classifier.fit(supportVectors,classesList)
############Test our sets through our logisitc model##################
print("--------------------LOGISTIC------------------------")
logistic(classifier,testArray,"TEST")
print("--------------------SVM------------------------")
#Here we set up the svm
classifier = svm.SVC()
classifier.fit(supportVectors,classesList)
classifier.kernel="linear"
############Test our sets through our SVM model##################
SVC(classifier,testArray,"TEST")
sys.exit(0)
def SVC(classifier, testArray,t):
testpredictionarray = []
for row in testArray:
predictionvector = row[:-1]
if 1 in predictionvector:
predictionvector = [predictionvector]
prediction = classifier.predict(predictionvector)
pre = int(prediction[0])
else:
pre = -1
testpredictionarray.append(pre)
totalAccuray(testArray,testpredictionarray,t)
featAccuracy(testArray,testpredictionarray,t,1)
featAccuracy(testArray,testpredictionarray,t,2)
def logistic(classifier, testArray,t):
testpredictionarray = []
for row in testArray:
predictionvector = row[:-1]
if 1 in predictionvector:
predictionvector = [predictionvector]
prediction = classifier.predict(predictionvector)
pre = int(prediction[0])
else:
pre = -1
testpredictionarray.append(pre)
totalAccuray(testArray,testpredictionarray,t)
featAccuracy(testArray,testpredictionarray,t,1)
featAccuracy(testArray,testpredictionarray,t,2)
def totalAccuray(testArray,testpredictionarray,t):
testcounter = 0
#here we test for accuracy in the test set results.
for x in range(0,len(testArray)):
t1= testArray[x][len(testArray[x])-1]
t1 = int(t1)
if t1 == testpredictionarray[x]:
testcounter = testcounter + 1
accuracy= testcounter/len(testArray)
print(t+" set accuracy = " + str(accuracy))
def featAccuracy(testArray,testpredictionarray,t,y):
actual = 0
testcounter = 0
for x in range(0,len(testArray)):
l = list(testArray[x])
c = l.count(1)
if c == y:
actual+=1
t1= testArray[x][len(testArray[x])-1]
t1 = int(t1)
if t1 == testpredictionarray[x]:
testcounter = testcounter + 1
try:
accuracy= testcounter/actual
except ZeroDivisionError:
print(t+" set accuracy for only "+str(y)+" feature vectors = UNDEFINED")
return
print(t+" set accuracy for only "+str(y)+" feature vectors = " + str(accuracy))
main(sys.argv)
#
