import pandas
# from pattern.en import sentiment
# import HTMLParser
import re
import pandas as pd
from collections import Counter
from nltk.corpus import stopwords
import string
from collections import OrderedDict
from nltk import bigrams
from nltk.tokenize import word_tokenize
import matplotlib.pyplot as plt
import numpy as np
# import plotly.plotly as py

# import pandas as pd
# import matplotlib.pyplot as plt
import numpy as np
from sklearn.metrics import recall_score, precision_score, accuracy_score
import math
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.model_selection import train_test_split
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.naive_bayes import MultinomialNB
from sklearn.metrics import confusion_matrix
from sklearn.feature_selection import RFE
import requests
from bs4 import BeautifulSoup
# import numpy as np
# import matplotlib.pyplot as plt
# from matplotlib import style
# style.use("ggplot")
import os

data_outcome = pd.read_csv("C:\Shashank Reddy\Outcome.csv",sep='\s*,\s*',header=0, encoding='ascii', engine='python')



#print(data_outcome)

data_outcome = data_outcome.fillna("zero")





# outcome dictionary
outcome = {'removed': 1, 'not removed': 2, 'retrieval': 3, 'non retrieval': 4, 'zero' : 0}
data_outcome["Outcome"] = [outcome[item] for item in data_outcome["Outcome"]]

list_outcome = pd.DataFrame(list(data_outcome["Outcome"]))

list_outcome.to_csv(r"C:\Shashank Reddy\FinalOutcome.csv",sep='\t', index=False)

#print(data_outcome)




#data_outcome.to_dense().to_csv(r"C:\Shashank Reddy\FinalOutcome.csv")


sessilelocation = pd.read_csv("C:\Shashank Reddy\SessileLocation.csv",sep='\s*,\s*',header=0, encoding='ascii', engine='python').fillna("zero")
#print(sessilelocation.columns.tolist())
#print(sessilelocation)

location = {'cecal': 1, 'ascending': 2, 'ileum': 3, 'ileocecal': 3, 'hepatic': 4, 'transverse': 5, 'splenic': 6, 'descending': 7, 'sigmoid': 8, 'recto-sigmoid': 9, 'rectal': 10, 'appendix': 11,'zero': 0}
sessilelocation["PositionA"] = [location[item] for item in sessilelocation["PositionA"]]
sessilelocation["PositionB"] = [location[item] for item in sessilelocation["PositionB"]]
sessilelocation["PositionC"] = [location[item] for item in sessilelocation["PositionC"]]
sessilelocation["PositionD"] = [location[item] for item in sessilelocation["PositionD"]]
sessilelocation["PositionE"] = [location[item] for item in sessilelocation["PositionE"]]
sessilelocation["PositionF"] = [location[item] for item in sessilelocation["PositionF"]]
sessilelocation["PositionG"] = [location[item] for item in sessilelocation["PositionG"]]

#print(sessilelocation)


sessileshape = pd.read_csv("C:\Shashank Reddy\SessileShape.csv",sep='\s*,\s*',header=0, encoding='ascii', engine='python').fillna("zero")
#print(sessileshape)
shape = {'zero':0,'sessile':1,'pedunculated':2,'flat':3,'mass':4,'smooth':5,'serrated':6}

sessileshape["Shape"] = [shape[item] for item in sessileshape["Shape"]]
#print(sessileshape)


sessilesize = pd.read_csv("C:\Shashank Reddy\SessileSize.csv",sep='\s*,\s*',header=0, encoding='ascii', engine='python').fillna("zero")
#print(sessilesize)

size = {'zero':0,'diminutive':1,'small':2,'medium':3,'large':4}
sessilesize["Size"] = [size[item] for item in sessilesize["Size"]]
#print(sessilesize)


sessileside = pd.read_csv("C:\Shashank Reddy\Sides.csv",sep='\s*,\s*',header=0, encoding='ascii', engine='python').fillna("zero")
#print(sessileside)

side = {'zero':0,'left':1,'right':2}
sessileside["Sides"] = [side[item] for item in sessileside["Sides"]]
#print(sessileside)


cancer_treatment = pd.read_csv("C:\Shashank Reddy\Treatment.csv",sep='\s*,\s*',header=0, encoding='ascii', engine='python').fillna("zero")
#print(cancer_treatment)

treatment = {'zero':0,'piermeal':1,'cold snare':2,'hot snare':3,'snare':4,'electocautery snare':5,'excisional biopsy':6,'biopsy forcep':7,'cold biopsy':8}
cancer_treatment["Treatment"] = [treatment[item] for item in cancer_treatment["Treatment"]]

list_treatment = pd.DataFrame(list(cancer_treatment["Treatment"]))

list_treatment.to_csv(r"C:\Shashank Reddy\FinalTreatment.csv",sep='\t', index=False)
#print(cancer_treatment)


sessile_number = pd.read_csv("C:\Shashank Reddy\SessileNumber.csv",sep='\s*,\s*',header=0, encoding='ascii', engine='python').fillna("zero")
#print(sessile_number)

number = {'zero':0,'one':1,'two':2,'three':3,'four':4,'five':5,'six':7,'eight':8,'nine':9,'ten':10}

sessile_number["Number1"] = [number[item] for item in sessile_number["Number1"]]
sessile_number["Number2"] = [number[item] for item in sessile_number["Number2"]]
sessile_number["Number3"] = [number[item] for item in sessile_number["Number3"]]
sessile_number["Number4"] = [number[item] for item in sessile_number["Number4"]]

#print(sessile_number)

#************************************* Data Union *********************************************************************************

list_mat1= pd.DataFrame(list(sessilelocation["PositionA"]))
list_mat2= pd.DataFrame(list(sessilelocation["PositionB"]))
list_mat3= pd.DataFrame(list(sessilelocation["PositionC"]))
list_mat4= pd.DataFrame(list(sessilelocation["PositionD"]))
list_mat5= pd.DataFrame(list(sessilelocation["PositionE"]))
list_mat6= pd.DataFrame(list(sessilelocation["PositionF"]))
list_mat7= pd.DataFrame(list(sessilelocation["PositionG"]))

list_mat8= pd.DataFrame(list(sessileshape["Shape"]))
list_mat9= pd.DataFrame(list(sessilesize["Size"]))
list_mat10= pd.DataFrame(list(sessileside["Sides"]))

list_mat11= pd.DataFrame(list(sessile_number["Number1"]))
list_mat12= pd.DataFrame(list(sessile_number["Number2"]))
list_mat13= pd.DataFrame(list(sessile_number["Number3"]))
list_mat14= pd.DataFrame(list(sessile_number["Number4"]))


Final_Data = pd.concat([list_mat1,list_mat2,list_mat3,list_mat4,list_mat5,list_mat6,list_mat7,list_mat8,list_mat9,list_mat10,list_mat11
                        ,list_mat12,list_mat13,list_mat14],axis = 1)


print(Final_Data)


Final_Data.to_csv(r"C:\Shashank Reddy\DataSet_Final.csv",index = False)


#print(Final_Data)