import sys
#from nltk.stem import WordNetLemmatizer
#import nltk
#from nltk.corpus import stopwords
from nltk.tokenize import word_tokenize
import math

#stop_words = set(stopwords.words('english'))


#raw_word_count=dict()
#lemma_word_count=dict()
#lemma_raw_connect=dict()
id_uniqueWords=dict()
ngram_counts=dict()

gram2_count=0
gram3_count=0
gram4_count=0
gram5_count=0
gram6_count=0
gram7_count=0
gram7_count=0
gram9_count=0
gram10_count=0

id_count=0



with open(sys.argv[1], encoding="utf8", errors='ignore') as in_file:
	for line in in_file:
		if line.strip()=="":
			break
		id_count+=1
		id_key=line.split("\t")[0]
		fields=line.split("\t")[1:]
		id_uniqueWords[id_key]=set([])
		for f in fields:
			#id_uniqueWords[fields[0]].add(set(word_tokenize(f.strip("True:").strip("False:"))))
			words=word_tokenize(f.lstrip("True:").lstrip("False:").lower())
			max_len=len(words)
			for i,w in enumerate(words):
				if i+1<max_len:
					gram2=tuple([w,words[i+1]])
					gram2_count+=1
					if gram2 in ngram_counts:
						ngram_counts[gram2]+=1
					else:
						ngram_counts[gram2]=1
				if i+2<max_len:
					gram3=tuple([w,words[i+1],words[i+2]])
					gram3_count+=1
					if gram3 in ngram_counts:
						ngram_counts[gram3]+=1
					else:
						ngram_counts[gram3]=1
				if i+3<max_len:
					gram4=tuple([w,words[i+1],words[i+2],words[i+3]])
					gram4_count+=1
					if gram4 in ngram_counts:
						ngram_counts[gram4]+=1
					else:
						ngram_counts[gram4]=1
				if i+4<max_len:
					gram5=tuple([w,words[i+1],words[i+2],words[i+3],words[i+4]])
					gram5_count+=1
					if gram5 in ngram_counts:
						ngram_counts[gram5]+=1
					else:
						ngram_counts[gram5]=1
				if i+5<max_len:
					gram6=tuple([w,words[i+1],words[i+2],words[i+3],words[i+4],words[i+5]])
					gram6_count+=1
					if gram6 in ngram_counts:
						ngram_counts[gram6]+=1
					else:
						ngram_counts[gram6]=1
				if i+6<max_len:
					gram7=tuple([w,words[i+1],words[i+2],words[i+3],words[i+4],words[i+5],words[i+6]])
					gram7_count+=1
					if gram6 in ngram_counts:
						ngram_counts[gram7]+=1
					else:
						ngram_counts[gram7]=1
				if i+7<max_len:
					gram8=tuple([w,words[i+1],words[i+2],words[i+3],words[i+4],words[i+5],words[i+6],words[i+7]])
					gram8_count+=1
					if gram8 in ngram_counts:
						ngram_counts[gram8]+=1
					else:
						ngram_counts[gram8]=1

for k,v in ngram_counts.items():
	if v>1:
		print (str(math.ceil(float(v)/id_count))+"\t"+"\t".join(k))
	else:
		continue
	#if len(k)==2:
		#print ("\t".join(k)+"\t"+str(math.ceil(float(v)/gram2_count)))
		#print (str(math.ceil(float(v)/gram2_count))+"\t" +"\t".join(k))
	#elif len(k)==3:
		#print ("\t".join(k)+"\t"+str(math.ceil(float(v)/gram3_count)))
		#print (str(math.ceil(float(v)/gram2_count))+"\t" +"\t".join(k))
	#elif len(k)==4:
		#print ("\t".join(k)+"\t"+str(math.ceil(float(v)/gram4_count)))
		#print (str(math.ceil(float(v)/gram2_count))+"\t" +"\t".join(k))
	#elif len(k)==5:
		#print ("\t".join(k)+"\t"+str(math.ceil(float(v)/gram5_count)))
		#print (str(math.ceil(float(v)/gram2_count))+"\t" +"\t".join(k))
	#elif len(k)==6:
		#print ("\t".join(k)+"\t"+str(math.ceil(float(v)/gram6_count)))
		#print (str(math.ceil(float(v)/gram2_count))+"\t" +"\t".join(k))



exit()
print ("id\t"+"\t".join(id_uniqueWords.keys()))
for k1,v1 in id_uniqueWords.items():
	avg_list=[]
	for k2,v2 in id_uniqueWords.items():
		avg=(len(v1)+len(v2))/2.0

		avg_list.append(str(int(100*(len(v1.intersection(v2))/avg))))
	print (k2+"\t".join(avg_list))