from __future__ import print_function

from sklearn.decomposition import TruncatedSVD

from sklearn.feature_extraction.text import TfidfVectorizer

from sklearn.feature_extraction.text import HashingVectorizer

from sklearn.feature_extraction.text import TfidfTransformer

from sklearn.preprocessing import Normalizer

from sklearn import metrics

from sklearn.cluster import KMeans, MiniBatchKMeans

import logging

import sys

from time import time

import numpy as np

import pandas as pd

data = pd.read_csv("cleanedsmplinds.csv", sep = ",", quoting = 1, quotechar = '"')

dataset = []

data = np.array(data)

for x in data :

    count = 0

    placeholder = ""

    for y in x :

        if (count == 2) :

            placeholder = y + "";

        elif (count == 3) :

            if y == True :

                #dataset.append(str(1) + " " + placeholder)

                dataset.append(placeholder)

            else :

                dataset.append(placeholder)

                #dataset.append(str(0) + " " + placeholder)

        count=count+1

nb_clust = 300

vectorizer = TfidfVectorizer(max_df=0.5, max_features=10000, min_df=2, stop_words='english', use_idf=True)

X = vectorizer.fit_transform(dataset)

print(X.shape)

km = KMeans(n_clusters=nb_clust, init='k-means++', max_iter=100, random_state=1337, n_init=1,verbose=True)

km.fit(X)

data2 = pd.read_csv("cleansmplinds.csv", sep = ",", quoting = 1, quotechar = '"')

data2 = np.array(data2)

cluster_map = pd.DataFrame()

cluster_map['cluster'] = km.labels_

for x in range(nb_clust) :

    f1 = open('./final340numclusters2/clust_' + str(x) + '.txt', 'a')

    y = cluster_map[cluster_map.cluster == x]['cluster'].index

    for n in y :

        f1.write(data2[n][2])

        f1.write("\n")

f2 = open('./final340numclusters2/centers.txt', 'a')

centers = km.cluster_centers_.argsort()[:, ::-1]

terms = vectorizer.get_feature_names()

for i in range(nb_clust):

    f2.write("Cluster %d:" % i)

    for ind in centers[i, :10]:

        f2.write(' %s' % terms[ind])

    f2.write("\n")