#!/usr/bin/env python

# coding: utf-8

# In[1]:

import pandas as pd

import numpy as np

from sklearn.feature_extraction.text import TfidfVectorizer, HashingVectorizer, CountVectorizer

from gensim.models import Word2Vec

import warnings

warnings.filterwarnings('ignore')

#read cleaned data

Text = pd.read_csv('cleaned_data.csv')

# Create and fit tf_idf model

text_vectorize = TfidfVectorizer()

X_tf_idf = text_vectorize.fit_transform(Text["new_text"])

dense_list = X_tf_idf.todense().tolist()

feature_names = text_vectorize.get_feature_names()

df_tf_idf = pd.DataFrame(dense_list, columns = feature_names)

# concatenate prompt column with tf_idf matrix

text_tf_idf = pd.concat([Text["prompt"], df_tf_idf], axis = 1)

text_tf_idf.to_csv(f"tf_idf.csv", index=False)

# Create and fit hashvector model

n = Text['prompt'].nunique()

text_hashvectorize = HashingVectorizer(n_features = n*3)

X_hash = text_hashvectorize.fit_transform(Text["new_text"])

df_hash_vectorize = pd.DataFrame(X_hash.toarray())

# concatenate prompt column with hash vectorized matrix

text_hash_vectorize = pd.concat([Text["prompt"], df_hash_vectorize], axis = 1)

text_hash_vectorize.to_csv(f"hash_vectorize.csv", index=False)

# extract feature using bag_of_words

bag_word = CountVectorizer()

feature_bow = bag_word.fit_transform(Text["new_text"].values)

# maping feature 

df_bow = pd.DataFrame(feature_bow.todense().tolist(), columns = bag_word.get_feature_names())

# concatenate prompt column with bow matrix

bag_word_df = pd.concat([Text['prompt'], df_bow], axis = 1)

bag_word_df.to_csv('bag_word_df.csv',index=False)

# Create the list of list format for gensim w2v modeling 

Text['new_text_clean'] = Text['new_text'].apply(lambda x: x.split(" "))

# Train the word2vec model

w2v_model = Word2Vec(Text['new_text_clean'], min_count = 1,vector_size = 100, window = 5)

# Take the average of the word vectors for the words contained in each sentence

def word_avg_vect(data, model, num_features):

    words = set(model.wv.index_to_key)

    X_vect = np.array([np.array([model.wv[i] for i in s if i in words]) for s in data])

    X_vect_avg = []

    for v in X_vect:

        if v.size:

            X_vect_avg.append(v.mean(axis = 0))

        else:

            X_vect_avg.append(np.zeros(num_features, dtype = float))

    df_vect_avg = pd.DataFrame(X_vect_avg)

    return df_vect_avg

X_w2v = word_avg_vect(Text['new_text_clean'], w2v_model, 100)

# concatenate prompt column with averaged w2v matrix

df_w2v = pd.concat([Text["prompt"], X_w2v], axis = 1)

df_w2v.to_csv(f"df_w2v.csv", index=False)

# In[ ]: