#!/usr/bin/env python
# coding: utf-8

# In[1]:


import pandas as pd
import numpy as np
from sklearn.decomposition import PCA
import warnings
warnings.filterwarnings('ignore')


# fucntion for PCA as feature selection 
# set cutoff value is number of components that represents 99% of variance 
# return reduced dataset with appropriate PCA components represented 99% variance
def PCA_project(data, data_name="", threshold = 99):
    max_component = data.shape[1]
    cutoff = threshold
    covar_matrix = PCA(n_components = max_component)
    covar_matrix.fit(data)
    variance = covar_matrix.explained_variance_ratio_
    var = np.cumsum(np.round(variance, decimals = 4)*100)
    index = 0
    for i in range(len(var)):
        
        if np.round(var[i]) < cutoff:
            index += 1
        else:
            break
    principal=PCA(n_components=index)
    principal.fit(data)
    print('%s reduce features from %d to %d'% (data_name,max_component, index))
    return pd.DataFrame(principal.transform(data))

#read apply PCA on our 4 dataset:bag of words, tf_idf, hash, word2vec
df_bow = pd.read_csv('bag_word_df.csv')
bow_P = PCA_project(df_bow.drop('prompt', axis=1), 'bag of words')

df_tf_idf = pd.read_csv('tf_idf.csv')
tf_idf_P= PCA_project(df_tf_idf.drop('prompt', axis=1), 'tf_idf')

df_hash_vectorize = pd.read_csv('hash_vectorize.csv')
hash_P= PCA_project(df_hash_vectorize.drop('prompt', axis=1), 'hash_vectorize')

df_w2v = pd.read_csv('df_w2v.csv')
w2v_P= PCA_project(df_w2v.drop('prompt',axis =1), 'word2vec')

#save these transformed data
bow_P.to_csv('bow_P.csv', index = False)
tf_idf_P.to_csv('tf_idf_P.csv', index = False)
hash_P.to_csv('hash_P.csv', index = False)
w2v_P.to_csv('w2v_P.csv', index = False)


# In[2]:


from sklearn.preprocessing import LabelEncoder
from sklearn.model_selection import train_test_split
Text = pd.read_csv('cleaned_data.csv')
y = Text["prompt"]

X_bow_train, X_bow_test, y_bow_train, y_bow_test = train_test_split(bow_P,y,test_size = 0.2, random_state =0, stratify = y)
X_tf_train, X_tf_test, y_tf_train, y_tf_test = train_test_split(tf_idf_P,y,test_size = 0.2, random_state =0, stratify = y)
X_hash_train, X_hash_test, y_hash_train, y_hash_test = train_test_split(hash_P,y,test_size = 0.2, random_state =0, stratify = y)
X_w2v_train, X_w2v_test, y_w2v_train, y_w2v_test = train_test_split(w2v_P,y,test_size = 0.2, random_state =0, stratify = y)

pd.DataFrame(X_bow_train).to_csv('X_bow_train.csv', index = False) 
pd.DataFrame(X_bow_test).to_csv('X_bow_test.csv', index = False)
pd.DataFrame(y_bow_train).to_csv('y_bow_train.csv', index = False) 
pd.DataFrame(y_bow_test).to_csv('y_bow_test.csv', index = False)

pd.DataFrame(X_tf_train).to_csv('X_tf_train.csv', index = False) 
pd.DataFrame(X_tf_test).to_csv('X_tf_test.csv', index = False)
pd.DataFrame(y_tf_train).to_csv('y_tf_train.csv', index = False) 
pd.DataFrame(y_tf_test).to_csv('y_tf_test.csv', index = False)

pd.DataFrame(X_hash_train).to_csv('X_hash_train.csv', index = False) 
pd.DataFrame(X_hash_test).to_csv('X_hash_test.csv', index = False)
pd.DataFrame(y_hash_train).to_csv('y_hash_train.csv', index = False) 
pd.DataFrame(y_hash_test).to_csv('y_hash_test.csv', index = False)

pd.DataFrame(X_w2v_train).to_csv('X_w2v_train.csv', index = False) 
pd.DataFrame(X_w2v_test).to_csv('X_w2v_test.csv', index = False)
pd.DataFrame(y_w2v_train).to_csv('y_w2v_train.csv', index = False) 
pd.DataFrame(y_w2v_test).to_csv('y_w2v_test.csv', index = False)


# In[ ]: