from neural_network import NeuralNetwork, NeuralLogistic

from mix_of_exp import MixtureOfExperts

from neural_network import NeuralNetwork, NeuralLogistic

from sklearn.linear_model import LogisticRegression, LinearRegression, SGDClassifier, SGDRegressor

import random

import numpy as np

import matplotlib.colors as mcolors

import matplotlib.pyplot as plt

from sklearn.ensemble import AdaBoostClassifier

from utils import generate_data, plot_predictions, prediction_accuracy, sparsify_data

from sklearn.pipeline import FeatureUnion, Pipeline

from model_builder import build_model, regex_baseline

from model_tester import execute_test

import sys

import profile

logistic = NeuralLogistic(restarts = 2, regularization = 0.00)

nn = NeuralNetwork([(8, 'logistic'), (8, 'tanh'), (None, 'softmax')], 'maxent', regularization = 1e0, restarts = 30, max_iter = 10000, init_size = 1, step_size = 1e-2)

#nn = NeuralNetwork([(None, 'softmax')], 'maxent', include_offset = True, max_iter = 5000)

me = MixtureOfExperts([NeuralLogistic(regularization = 1e-1), NeuralLogistic(regularization = 1e-1), NeuralLogistic(regularization = 1e-1)], 

                       NeuralLogistic(regularization = 1e-1), max_iter = 25)

                       #NeuralNetwork([(3, 'tanh'), (None, 'softmax')], 'maxent'))

adaboost = AdaBoostClassifier(n_estimators = 500)

mix_ex_args = {'experts' :[NeuralLogistic(regularization = 1e-1), NeuralLogistic(regularization = 1e-1), NeuralLogistic(regularization = 1e-1)], 

               'gate'    : NeuralLogistic(regularization = 1e-1),

               'max_iter': 15}

model = build_model(regex_baseline, method = 'me', model_args = mix_ex_args)

print execute_test(model, 25, 5)