/*

 * Select only models getting specific min MCC or max RMSE

 * Select all features from selected models

 * Retrain a model (best classifier of all tested) with the unique selected features with LOOCV 75/25.

 * Do it 10 times with various seeds, report the average scores (ex: AUC)

 * Explore biology behind the set of selected features

 */

package biodiscml;

import java.io.BufferedReader;

import java.io.File;

import java.io.FileReader;

import java.io.FileWriter;

import java.io.PrintWriter;

import java.text.DecimalFormat;

import java.text.DecimalFormatSymbols;

import java.util.ArrayList;

import java.util.Arrays;

import java.util.Collections;

import java.util.HashMap;

import java.util.LinkedHashMap;

import java.util.TreeMap;

import org.apache.commons.math3.stat.descriptive.moment.Mean;

import org.apache.commons.math3.stat.descriptive.moment.StandardDeviation;

import utils.UpSetR;

import utils.Weka_module;

import utils.utils;

import weka.core.SerializationHelper;

/**

 *

 * @author Mickael

 */

public class BestModelSelectionAndReport {

    public static String wd = Main.wd;

    public static Weka_module weka = new Weka_module();

    public static HashMap<String, Integer> hmResultsHeaderNames = new HashMap<>();

    public static HashMap< Integer, String> hmResultsHeaderIndexes = new HashMap<>();

    public static DecimalFormat df = new DecimalFormat();

    public static String trainFileName;

    public static String featureSelectionFile;

    public static String predictionsResultsFile;

    public static String correlatedFeatures;

    /**

     *

     * @param trainFilName

     * @param featureSelFile

     * @param predictionsResFile

     * @param type

     */

    public BestModelSelectionAndReport(String trainFilName,

            String featureSelFile,

            String predictionsResFile,

            String type

    ) {

        trainFileName = trainFilName;

        if (Main.noFeatureSelection) {

            featureSelectionFile = trainFilName;

        } else {

            featureSelectionFile = featureSelFile;

        }

        predictionsResultsFile = predictionsResFile;

        df.setMaximumFractionDigits(3);

        DecimalFormatSymbols dfs = new DecimalFormatSymbols();

        dfs.setDecimalSeparator('.');

        df.setDecimalFormatSymbols(dfs);

        String bestOrCombine = "Select best ";

        if (Main.combineModels) {

            bestOrCombine = "Combine ";

        }

        String sign = " >= ";

        boolean metricToMinimize = (Main.bestModelsSortingMetric.contains("RMSE")

                || Main.bestModelsSortingMetric.contains("BER")

                || Main.bestModelsSortingMetric.contains("FPR")

                || Main.bestModelsSortingMetric.contains("FNR")

                || Main.bestModelsSortingMetric.contains("FDR")

                || Main.bestModelsSortingMetric.contains("MAE")

                || Main.bestModelsSortingMetric.contains("RAE")

                || Main.bestModelsSortingMetric.contains("RRSE"));

        if (metricToMinimize) {

            sign = " <= ";

        }

        System.out.println("## " + bestOrCombine + " models using " + Main.bestModelsSortingMetric + " as sorting metric.\n"

                + "## Parameters: " + Main.numberOfBestModels + " best models and "

                + Main.bestModelsSortingMetric + sign + Main.bestModelsSortingMetricThreshold);

        //Read results file

        boolean classification = type.equals("classification");

        try {

            BufferedReader br = new BufferedReader(new FileReader(predictionsResultsFile));

            TreeMap<String, Object> tmModels = new TreeMap<>(); //<metric modelID, classification/regression Object>

            HashMap<String, Object> hmModels = new HashMap<>(); //<modelID, classification/regression Object>

            //in case of RMSE or BER, we want the minimum value instead of the maximal one

            if (!metricToMinimize) {

                tmModels = new TreeMap<>(Collections.reverseOrder());

            }

            String line = br.readLine();

            //fill header mapping

            String header = line;

            int cpt = 0;

            for (String s : header.split("\t")) {

                hmResultsHeaderNames.put(s, cpt);

                hmResultsHeaderIndexes.put(cpt, s);

                cpt++;

            }

            if (!hmResultsHeaderNames.containsKey(Main.bestModelsSortingMetric)) {

                System.err.println("[error] " + Main.bestModelsSortingMetric + " column does not exist in the results file.");

                if (Main.doRegression) {

                    System.out.println("Use AVG_CC instead since we are in regression mode");

                    Main.bestModelsSortingMetric = "AVG_CC";

                } else {

                    System.exit(0);

                }

            }

            //read results

            while (br.ready()) {

                line = br.readLine();

                if (!line.trim().isEmpty() && !line.contains("[model error]")) {

                    if (classification) {

                        try {

                            classificationObject co = new classificationObject(line);

                            tmModels.put(Double.valueOf(co.hmValues.get(Main.bestModelsSortingMetric)) + " " + co.hmValues.get("ID"), co);

                            hmModels.put(co.hmValues.get("ID"), co);

                        } catch (Exception e) {

                            if (Main.debug) {

                                e.printStackTrace();

                            }

                        }

                    } else {

                        try {

                            regressionObject ro = new regressionObject(line);

                            tmModels.put(Double.valueOf(ro.hmValues.get(Main.bestModelsSortingMetric)) + " " + ro.hmValues.get("ID"), ro);

                            hmModels.put(ro.hmValues.get("ID"), ro);

                        } catch (Exception e) {

                            if (Main.debug) {

                                e.printStackTrace();

                            }

                        }

                    }

                }

            }

            br.close();

            //control available models

            if (Main.numberOfBestModels > tmModels.size()) {

                System.out.println("Only " + tmModels.size() + " available models. You have configured " + Main.numberOfBestModels + " best models");

                Main.numberOfBestModels = tmModels.size();

            }

            // get best models list

            ArrayList<Object> alBestClassifiers = new ArrayList<>();

            cpt = 0;

            if (Main.hmTrainingBestModelList.isEmpty()) {

                for (String metricAndModel : tmModels.keySet()) {

                    cpt++;

                    boolean condition = false;

                    if (metricToMinimize) {

                        condition = Double.valueOf(metricAndModel.split(" ")[0]) < Main.bestModelsSortingMetricThreshold;

                    } else {

                        condition = Double.valueOf(metricAndModel.split(" ")[0]) > Main.bestModelsSortingMetricThreshold;

                    }

                    if (condition && cpt <= Main.numberOfBestModels) {

                        if (classification) {

                            alBestClassifiers.add(((classificationObject) tmModels.get(metricAndModel)));

                        } else {

                            alBestClassifiers.add(((regressionObject) tmModels.get(metricAndModel)));

                        }

                    }

                }

            } else {

                for (String modelID : Main.hmTrainingBestModelList.keySet()) {

                    if (classification) {

                        alBestClassifiers.add(((classificationObject) hmModels.get(modelID)));

                    } else {

                        alBestClassifiers.add(((regressionObject) hmModels.get(modelID)));

                    }

                }

            }

            //if model combination vote

            if (Main.combineModels) {

                if (classification) {

                    classificationObject co = new classificationObject();

                    co.buildVoteClassifier(alBestClassifiers);

                    alBestClassifiers = new ArrayList<>();

                    alBestClassifiers.add(co);

                } else {

                    regressionObject ro = new regressionObject();

                    ro.buildVoteClassifier(alBestClassifiers);

                    alBestClassifiers = new ArrayList<>();

                    alBestClassifiers.add(ro);

                }

            }

            //perform evaluations and create models

            PrintWriter pw = null;

            for (Object classifier : alBestClassifiers) {

                // initialize weka module

                if (classification) {

                    init(featureSelectionFile.replace("infoGain.csv", "infoGain.arff"), classification);

                } else {

                    init(featureSelectionFile.replace("RELIEFF.csv", "RELIEFF.arff"), classification);

                }

                createBestModel(classifier, classification, pw, br, false);

                if (Main.generateModelWithCorrelatedGenes) {

                    init(trainFilName, classification);

                    createBestModel(classifier, classification, pw, br, true);

                    correlatedFeatures = null;

                }

            }

        } catch (ClassCastException e) {

            e.printStackTrace();

            System.err.println("Unable to train selected best model(s). Check input files. ");

        } catch (Exception e) {

            e.printStackTrace();

        }

    }

    private void createBestModel(Object classifier,

            Boolean classification,

            PrintWriter pw, BufferedReader br, Boolean correlatedFeaturesMode) throws Exception {

        String corrMode = "";

        if (correlatedFeaturesMode) {

            corrMode = "_corr";

            System.out.print("\n# Model with correlated features ");

            ((classificationObject) classifier).featuresSeparatedByCommas = correlatedFeatures;

        } else {

            System.out.print("\n# Model ");

        }

        ArrayList<Double> alMCCs = new ArrayList<>();

        ArrayList<Double> alMAEs = new ArrayList<>();

        ArrayList<Double> alCCs = new ArrayList<>();

        if (Main.debug) {

            System.out.println("Save model ");

        }

        String modelFilename;

        Weka_module.ClassificationResultsObject cr = null;

        Weka_module.RegressionResultsObject rr = null;

        classificationObject co = null;

        regressionObject ro = null;

        String classifierName = "";

        //saving files

        if (classification) {

            co = (classificationObject) classifier;

            //train model

            modelFilename = Main.wd + Main.project

                    + "d." + co.classifier + "_" + co.printOptions() + "_"

                    + co.optimizer.toUpperCase().trim() + "_" + co.mode + corrMode;

            Object trainingOutput = weka.trainClassifier(co.classifier, co.options,

                    co.featuresSeparatedByCommas, classification, 10);

            cr = (Weka_module.ClassificationResultsObject) trainingOutput;

            classifierName = co.classifier + "_" + co.printOptions() + "_"

                    + co.optimizer.toUpperCase().trim() + "_" + co.mode;

            //save feature file

            weka.saveFilteredDataToCSV(co.featuresSeparatedByCommas, classification, modelFilename + ".train_features.csv");

            //call ranking function

            cr.featuresRankingResults = weka.featureRankingForClassification(modelFilename + ".train_features.csv");

            //save model

            try {

                SerializationHelper.write(modelFilename + ".model", cr.model);

            } catch (Exception e) {

                e.printStackTrace();

            }

            System.out.println(modelFilename);

        } else {

            ro = (regressionObject) classifier;

            modelFilename = Main.wd + Main.project

                    + "d." + ro.classifier + "_" + ro.printOptions() + "_"

                    + ro.optimizer.toUpperCase().trim() + "_" + ro.mode;

            Object trainingOutput = weka.trainClassifier(ro.classifier, ro.options,

                    ro.featuresSeparatedByCommas, classification, 10);

            rr = (Weka_module.RegressionResultsObject) trainingOutput;

            classifierName = ro.classifier + "_" + ro.printOptions() + "_"

                    + ro.optimizer.toUpperCase().trim() + "_" + ro.mode;

            //save model and features

            try {

                SerializationHelper.write(modelFilename + ".model", rr.model);

            } catch (Exception e) {

                e.printStackTrace();

            }

            //save feature file

            weka.saveFilteredDataToCSV(ro.featuresSeparatedByCommas, classification, modelFilename + ".train_features.csv");

            //call ranking function

            rr.featuresRankingResults = weka.featureRankingForRegression(modelFilename + ".train_features.csv");

            System.out.println(modelFilename);

        }

        //header

        pw = new PrintWriter(new FileWriter(modelFilename + ".details.txt"));

        pw.println("## Generated by BioDiscML (Leclercq et al. 2019)##");

        pw.println("# Project: " + Main.project.substring(0, Main.project.length() - 1));

        if (classification) {

            pw.println("# ID: " + co.identifier);

            System.out.println("# ID: " + co.identifier);

            pw.println("# Classifier: " + co.classifier + " " + co.options

                    + "\n# Optimizer: " + co.optimizer.toUpperCase()

                    + "\n# Feature search mode: " + co.mode);

        } else {

            pw.println("# ID: " + ro.identifier);

            System.out.println("# ID: " + ro.identifier);

            pw.println("# Classifier: " + ro.classifier + " " + ro.options

                    + "\n# Optimizer: " + ro.optimizer.toUpperCase()

                    + "\n# Feature search mode: " + ro.mode);

        }

        //show combined models in case of combined vote

        if (Main.combineModels) {

            pw.println("# Combined classifiers:");

            String combOpt = "";

            if (classification) {

                combOpt = co.options;

            } else {

                combOpt = ro.options;

            }

            for (String s : combOpt.split("-B ")) {

                if (s.startsWith("\"weka.classifiers.meta.FilteredClassifier")) {

                    String usedFeatures = s.split("Remove -V -R ")[1];

                    usedFeatures = usedFeatures.split("\\\\")[0];

                    String model = s.substring(s.indexOf("-W ") + 2).trim()

                            .replace("-- ", "")

                            .replace("\\\"", "\"")

                            .replace("\\\\\"", "\\\"");

                    model = model.substring(0, model.length() - 1);

                    pw.println(model + " (features: " + usedFeatures + ")");

                }

            }

        }

        pw.flush();

        //UpSetR

        if (Main.UpSetR) {

            UpSetR up = new UpSetR();

            up.creatUpSetRDatasetFromSignature(co, featureSelectionFile, predictionsResultsFile);

        }

        //10CV performance

        System.out.println("# 10 fold cross validation performance");

        pw.println("\n# 10 fold cross validation performance");

        if (classification) {

            System.out.println(cr.toStringDetails());

            alMCCs.add(Double.valueOf(cr.MCC));

            alMAEs.add(Double.valueOf(cr.MAE));

            pw.println(cr.toStringDetails().replace("[score_training] ", ""));

        } else {

            System.out.println(rr.toStringDetails());

            alCCs.add(Double.valueOf(rr.CC));

            alMAEs.add(Double.valueOf(rr.MAE));

            pw.println(rr.toStringDetails().replace("[score_training] ", ""));

        }

        pw.flush();

        //LOOCV performance

        if (Main.loocv) {

            System.out.println("# LOOCV (Leave-One-Out cross validation) performance");

            pw.println("\n# LOOCV (Leave-One-Out Cross Validation) performance");

            if (classification) {

                Weka_module.ClassificationResultsObject cr2 = (Weka_module.ClassificationResultsObject) weka.trainClassifier(co.classifier, co.options,

                        co.featuresSeparatedByCommas, classification, weka.myData.numInstances());

                System.out.println(cr2.toStringDetails());

                alMCCs.add(Double.valueOf(cr2.MCC));

                alMAEs.add(Double.valueOf(cr2.MAE));

                pw.println(cr2.toStringDetails().replace("[score_training] ", ""));

            } else {

                Weka_module.RegressionResultsObject rr2 = (Weka_module.RegressionResultsObject) weka.trainClassifier(ro.classifier, ro.options,

                        ro.featuresSeparatedByCommas, classification, weka.myData.numInstances());

                System.out.println(rr2.toStringDetails());

                alCCs.add(Double.valueOf(rr2.CC));

                alMAEs.add(Double.valueOf(rr2.MAE));

                pw.println(rr2.toStringDetails().replace("[score_training] ", ""));

            }

            pw.flush();

        }

        //REPEATED HOLDOUT performance TRAIN set

        ArrayList<Object> alROCs = new ArrayList<>();

        Weka_module.evaluationPerformancesResultsObject eproRHTrain = new Weka_module.evaluationPerformancesResultsObject();

        if (classification) {

            System.out.println("Repeated Holdout evaluation on TRAIN set of " + co.classifier + " " + co.options

                    + " optimized by " + co.optimizer + "...");

            pw.println("\n#Repeated Holdout evaluation performance on TRAIN set, "

                    + Main.bootstrapAndRepeatedHoldoutFolds + " times weighted average (and standard deviation) on random seeds");

            for (int i = 0; i < Main.bootstrapAndRepeatedHoldoutFolds; i++) {

                Weka_module.ClassificationResultsObject cro

                        = (Weka_module.ClassificationResultsObject) weka.trainClassifierHoldOutValidation(co.classifier, co.options,

                                co.featuresSeparatedByCommas, classification, i);

                eproRHTrain.alAUCs.add(Double.valueOf(cro.AUC));

                eproRHTrain.alpAUCs.add(Double.valueOf(cro.pAUC));

                eproRHTrain.alAUPRCs.add(Double.valueOf(cro.AUPRC));

                eproRHTrain.alACCs.add(Double.valueOf(cro.ACC));

                eproRHTrain.alSEs.add(Double.valueOf(cro.TPR));

                eproRHTrain.alSPs.add(Double.valueOf(cro.TNR));

                eproRHTrain.alMCCs.add(Double.valueOf(cro.MCC));

                eproRHTrain.alMAEs.add(Double.valueOf(cro.MAE));

                eproRHTrain.alBERs.add(Double.valueOf(cro.BER));

                alROCs.add(cro);

                // System.out.println(i+"\t"+Double.valueOf(cro.AUC));

            }

            eproRHTrain.computeMeans();

            System.out.println(eproRHTrain.toStringClassificationDetails());

            alMCCs.add(Double.valueOf(eproRHTrain.meanMCCs));

            alMAEs.add(Double.valueOf(eproRHTrain.meanMAEs));

            pw.println(eproRHTrain.toStringClassificationDetails().replace("[score_training] ", ""));

            if (Main.ROCcurves) {

                rocCurveGraphs.createRocCurvesWithConfidence(alROCs, classification, modelFilename, ".roc_train.png");

            }

        } else {

            System.out.println("Repeated Holdout evaluation on TRAIN set of " + ro.classifier + " "

                    + ro.options + "optimized by " + ro.optimizer.toUpperCase());

            pw.println("\n\n#Repeated Holdout evaluation performance on TRAIN set, " + Main.bootstrapAndRepeatedHoldoutFolds + " times average on random seeds");

            for (int i = 0; i < Main.bootstrapAndRepeatedHoldoutFolds; i++) {

                Weka_module.RegressionResultsObject rro

                        = (Weka_module.RegressionResultsObject) weka.trainClassifierHoldOutValidation(ro.classifier, ro.options,

                                ro.featuresSeparatedByCommas, classification, i);

                eproRHTrain.alCCs.add(Double.valueOf(rro.CC));

                eproRHTrain.alMAEs.add(Double.valueOf(rro.MAE));

                eproRHTrain.alRMSEs.add(Double.valueOf(rro.RMSE));

                eproRHTrain.alRAEs.add(Double.valueOf(rro.RAE));

                eproRHTrain.alRRSEs.add(Double.valueOf(rro.RRSE));

            }

            eproRHTrain.computeMeans();

            alCCs.add(Double.valueOf(eproRHTrain.meanCCs));

            alMAEs.add(Double.valueOf(eproRHTrain.meanMAEs));

            pw.println(eproRHTrain.toStringRegressionDetails().replace("[score_training] ", ""));

        }

        pw.flush();

        //BOOTSTRAP performance TRAIN set

        double bootstrapTrain632plus = -1;

        Weka_module.evaluationPerformancesResultsObject eproBSTrain = new Weka_module.evaluationPerformancesResultsObject();

        if (classification) {

            System.out.println("Bootstrap evaluation on TRAIN set of " + co.classifier + " " + co.options

                    + " optimized by " + co.optimizer + "...");

            pw.println("\n#Bootstrap evaluation performance on TRAIN set, "

                    + Main.bootstrapAndRepeatedHoldoutFolds + " times weighted average (and standard deviation) on random seeds");

            for (int i = 0; i < Main.bootstrapAndRepeatedHoldoutFolds; i++) {

                Weka_module.ClassificationResultsObject cro

                        = (Weka_module.ClassificationResultsObject) weka.trainClassifierBootstrap(co.classifier, co.options,

                                co.featuresSeparatedByCommas, classification, i);

                eproBSTrain.alAUCs.add(Double.valueOf(cro.AUC));

                eproBSTrain.alpAUCs.add(Double.valueOf(cro.pAUC));

                eproBSTrain.alAUPRCs.add(Double.valueOf(cro.AUPRC));

                eproBSTrain.alACCs.add(Double.valueOf(cro.ACC));

                eproBSTrain.alSEs.add(Double.valueOf(cro.TPR));

                eproBSTrain.alSPs.add(Double.valueOf(cro.TNR));

                eproBSTrain.alMCCs.add(Double.valueOf(cro.MCC));

                eproBSTrain.alMAEs.add(Double.valueOf(cro.MAE));

                eproBSTrain.alBERs.add(Double.valueOf(cro.BER));

                alROCs.add(cro);

                // System.out.println(i+"\t"+Double.valueOf(cro.AUC));

            }

            eproBSTrain.computeMeans();

            alMCCs.add(Double.valueOf(eproBSTrain.meanMCCs));

            alMAEs.add(Double.valueOf(eproBSTrain.meanMAEs));

            System.out.println(eproBSTrain.toStringClassificationDetails());

            pw.println(eproBSTrain.toStringClassificationDetails().replace("[score_training] ", ""));

            //632+ rule

            System.out.println("Bootstrap .632+ rule calculated on TRAIN set of " + co.classifier + " " + co.options

                    + " optimized by " + co.optimizer + "...");

            pw.println("\n#Bootstrap .632+ rule calculated on TRAIN set, "

                    + Main.bootstrapAndRepeatedHoldoutFolds + " folds with random seeds");

            bootstrapTrain632plus = weka.trainClassifierBootstrap632plus(co.classifier, co.options,

                    co.featuresSeparatedByCommas);

            System.out.println(df.format(bootstrapTrain632plus));

            pw.println(df.format(bootstrapTrain632plus));

            if (Main.ROCcurves) {

                rocCurveGraphs.createRocCurvesWithConfidence(alROCs, classification, modelFilename, ".roc_train.png");

            }

        } else {

            System.out.println("Bootstrap evaluation on TRAIN set of " + ro.classifier + " "

                    + ro.options + "optimized by " + ro.optimizer.toUpperCase());

            pw.println("\n#Bootstrap evaluation performance on TRAIN set, "

                    + Main.bootstrapAndRepeatedHoldoutFolds + " times average on random seeds");

            for (int i = 0; i < Main.bootstrapAndRepeatedHoldoutFolds; i++) {

                Weka_module.RegressionResultsObject rro

                        = (Weka_module.RegressionResultsObject) weka.trainClassifierBootstrap(ro.classifier, ro.options,

                                ro.featuresSeparatedByCommas, classification, i);

                eproBSTrain.alCCs.add(Double.valueOf(rro.CC));

                eproBSTrain.alMAEs.add(Double.valueOf(rro.MAE));

                eproBSTrain.alRMSEs.add(Double.valueOf(rro.RMSE));

                eproBSTrain.alRAEs.add(Double.valueOf(rro.RAE));

                eproBSTrain.alRRSEs.add(Double.valueOf(rro.RRSE));

            }

            eproBSTrain.computeMeans();

            alCCs.add(Double.valueOf(eproBSTrain.meanCCs));

            alMAEs.add(Double.valueOf(eproBSTrain.meanMAEs));

            pw.println(eproBSTrain.toStringRegressionDetails().replace("[score_training] ", ""));

        }

        pw.flush();

        // IF TEST SET

        try {

            if (Main.doSampling) {

                alROCs = new ArrayList<>();

                System.out.println("Evaluation performance on test set");

                pw.println("\n#Evaluation performance on test set");

                //get arff test filename generated before training

                String arffTestFile = trainFileName.replace("data_to_train.csv", "data_to_test.arff");

                //set the outfile of the extracted features needed to test the current model

                String arffTestFileWithExtractedModelFeatures = modelFilename + ".test_features.arff";

                //check if test set is here

                if (!new File(arffTestFile).exists()) {

                    pw.println("Test file " + arffTestFile + " not found");

                }

                //adapt test file to model (extract the needed features)

                //test file come from original dataset preprocessed in AdaptDatasetToTraining, so the arff is compatible

                Weka_module weka2 = new Weka_module();

                weka2.setARFFfile(arffTestFile);

                weka2.setDataFromArff();

                // create compatible test file

                if (Main.combineModels) {

                    //combined model contains the filters, we need to keep the same

                    //features indexes as the b.featureSelection.infoGain.arff

                    weka2.extractFeaturesFromArffFileBasedOnSelectedFeatures(weka.myData,

                            weka2.myData, arffTestFileWithExtractedModelFeatures);

                } else {

                    weka2.extractFeaturesFromTestFileBasedOnModel(modelFilename + ".model",

                            weka2.myData, arffTestFileWithExtractedModelFeatures);

                }

                //TESTING

                // reload compatible test file in weka2

                weka2 = new Weka_module();

                weka2.setARFFfile(arffTestFileWithExtractedModelFeatures);

                weka2.setDataFromArff();

                if (classification) {

                    Weka_module.ClassificationResultsObject cr2

                            = (Weka_module.ClassificationResultsObject) weka2.testClassifierFromFileSource(new File(weka2.ARFFfile),

                                    modelFilename + ".model", true);

                    alROCs.add(cr2);

                    alROCs.add(cr2);

                    System.out.println("[score_testing] ACC: " + cr2.ACC);

                    System.out.println("[score_testing] AUC: " + cr2.AUC);

                    System.out.println("[score_testing] AUPRC: " + cr2.AUPRC);

                    System.out.println("[score_testing] SEN: " + cr2.TPR);

                    System.out.println("[score_testing] SPE: " + cr2.TNR);

                    System.out.println("[score_testing] MCC: " + cr2.MCC);

                    System.out.println("[score_testing] MAE: " + cr2.MAE);

                    System.out.println("[score_testing] BER: " + cr2.BER);

                    pw.println("AUC: " + cr2.AUC);

                    pw.println("ACC: " + cr2.ACC);

                    pw.println("AUPRC: " + cr2.AUPRC);

                    pw.println("SEN: " + cr2.TPR);

                    pw.println("SPE: " + cr2.TNR);

                    pw.println("MCC: " + cr2.MCC);

                    pw.println("MAE: " + cr2.MAE);

                    pw.println("BER: " + cr2.BER);

                    alMCCs.add(Double.valueOf(cr2.MCC));

                    alMAEs.add(Double.valueOf(cr2.MAE));

                    if (Main.ROCcurves) {

                        rocCurveGraphs.createRocCurvesWithConfidence(alROCs, classification, modelFilename, ".roc_test.png");

                    }

                } else {

                    Weka_module.RegressionResultsObject rr2

                            = (Weka_module.RegressionResultsObject) weka2.testClassifierFromFileSource(new File(weka2.ARFFfile),

                                    modelFilename + ".model", false);

                    System.out.println("[score_testing] Average CC: " + rr2.CC);

                    System.out.println("[score_testing] Average RMSE: " + rr2.RMSE);

                    //

                    pw.println("Average CC: " + rr2.CC);

                    pw.println("Average RMSE: " + rr2.RMSE);

                    alCCs.add(Double.valueOf(rr2.CC));

                    alMAEs.add(Double.valueOf(rr2.MAE));

                }

                new File(arffTestFileWithExtractedModelFeatures).delete();

                //REPEATED HOLDOUT TRAIN_TEST

                arffTestFileWithExtractedModelFeatures = arffTestFileWithExtractedModelFeatures.replace(".test_features.arff", ".RH_features.arff");

                Weka_module.evaluationPerformancesResultsObject eproRHTrainTest = new Weka_module.evaluationPerformancesResultsObject();

                try {

                    alROCs = new ArrayList<>();

                    // adapt original dataset file to model (extract the needed features)

                    Weka_module weka3 = new Weka_module();

                    weka3.setARFFfile(trainFileName.replace("data_to_train.csv", "all_data.arff"));

                    weka3.setDataFromArff();

                    // create compatible  file

                    if (Main.combineModels) {

                        //combined model contains the filters, we need to keep the same

                        //features indexes as the b.featureSelection.infoGain.arff

                        weka3.extractFeaturesFromArffFileBasedOnSelectedFeatures(weka.myData,

                                weka3.myData, arffTestFileWithExtractedModelFeatures);

                    } else {

                        weka3.extractFeaturesFromTestFileBasedOnModel(modelFilename + ".model",

                                weka3.myData, arffTestFileWithExtractedModelFeatures);

                    }

                    // reload compatible file in weka2

                    weka3 = new Weka_module();

                    weka3.setARFFfile(arffTestFileWithExtractedModelFeatures);

                    weka3.setDataFromArff();

                    if (classification) {

                        if (!Main.combineModels) {

                            weka3.myData = weka3.extractFeaturesFromDatasetBasedOnModel(cr.model, weka3.myData);

                        }

                        System.out.println("Repeated Holdout evaluation on TRAIN AND TEST sets of " + co.classifier + " " + co.options

                                + " optimized by " + co.optimizer);

                        pw.println("\n#Repeated Holdout evaluation performance on TRAIN AND TEST set, "

                                + Main.bootstrapAndRepeatedHoldoutFolds + " times weighted average (and standard deviation) on random seeds");

                        for (int i = 0; i < Main.bootstrapAndRepeatedHoldoutFolds; i++) {

                            Weka_module.ClassificationResultsObject cro

                                    = (Weka_module.ClassificationResultsObject) weka3.trainClassifierHoldOutValidation(co.classifier, co.options,

                                            null, classification, i);

                            eproRHTrainTest.alAUCs.add(Double.valueOf(cro.AUC));

                            eproRHTrainTest.alpAUCs.add(Double.valueOf(cro.pAUC));

                            eproRHTrainTest.alAUPRCs.add(Double.valueOf(cro.AUPRC));

                            eproRHTrainTest.alACCs.add(Double.valueOf(cro.ACC));

                            eproRHTrainTest.alSEs.add(Double.valueOf(cro.TPR));

                            eproRHTrainTest.alSPs.add(Double.valueOf(cro.TNR));

                            eproRHTrainTest.alMCCs.add(Double.valueOf(cro.MCC));

                            eproRHTrainTest.alMAEs.add(Double.valueOf(cro.MAE));

                            eproRHTrainTest.alBERs.add(Double.valueOf(cro.BER));

                            alROCs.add(cro);

                        }

                        eproRHTrainTest.computeMeans();

                        alMCCs.add(Double.valueOf(eproRHTrainTest.meanMCCs));

                        alMAEs.add(Double.valueOf(eproRHTrainTest.meanMAEs));

                        System.out.println(eproRHTrainTest.toStringClassificationDetails());

                        pw.println(eproRHTrainTest.toStringClassificationDetails().replace("[score_training] ", ""));

                        if (Main.ROCcurves) {

                            rocCurveGraphs.createRocCurvesWithConfidence(alROCs, classification, modelFilename, ".roc.png");

                        }

                    } else {

                        if (!Main.combineModels) {

                            weka3.myData = weka3.extractFeaturesFromDatasetBasedOnModel(rr.model, weka3.myData);

                        }

                        System.out.println("Repeated Holdout evaluation on TRAIN AND TEST sets of " + ro.classifier + " "

                                + ro.options + "optimized by " + ro.optimizer.toUpperCase());

                        pw.println("\n#Repeated Holdout evaluation performance on TRAIN AND TEST set, "

                                + Main.bootstrapAndRepeatedHoldoutFolds + " times weighted average (and standard deviation) on random seeds");

                        for (int i = 0; i < Main.bootstrapAndRepeatedHoldoutFolds; i++) {

                            Weka_module.RegressionResultsObject rro

                                    = (Weka_module.RegressionResultsObject) weka3.trainClassifierHoldOutValidation(ro.classifier, ro.options,

                                            null, classification, i);

                            eproRHTrainTest.alCCs.add(Double.valueOf(rro.CC));

                            eproRHTrainTest.alMAEs.add(Double.valueOf(rro.MAE));

                            eproRHTrainTest.alRMSEs.add(Double.valueOf(rro.RMSE));

                            eproRHTrainTest.alRAEs.add(Double.valueOf(rro.RAE));

                            eproRHTrainTest.alRRSEs.add(Double.valueOf(rro.RRSE));

                        }

                        eproRHTrainTest.computeMeans();

                        alCCs.add(Double.valueOf(eproRHTrainTest.meanCCs));

                        alMAEs.add(Double.valueOf(eproRHTrainTest.meanMAEs));

                        System.out.println(eproRHTrainTest.toStringRegressionDetails());

                        pw.println(eproRHTrainTest.toStringRegressionDetails().replace("[score_training] ", ""));

                    }

                    eproRHTrainTest.computeMeans();

                } catch (Exception e) {

                    if (Main.debug) {

                        e.printStackTrace();

                    }

                }

                new File(arffTestFileWithExtractedModelFeatures).delete();

                //BOOTSRAP TRAIN_TEST

                arffTestFileWithExtractedModelFeatures = arffTestFileWithExtractedModelFeatures.replace(".RH_features.arff", ".BS_features.arff");

                Weka_module.evaluationPerformancesResultsObject eproBSTrainTest = new Weka_module.evaluationPerformancesResultsObject();

                try {

                    alROCs = new ArrayList<>();

                    Weka_module weka4 = new Weka_module();

                    weka4.setARFFfile(trainFileName.replace("data_to_train.csv", "all_data.arff"));

                    weka4.setDataFromArff();

                    // create compatible  file

                    if (Main.combineModels) {

                        //combined model contains the filters, we need to keep the same

                        //features indexes as the b.featureSelection.infoGain.arff

                        weka4.extractFeaturesFromArffFileBasedOnSelectedFeatures(weka.myData,

                                weka4.myData, arffTestFileWithExtractedModelFeatures);

                    } else {

                        weka4.extractFeaturesFromTestFileBasedOnModel(modelFilename + ".model",

                                weka4.myData, arffTestFileWithExtractedModelFeatures);

                    }

                    // reload compatible file in weka2

                    weka4 = new Weka_module();

                    weka4.setARFFfile(arffTestFileWithExtractedModelFeatures);

                    weka4.setDataFromArff();

                    if (classification) {

                        if (!Main.combineModels) {

                            weka4.myData = weka4.extractFeaturesFromDatasetBasedOnModel(cr.model, weka4.myData);

                        }

                        System.out.println("Bootstrap evaluation on TRAIN AND TEST sets of " + co.classifier + " " + co.options

                                + " optimized by " + co.optimizer);

                        pw.println("\n#Bootstrap evaluation performance on TRAIN AND TEST set, "

                                + Main.bootstrapAndRepeatedHoldoutFolds + " times weighted average (and standard deviation) on random seeds");

                        for (int i = 0; i < Main.bootstrapAndRepeatedHoldoutFolds; i++) {

                            Weka_module.ClassificationResultsObject cro

                                    = (Weka_module.ClassificationResultsObject) weka4.trainClassifierBootstrap(co.classifier, co.options,

                                            null, classification, i);

                            eproBSTrainTest.alAUCs.add(Double.valueOf(cro.AUC));

                            eproBSTrainTest.alpAUCs.add(Double.valueOf(cro.pAUC));

                            eproBSTrainTest.alAUPRCs.add(Double.valueOf(cro.AUPRC));

                            eproBSTrainTest.alACCs.add(Double.valueOf(cro.ACC));

                            eproBSTrainTest.alSEs.add(Double.valueOf(cro.TPR));

                            eproBSTrainTest.alSPs.add(Double.valueOf(cro.TNR));

                            eproBSTrainTest.alMCCs.add(Double.valueOf(cro.MCC));

                            eproBSTrainTest.alMAEs.add(Double.valueOf(cro.MAE));

                            eproBSTrainTest.alBERs.add(Double.valueOf(cro.BER));

                            alROCs.add(cro);

                        }

                        eproBSTrainTest.computeMeans();

                        alMCCs.add(Double.valueOf(eproBSTrainTest.meanMCCs));

                        alMAEs.add(Double.valueOf(eproBSTrainTest.meanMAEs));

                        System.out.println(eproBSTrainTest.toStringClassificationDetails());

                        pw.println(eproBSTrainTest.toStringClassificationDetails().replace("[score_training] ", ""));

                        //632+ rule

                        System.out.println("Bootstrap .632+ rule calculated on TRAIN AND TEST set of " + co.classifier + " " + co.options

                                + " optimized by " + co.optimizer + "...");

                        pw.println("\n#Bootstrap .632+ rule calculated on TRAIN AND TEST set, "

                                + Main.bootstrapAndRepeatedHoldoutFolds + " folds with random seeds");

                        bootstrapTrain632plus = weka4.trainClassifierBootstrap632plus(co.classifier, co.options,

                                null);

                        System.out.println(df.format(bootstrapTrain632plus));

                        pw.println(df.format(bootstrapTrain632plus));

                        if (Main.ROCcurves) {

                            rocCurveGraphs.createRocCurvesWithConfidence(alROCs, classification, modelFilename, ".roc.png");

                        }

                    } else {

                        if (!Main.combineModels) {

                            weka4.myData = weka4.extractFeaturesFromDatasetBasedOnModel(rr.model, weka4.myData);

                        }

                        System.out.println("Bootstrap evaluation on TRAIN AND TEST sets of " + ro.classifier + " "

                                + ro.options + "optimized by " + ro.optimizer.toUpperCase());

                        pw.println("\n#Bootstrap evaluation performance on TRAIN AND TEST set, "

                                + Main.bootstrapAndRepeatedHoldoutFolds + " times weighted average (and standard deviation) on random seeds");

                        for (int i = 0; i < Main.bootstrapAndRepeatedHoldoutFolds; i++) {

                            Weka_module.RegressionResultsObject rro

                                    = (Weka_module.RegressionResultsObject) weka4.trainClassifierBootstrap(ro.classifier, ro.options,

                                            null, classification, i);

                            eproBSTrainTest.alCCs.add(Double.valueOf(rro.CC));

                            eproBSTrainTest.alMAEs.add(Double.valueOf(rro.MAE));

                            eproBSTrainTest.alRMSEs.add(Double.valueOf(rro.RMSE));

                            eproBSTrainTest.alRAEs.add(Double.valueOf(rro.RAE));

                            eproBSTrainTest.alRRSEs.add(Double.valueOf(rro.RRSE));

                        }

                        eproBSTrainTest.computeMeans();

                        alCCs.add(Double.valueOf(eproBSTrainTest.meanCCs));

                        alMAEs.add(Double.valueOf(eproBSTrainTest.meanMAEs));

                        System.out.println(eproBSTrainTest.toStringRegressionDetails());

                        pw.println(eproBSTrainTest.toStringRegressionDetails().replace("[score_training] ", ""));

                    }

                    eproBSTrainTest.computeMeans();

                } catch (Exception e) {

                    if (Main.debug) {

                        e.printStackTrace();

                    }

                }

                //remove test file arff once done

                new File(arffTestFileWithExtractedModelFeatures).delete();

            }

        } catch (Exception e) {

            if (Main.debug) {

                e.printStackTrace();

            }

        }

        pw.flush();

        // show average metrics and standard deviation

        if (classification) {

            pw.println("\n# Average MCC: " + utils.getMean(alMCCs)

                    + "\t(" + utils.getStandardDeviation(alMCCs) + ")");

            System.out.println("\n# Average MCC: " + utils.getMean(alMCCs));

            pw.println("# Average MAE: " + utils.getMean(alMAEs)

                    + "\t(" + utils.getStandardDeviation(alMAEs) + ")");

            System.out.println("# Average MAE: " + utils.getMean(alMAEs));

        } else {

            pw.println("\n# Average CC: " + utils.getMean(alCCs)

                    + "\t(" + utils.getStandardDeviation(alCCs) + ")");

            System.out.println("\n# Average CC: " + utils.getMean(alCCs));

            pw.println("# Average MAE: " + utils.getMean(alMAEs)

                    + "\t(" + utils.getStandardDeviation(alMAEs) + ")");

            System.out.println("# Average MAE: " + utils.getMean(alMAEs));

        }

        //output features

        if (classification) {

            try {

                pw.print("\n# Selected Attributes (Total attributes:" + cr.numberOfFeatures + "). "

                        + "Occurrences are shown if you chose combined model\n");

                pw.print(cr.features);

                pw.println("\n# Attribute ranking by merit calculated by information gain");

                pw.print(cr.getFeatureRankingResults());

            } catch (Exception e) {

                e.printStackTrace();

            }

        } else {

            try {

                pw.print("\n# Selected Attributes\t(Total attributes:" + rr.numberOfFeatures + "). "

                        + "Occurrences are shown if you chose combined model\n");

                pw.print(rr.features);

                pw.println("\n# Attribute ranking by merit calculated by RELIEFF");

                pw.print(rr.getFeatureRankingResults());

            } catch (Exception e) {

                e.printStackTrace();

            }

        }

        pw.flush();

        //retrieve correlated features

        // do not retreive correlated features if we are already computing a model for the long signature

        if (Main.retrieveCorrelatedGenes && !correlatedFeaturesMode) {

            if (new File(trainFileName).exists()) {

                System.out.print("Search correlated features (spearman)...");

                pw.println("\n# Correlated features (Spearman)");

                pw.println("FeatureInSignature\tSpearmanCorrelationScore\tCorrelatedFeature");

                TreeMap<String, Double> tmsCorrelatedgenes

                        = RetreiveCorrelatedGenes.spearmanCorrelation(modelFilename + ".train_features.csv", trainFileName);

                for (String correlation : tmsCorrelatedgenes.keySet()) {

                    pw.println(correlation);

                }

                if (tmsCorrelatedgenes.isEmpty()) {

                    pw.println("#nothing found !");

                }

                System.out.println("[done]");

                System.out.print("Search correlated features (pearson)...");

                pw.println("\n# Correlated features (Pearson)");

                pw.println("FeatureInSignature\tPearsonCorrelationScore\tCorrelatedFeature");

                TreeMap<String, Double> tmpCorrelatedgenes

                        = RetreiveCorrelatedGenes.pearsonCorrelation(modelFilename + ".train_features.csv", trainFileName);

                for (String correlation : tmpCorrelatedgenes.keySet()) {

                    pw.println(correlation);

                }

                if (tmpCorrelatedgenes.isEmpty()) {

                    pw.println("#nothing found !");

                }

                System.out.println("[done]");

            } else {

                System.out.println("Feature file " + trainFileName + " not found. Unable to calculate correlated genes");

            }

            pw.flush();

            //retreive rankings

            String ranking = "";

            if (classification) {

                ranking = weka.featureRankingForClassification(trainFileName.replace("csv", "arff"));

            } else {

                ranking = weka.featureRankingForRegression(trainFileName.replace("csv", "arff"));

            }

            String lines[] = ranking.split("\n");

            HashMap<String, ArrayList<RankerObject>> hmRanks = new HashMap();

            try {

                for (String s : lines) {

                    s = s.replaceAll(" +", " ");

                    if (!s.startsWith("\t") && !s.trim().isEmpty() && s.trim().split(" ").length == 3) {

                        RankerObject rankero = new RankerObject(s.trim());

                        if (hmRanks.containsKey(rankero.roundedScore)) {

                            ArrayList<RankerObject> alRankero = hmRanks.get(rankero.roundedScore);

                            alRankero.add(rankero);

                            hmRanks.put(rankero.roundedScore, alRankero);

                        } else {

                            ArrayList<RankerObject> alRankero = new ArrayList<>();

                            alRankero.add(rankero);

                            hmRanks.put(rankero.roundedScore, alRankero);

                        }

                    }

                }

            } catch (Exception e) {

                if (Main.debug) {

                    e.printStackTrace();

                }

            }

            if (Main.retreiveCorrelatedGenesByRankingScore) {

                System.out.print("Search similar ranking scores (infogain for classification or relieFf) in the original dataset...");

                pw.println("\n# Similar ranking score (maximal difference: " + Main.maxRankingScoreDifference + ")");

                pw.println("FeatureInSignature\tRankingScore\tFeatureInDataset\tRankingScore");

                String rankedFeaturesSign[];

                if (classification) {

                    rankedFeaturesSign = cr.getFeatureRankingResults().split("\n");

                } else {

                    rankedFeaturesSign = rr.getFeatureRankingResults().split("\n");

                }

                for (String featureSign : rankedFeaturesSign) {

                    String featureSignIG = featureSign.split("\t")[0];

                    String featureSignIGrounded = df.format(Double.valueOf(featureSignIG));

                    if (hmRanks.containsKey(featureSignIGrounded)) {

                        for (RankerObject alio : hmRanks.get(featureSignIGrounded)) {

                            if (!featureSign.contains(alio.feature) && !alio.feature.equals(Main.mergingID)) {

                                //max difference between infogains: 0.005

                                if (Math.abs(Double.parseDouble(featureSignIG) - Double.parseDouble(alio.infogain))

                                        <= Main.maxRankingScoreDifference) {

                                    pw.println(featureSign.split("\t")[1] + "\t"

                                            + featureSignIG + "\t"

                                            + alio.feature + "\t"

                                            + alio.infogain);

                                }

                            }

                        }

                    }

                }

                System.out.println("[done]");

            }

            //close file

            pw.println("\n\n## End of file ##");

            pw.close();

            //export enriched signature

            LinkedHashMap<String, String> lhmCorrFeaturesNames = new LinkedHashMap<>();//signature + correlated features

            LinkedHashMap<String, String> lhmFeaturesNames = new LinkedHashMap<>();//signature only

            try {

                br = new BufferedReader(new FileReader(modelFilename + ".details.txt"));

                String line = "";

                //go to selected attributes

                while (!line.startsWith("# Attribute ranking by")) {

                    line = br.readLine();

                }

                line = br.readLine();

                //add attributes to hashmap

                while (!line.startsWith("#")) {

                    if (!line.isEmpty()) {

                        lhmCorrFeaturesNames.put(line.split("\t")[1].trim(), "");

                        lhmFeaturesNames.put(line.split("\t")[1].trim(), "");

                    }

                    line = br.readLine();

                }

                //go to spearman correlated attributes

                while (!line.startsWith("FeatureInSignature")) {

                    line = br.readLine();

                }

                line = br.readLine();

                //add attributes to hashmap

                while (!line.startsWith("#")) {

                    if (!line.isEmpty()) {

                        lhmCorrFeaturesNames.put(line.split("\t")[2].trim(), "");

                    }

                    line = br.readLine();

                }

                //go to pearson correlated attributes

                while (!line.startsWith("FeatureInSignature")) {

                    line = br.readLine();

                }

                line = br.readLine();

                //add attributes to hashmap

                while (!line.startsWith("#")) {

                    if (!line.isEmpty()) {

                        lhmCorrFeaturesNames.put(line.split("\t")[2].trim(), "");

                    }

                    line = br.readLine();

                }

                if (Main.retreiveCorrelatedGenesByRankingScore) {

                    //go to infogain correlated attributes

                    while (!line.startsWith("FeatureInSignature")) {

                        line = br.readLine();

                    }

                    line = br.readLine();

                    //add attributes to hashmap

                    while (br.ready()) {

                        if (!line.isEmpty()) {

                            lhmCorrFeaturesNames.put(line.split("\t")[0].trim(), "");

                            lhmCorrFeaturesNames.put(line.split("\t")[2].trim(), "");

                        }

                        line = br.readLine();

                    }

                }

                br.close();

                //write correlated feature file from training file

                correlatedFeatures = writeFeaturesFile(lhmCorrFeaturesNames, trainFileName,

                        classification, modelFilename + ".train_corrFeatures.csv");

                if (Main.doSampling) {

                    //write correlated feature file from all data file

                    //if we have done a sampling, then we can't go from trainFeaturesFile

                    //but to allFeaturesFile, which contain test data

                    correlatedFeatures = writeFeaturesFile(lhmCorrFeaturesNames, trainFileName.replace("data_to_train", "all_data"),

                            classification, modelFilename + ".all_corrFeatures.csv");

                    //short signature

                    writeFeaturesFile(lhmFeaturesNames, trainFileName.replace("data_to_train", "all_data"),

                            classification, modelFilename + ".all_features.csv");

                    System.out.println("");

                }

            } catch (Exception e) {

                e.printStackTrace();

            }

        }

        // delete useless files

        if (Main.doSampling) {

            new File(modelFilename + ".train_features.csv").delete();

            new File(modelFilename + ".train_corrFeatures.csv").delete();

        }

        new File(modelFilename + ".test_features.arff").delete();

        new File(modelFilename + ".RH_features.arff").delete();

        new File(modelFilename + ".BS_features.arff").delete();

    }

    /**

     *

     * @param lhm feature names in order

     * @param originFile the training file or all data file

     * @param classification if we are doing a classification

     * @param outfile outfile name

     */

    private String writeFeaturesFile(LinkedHashMap<String, String> lhm, String originFile, boolean classification, String outfile) {

        //find columns indices

        String featuresSeparatedByCommas = "1";

        try {

            BufferedReader br = new BufferedReader(new FileReader(originFile));

            String header = br.readLine();

            String features[] = header.split(utils.detectSeparator(originFile));

            for (int i = 0; i < features.length; i++) {

                String feature = features[i];

                if (lhm.containsKey(feature)) {

                    featuresSeparatedByCommas += "," + (i + 1);

                }

            }

            featuresSeparatedByCommas += "," + features.length;

        } catch (Exception e) {

            e.printStackTrace();

        }

        //extract columns using weka filter

        Weka_module weka2 = new Weka_module();

        weka2.setARFFfile(originFile.replace("csv", "arff"));

        weka2.setDataFromArff();

        weka2.saveFilteredDataToCSV(featuresSeparatedByCommas,

                classification, outfile);

        return featuresSeparatedByCommas;

    }

    /**

     * initialize weka

     *

     * @param infile

     * @param classification

     */

    private static void init(String infile, boolean classification) {

        //convert csv to arff

        if (infile.endsWith(".csv")) {

            weka.setCSVFile(new File(infile));

            weka.csvToArff(classification);

        } else {

            weka.setARFFfile(infile.replace(".csv", ".arff"));

        }

        //set local variable of weka object from ARFFfile

        weka.setDataFromArff();

        weka.myData = weka.convertStringsToNominal(weka.myData);

//        // check if class has numeric values, hence regression, instead of nominal class (classification)

        classification = weka.isClassification();

    }

    /**

     * calculate mean and standard deviation of an array of doubles

     *

     * @param al

     * @return

     */

    private static String getMeanAndStandardDeviation(ArrayList<Double> al) {

        double d[] = new double[al.size()];

        for (int i = 0; i < al.size(); i++) {

            d[i] = (double) al.get(i);

        }

        StandardDeviation sd = new StandardDeviation();

        Mean m = new Mean();

        return df.format(m.evaluate(d)) + " (" + df.format(sd.evaluate(d)) + ")";

    }

    /**

     * classification object

     */

    public static class classificationObject {

        public ArrayList<String> featureList = new ArrayList<>();

        public String featuresSeparatedByCommas = "";

        public String optimizer = "";

        public String mode = "";

        public String classifier = "";

        public String options = "";

        public String identifier = "";

        public TreeMap<Integer, Integer> tmFeatures;

        public HashMap<String, String> hmValues = new HashMap<>(); //Column name, value

        public classificationObject() {

        }

        public classificationObject(String line) {

            identifier = line.split("\t")[hmResultsHeaderNames.get("ID")];

            classifier = line.split("\t")[hmResultsHeaderNames.get("classifier")];

            options = line.split("\t")[hmResultsHeaderNames.get("Options")];