Datasets
Models
Diff of
/src/biodiscml/Training.java
[000000]
..
[ce076b]
Switch to side-by-side view
--- a +++ b/src/biodiscml/Training.java @@ -0,0 +1,1511 @@ +/* + * execute machine learning algorithms in parrallel + * Works with regression and classification + */ +package biodiscml; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.FileWriter; +import java.io.PrintWriter; +import java.text.DecimalFormat; +import java.text.DecimalFormatSymbols; +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Random; +import org.apache.commons.math3.stat.descriptive.moment.*; +import utils.Weka_module; +import utils.utils; +import weka.attributeSelection.AttributeSelection; +import weka.classifiers.Classifier; + +/** + * + * @author Mickael + */ +public class Training { + + public static Weka_module weka = new Weka_module(); + public static AttributeSelection ranking; + public static ArrayList<String[]> alClassifiers = new ArrayList<>(); + public static boolean isClassification = true; + public static String resultsSummaryHeader = ""; + public static int cptPassed = 0; + public static int cptFailed = 0; + public boolean parrallel = true; + public static String trainFileName = ""; + public static DecimalFormat df = new DecimalFormat(); + public static PrintWriter pw; + + public Training() { + } + + /** + * @param dataToTrainModel + * @param resultsFile + * @param featureSelectionFile + * @param type. Is "class" for classification, else "reg" for regression + */ + public Training(String dataToTrainModel, String resultsFile, + String featureSelectionFile, String type) { + // deleting previous run + if (!Main.restoreRun && !Main.resumeTraining) { + if (new File(featureSelectionFile).exists()) { + System.out.println("\t" + featureSelectionFile + " exist... deleting..."); + new File(featureSelectionFile).delete(); + new File(featureSelectionFile.replace(".csv", ".arff")).delete(); + } + if (new File(resultsFile).exists()) { + System.out.println("\t" + resultsFile + " exist... deleting..."); + new File(resultsFile).delete(); + new File(resultsFile.replace(".csv", ".arff")).delete(); + } + } + + df.setMaximumFractionDigits(3); + DecimalFormatSymbols dfs = new DecimalFormatSymbols(); + dfs.setDecimalSeparator('.'); + df.setDecimalFormatSymbols(dfs); + if (Main.cpus.equals("1")) { + parrallel = false; + } + System.out.println("Training on " + dataToTrainModel + ". All tested models results will be in " + resultsFile); + alClassifiers = new ArrayList<>(); + isClassification = type.equals("class"); + trainFileName = dataToTrainModel; +// File f = new File(Main.CVfolder);//cross validation folder outputs +// if (!f.exists()) { +// f.mkdir(); +// } + + //convert csv to arff + if (dataToTrainModel.endsWith(".csv") && !new File(dataToTrainModel.replace(".csv", ".arff")).exists()) { + weka.setCSVFile(new File(dataToTrainModel)); + weka.csvToArff(isClassification); + } else { + weka.setARFFfile(dataToTrainModel.replace(".csv", ".arff")); + } + + //set local variable of weka object from ARFFfile + weka.setDataFromArff(); + weka.myData = weka.convertStringsToNominal(weka.myData); + +// // check if class has numeric values, hence regression, instead of nominal class (classification) + //classification = weka.isClassification(); + //CLASSIFICATION + if (isClassification) { + if (Main.debug) { + System.out.println("Only use classification algorithms"); + } + //HEADER FOR SUMMARY OUTPUT + resultsSummaryHeader = "ID" + + "\tclassifier" + + "\tOptions" + + "\tOptimizedValue" + + "\tSearchMode" + + "\tnbrOfFeatures" + //10CV + + "\tTRAIN_10CV_ACC" + + "\tTRAIN_10CV_AUC" + + "\tTRAIN_10CV_AUPRC" + + "\tTRAIN_10CV_SEN" + + "\tTRAIN_10CV_SPE" + + "\tTRAIN_10CV_MCC" + + "\tTRAIN_10CV_MAE" + + "\tTRAIN_10CV_BER" + + "\tTRAIN_10CV_FPR" + + "\tTRAIN_10CV_FNR" + + "\tTRAIN_10CV_PPV" + + "\tTRAIN_10CV_FDR" + + "\tTRAIN_10CV_Fscore" + + "\tTRAIN_10CV_kappa" + + "\tTRAIN_matrix" + //LOOCV + + "\tTRAIN_LOOCV_ACC" + + "\tTRAIN_LOOCV_AUC" + + "\tTRAIN_LOOCV_AUPRC" + + "\tTRAIN_LOOCV_SEN" + + "\tTRAIN_LOOCV_SPE" + + "\tTRAIN_LOOCV_MCC" + + "\tTRAIN_LOOCV_MAE" + + "\tTRAIN_LOOCV_BER" + //Repeated Holdout TRAIN + + "\tTRAIN_RH_ACC" + + "\tTRAIN_RH_AUC" + + "\tTRAIN_RH_AUPRC" + + "\tTRAIN_RH_SEN" + + "\tTRAIN_RH_SPE" + + "\tTRAIN_RH_MCC" + + "\tTRAIN_RH_MAE" + + "\tTRAIN_RH_BER" + //Bootstrap TRAIN + + "\tTRAIN_BS_ACC" + + "\tTRAIN_BS_AUC" + + "\tTRAIN_BS_AUPRC" + + "\tTRAIN_BS_SEN" + + "\tTRAIN_BS_SPE" + + "\tTRAIN_BS_MCC" + + "\tTRAIN_BS_MAE" + + "\tTRAIN_BS_BER" + //Bootstrap .632+ TRAIN + + "\tTRAIN_BS.632+" + //test + + "\tTEST_ACC" + + "\tTEST_AUC" + + "\tTEST_AUPRC" + + "\tTEST_SEN" + + "\tTEST_SPE" + + "\tTEST_MCC" + + "\tTEST_MAE" + + "\tTEST_BER" + //Repeated Holdout TRAIN_TEST + + "\tTRAIN_TEST_RH_ACC" + + "\tTRAIN_TEST_RH_AUC" + + "\tTRAIN_TEST_RH_AUPRC" + + "\tTRAIN_TEST_RH_SEN" + + "\tTRAIN_TEST_RH_SPE" + + "\tTRAIN_TEST_RH_MCC" + + "\tTRAIN_TEST_RH_MAE" + + "\tTRAIN_TEST_RH_BER" + //Bootstrap TRAIN_TEST + + "\tTRAIN_TEST_BS_ACC" + + "\tTRAIN_TEST_BS_AUC" + + "\tTRAIN_TEST_BS_AUPRC" + + "\tTRAIN_TEST_BS_SEN" + + "\tTRAIN_TEST_BS_SPE" + + "\tTRAIN_TEST_BS_MCC" + + "\tTRAIN_TEST_BS_MAE" + + "\tTRAIN_TEST_BS_BER" + //Bootstrap .632+ TRAIN + + "\tTRAIN_TEST_BS.632+" + //stats + + "\tAVG_BER" + + "\tSTD_BER" + + "\tAVG_MAE" + + "\tSTD_MAE" + + "\tAVG_MCC" + + "\tSTD_MCC" + + "\tAttributeList"; + + //fast way classification. Predetermined commands. + if (Main.classificationFastWay) { + //hmclassifiers.put(new String[]{"misc.VFI", "-B 0.6", "AUC"}, ""); + //hmclassifiers.put(new String[]{"meta.CostSensitiveClassifier", "-cost-matrix \"[0.0 ratio; 100.0 0.0]\" -S 1 -W weka.classifiers.misc.VFI -- -B 0.4", "MCC"}, ""); + + for (String cmd : Main.classificationFastWayCommands) { + String optimizer = cmd.split(":")[1]; + String searchmode = cmd.split(":")[2]; + String classifier = cmd.split(":")[0].split(" ")[0]; + String options = cmd.split(":")[0].replace(classifier, ""); + //case where optimizer AND search modes are empty + if (optimizer.equals("allopt") && searchmode.equals("allsearch")) { + for (String allOptimizers : Main.classificationOptimizers.split(",")) { + for (String allSearchModes : Main.searchmodes.split(",")) { + alClassifiers.add(new String[]{classifier, options, allOptimizers.trim(), allSearchModes.trim()}); + } + } + //Case where only searchmode is empty + } else if (!optimizer.equals("allopt") && searchmode.equals("allsearch")) { + for (String allSearchModes : Main.searchmodes.split(",")) { + alClassifiers.add(new String[]{classifier, options, optimizer.trim(), allSearchModes.trim()}); + } + //case where only optimizer is empty + } else if (optimizer.equals("allopt") && !searchmode.equals("allsearch")) { + for (String allOptimizers : Main.classificationOptimizers.split(",")) { + alClassifiers.add(new String[]{classifier, options, allOptimizers.trim(), searchmode.trim()}); + } + //case where optimizer and searchmode are provided + } else { + alClassifiers.add(new String[]{classifier, options, optimizer, searchmode}); + } + + } + } else { + // if no feature selection is asked + if (Main.noFeatureSelection) { + Main.maxNumberOfSelectedFeatures = weka.myData.numAttributes(); + Main.maxNumberOfFeaturesInModel = weka.myData.numAttributes(); + for (String cmd : Main.classificationBruteForceCommands) { + String classifier = cmd.split(" ")[0]; + String options = cmd.replace(classifier, "").trim(); + addClassificationToQueue(classifier, options); + } + + } else { + //brute force classification, try everything in the provided classifiers and optimizers + for (String cmd : Main.classificationBruteForceCommands) { + String classifier = cmd.split(" ")[0]; + String options = cmd.replace(classifier, "").trim(); + addClassificationToQueue(classifier, options); + } + } + } + + if (!Main.noFeatureSelection) { + System.out.print("Feature selection and ranking..."); + if (new File(featureSelectionFile).exists() && Main.resumeTraining) { + System.out.print("\nFeature selection and ranking already done... skipping"); + } else { + //ATTRIBUTE SELECTION for classification + weka.attributeSelectionByInfoGainRankingAndSaveToCSV(featureSelectionFile); + //get the rank of attributes + ranking = weka.featureRankingForClassification(); + System.out.println("[done]"); + //reset arff and keep compatible header + weka.setCSVFile(new File(featureSelectionFile)); + weka.csvToArff(isClassification); + weka.makeCompatibleARFFheaders(dataToTrainModel.replace("data_to_train.csv", "data_to_train.arff"), + featureSelectionFile.replace("infoGain.csv", "infoGain.arff")); + } + weka.setARFFfile(featureSelectionFile.replace("infoGain.csv", "infoGain.arff")); + weka.setDataFromArff(); + } + + } else { + //REGRESSION + if (Main.debug) { + System.out.println("Only use regression algorithms"); + } + resultsSummaryHeader = "ID" + + "\tclassifier" + + "\tOptions" + + "\tOptimizedValue" + + "\tSearchMode" + + "\tnbrOfFeatures" + //10cv + + "\tTRAIN_10CV_CC" + + "\tTRAIN_10CV_MAE" + + "\tTRAIN_10CV_RMSE" + + "\tTRAIN_10CV_RAE" + + "\tTRAIN_10CV_RRSE" + //LOOCV + + "\tTRAIN_LOOCV_CC" + + "\tTRAIN_LOOCV_MAE" + + "\tTRAIN_LOOCV_RMSE" + + "\tTRAIN_LOOCV_RAE" + + "\tTRAIN_LOOCV_RRSE" + //Repeated Holdout + + "\tTRAIN_RH_CC" + + "\tTRAIN_RH_MAE" + + "\tTRAIN_RH_RMSE" + + "\tTRAIN_RH_RAE" + + "\tTRAIN_RH_RRSE" + //Bootstrap + + "\tTRAIN_BS_CC" + + "\tTRAIN_BS_MAE" + + "\tTRAIN_BS_RMSE" + + "\tTRAIN_BS_RAE" + + "\tTRAIN_BS_RRSE" + //TEST SET + + "\tTEST_CC" + + "\tTEST_MAE" + + "\tTEST_RMSE" + + "\tTEST_RAE" + + "\tTEST_RRSE" + //Repeated Holdout TRAIN_TEST + + "\tTRAIN_TEST_RH_CC" + + "\tTRAIN_TEST_RH_MAE" + + "\tTRAIN_TEST_RH_RMSE" + + "\tTRAIN_TEST_RH_RAE" + + "\tTRAIN_TEST_RH_RRSE" + //Bootstrap TRAIN_TEST + + "\tTRAIN_TEST_BS_CC" + + "\tTRAIN_TEST_BS_MAE" + + "\tTRAIN_TEST_BS_RMSE" + + "\tTRAIN_TEST_BS_RAE" + + "\tTRAIN_TEST_BS_RRSE" + //stats + + "\tAVG_CC" + + "\tSTD_CC" + + "\tAVG_MAE" + + "\tSTD_MAE" + + "\tAVG_RMSE" + + "\tSTD_RMSE" + + "\tAttributeList"; + + //fast way regression. Predetermined commands. + if (Main.regressionFastWay) { + //hmclassifiers.put(new String[]{"functions.GaussianProcesses", "-L 1.0 -N 0 -K \"weka.classifiers.functions.supportVector.RBFKernel -C 250007 -G 1.0\"", "CC"}, ""); + //hmclassifiers.put(new String[]{"meta.AdditiveRegression", + // "-S 1.0 -I 10 -W weka.classifiers.functions.GaussianProcesses -- -L 1.0 -N 0 -K \"weka.classifiers.functions.supportVector.PolyKernel -C 250007 -E 1.0 -L\"", "CC"}, ""); + + for (String cmd : Main.regressionFastWayCommands) { + String optimizer = cmd.split(":")[1]; + String searchmode = cmd.split(":")[2]; + String classifier = cmd.split(":")[0].split(" ")[0]; + String options = cmd.split(":")[0].replace(classifier, ""); + + //case where optimizer AND search modes are empty + if (optimizer.equals("allopt") && searchmode.equals("allsearch")) { + for (String allOptimizers : Main.regressionOptimizers.split(",")) { + for (String allSearchModes : Main.searchmodes.split(",")) { + alClassifiers.add(new String[]{classifier, options, allOptimizers.trim(), allSearchModes.trim()}); + } + } + //Case where only searchmode is empty + } else if (!optimizer.equals("allopt") && searchmode.equals("allsearch")) { + for (String allSearchModes : Main.searchmodes.split(",")) { + alClassifiers.add(new String[]{classifier, options, optimizer.trim(), allSearchModes.trim()}); + } + //case where only optimizer is empty + } else if (optimizer.equals("allopt") && !searchmode.equals("allsearch")) { + for (String allOptimizers : Main.regressionOptimizers.split(",")) { + alClassifiers.add(new String[]{classifier, options, allOptimizers.trim(), searchmode.trim()}); + } + //case where optimizer and searchmode are provided + } else { + alClassifiers.add(new String[]{classifier, options, optimizer, searchmode}); + } + } + } else { + //brute force classification, try everything in the provided classifiers and optimizers + for (String cmd : Main.regressionBruteForceCommands) { + String classifier = cmd.split(" ")[0]; + String options = cmd.replace(classifier, ""); + addRegressionToQueue(classifier, options); + } + } + + //ATTRIBUTE SELECTION for regression + if (!Main.noFeatureSelection) { + System.out.print("Selecting attributes and ranking by RelieFF..."); + if (new File(featureSelectionFile).exists() && Main.resumeTraining) { + System.out.print("Selecting attributes and ranking by RelieFF already done... skipped by resumeTraining"); + } else { + weka.attributeSelectionByRelieFFAndSaveToCSV(featureSelectionFile); + + System.out.println("[done]"); + + //reset arff and keep compatible header + weka.setCSVFile(new File(featureSelectionFile)); + weka.csvToArff(isClassification); + weka.makeCompatibleARFFheaders(dataToTrainModel.replace("data_to_train.csv", "data_to_train.arff"), + featureSelectionFile.replace("RELIEFF.csv", "RELIEFF.arff")); + weka.setARFFfile(featureSelectionFile.replace("RELIEFF.csv", "RELIEFF.arff")); + weka.setDataFromArff(); + } + } + + } + //resume training, remove from alClassifier all classifiers already trained + if (Main.resumeTraining && !Main.restoreRun) { + HashMap<String, String> hm = new HashMap<>(); + //get data + try { + BufferedReader br = new BufferedReader(new FileReader(resultsFile)); + br.readLine(); //skip header + while (br.ready()) { + String line[] = br.readLine().split("\t"); + if (!line[0].startsWith("ERROR")) { + String s[] = new String[4]; + s[0] = line[1]; + s[1] = line[2]; + s[2] = line[3].toLowerCase(); + s[3] = line[4].toLowerCase(); + hm.put(line[1] + "\t" + line[2] + "\t" + line[3].toLowerCase() + "\t" + line[4].toLowerCase(), ""); + } + } + } catch (Exception e) { + if (Main.debug) { + e.printStackTrace(); + } + } + //remove from alClassifiers + int alClassifiersBeforeRemoval = alClassifiers.size(); + for (int i = 0; i < alClassifiers.size(); i++) { + String s = alClassifiers.get(i)[0] + "\t" + alClassifiers.get(i)[1] + "\t" + alClassifiers.get(i)[2] + "\t" + alClassifiers.get(i)[3]; + if (hm.containsKey(s)) { + alClassifiers.remove(i); + } + } + int alClassifiersAfterRemoval = alClassifiers.size(); + int totalRemoved = alClassifiersBeforeRemoval - alClassifiersAfterRemoval; + if (Main.debug) { + System.out.println("Total removed from alClassifier after resumeTraining = " + totalRemoved); + } + + System.out.println("ResumeTraining: Remains " + alClassifiersAfterRemoval + + " classifiers to train on the " + alClassifiersBeforeRemoval); + + } + + try { + //PREPARE OUTPUT + if (Main.resumeTraining && !Main.restoreRun) { + pw = new PrintWriter(new FileWriter(resultsFile, true)); + //pw.println("Resumed here"); + pw.flush(); + } else { + pw = new PrintWriter(new FileWriter(resultsFile)); + pw.println(resultsSummaryHeader); + } + + //EXECUTE IN PARRALLEL + System.out.println("Total classifiers to test: " + alClassifiers.size()); + + if (parrallel) { + alClassifiers + .parallelStream() + .map((classif) -> StepWiseFeatureSelectionTraining(classif[0], classif[1], classif[2], classif[3])) + .map((s) -> { + if (!s.toLowerCase().contains("error")) { + pw.println(s); + pw.flush(); + } else if (Main.printFailedModels) { + pw.println(s); + pw.flush(); + } + return s; + }) + .sorted() + .forEach((_item) -> { + pw.flush(); + }); + + } else { + alClassifiers.stream().map((classif) -> { + String s = StepWiseFeatureSelectionTraining(classif[0], classif[1], classif[2], classif[3]); + if (!s.toLowerCase().contains("error")) { + pw.println(s); + } else if (Main.printFailedModels) { + pw.println(s); + } + return classif; + }).forEach((_item) -> { + pw.flush(); + }); + } + pw.close(); + + //END + System.out.println("Total model tested: " + cptPassed + "/" + alClassifiers.size() + + ", including " + cptFailed + " incompatible models"); + + } catch (Exception e) { + if (Main.debug) { + e.printStackTrace(); + } + System.out.println("Total model tested: " + cptPassed + "/" + alClassifiers.size() + + ", including " + cptFailed + " incompatible models"); + pw.close(); + + } catch (Error err) { + if (Main.debug) { + err.printStackTrace(); + } + System.out.println("Total model tested: " + cptPassed + "/" + alClassifiers.size() + + ", including " + cptFailed + " incompatible models"); + pw.close(); + } + } + + /** + * StepWise training. We add the features one by one in the model if they + * improve the value to maximize We only test the first 500 attributes (or + * less if the number of attributes is less than 500) + * + * @param classifier + * @param classifier_options + * @param valueToMaximizeOrMinimize + * @return + */ + private static String StepWiseFeatureSelectionTraining(String classifier, String classifier_options, + String valueToMaximizeOrMinimize, String searchMethod) { + String out = classifier + "\t" + classifier_options + "\t" + valueToMaximizeOrMinimize.toUpperCase() + "\t" + searchMethod; + Instant start = Instant.now(); + System.out.println("[model] (" + (cptPassed++) + "/" + alClassifiers.size() + ")" + out); + String lastOutput = ""; + + boolean minimize = valueToMaximizeOrMinimize.equals("fdr") + || valueToMaximizeOrMinimize.equals("mae") + || valueToMaximizeOrMinimize.equals("rmse") + || valueToMaximizeOrMinimize.equals("ber") + || valueToMaximizeOrMinimize.equals("rae") + || valueToMaximizeOrMinimize.equals("rrse"); + + try { + Object o = null; + int numberOfAttributes = 0; + double previousMeasureToMaximize = -1000.0; + double previousMeasureToMinimize = 1000.0; + ArrayList<Integer> alAttributes = new ArrayList<>(); + int classIndex = weka.myData.numAttributes(); + for (int i = 1; i <= classIndex; i++) { // weka starts at 1, not 0 + alAttributes.add(i); + } + AttributeOject ao = new AttributeOject(alAttributes); + int cpt = 0; + Weka_module.ClassificationResultsObject cr = null; + Weka_module.RegressionResultsObject rr = null; + + // SHORT TEST to ensure compatibility of the model + if (Main.performShortTest) { + o = weka.shortTestTrainClassifier(classifier, classifier_options, + ao.getAttributesIdClassInString(), isClassification); + if (o.getClass().getName().equals("java.lang.String")) { + if (Main.debug) { + System.out.println("SHORT TEST FAILED"); + } + return "ERROR\t" + o.toString(); + } + } + if (Main.debug) { + System.out.println("\tGoing to training"); + } + + //all features search, no feature selection + if (searchMethod.startsWith("all")) { + int top = ao.alAttributes.size(); + //Create attribute list + for (int i = 0; i < top; i++) { //from ID to class (excluded) + //add new attribute to the set of retainedAttributes + ao.addNewAttributeToRetainedAttributes(i); + } + //TRAIN 10CV + o = weka.trainClassifier(classifier, classifier_options, + ao.getRetainedAttributesIdClassInString(), isClassification, 10); + + if (!(o instanceof String) || o == null) { + if (isClassification) { + cr = (Weka_module.ClassificationResultsObject) o; + } else { + rr = (Weka_module.RegressionResultsObject) o; + } + } + } + + //TOP X features search + if (searchMethod.startsWith("top")) { + int top = Integer.valueOf(searchMethod.replace("top", "")); + if (top <= ao.alAttributes.size()) { + //Create attribute list + for (int i = 0; i < top; i++) { //from ID to class (excluded) + //add new attribute to the set of retainedAttributes + ao.addNewAttributeToRetainedAttributes(i); + } + //TRAIN 10CV + o = weka.trainClassifier(classifier, classifier_options, + ao.getRetainedAttributesIdClassInString(), isClassification, 10); + + if (!(o instanceof String) || o == null) { + if (isClassification) { + cr = (Weka_module.ClassificationResultsObject) o; + } else { + rr = (Weka_module.RegressionResultsObject) o; + } + } + } else { + o = null; + } + } + + //FORWARD AND FORWARD-BACKWARD search AND BACKWARD AND BACKWARD-FORWARD search + if (searchMethod.startsWith("f") || searchMethod.startsWith("b")) { + boolean doForwardBackward_OR_BackwardForward = searchMethod.equals("fb") || searchMethod.equals("bf"); + boolean StartBackwardInsteadForward = searchMethod.startsWith("b"); + if (StartBackwardInsteadForward) { + Collections.reverse(ao.alAttributes); + } + + for (int i = 0; i < ao.alAttributes.size(); i++) { //from ID to class (excluded) + cpt++; + //add new attribute to the set of retainedAttributes + ao.addNewAttributeToRetainedAttributes(i); + Weka_module.ClassificationResultsObject oldcr = cr; + //do feature selection by forward(-backward) + if (ao.retainedAttributesOnly.size() <= Main.maxNumberOfFeaturesInModel) { + o = weka.trainClassifier(classifier, classifier_options, + ao.getRetainedAttributesIdClassInString(), isClassification, 10); + + if (o instanceof String || o == null) { + return (String) o; + } else if (isClassification) { + cr = (Weka_module.ClassificationResultsObject) o; + } else { + rr = (Weka_module.RegressionResultsObject) o; + } + + //choose what we want to maximize or minimize (such as error rates) + //this will crash if model had an error + double currentMeasure = getValueToMaximize(valueToMaximizeOrMinimize, cr, rr); + + //Report results + boolean modelIsImproved = false; + //i<2 is to avoid return no attribute at all + // test if model is improved + if (minimize) { + modelIsImproved = (i < 2 && currentMeasure <= previousMeasureToMinimize) || currentMeasure < previousMeasureToMinimize; + } else { + modelIsImproved = (i < 2 && currentMeasure >= previousMeasureToMaximize) || currentMeasure > previousMeasureToMaximize; + } + + if (modelIsImproved) { + if (!minimize) { + previousMeasureToMaximize = currentMeasure; + } else { + previousMeasureToMinimize = currentMeasure; + } + + // do backward OR forward, check if we have an improvement if we remove previously chosen features + if (doForwardBackward_OR_BackwardForward && numberOfAttributes > 1) { + oldcr = cr; + ArrayList<Integer> attributesToTestInBackward = ao.getRetainedAttributesIdClassInArrayList(); + for (int j = 1/*skip ID*/; + j < attributesToTestInBackward.size() - 2/*skip last attribute we added by forward and class*/; j++) { + attributesToTestInBackward.remove(j); + + String featuresToTest = utils.arrayToString(attributesToTestInBackward, ","); + //train + if (isClassification) { + cr = (Weka_module.ClassificationResultsObject) weka.trainClassifier(classifier, classifier_options, + featuresToTest, isClassification, 10); + } else { + rr = (Weka_module.RegressionResultsObject) weka.trainClassifier(classifier, classifier_options, + featuresToTest, isClassification, 10); + } + //get measure + double measureWithRemovedFeature = getValueToMaximize(valueToMaximizeOrMinimize, cr, rr); + //check if we have improvement + if (minimize) { + modelIsImproved = (measureWithRemovedFeature <= currentMeasure) + || measureWithRemovedFeature < currentMeasure; + } else { + modelIsImproved = (measureWithRemovedFeature >= currentMeasure) + || measureWithRemovedFeature > currentMeasure; + } + + if (modelIsImproved) { + //if model is improved definitly discard feature is improvement + ao.changeRetainedAttributes(featuresToTest); + oldcr = cr; + currentMeasure = measureWithRemovedFeature; + } else { + //restore the feature + attributesToTestInBackward = ao.getRetainedAttributesIdClassInArrayList(); + cr = oldcr; + } + } + } +// + //modify results summary output + //only for DEBUG purposes + if (isClassification) { + lastOutput = out + + "\t" + cr.numberOfFeatures + "\t" + cr.toString() + "\t" + ao.getRetainedAttributesIdClassInString(); + + } else { + lastOutput = out + + "\t" + rr.numberOfFeatures + "\t" + rr.toString() + "\t" + ao.getRetainedAttributesIdClassInString(); + } + + } else { + //back to previous attribute if no improvement with the new attribute and go to the next + ao.retainedAttributesOnly.remove(ao.retainedAttributesOnly.size() - 1); + cr = oldcr; + } + } + } + } + + //now we have a good model, evaluate the performance using various approaches + if (!(o instanceof String) && o != null) { + //LOOCV + Weka_module.ClassificationResultsObject crLoocv = null; + Weka_module.RegressionResultsObject rrLoocv = null; + String loocvOut = ""; + if (Main.loocv) { + if (Main.debug) { + System.out.println("\tLOOCV Train set " + Main.bootstrapAndRepeatedHoldoutFolds + " times"); + } + Object oLoocv; + if (isClassification) { + oLoocv = weka.trainClassifier(classifier, classifier_options, + ao.getRetainedAttributesIdClassInString(), isClassification, + cr.dataset.numInstances()); + } else { + oLoocv = weka.trainClassifier(classifier, classifier_options, + ao.getRetainedAttributesIdClassInString(), isClassification, + rr.dataset.numInstances()); + } + + if (oLoocv instanceof String || oLoocv == null) { + if (Main.debug) { + System.err.println("[error] LOOCV failed"); + } + } else if (isClassification) { + crLoocv = (Weka_module.ClassificationResultsObject) oLoocv; + loocvOut = crLoocv.toStringShort(); + } else { + rrLoocv = (Weka_module.RegressionResultsObject) oLoocv; + loocvOut = rrLoocv.toString(); + } + + } else { + if (isClassification) { + loocvOut = "\t" + "\t" + "\t" + "\t" + "\t" + "\t" + "\t"; + } else { + loocvOut = "\t" + "\t" + "\t" + "\t"; + } + } + //Repeated Holdout + Weka_module.evaluationPerformancesResultsObject eproRHTrain = new Weka_module.evaluationPerformancesResultsObject(); + if (Main.repeatedHoldout) { + if (Main.debug) { + System.out.println("\tRepeated Holdout on Train set " + Main.bootstrapAndRepeatedHoldoutFolds + " times"); + } + if (isClassification) { + for (int i = 0; i < Main.bootstrapAndRepeatedHoldoutFolds; i++) { + Weka_module.ClassificationResultsObject cro + = (Weka_module.ClassificationResultsObject) weka.trainClassifierHoldOutValidation(classifier, classifier_options, + ao.getRetainedAttributesIdClassInString(), isClassification, i); + eproRHTrain.alAUCs.add(Double.valueOf(cro.AUC)); + eproRHTrain.alpAUCs.add(Double.valueOf(cro.pAUC)); + eproRHTrain.alAUPRCs.add(Double.valueOf(cro.AUPRC)); + eproRHTrain.alACCs.add(Double.valueOf(cro.ACC)); + eproRHTrain.alSEs.add(Double.valueOf(cro.TPR)); + eproRHTrain.alSPs.add(Double.valueOf(cro.TNR)); + eproRHTrain.alMCCs.add(Double.valueOf(cro.MCC)); + eproRHTrain.alMAEs.add(Double.valueOf(cro.MAE)); + eproRHTrain.alBERs.add(Double.valueOf(cro.BER)); + } + } else { + for (int i = 0; i < Main.bootstrapAndRepeatedHoldoutFolds; i++) { + Weka_module.RegressionResultsObject rro + = (Weka_module.RegressionResultsObject) weka.trainClassifierHoldOutValidation(classifier, classifier_options, + ao.getRetainedAttributesIdClassInString(), isClassification, i); + eproRHTrain.alCCs.add(Double.valueOf(rro.CC)); + eproRHTrain.alMAEs.add(Double.valueOf(rro.MAE)); + eproRHTrain.alRMSEs.add(Double.valueOf(rro.RMSE)); + eproRHTrain.alRAEs.add(Double.valueOf(rro.RAE)); + eproRHTrain.alRRSEs.add(Double.valueOf(rro.RRSE)); + } + } + eproRHTrain.computeMeans(); + } + + //BOOTSTRAP AND BOOTSTRAP .632+ rule TRAIN + if (Main.debug) { + System.out.println("\tBootstrapping on Train set " + Main.bootstrapAndRepeatedHoldoutFolds + " times"); + } + double bootstrapTrain632plus = 1000; + Weka_module.evaluationPerformancesResultsObject eproBSTrain + = new Weka_module.evaluationPerformancesResultsObject(); + if (Main.bootstrap) { + if (isClassification) { + bootstrapTrain632plus = weka.trainClassifierBootstrap632plus(classifier, classifier_options, + ao.getRetainedAttributesIdClassInString()); + for (int i = 0; i < Main.bootstrapAndRepeatedHoldoutFolds; i++) { + Weka_module.ClassificationResultsObject cro + = (Weka_module.ClassificationResultsObject) weka.trainClassifierBootstrap(classifier, classifier_options, + ao.getRetainedAttributesIdClassInString(), isClassification, i); + eproBSTrain.alAUCs.add(Double.valueOf(cro.AUC)); + eproBSTrain.alpAUCs.add(Double.valueOf(cro.pAUC)); + eproBSTrain.alAUPRCs.add(Double.valueOf(cro.AUPRC)); + eproBSTrain.alACCs.add(Double.valueOf(cro.ACC)); + eproBSTrain.alSEs.add(Double.valueOf(cro.TPR)); + eproBSTrain.alSPs.add(Double.valueOf(cro.TNR)); + eproBSTrain.alMCCs.add(Double.valueOf(cro.MCC)); + eproBSTrain.alMAEs.add(Double.valueOf(cro.MAE)); + eproBSTrain.alBERs.add(Double.valueOf(cro.BER)); + } + } else { + for (int i = 0; i < Main.bootstrapAndRepeatedHoldoutFolds; i++) { + Weka_module.RegressionResultsObject rro + = (Weka_module.RegressionResultsObject) weka.trainClassifierBootstrap(classifier, classifier_options, + ao.getRetainedAttributesIdClassInString(), isClassification, i); + + eproBSTrain.alCCs.add(Double.valueOf(rro.CC)); + eproBSTrain.alMAEs.add(Double.valueOf(rro.MAE)); + eproBSTrain.alRMSEs.add(Double.valueOf(rro.RMSE)); + eproBSTrain.alRAEs.add(Double.valueOf(rro.RAE)); + eproBSTrain.alRRSEs.add(Double.valueOf(rro.RRSE)); + } + } + eproBSTrain.computeMeans(); + } + + // TEST SET + if (Main.debug) { + System.out.println("\tEvalutation on Test set "); + } + Weka_module.testResultsObject tro = new Weka_module.testResultsObject(); + String testResults = null; + if (isClassification) { + testResults = "\t" + "\t" + "\t" + "\t" + "\t" + "\t" + "\t"; + } else { + testResults = "\t" + "\t" + "\t" + "\t"; + } + try { + if (Main.doSampling) { + //testing + Weka_module weka2 = new Weka_module(); + if (isClassification) { + Weka_module.ClassificationResultsObject cr2 = null; + try { + cr2 = (Weka_module.ClassificationResultsObject) weka2.testClassifierFromModel(cr.model, + trainFileName.replace("data_to_train.csv", "data_to_test.csv"),//test file + Main.isClassification, out); + } catch (Exception e) { + if (Main.debug) { + e.printStackTrace(); + } + } + tro.ACC = cr2.ACC; + tro.AUC = cr2.AUC; + tro.AUPRC = cr2.AUPRC; + tro.SE = cr2.TPR; + tro.SP = cr2.TNR; + tro.MCC = cr2.MCC; + tro.MAE = cr2.MAE; + tro.BER = cr2.BER; + testResults = tro.toStringClassification(); + + } else { + Weka_module.RegressionResultsObject rr2 + = (Weka_module.RegressionResultsObject) weka2.testClassifierFromModel(rr.model, + trainFileName.replace("data_to_train.csv", "data_to_test.csv"),//test file + Main.isClassification, out); + tro.CC = rr2.CC; + tro.RRSE = rr2.RRSE; + tro.RMSE = rr2.RMSE; + tro.MAE = rr2.MAE; + tro.RAE = rr2.RAE; + testResults = tro.toStringRegression(); + } + } + } catch (Exception e) { + if (Main.debug) { + e.printStackTrace(); + } + } + + //REPEATED HOLDOUT TRAIN_TEST + if (Main.debug) { + System.out.println("\tRepeated Holdout on Train AND Test set " + + Main.bootstrapAndRepeatedHoldoutFolds + " times"); + } + Weka_module.evaluationPerformancesResultsObject eproRHTrainTest + = new Weka_module.evaluationPerformancesResultsObject(); + try { + if (Main.doSampling && Main.repeatedHoldout) { + Weka_module weka2 = new Weka_module(); + weka2.setARFFfile(trainFileName.replace("data_to_train.csv", "all_data.arff")); + weka2.setDataFromArff(); + if (isClassification) { + weka2.myData = weka2.extractFeaturesFromDatasetBasedOnModel(cr.model, weka2.myData); + for (int i = 0; i < Main.bootstrapAndRepeatedHoldoutFolds; i++) { + Weka_module.ClassificationResultsObject cro + = (Weka_module.ClassificationResultsObject) weka2.trainClassifierHoldOutValidation( + classifier, classifier_options, + null, isClassification, i); + + eproRHTrainTest.alAUCs.add(Double.valueOf(cro.AUC)); + eproRHTrainTest.alpAUCs.add(Double.valueOf(cro.pAUC)); + eproRHTrainTest.alAUPRCs.add(Double.valueOf(cro.AUPRC)); + eproRHTrainTest.alACCs.add(Double.valueOf(cro.ACC)); + eproRHTrainTest.alSEs.add(Double.valueOf(cro.TPR)); + eproRHTrainTest.alSPs.add(Double.valueOf(cro.TNR)); + eproRHTrainTest.alMCCs.add(Double.valueOf(cro.MCC)); + eproRHTrainTest.alMAEs.add(Double.valueOf(cro.MAE)); + eproRHTrainTest.alBERs.add(Double.valueOf(cro.BER)); + } + } else { + weka2.myData = weka2.extractFeaturesFromDatasetBasedOnModel(rr.model, weka2.myData); + for (int i = 0; i < Main.bootstrapAndRepeatedHoldoutFolds; i++) { + Weka_module.RegressionResultsObject rro + = (Weka_module.RegressionResultsObject) weka2.trainClassifierHoldOutValidation( + classifier, classifier_options, + null, isClassification, i); + eproRHTrainTest.alCCs.add(Double.valueOf(rro.CC)); + eproRHTrainTest.alMAEs.add(Double.valueOf(rro.MAE)); + eproRHTrainTest.alRMSEs.add(Double.valueOf(rro.RMSE)); + eproRHTrainTest.alRAEs.add(Double.valueOf(rro.RAE)); + eproRHTrainTest.alRRSEs.add(Double.valueOf(rro.RRSE)); + } + } + eproRHTrainTest.computeMeans(); + } + } catch (Exception e) { + if (Main.debug) { + e.printStackTrace(); + } + } + + //BOOTSTRAP TRAIN_TEST AND BOOTSTRAP .632+ rule TRAIN_TEST + if (Main.debug) { + System.out.println("\tBootstrapping on Train AND Test set " + Main.bootstrapAndRepeatedHoldoutFolds + " times"); + } + double bootstrapTrainTest632plus = 1000; + Weka_module.evaluationPerformancesResultsObject eproBSTrainTest = new Weka_module.evaluationPerformancesResultsObject(); + try { + if (Main.doSampling && Main.bootstrap) { + Weka_module weka2 = new Weka_module(); + weka2.setARFFfile(trainFileName.replace("data_to_train.csv", "all_data.arff")); + weka2.setDataFromArff(); + if (isClassification) { + weka2.myData = weka2.extractFeaturesFromDatasetBasedOnModel(cr.model, weka2.myData); + bootstrapTrainTest632plus = weka2.trainClassifierBootstrap632plus(classifier, classifier_options, + null); + for (int i = 0; i < Main.bootstrapAndRepeatedHoldoutFolds; i++) { + Weka_module.ClassificationResultsObject cro + = (Weka_module.ClassificationResultsObject) weka2.trainClassifierBootstrap( + classifier, classifier_options, + null, isClassification, i); + + eproBSTrainTest.alAUCs.add(Double.valueOf(cro.AUC)); + eproBSTrainTest.alpAUCs.add(Double.valueOf(cro.pAUC)); + eproBSTrainTest.alAUPRCs.add(Double.valueOf(cro.AUPRC)); + eproBSTrainTest.alACCs.add(Double.valueOf(cro.ACC)); + eproBSTrainTest.alSEs.add(Double.valueOf(cro.TPR)); + eproBSTrainTest.alSPs.add(Double.valueOf(cro.TNR)); + eproBSTrainTest.alMCCs.add(Double.valueOf(cro.MCC)); + eproBSTrainTest.alMAEs.add(Double.valueOf(cro.MAE)); + eproBSTrainTest.alBERs.add(Double.valueOf(cro.BER)); + } + } else { + weka2.myData = weka2.extractFeaturesFromDatasetBasedOnModel(rr.model, weka2.myData); + for (int i = 0; i < Main.bootstrapAndRepeatedHoldoutFolds; i++) { + Weka_module.RegressionResultsObject rro + = (Weka_module.RegressionResultsObject) weka2.trainClassifierBootstrap( + classifier, classifier_options, + null, isClassification, i); + eproBSTrainTest.alCCs.add(Double.valueOf(rro.CC)); + eproBSTrainTest.alMAEs.add(Double.valueOf(rro.MAE)); + eproBSTrainTest.alRMSEs.add(Double.valueOf(rro.RMSE)); + eproBSTrainTest.alRAEs.add(Double.valueOf(rro.RAE)); + eproBSTrainTest.alRRSEs.add(Double.valueOf(rro.RRSE)); + } + } + eproBSTrainTest.computeMeans(); + } + } catch (Exception e) { + if (Main.debug) { + e.printStackTrace(); + } + } + + //STATISTICS AND OUTPUT + if (isClassification) { + //average BER + List<Double> alBERs = new ArrayList<>(); + try { + alBERs.add(Double.valueOf(cr.BER)); + } catch (Exception e) { + } + try { + alBERs.add(Double.valueOf(eproRHTrain.meanBERs)); + } catch (Exception e) { + } + try { + alBERs.add(Double.valueOf(eproBSTrain.meanBERs)); + } catch (Exception e) { + } + try { + alBERs.add(Double.valueOf(crLoocv.BER)); + } catch (Exception e) { + } + try { + alBERs.add(Double.valueOf(tro.BER)); + } catch (Exception e) { + } + try { + alBERs.add(Double.valueOf(eproRHTrainTest.meanBERs)); + } catch (Exception e) { + } + try { + alBERs.add(Double.valueOf(eproBSTrainTest.meanBERs)); + } catch (Exception e) { + } + + double BERs[] = alBERs.stream().mapToDouble(Double::doubleValue).toArray(); + + StandardDeviation std = new StandardDeviation(); + std.setData(BERs); + double StdBER = std.evaluate(); + + Mean m = new Mean(); + m.setData(BERs); + double averageBER = m.evaluate(); + + //average MAE + List<Double> alMAEs = new ArrayList<>(); + try { + alMAEs.add(Double.valueOf(cr.MAE)); + } catch (Exception e) { + } + try { + alMAEs.add(Double.valueOf(eproRHTrain.meanMAEs)); + } catch (Exception e) { + } + try { + alMAEs.add(Double.valueOf(eproBSTrain.meanMAEs)); + } catch (Exception e) { + } + try { + alMAEs.add(Double.valueOf(crLoocv.MAE)); + } catch (Exception e) { + } + try { + alMAEs.add(Double.valueOf(tro.MAE)); + } catch (Exception e) { + } + try { + alMAEs.add(Double.valueOf(eproRHTrainTest.meanMAEs)); + } catch (Exception e) { + } + try { + alMAEs.add(Double.valueOf(eproBSTrainTest.meanMAEs)); + } catch (Exception e) { + } + + double MAEs[] = alMAEs.stream().mapToDouble(Double::doubleValue).toArray(); + + std = new StandardDeviation(); + std.setData(MAEs); + double StdMAE = std.evaluate(); + + m = new Mean(); + m.setData(MAEs); + double averageMAE = m.evaluate(); + + //average MCC + List<Double> alMCCs = new ArrayList<>(); + try { + alMCCs.add(Double.valueOf(cr.MCC)); + } catch (Exception e) { + } + try { + alMCCs.add(Double.valueOf(eproRHTrain.meanMCCs)); + } catch (Exception e) { + } + try { + alMCCs.add(Double.valueOf(eproBSTrain.meanMCCs)); + } catch (Exception e) { + } + try { + alMCCs.add(Double.valueOf(crLoocv.MCC)); + } catch (Exception e) { + } + try { + alMCCs.add(Double.valueOf(tro.MCC)); + } catch (Exception e) { + } + try { + alMCCs.add(Double.valueOf(eproRHTrainTest.meanMCCs)); + } catch (Exception e) { + } + try { + alMCCs.add(Double.valueOf(eproBSTrainTest.meanMCCs)); + } catch (Exception e) { + } + + double MCCs[] = alMCCs.stream().mapToDouble(Double::doubleValue).toArray(); + + std = new StandardDeviation(); + std.setData(MCCs); + double StdMCC = std.evaluate(); + + m = new Mean(); + m.setData(MCCs); + double averageMCC = m.evaluate(); + + String stats = df.format(averageBER) + "\t" + + df.format(StdBER) + "\t" + + df.format(averageMAE) + "\t" + + df.format(StdMAE) + "\t" + + df.format(averageMCC) + "\t" + + df.format(StdMCC); + + //output + String bt632 = df.format(bootstrapTrain632plus); + if (bt632.equals(1000)) { + bt632 = ""; + } + String btt632 = df.format(bootstrapTrainTest632plus); + if (bootstrapTrainTest632plus == 1000) { + btt632 = ""; + } + + lastOutput = out + + "\t" + cr.numberOfFeatures + + "\t" + cr.toString() + + "\t" + loocvOut + + "\t" + eproRHTrain.toStringClassification() + + "\t" + eproBSTrain.toStringClassification() + + "\t" + bt632 + + "\t" + testResults + + "\t" + eproRHTrainTest.toStringClassification() + + "\t" + eproBSTrainTest.toStringClassification() + + "\t" + btt632 + + "\t" + stats + "\t" + ao.getRetainedAttributesIdClassInString(); + } else { + //statistics + + //average CC + List<Double> alCCs = new ArrayList<>(); + try { + alCCs.add(Double.valueOf(rr.CC)); + } catch (Exception e) { + } + try { + alCCs.add(Double.valueOf(eproRHTrain.meanCCs)); + } catch (Exception e) { + } + try { + alCCs.add(Double.valueOf(eproBSTrain.meanCCs)); + } catch (Exception e) { + } + try { + alCCs.add(Double.valueOf(rrLoocv.CC)); + } catch (Exception e) { + } + try { + alCCs.add(Double.valueOf(tro.CC)); + } catch (Exception e) { + } + try { + alCCs.add(Double.valueOf(eproRHTrainTest.meanCCs)); + } catch (Exception e) { + } + try { + alCCs.add(Double.valueOf(eproBSTrainTest.meanCCs)); + } catch (Exception e) { + } + + double CCs[] = alCCs.stream().mapToDouble(Double::doubleValue).toArray(); + + StandardDeviation std = new StandardDeviation(); + std.setData(CCs); + double StdCC = std.evaluate(); + + Mean m = new Mean(); + m.setData(CCs); + double averageCC = m.evaluate(); + + //average MAE + List<Double> alMAEs = new ArrayList<>(); + try { + alMAEs.add(Double.valueOf(rr.MAE)); + } catch (Exception e) { + } + try { + alMAEs.add(Double.valueOf(eproRHTrain.meanMAEs)); + } catch (Exception e) { + } + try { + alMAEs.add(Double.valueOf(eproBSTrain.meanMAEs)); + } catch (Exception e) { + } + try { + alMAEs.add(Double.valueOf(rrLoocv.MAE)); + } catch (Exception e) { + } + try { + alMAEs.add(Double.valueOf(tro.MAE)); + } catch (Exception e) { + } + try { + alMAEs.add(Double.valueOf(eproRHTrainTest.meanMAEs)); + } catch (Exception e) { + } + try { + alMAEs.add(Double.valueOf(eproBSTrainTest.meanMAEs)); + } catch (Exception e) { + } + + double MAEs[] = alMAEs.stream().mapToDouble(Double::doubleValue).toArray(); + + std = new StandardDeviation(); + std.setData(MAEs); + double StdMAE = std.evaluate(); + + m = new Mean(); + m.setData(MAEs); + double averageMAE = m.evaluate(); + + //average RMSE + List<Double> alRMSEs = new ArrayList<>(); + try { + alRMSEs.add(Double.valueOf(rr.RMSE)); + } catch (Exception e) { + } + try { + alRMSEs.add(Double.valueOf(eproRHTrain.meanRMSEs)); + } catch (Exception e) { + } + try { + alRMSEs.add(Double.valueOf(eproBSTrain.meanRMSEs)); + } catch (Exception e) { + } + try { + alRMSEs.add(Double.valueOf(rrLoocv.RMSE)); + } catch (Exception e) { + } + try { + alRMSEs.add(Double.valueOf(tro.RMSE)); + } catch (Exception e) { + } + try { + alRMSEs.add(Double.valueOf(eproRHTrainTest.meanRMSEs)); + } catch (Exception e) { + } + try { + alRMSEs.add(Double.valueOf(eproBSTrainTest.meanRMSEs)); + } catch (Exception e) { + } + + double RMSEs[] = alRMSEs.stream().mapToDouble(Double::doubleValue).toArray(); + + std = new StandardDeviation(); + std.setData(RMSEs); + double StdRMSE = std.evaluate(); + + m = new Mean(); + m.setData(RMSEs); + double averageRMSE = m.evaluate(); + + String stats = df.format(averageCC) + "\t" + + df.format(StdCC) + "\t" + + df.format(averageMAE) + "\t" + + df.format(StdMAE) + "\t" + + df.format(averageRMSE) + "\t" + + df.format(StdRMSE); + lastOutput = out + + "\t" + rr.numberOfFeatures + "\t" + rr.toString() + + "\t" + loocvOut + + "\t" + eproRHTrain.toStringRegression() + + "\t" + eproBSTrain.toStringRegression() + + "\t" + testResults + + "\t" + eproRHTrainTest.toStringRegression() + + "\t" + eproBSTrainTest.toStringRegression() + + "\t" + stats + + "\t" + ao.getRetainedAttributesIdClassInString(); + } + + //CREATE ID + Random r = new Random(); + int randomNumber = r.nextInt(10000 - 10) + 10; + out = (lastOutput.split("\t")[0] + "_" + lastOutput.split("\t")[2] + + "_" + lastOutput.split("\t")[3] + "_" + lastOutput.split("\t")[4] + "_" + lastOutput.split("\t")[5] + "_" + randomNumber); + + // temporary models output +// String filename = Main.CVfolder + File.separatorChar + out; +// PrintWriter pw = new PrintWriter(new FileWriter(filename + ".txt")); +// pw.println("#" + classifier + " " + classifier_options + "\n#Value to maximize or minimize:" + valueToMaximizeOrMinimize); +// if (isClassification) { +// pw.print("#ID\tactual\tpredicted\terror\tprobability\n"); +// } else { +// pw.print("#ID\tactual\tpredicted\terror\n"); +// } +// pw.print(predictions + "\n\n"); +// pw.print("#Selected Attributes\t(Total attributes:" + numberOfAttributes + ")\n"); +// pw.print(selectedAttributes); +// pw.close(); +// //save model +// SerializationHelper.write(filename + ".model", model); + } else { + out = "ERROR"; + if (Main.debug) { + System.err.println(o); + } + if (o.toString().equals("null")) { + out += "\t " + classifier + " " + classifier_options + " | " + searchMethod + " | Error probably because of number of features inferior to topX"; + } + } + + } catch (Exception e) { + out = "ERROR\t" + classifier + " " + classifier_options + " | " + searchMethod + " | " + e.getMessage(); + cptFailed++; + if (Main.debug) { + e.printStackTrace(); + } + } + Instant finish = Instant.now(); + long s = Duration.between(start, finish).toMillis() / 1000; + if (Main.debug) { + System.out.println("model created in [" + s + "s]"); + } + return out + "\t" + lastOutput; + } + + /** + * get value of interest + * + * @param valueWanted + * @param cr + * @param rr + * @return + */ + private static double getValueToMaximize(String valueWanted, Weka_module.ClassificationResultsObject cr, + Weka_module.RegressionResultsObject rr) { + + switch (valueWanted.toLowerCase()) { + //classification + case "auc": + return Double.parseDouble(cr.AUC); + case "pauc": + return Double.parseDouble(cr.pAUC); + case "acc": + return Double.parseDouble(cr.ACC); + case "sen": + return Double.parseDouble(cr.TPR); + case "spe": + return Double.parseDouble(cr.TNR); + case "mcc": + return Double.parseDouble(cr.MCC); + case "kappa": + return Double.parseDouble(cr.kappa); + case "aupcr": + return Double.parseDouble(cr.AUPRC); + case "fscore": + return Double.parseDouble(cr.Fscore); + case "precision": + return Double.parseDouble(cr.precision); + case "recall": + return Double.parseDouble(cr.recall); + case "fdr": + return Double.parseDouble(cr.FDR);//to minimize + case "ber": + return Double.parseDouble(cr.BER);//to minimize + case "tp+fn": + double sumOfTP_TN = Double.parseDouble(cr.TPR) + Double.parseDouble(cr.TNR); + return sumOfTP_TN; + //regression + case "cc": + return Double.parseDouble(rr.CC); + case "mae": + return Double.parseDouble(rr.MAE);//to minimize + case "rmse": + return Double.parseDouble(rr.RMSE);//to minimize + case "rae": + return Double.parseDouble(rr.RAE);//to minimize + case "rrse": + return Double.parseDouble(rr.RRSE);//to minimize + } + return 0; + } + + /** + * add to hm weka configurations + * <["misc.VFI", "-B 0.6", "AUC"],""> + * <["CostSensitiveClassifier", "-cost-matrix "[0.0 ratio; 100.0 0.0]" -S 1 -W weka.classifiers.misc.VFI -- -B 0.6", "AUC"],""> + * + * @param classifier + * @param options + * @param costSensitive + */ + private static void addClassificationToQueue(String classifier, String options) { + String optimizers[] = Main.classificationOptimizers.split(","); + String searchmodes[] = Main.searchmodes.split(","); + if (Main.noFeatureSelection) { + searchmodes = new String[]{"all"}; + } +// else if(weka.myData.numAttributes()>500){ +// +// } + + + for (String optimizer : optimizers) {//AUC, ACC, SEN, SPE, MCC, TP+FN, kappa + for (String searchmode : searchmodes) {//F, FB, B, BF, topk + alClassifiers.add(new String[]{classifier, options, optimizer.trim(), searchmode.trim()}); + } + } + if (Main.metaCostSensitiveClassifier) { + options = "-cost-matrix \"[0.0 ratio; 100.0 0.0]\" -S 1 -W weka.classifiers." + classifier + " -- " + options; + for (String value : optimizers) { + alClassifiers.add(new String[]{"meta.CostSensitiveClassifier", options, value.trim(), "f"}); + alClassifiers.add(new String[]{"meta.CostSensitiveClassifier", options, value.trim(), "bf"}); + } + } + } + + private static void addRegressionToQueue(String classifier, String options) { + //String valuesToMaximizeOrMinimize[] = new String[]{"CC", "MAE", "RMSE", "RAE", "RRSE"}; + String optimizers[] = Main.regressionOptimizers.split(","); + String searchmodes[] = Main.searchmodes.split(","); + if (Main.noFeatureSelection) { + searchmodes = new String[]{"all"}; + } + for (String optimizer : optimizers) {//only CC is maximized here + for (String searchmode : searchmodes) { + alClassifiers.add(new String[]{classifier, options, optimizer.trim(), searchmode.trim()}); + } + + } + if (Main.metaAdditiveRegression) { + options = "-S 1.0 -I 10 -W weka.classifiers." + classifier + " -- " + options; + for (String value : optimizers) { + alClassifiers.add(new String[]{"meta.AdditiveRegression", options, value, "F"}); + alClassifiers.add(new String[]{"meta.AdditiveRegression", options, value, "FB"}); + } + } + } + + private static class AttributeOject { + + public Integer ID; + public ArrayList<Integer> alAttributes = new ArrayList<>(); + public Integer Class; + public ArrayList<Integer> retainedAttributesOnly = new ArrayList<>(); + + public AttributeOject() { + + } + + private AttributeOject(ArrayList<Integer> attributes) { + ID = attributes.get(0); + Class = attributes.get(attributes.size() - 1); + attributes.remove(attributes.size() - 1); + attributes.remove(0); + alAttributes = attributes; + } + + private void addNewAttributeToRetainedAttributes(int index) { + retainedAttributesOnly.add(alAttributes.get(index)); + } + + private String getRetainedAttributesIdClassInString() { + String attributes = ""; + attributes += (ID) + ","; + for (Integer retainedAttribute : retainedAttributesOnly) { + attributes += (retainedAttribute) + ","; + } + attributes += Class; + return attributes; + } + + private ArrayList getRetainedAttributesIdClassInArrayList() { + ArrayList<Integer> al = new ArrayList<>(); + al.add(ID); + for (Integer retainedAttribute : retainedAttributesOnly) { + al.add(retainedAttribute); + } + al.add(Class); + return al; + } + + private void changeRetainedAttributes(String featuresToTest) { + String features[] = featuresToTest.split(","); + retainedAttributesOnly = new ArrayList<>(); + for (int i = 1; i < features.length - 1; i++) {//skip ID and class indexes + retainedAttributesOnly.add(Integer.valueOf(features[i])); + } + } + + private String getAttributesIdClassInString() { + String attributes = ""; + attributes += (ID) + ","; + for (Integer att : alAttributes) { + attributes += (att) + ","; + } + attributes += Class; + return attributes; + } + } + +}