/*
* execute machine learning algorithms in parrallel
* Works with regression and classification
*/
package biodiscml;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.PrintWriter;
import java.text.DecimalFormat;
import java.text.DecimalFormatSymbols;
import java.time.Duration;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Random;
import org.apache.commons.math3.stat.descriptive.moment.*;
import utils.Weka_module;
import utils.utils;
import weka.attributeSelection.AttributeSelection;
import weka.classifiers.Classifier;
/**
*
* @author Mickael
*/
public class Training {
public static Weka_module weka = new Weka_module();
public static AttributeSelection ranking;
public static ArrayList<String[]> alClassifiers = new ArrayList<>();
public static boolean isClassification = true;
public static String resultsSummaryHeader = "";
public static int cptPassed = 0;
public static int cptFailed = 0;
public boolean parrallel = true;
public static String trainFileName = "";
public static DecimalFormat df = new DecimalFormat();
public static PrintWriter pw;
public Training() {
}
/**
* @param dataToTrainModel
* @param resultsFile
* @param featureSelectionFile
* @param type. Is "class" for classification, else "reg" for regression
*/
public Training(String dataToTrainModel, String resultsFile,
String featureSelectionFile, String type) {
// deleting previous run
if (!Main.restoreRun && !Main.resumeTraining) {
if (new File(featureSelectionFile).exists()) {
System.out.println("\t" + featureSelectionFile + " exist... deleting...");
new File(featureSelectionFile).delete();
new File(featureSelectionFile.replace(".csv", ".arff")).delete();
}
if (new File(resultsFile).exists()) {
System.out.println("\t" + resultsFile + " exist... deleting...");
new File(resultsFile).delete();
new File(resultsFile.replace(".csv", ".arff")).delete();
}
}
df.setMaximumFractionDigits(3);
DecimalFormatSymbols dfs = new DecimalFormatSymbols();
dfs.setDecimalSeparator('.');
df.setDecimalFormatSymbols(dfs);
if (Main.cpus.equals("1")) {
parrallel = false;
}
System.out.println("Training on " + dataToTrainModel + ". All tested models results will be in " + resultsFile);
alClassifiers = new ArrayList<>();
isClassification = type.equals("class");
trainFileName = dataToTrainModel;
// File f = new File(Main.CVfolder);//cross validation folder outputs
// if (!f.exists()) {
// f.mkdir();
// }
//convert csv to arff
if (dataToTrainModel.endsWith(".csv") && !new File(dataToTrainModel.replace(".csv", ".arff")).exists()) {
weka.setCSVFile(new File(dataToTrainModel));
weka.csvToArff(isClassification);
} else {
weka.setARFFfile(dataToTrainModel.replace(".csv", ".arff"));
}
//set local variable of weka object from ARFFfile
weka.setDataFromArff();
weka.myData = weka.convertStringsToNominal(weka.myData);
// // check if class has numeric values, hence regression, instead of nominal class (classification)
//classification = weka.isClassification();
//CLASSIFICATION
if (isClassification) {
if (Main.debug) {
System.out.println("Only use classification algorithms");
}
//HEADER FOR SUMMARY OUTPUT
resultsSummaryHeader = "ID"
+ "\tclassifier"
+ "\tOptions"
+ "\tOptimizedValue"
+ "\tSearchMode"
+ "\tnbrOfFeatures"
//10CV
+ "\tTRAIN_10CV_ACC"
+ "\tTRAIN_10CV_AUC"
+ "\tTRAIN_10CV_AUPRC"
+ "\tTRAIN_10CV_SEN"
+ "\tTRAIN_10CV_SPE"
+ "\tTRAIN_10CV_MCC"
+ "\tTRAIN_10CV_MAE"
+ "\tTRAIN_10CV_BER"
+ "\tTRAIN_10CV_FPR"
+ "\tTRAIN_10CV_FNR"
+ "\tTRAIN_10CV_PPV"
+ "\tTRAIN_10CV_FDR"
+ "\tTRAIN_10CV_Fscore"
+ "\tTRAIN_10CV_kappa"
+ "\tTRAIN_matrix"
//LOOCV
+ "\tTRAIN_LOOCV_ACC"
+ "\tTRAIN_LOOCV_AUC"
+ "\tTRAIN_LOOCV_AUPRC"
+ "\tTRAIN_LOOCV_SEN"
+ "\tTRAIN_LOOCV_SPE"
+ "\tTRAIN_LOOCV_MCC"
+ "\tTRAIN_LOOCV_MAE"
+ "\tTRAIN_LOOCV_BER"
//Repeated Holdout TRAIN
+ "\tTRAIN_RH_ACC"
+ "\tTRAIN_RH_AUC"
+ "\tTRAIN_RH_AUPRC"
+ "\tTRAIN_RH_SEN"
+ "\tTRAIN_RH_SPE"
+ "\tTRAIN_RH_MCC"
+ "\tTRAIN_RH_MAE"
+ "\tTRAIN_RH_BER"
//Bootstrap TRAIN
+ "\tTRAIN_BS_ACC"
+ "\tTRAIN_BS_AUC"
+ "\tTRAIN_BS_AUPRC"
+ "\tTRAIN_BS_SEN"
+ "\tTRAIN_BS_SPE"
+ "\tTRAIN_BS_MCC"
+ "\tTRAIN_BS_MAE"
+ "\tTRAIN_BS_BER"
//Bootstrap .632+ TRAIN
+ "\tTRAIN_BS.632+"
//test
+ "\tTEST_ACC"
+ "\tTEST_AUC"
+ "\tTEST_AUPRC"
+ "\tTEST_SEN"
+ "\tTEST_SPE"
+ "\tTEST_MCC"
+ "\tTEST_MAE"
+ "\tTEST_BER"
//Repeated Holdout TRAIN_TEST
+ "\tTRAIN_TEST_RH_ACC"
+ "\tTRAIN_TEST_RH_AUC"
+ "\tTRAIN_TEST_RH_AUPRC"
+ "\tTRAIN_TEST_RH_SEN"
+ "\tTRAIN_TEST_RH_SPE"
+ "\tTRAIN_TEST_RH_MCC"
+ "\tTRAIN_TEST_RH_MAE"
+ "\tTRAIN_TEST_RH_BER"
//Bootstrap TRAIN_TEST
+ "\tTRAIN_TEST_BS_ACC"
+ "\tTRAIN_TEST_BS_AUC"
+ "\tTRAIN_TEST_BS_AUPRC"
+ "\tTRAIN_TEST_BS_SEN"
+ "\tTRAIN_TEST_BS_SPE"
+ "\tTRAIN_TEST_BS_MCC"
+ "\tTRAIN_TEST_BS_MAE"
+ "\tTRAIN_TEST_BS_BER"
//Bootstrap .632+ TRAIN
+ "\tTRAIN_TEST_BS.632+"
//stats
+ "\tAVG_BER"
+ "\tSTD_BER"
+ "\tAVG_MAE"
+ "\tSTD_MAE"
+ "\tAVG_MCC"
+ "\tSTD_MCC"
+ "\tAttributeList";
//fast way classification. Predetermined commands.
if (Main.classificationFastWay) {
//hmclassifiers.put(new String[]{"misc.VFI", "-B 0.6", "AUC"}, "");
//hmclassifiers.put(new String[]{"meta.CostSensitiveClassifier", "-cost-matrix \"[0.0 ratio; 100.0 0.0]\" -S 1 -W weka.classifiers.misc.VFI -- -B 0.4", "MCC"}, "");
for (String cmd : Main.classificationFastWayCommands) {
String optimizer = cmd.split(":")[1];
String searchmode = cmd.split(":")[2];
String classifier = cmd.split(":")[0].split(" ")[0];
String options = cmd.split(":")[0].replace(classifier, "");
//case where optimizer AND search modes are empty
if (optimizer.equals("allopt") && searchmode.equals("allsearch")) {
for (String allOptimizers : Main.classificationOptimizers.split(",")) {
for (String allSearchModes : Main.searchmodes.split(",")) {
alClassifiers.add(new String[]{classifier, options, allOptimizers.trim(), allSearchModes.trim()});
}
}
//Case where only searchmode is empty
} else if (!optimizer.equals("allopt") && searchmode.equals("allsearch")) {
for (String allSearchModes : Main.searchmodes.split(",")) {
alClassifiers.add(new String[]{classifier, options, optimizer.trim(), allSearchModes.trim()});
}
//case where only optimizer is empty
} else if (optimizer.equals("allopt") && !searchmode.equals("allsearch")) {
for (String allOptimizers : Main.classificationOptimizers.split(",")) {
alClassifiers.add(new String[]{classifier, options, allOptimizers.trim(), searchmode.trim()});
}
//case where optimizer and searchmode are provided
} else {
alClassifiers.add(new String[]{classifier, options, optimizer, searchmode});
}
}
} else {
// if no feature selection is asked
if (Main.noFeatureSelection) {
Main.maxNumberOfSelectedFeatures = weka.myData.numAttributes();
Main.maxNumberOfFeaturesInModel = weka.myData.numAttributes();
for (String cmd : Main.classificationBruteForceCommands) {
String classifier = cmd.split(" ")[0];
String options = cmd.replace(classifier, "").trim();
addClassificationToQueue(classifier, options);
}
} else {
//brute force classification, try everything in the provided classifiers and optimizers
for (String cmd : Main.classificationBruteForceCommands) {
String classifier = cmd.split(" ")[0];
String options = cmd.replace(classifier, "").trim();
addClassificationToQueue(classifier, options);
}
}
}
if (!Main.noFeatureSelection) {
System.out.print("Feature selection and ranking...");
if (new File(featureSelectionFile).exists() && Main.resumeTraining) {
System.out.print("\nFeature selection and ranking already done... skipping");
} else {
//ATTRIBUTE SELECTION for classification
weka.attributeSelectionByInfoGainRankingAndSaveToCSV(featureSelectionFile);
//get the rank of attributes
ranking = weka.featureRankingForClassification();
System.out.println("[done]");
//reset arff and keep compatible header
weka.setCSVFile(new File(featureSelectionFile));
weka.csvToArff(isClassification);
weka.makeCompatibleARFFheaders(dataToTrainModel.replace("data_to_train.csv", "data_to_train.arff"),
featureSelectionFile.replace("infoGain.csv", "infoGain.arff"));
}
weka.setARFFfile(featureSelectionFile.replace("infoGain.csv", "infoGain.arff"));
weka.setDataFromArff();
}
} else {
//REGRESSION
if (Main.debug) {
System.out.println("Only use regression algorithms");
}
resultsSummaryHeader = "ID"
+ "\tclassifier"
+ "\tOptions"
+ "\tOptimizedValue"
+ "\tSearchMode"
+ "\tnbrOfFeatures"
//10cv
+ "\tTRAIN_10CV_CC"
+ "\tTRAIN_10CV_MAE"
+ "\tTRAIN_10CV_RMSE"
+ "\tTRAIN_10CV_RAE"
+ "\tTRAIN_10CV_RRSE"
//LOOCV
+ "\tTRAIN_LOOCV_CC"
+ "\tTRAIN_LOOCV_MAE"
+ "\tTRAIN_LOOCV_RMSE"
+ "\tTRAIN_LOOCV_RAE"
+ "\tTRAIN_LOOCV_RRSE"
//Repeated Holdout
+ "\tTRAIN_RH_CC"
+ "\tTRAIN_RH_MAE"
+ "\tTRAIN_RH_RMSE"
+ "\tTRAIN_RH_RAE"
+ "\tTRAIN_RH_RRSE"
//Bootstrap
+ "\tTRAIN_BS_CC"
+ "\tTRAIN_BS_MAE"
+ "\tTRAIN_BS_RMSE"
+ "\tTRAIN_BS_RAE"
+ "\tTRAIN_BS_RRSE"
//TEST SET
+ "\tTEST_CC"
+ "\tTEST_MAE"
+ "\tTEST_RMSE"
+ "\tTEST_RAE"
+ "\tTEST_RRSE"
//Repeated Holdout TRAIN_TEST
+ "\tTRAIN_TEST_RH_CC"
+ "\tTRAIN_TEST_RH_MAE"
+ "\tTRAIN_TEST_RH_RMSE"
+ "\tTRAIN_TEST_RH_RAE"
+ "\tTRAIN_TEST_RH_RRSE"
//Bootstrap TRAIN_TEST
+ "\tTRAIN_TEST_BS_CC"
+ "\tTRAIN_TEST_BS_MAE"
+ "\tTRAIN_TEST_BS_RMSE"
+ "\tTRAIN_TEST_BS_RAE"
+ "\tTRAIN_TEST_BS_RRSE"
//stats
+ "\tAVG_CC"
+ "\tSTD_CC"
+ "\tAVG_MAE"
+ "\tSTD_MAE"
+ "\tAVG_RMSE"
+ "\tSTD_RMSE"
+ "\tAttributeList";
//fast way regression. Predetermined commands.
if (Main.regressionFastWay) {
//hmclassifiers.put(new String[]{"functions.GaussianProcesses", "-L 1.0 -N 0 -K \"weka.classifiers.functions.supportVector.RBFKernel -C 250007 -G 1.0\"", "CC"}, "");
//hmclassifiers.put(new String[]{"meta.AdditiveRegression",
// "-S 1.0 -I 10 -W weka.classifiers.functions.GaussianProcesses -- -L 1.0 -N 0 -K \"weka.classifiers.functions.supportVector.PolyKernel -C 250007 -E 1.0 -L\"", "CC"}, "");
for (String cmd : Main.regressionFastWayCommands) {
String optimizer = cmd.split(":")[1];
String searchmode = cmd.split(":")[2];
String classifier = cmd.split(":")[0].split(" ")[0];
String options = cmd.split(":")[0].replace(classifier, "");
//case where optimizer AND search modes are empty
if (optimizer.equals("allopt") && searchmode.equals("allsearch")) {
for (String allOptimizers : Main.regressionOptimizers.split(",")) {
for (String allSearchModes : Main.searchmodes.split(",")) {
alClassifiers.add(new String[]{classifier, options, allOptimizers.trim(), allSearchModes.trim()});
}
}
//Case where only searchmode is empty
} else if (!optimizer.equals("allopt") && searchmode.equals("allsearch")) {
for (String allSearchModes : Main.searchmodes.split(",")) {
alClassifiers.add(new String[]{classifier, options, optimizer.trim(), allSearchModes.trim()});
}
//case where only optimizer is empty
} else if (optimizer.equals("allopt") && !searchmode.equals("allsearch")) {
for (String allOptimizers : Main.regressionOptimizers.split(",")) {
alClassifiers.add(new String[]{classifier, options, allOptimizers.trim(), searchmode.trim()});
}
//case where optimizer and searchmode are provided
} else {
alClassifiers.add(new String[]{classifier, options, optimizer, searchmode});
}
}
} else {
//brute force classification, try everything in the provided classifiers and optimizers
for (String cmd : Main.regressionBruteForceCommands) {
String classifier = cmd.split(" ")[0];
String options = cmd.replace(classifier, "");
addRegressionToQueue(classifier, options);
}
}
//ATTRIBUTE SELECTION for regression
if (!Main.noFeatureSelection) {
System.out.print("Selecting attributes and ranking by RelieFF...");
if (new File(featureSelectionFile).exists() && Main.resumeTraining) {
System.out.print("Selecting attributes and ranking by RelieFF already done... skipped by resumeTraining");
} else {
weka.attributeSelectionByRelieFFAndSaveToCSV(featureSelectionFile);
System.out.println("[done]");
//reset arff and keep compatible header
weka.setCSVFile(new File(featureSelectionFile));
weka.csvToArff(isClassification);
weka.makeCompatibleARFFheaders(dataToTrainModel.replace("data_to_train.csv", "data_to_train.arff"),
featureSelectionFile.replace("RELIEFF.csv", "RELIEFF.arff"));
weka.setARFFfile(featureSelectionFile.replace("RELIEFF.csv", "RELIEFF.arff"));
weka.setDataFromArff();
}
}
}
//resume training, remove from alClassifier all classifiers already trained
if (Main.resumeTraining && !Main.restoreRun) {
HashMap<String, String> hm = new HashMap<>();
//get data
try {
BufferedReader br = new BufferedReader(new FileReader(resultsFile));
br.readLine(); //skip header
while (br.ready()) {
String line[] = br.readLine().split("\t");
if (!line[0].startsWith("ERROR")) {
String s[] = new String[4];
s[0] = line[1];
s[1] = line[2];
s[2] = line[3].toLowerCase();
s[3] = line[4].toLowerCase();
hm.put(line[1] + "\t" + line[2] + "\t" + line[3].toLowerCase() + "\t" + line[4].toLowerCase(), "");
}
}
} catch (Exception e) {
if (Main.debug) {
e.printStackTrace();
}
}
//remove from alClassifiers
int alClassifiersBeforeRemoval = alClassifiers.size();
for (int i = 0; i < alClassifiers.size(); i++) {
String s = alClassifiers.get(i)[0] + "\t" + alClassifiers.get(i)[1] + "\t" + alClassifiers.get(i)[2] + "\t" + alClassifiers.get(i)[3];
if (hm.containsKey(s)) {
alClassifiers.remove(i);
}
}
int alClassifiersAfterRemoval = alClassifiers.size();
int totalRemoved = alClassifiersBeforeRemoval - alClassifiersAfterRemoval;
if (Main.debug) {
System.out.println("Total removed from alClassifier after resumeTraining = " + totalRemoved);
}
System.out.println("ResumeTraining: Remains " + alClassifiersAfterRemoval
+ " classifiers to train on the " + alClassifiersBeforeRemoval);
}
try {
//PREPARE OUTPUT
if (Main.resumeTraining && !Main.restoreRun) {
pw = new PrintWriter(new FileWriter(resultsFile, true));
//pw.println("Resumed here");
pw.flush();
} else {
pw = new PrintWriter(new FileWriter(resultsFile));
pw.println(resultsSummaryHeader);
}
//EXECUTE IN PARRALLEL
System.out.println("Total classifiers to test: " + alClassifiers.size());
if (parrallel) {
alClassifiers
.parallelStream()
.map((classif) -> StepWiseFeatureSelectionTraining(classif[0], classif[1], classif[2], classif[3]))
.map((s) -> {
if (!s.toLowerCase().contains("error")) {
pw.println(s);
pw.flush();
} else if (Main.printFailedModels) {
pw.println(s);
pw.flush();
}
return s;
})
.sorted()
.forEach((_item) -> {
pw.flush();
});
} else {
alClassifiers.stream().map((classif) -> {
String s = StepWiseFeatureSelectionTraining(classif[0], classif[1], classif[2], classif[3]);
if (!s.toLowerCase().contains("error")) {
pw.println(s);
} else if (Main.printFailedModels) {
pw.println(s);
}
return classif;
}).forEach((_item) -> {
pw.flush();
});
}
pw.close();
//END
System.out.println("Total model tested: " + cptPassed + "/" + alClassifiers.size()
+ ", including " + cptFailed + " incompatible models");
} catch (Exception e) {
if (Main.debug) {
e.printStackTrace();
}
System.out.println("Total model tested: " + cptPassed + "/" + alClassifiers.size()
+ ", including " + cptFailed + " incompatible models");
pw.close();
} catch (Error err) {
if (Main.debug) {
err.printStackTrace();
}
System.out.println("Total model tested: " + cptPassed + "/" + alClassifiers.size()
+ ", including " + cptFailed + " incompatible models");
pw.close();
}
}
/**
* StepWise training. We add the features one by one in the model if they
* improve the value to maximize We only test the first 500 attributes (or
* less if the number of attributes is less than 500)
*
* @param classifier
* @param classifier_options
* @param valueToMaximizeOrMinimize
* @return
*/
private static String StepWiseFeatureSelectionTraining(String classifier, String classifier_options,
String valueToMaximizeOrMinimize, String searchMethod) {
String out = classifier + "\t" + classifier_options + "\t" + valueToMaximizeOrMinimize.toUpperCase() + "\t" + searchMethod;
Instant start = Instant.now();
System.out.println("[model] (" + (cptPassed++) + "/" + alClassifiers.size() + ")" + out);
String lastOutput = "";
boolean minimize = valueToMaximizeOrMinimize.equals("fdr")
|| valueToMaximizeOrMinimize.equals("mae")
|| valueToMaximizeOrMinimize.equals("rmse")
|| valueToMaximizeOrMinimize.equals("ber")
|| valueToMaximizeOrMinimize.equals("rae")
|| valueToMaximizeOrMinimize.equals("rrse");
try {
Object o = null;
int numberOfAttributes = 0;
double previousMeasureToMaximize = -1000.0;
double previousMeasureToMinimize = 1000.0;
ArrayList<Integer> alAttributes = new ArrayList<>();
int classIndex = weka.myData.numAttributes();
for (int i = 1; i <= classIndex; i++) { // weka starts at 1, not 0
alAttributes.add(i);
}
AttributeOject ao = new AttributeOject(alAttributes);
int cpt = 0;
Weka_module.ClassificationResultsObject cr = null;
Weka_module.RegressionResultsObject rr = null;
// SHORT TEST to ensure compatibility of the model
if (Main.performShortTest) {
o = weka.shortTestTrainClassifier(classifier, classifier_options,
ao.getAttributesIdClassInString(), isClassification);
if (o.getClass().getName().equals("java.lang.String")) {
if (Main.debug) {
System.out.println("SHORT TEST FAILED");
}
return "ERROR\t" + o.toString();
}
}
if (Main.debug) {
System.out.println("\tGoing to training");
}
//all features search, no feature selection
if (searchMethod.startsWith("all")) {
int top = ao.alAttributes.size();
//Create attribute list
for (int i = 0; i < top; i++) { //from ID to class (excluded)
//add new attribute to the set of retainedAttributes
ao.addNewAttributeToRetainedAttributes(i);
}
//TRAIN 10CV
o = weka.trainClassifier(classifier, classifier_options,
ao.getRetainedAttributesIdClassInString(), isClassification, 10);
if (!(o instanceof String) || o == null) {
if (isClassification) {
cr = (Weka_module.ClassificationResultsObject) o;
} else {
rr = (Weka_module.RegressionResultsObject) o;
}
}
}
//TOP X features search
if (searchMethod.startsWith("top")) {
int top = Integer.valueOf(searchMethod.replace("top", ""));
if (top <= ao.alAttributes.size()) {
//Create attribute list
for (int i = 0; i < top; i++) { //from ID to class (excluded)
//add new attribute to the set of retainedAttributes
ao.addNewAttributeToRetainedAttributes(i);
}
//TRAIN 10CV
o = weka.trainClassifier(classifier, classifier_options,
ao.getRetainedAttributesIdClassInString(), isClassification, 10);
if (!(o instanceof String) || o == null) {
if (isClassification) {
cr = (Weka_module.ClassificationResultsObject) o;
} else {
rr = (Weka_module.RegressionResultsObject) o;
}
}
} else {
o = null;
}
}
//FORWARD AND FORWARD-BACKWARD search AND BACKWARD AND BACKWARD-FORWARD search
if (searchMethod.startsWith("f") || searchMethod.startsWith("b")) {
boolean doForwardBackward_OR_BackwardForward = searchMethod.equals("fb") || searchMethod.equals("bf");
boolean StartBackwardInsteadForward = searchMethod.startsWith("b");
if (StartBackwardInsteadForward) {
Collections.reverse(ao.alAttributes);
}
for (int i = 0; i < ao.alAttributes.size(); i++) { //from ID to class (excluded)
cpt++;
//add new attribute to the set of retainedAttributes
ao.addNewAttributeToRetainedAttributes(i);
Weka_module.ClassificationResultsObject oldcr = cr;
//do feature selection by forward(-backward)
if (ao.retainedAttributesOnly.size() <= Main.maxNumberOfFeaturesInModel) {
o = weka.trainClassifier(classifier, classifier_options,
ao.getRetainedAttributesIdClassInString(), isClassification, 10);
if (o instanceof String || o == null) {
return (String) o;
} else if (isClassification) {
cr = (Weka_module.ClassificationResultsObject) o;
} else {
rr = (Weka_module.RegressionResultsObject) o;
}
//choose what we want to maximize or minimize (such as error rates)
//this will crash if model had an error
double currentMeasure = getValueToMaximize(valueToMaximizeOrMinimize, cr, rr);
//Report results
boolean modelIsImproved = false;
//i<2 is to avoid return no attribute at all
// test if model is improved
if (minimize) {
modelIsImproved = (i < 2 && currentMeasure <= previousMeasureToMinimize) || currentMeasure < previousMeasureToMinimize;
} else {
modelIsImproved = (i < 2 && currentMeasure >= previousMeasureToMaximize) || currentMeasure > previousMeasureToMaximize;
}
if (modelIsImproved) {
if (!minimize) {
previousMeasureToMaximize = currentMeasure;
} else {
previousMeasureToMinimize = currentMeasure;
}
// do backward OR forward, check if we have an improvement if we remove previously chosen features
if (doForwardBackward_OR_BackwardForward && numberOfAttributes > 1) {
oldcr = cr;
ArrayList<Integer> attributesToTestInBackward = ao.getRetainedAttributesIdClassInArrayList();
for (int j = 1/*skip ID*/;
j < attributesToTestInBackward.size() - 2/*skip last attribute we added by forward and class*/; j++) {
attributesToTestInBackward.remove(j);
String featuresToTest = utils.arrayToString(attributesToTestInBackward, ",");
//train
if (isClassification) {
cr = (Weka_module.ClassificationResultsObject) weka.trainClassifier(classifier, classifier_options,
featuresToTest, isClassification, 10);
} else {
rr = (Weka_module.RegressionResultsObject) weka.trainClassifier(classifier, classifier_options,
featuresToTest, isClassification, 10);
}
//get measure
double measureWithRemovedFeature = getValueToMaximize(valueToMaximizeOrMinimize, cr, rr);
//check if we have improvement
if (minimize) {
modelIsImproved = (measureWithRemovedFeature <= currentMeasure)
|| measureWithRemovedFeature < currentMeasure;
} else {
modelIsImproved = (measureWithRemovedFeature >= currentMeasure)
|| measureWithRemovedFeature > currentMeasure;
}
if (modelIsImproved) {
//if model is improved definitly discard feature is improvement
ao.changeRetainedAttributes(featuresToTest);
oldcr = cr;
currentMeasure = measureWithRemovedFeature;
} else {
//restore the feature
attributesToTestInBackward = ao.getRetainedAttributesIdClassInArrayList();
cr = oldcr;
}
}
}
//
//modify results summary output
//only for DEBUG purposes
if (isClassification) {
lastOutput = out
+ "\t" + cr.numberOfFeatures + "\t" + cr.toString() + "\t" + ao.getRetainedAttributesIdClassInString();
} else {
lastOutput = out
+ "\t" + rr.numberOfFeatures + "\t" + rr.toString() + "\t" + ao.getRetainedAttributesIdClassInString();
}
} else {
//back to previous attribute if no improvement with the new attribute and go to the next
ao.retainedAttributesOnly.remove(ao.retainedAttributesOnly.size() - 1);
cr = oldcr;
}
}
}
}
//now we have a good model, evaluate the performance using various approaches
if (!(o instanceof String) && o != null) {
//LOOCV
Weka_module.ClassificationResultsObject crLoocv = null;
Weka_module.RegressionResultsObject rrLoocv = null;
String loocvOut = "";
if (Main.loocv) {
if (Main.debug) {
System.out.println("\tLOOCV Train set " + Main.bootstrapAndRepeatedHoldoutFolds + " times");
}
Object oLoocv;
if (isClassification) {
oLoocv = weka.trainClassifier(classifier, classifier_options,
ao.getRetainedAttributesIdClassInString(), isClassification,
cr.dataset.numInstances());
} else {
oLoocv = weka.trainClassifier(classifier, classifier_options,
ao.getRetainedAttributesIdClassInString(), isClassification,
rr.dataset.numInstances());
}
if (oLoocv instanceof String || oLoocv == null) {
if (Main.debug) {
System.err.println("[error] LOOCV failed");
}
} else if (isClassification) {
crLoocv = (Weka_module.ClassificationResultsObject) oLoocv;
loocvOut = crLoocv.toStringShort();
} else {
rrLoocv = (Weka_module.RegressionResultsObject) oLoocv;
loocvOut = rrLoocv.toString();
}
} else {
if (isClassification) {
loocvOut = "\t" + "\t" + "\t" + "\t" + "\t" + "\t" + "\t";
} else {
loocvOut = "\t" + "\t" + "\t" + "\t";
}
}
//Repeated Holdout
Weka_module.evaluationPerformancesResultsObject eproRHTrain = new Weka_module.evaluationPerformancesResultsObject();
if (Main.repeatedHoldout) {
if (Main.debug) {
System.out.println("\tRepeated Holdout on Train set " + Main.bootstrapAndRepeatedHoldoutFolds + " times");
}
if (isClassification) {
for (int i = 0; i < Main.bootstrapAndRepeatedHoldoutFolds; i++) {
Weka_module.ClassificationResultsObject cro
= (Weka_module.ClassificationResultsObject) weka.trainClassifierHoldOutValidation(classifier, classifier_options,
ao.getRetainedAttributesIdClassInString(), isClassification, i);
eproRHTrain.alAUCs.add(Double.valueOf(cro.AUC));
eproRHTrain.alpAUCs.add(Double.valueOf(cro.pAUC));
eproRHTrain.alAUPRCs.add(Double.valueOf(cro.AUPRC));
eproRHTrain.alACCs.add(Double.valueOf(cro.ACC));
eproRHTrain.alSEs.add(Double.valueOf(cro.TPR));
eproRHTrain.alSPs.add(Double.valueOf(cro.TNR));
eproRHTrain.alMCCs.add(Double.valueOf(cro.MCC));
eproRHTrain.alMAEs.add(Double.valueOf(cro.MAE));
eproRHTrain.alBERs.add(Double.valueOf(cro.BER));
}
} else {
for (int i = 0; i < Main.bootstrapAndRepeatedHoldoutFolds; i++) {
Weka_module.RegressionResultsObject rro
= (Weka_module.RegressionResultsObject) weka.trainClassifierHoldOutValidation(classifier, classifier_options,
ao.getRetainedAttributesIdClassInString(), isClassification, i);
eproRHTrain.alCCs.add(Double.valueOf(rro.CC));
eproRHTrain.alMAEs.add(Double.valueOf(rro.MAE));
eproRHTrain.alRMSEs.add(Double.valueOf(rro.RMSE));
eproRHTrain.alRAEs.add(Double.valueOf(rro.RAE));
eproRHTrain.alRRSEs.add(Double.valueOf(rro.RRSE));
}
}
eproRHTrain.computeMeans();
}
//BOOTSTRAP AND BOOTSTRAP .632+ rule TRAIN
if (Main.debug) {
System.out.println("\tBootstrapping on Train set " + Main.bootstrapAndRepeatedHoldoutFolds + " times");
}
double bootstrapTrain632plus = 1000;
Weka_module.evaluationPerformancesResultsObject eproBSTrain
= new Weka_module.evaluationPerformancesResultsObject();
if (Main.bootstrap) {
if (isClassification) {
bootstrapTrain632plus = weka.trainClassifierBootstrap632plus(classifier, classifier_options,
ao.getRetainedAttributesIdClassInString());
for (int i = 0; i < Main.bootstrapAndRepeatedHoldoutFolds; i++) {
Weka_module.ClassificationResultsObject cro
= (Weka_module.ClassificationResultsObject) weka.trainClassifierBootstrap(classifier, classifier_options,
ao.getRetainedAttributesIdClassInString(), isClassification, i);
eproBSTrain.alAUCs.add(Double.valueOf(cro.AUC));
eproBSTrain.alpAUCs.add(Double.valueOf(cro.pAUC));
eproBSTrain.alAUPRCs.add(Double.valueOf(cro.AUPRC));
eproBSTrain.alACCs.add(Double.valueOf(cro.ACC));
eproBSTrain.alSEs.add(Double.valueOf(cro.TPR));
eproBSTrain.alSPs.add(Double.valueOf(cro.TNR));
eproBSTrain.alMCCs.add(Double.valueOf(cro.MCC));
eproBSTrain.alMAEs.add(Double.valueOf(cro.MAE));
eproBSTrain.alBERs.add(Double.valueOf(cro.BER));
}
} else {
for (int i = 0; i < Main.bootstrapAndRepeatedHoldoutFolds; i++) {
Weka_module.RegressionResultsObject rro
= (Weka_module.RegressionResultsObject) weka.trainClassifierBootstrap(classifier, classifier_options,
ao.getRetainedAttributesIdClassInString(), isClassification, i);
eproBSTrain.alCCs.add(Double.valueOf(rro.CC));
eproBSTrain.alMAEs.add(Double.valueOf(rro.MAE));
eproBSTrain.alRMSEs.add(Double.valueOf(rro.RMSE));
eproBSTrain.alRAEs.add(Double.valueOf(rro.RAE));
eproBSTrain.alRRSEs.add(Double.valueOf(rro.RRSE));
}
}
eproBSTrain.computeMeans();
}
// TEST SET
if (Main.debug) {
System.out.println("\tEvalutation on Test set ");
}
Weka_module.testResultsObject tro = new Weka_module.testResultsObject();
String testResults = null;
if (isClassification) {
testResults = "\t" + "\t" + "\t" + "\t" + "\t" + "\t" + "\t";
} else {
testResults = "\t" + "\t" + "\t" + "\t";
}
try {
if (Main.doSampling) {
//testing
Weka_module weka2 = new Weka_module();
if (isClassification) {
Weka_module.ClassificationResultsObject cr2 = null;
try {
cr2 = (Weka_module.ClassificationResultsObject) weka2.testClassifierFromModel(cr.model,
trainFileName.replace("data_to_train.csv", "data_to_test.csv"),//test file
Main.isClassification, out);
} catch (Exception e) {
if (Main.debug) {
e.printStackTrace();
}
}
tro.ACC = cr2.ACC;
tro.AUC = cr2.AUC;
tro.AUPRC = cr2.AUPRC;
tro.SE = cr2.TPR;
tro.SP = cr2.TNR;
tro.MCC = cr2.MCC;
tro.MAE = cr2.MAE;
tro.BER = cr2.BER;
testResults = tro.toStringClassification();
} else {
Weka_module.RegressionResultsObject rr2
= (Weka_module.RegressionResultsObject) weka2.testClassifierFromModel(rr.model,
trainFileName.replace("data_to_train.csv", "data_to_test.csv"),//test file
Main.isClassification, out);
tro.CC = rr2.CC;
tro.RRSE = rr2.RRSE;
tro.RMSE = rr2.RMSE;
tro.MAE = rr2.MAE;
tro.RAE = rr2.RAE;
testResults = tro.toStringRegression();
}
}
} catch (Exception e) {
if (Main.debug) {
e.printStackTrace();
}
}
//REPEATED HOLDOUT TRAIN_TEST
if (Main.debug) {
System.out.println("\tRepeated Holdout on Train AND Test set "
+ Main.bootstrapAndRepeatedHoldoutFolds + " times");
}
Weka_module.evaluationPerformancesResultsObject eproRHTrainTest
= new Weka_module.evaluationPerformancesResultsObject();
try {
if (Main.doSampling && Main.repeatedHoldout) {
Weka_module weka2 = new Weka_module();
weka2.setARFFfile(trainFileName.replace("data_to_train.csv", "all_data.arff"));
weka2.setDataFromArff();
if (isClassification) {
weka2.myData = weka2.extractFeaturesFromDatasetBasedOnModel(cr.model, weka2.myData);
for (int i = 0; i < Main.bootstrapAndRepeatedHoldoutFolds; i++) {
Weka_module.ClassificationResultsObject cro
= (Weka_module.ClassificationResultsObject) weka2.trainClassifierHoldOutValidation(
classifier, classifier_options,
null, isClassification, i);
eproRHTrainTest.alAUCs.add(Double.valueOf(cro.AUC));
eproRHTrainTest.alpAUCs.add(Double.valueOf(cro.pAUC));
eproRHTrainTest.alAUPRCs.add(Double.valueOf(cro.AUPRC));
eproRHTrainTest.alACCs.add(Double.valueOf(cro.ACC));
eproRHTrainTest.alSEs.add(Double.valueOf(cro.TPR));
eproRHTrainTest.alSPs.add(Double.valueOf(cro.TNR));
eproRHTrainTest.alMCCs.add(Double.valueOf(cro.MCC));
eproRHTrainTest.alMAEs.add(Double.valueOf(cro.MAE));
eproRHTrainTest.alBERs.add(Double.valueOf(cro.BER));
}
} else {
weka2.myData = weka2.extractFeaturesFromDatasetBasedOnModel(rr.model, weka2.myData);
for (int i = 0; i < Main.bootstrapAndRepeatedHoldoutFolds; i++) {
Weka_module.RegressionResultsObject rro
= (Weka_module.RegressionResultsObject) weka2.trainClassifierHoldOutValidation(
classifier, classifier_options,
null, isClassification, i);
eproRHTrainTest.alCCs.add(Double.valueOf(rro.CC));
eproRHTrainTest.alMAEs.add(Double.valueOf(rro.MAE));
eproRHTrainTest.alRMSEs.add(Double.valueOf(rro.RMSE));
eproRHTrainTest.alRAEs.add(Double.valueOf(rro.RAE));
eproRHTrainTest.alRRSEs.add(Double.valueOf(rro.RRSE));
}
}
eproRHTrainTest.computeMeans();
}
} catch (Exception e) {
if (Main.debug) {
e.printStackTrace();
}
}
//BOOTSTRAP TRAIN_TEST AND BOOTSTRAP .632+ rule TRAIN_TEST
if (Main.debug) {
System.out.println("\tBootstrapping on Train AND Test set " + Main.bootstrapAndRepeatedHoldoutFolds + " times");
}
double bootstrapTrainTest632plus = 1000;
Weka_module.evaluationPerformancesResultsObject eproBSTrainTest = new Weka_module.evaluationPerformancesResultsObject();
try {
if (Main.doSampling && Main.bootstrap) {
Weka_module weka2 = new Weka_module();
weka2.setARFFfile(trainFileName.replace("data_to_train.csv", "all_data.arff"));
weka2.setDataFromArff();
if (isClassification) {
weka2.myData = weka2.extractFeaturesFromDatasetBasedOnModel(cr.model, weka2.myData);
bootstrapTrainTest632plus = weka2.trainClassifierBootstrap632plus(classifier, classifier_options,
null);
for (int i = 0; i < Main.bootstrapAndRepeatedHoldoutFolds; i++) {
Weka_module.ClassificationResultsObject cro
= (Weka_module.ClassificationResultsObject) weka2.trainClassifierBootstrap(
classifier, classifier_options,
null, isClassification, i);
eproBSTrainTest.alAUCs.add(Double.valueOf(cro.AUC));
eproBSTrainTest.alpAUCs.add(Double.valueOf(cro.pAUC));
eproBSTrainTest.alAUPRCs.add(Double.valueOf(cro.AUPRC));
eproBSTrainTest.alACCs.add(Double.valueOf(cro.ACC));
eproBSTrainTest.alSEs.add(Double.valueOf(cro.TPR));
eproBSTrainTest.alSPs.add(Double.valueOf(cro.TNR));
eproBSTrainTest.alMCCs.add(Double.valueOf(cro.MCC));
eproBSTrainTest.alMAEs.add(Double.valueOf(cro.MAE));
eproBSTrainTest.alBERs.add(Double.valueOf(cro.BER));
}
} else {
weka2.myData = weka2.extractFeaturesFromDatasetBasedOnModel(rr.model, weka2.myData);
for (int i = 0; i < Main.bootstrapAndRepeatedHoldoutFolds; i++) {
Weka_module.RegressionResultsObject rro
= (Weka_module.RegressionResultsObject) weka2.trainClassifierBootstrap(
classifier, classifier_options,
null, isClassification, i);
eproBSTrainTest.alCCs.add(Double.valueOf(rro.CC));
eproBSTrainTest.alMAEs.add(Double.valueOf(rro.MAE));
eproBSTrainTest.alRMSEs.add(Double.valueOf(rro.RMSE));
eproBSTrainTest.alRAEs.add(Double.valueOf(rro.RAE));
eproBSTrainTest.alRRSEs.add(Double.valueOf(rro.RRSE));
}
}
eproBSTrainTest.computeMeans();
}
} catch (Exception e) {
if (Main.debug) {
e.printStackTrace();
}
}
//STATISTICS AND OUTPUT
if (isClassification) {
//average BER
List<Double> alBERs = new ArrayList<>();
try {
alBERs.add(Double.valueOf(cr.BER));
} catch (Exception e) {
}
try {
alBERs.add(Double.valueOf(eproRHTrain.meanBERs));
} catch (Exception e) {
}
try {
alBERs.add(Double.valueOf(eproBSTrain.meanBERs));
} catch (Exception e) {
}
try {
alBERs.add(Double.valueOf(crLoocv.BER));
} catch (Exception e) {
}
try {
alBERs.add(Double.valueOf(tro.BER));
} catch (Exception e) {
}
try {
alBERs.add(Double.valueOf(eproRHTrainTest.meanBERs));
} catch (Exception e) {
}
try {
alBERs.add(Double.valueOf(eproBSTrainTest.meanBERs));
} catch (Exception e) {
}
double BERs[] = alBERs.stream().mapToDouble(Double::doubleValue).toArray();
StandardDeviation std = new StandardDeviation();
std.setData(BERs);
double StdBER = std.evaluate();
Mean m = new Mean();
m.setData(BERs);
double averageBER = m.evaluate();
//average MAE
List<Double> alMAEs = new ArrayList<>();
try {
alMAEs.add(Double.valueOf(cr.MAE));
} catch (Exception e) {
}
try {
alMAEs.add(Double.valueOf(eproRHTrain.meanMAEs));
} catch (Exception e) {
}
try {
alMAEs.add(Double.valueOf(eproBSTrain.meanMAEs));
} catch (Exception e) {
}
try {
alMAEs.add(Double.valueOf(crLoocv.MAE));
} catch (Exception e) {
}
try {
alMAEs.add(Double.valueOf(tro.MAE));
} catch (Exception e) {
}
try {
alMAEs.add(Double.valueOf(eproRHTrainTest.meanMAEs));
} catch (Exception e) {
}
try {
alMAEs.add(Double.valueOf(eproBSTrainTest.meanMAEs));
} catch (Exception e) {
}
double MAEs[] = alMAEs.stream().mapToDouble(Double::doubleValue).toArray();
std = new StandardDeviation();
std.setData(MAEs);
double StdMAE = std.evaluate();
m = new Mean();
m.setData(MAEs);
double averageMAE = m.evaluate();
//average MCC
List<Double> alMCCs = new ArrayList<>();
try {
alMCCs.add(Double.valueOf(cr.MCC));
} catch (Exception e) {
}
try {
alMCCs.add(Double.valueOf(eproRHTrain.meanMCCs));
} catch (Exception e) {
}
try {
alMCCs.add(Double.valueOf(eproBSTrain.meanMCCs));
} catch (Exception e) {
}
try {
alMCCs.add(Double.valueOf(crLoocv.MCC));
} catch (Exception e) {
}
try {
alMCCs.add(Double.valueOf(tro.MCC));
} catch (Exception e) {
}
try {
alMCCs.add(Double.valueOf(eproRHTrainTest.meanMCCs));
} catch (Exception e) {
}
try {
alMCCs.add(Double.valueOf(eproBSTrainTest.meanMCCs));
} catch (Exception e) {
}
double MCCs[] = alMCCs.stream().mapToDouble(Double::doubleValue).toArray();
std = new StandardDeviation();
std.setData(MCCs);
double StdMCC = std.evaluate();
m = new Mean();
m.setData(MCCs);
double averageMCC = m.evaluate();
String stats = df.format(averageBER) + "\t"
+ df.format(StdBER) + "\t"
+ df.format(averageMAE) + "\t"
+ df.format(StdMAE) + "\t"
+ df.format(averageMCC) + "\t"
+ df.format(StdMCC);
//output
String bt632 = df.format(bootstrapTrain632plus);
if (bt632.equals(1000)) {
bt632 = "";
}
String btt632 = df.format(bootstrapTrainTest632plus);
if (bootstrapTrainTest632plus == 1000) {
btt632 = "";
}
lastOutput = out
+ "\t" + cr.numberOfFeatures
+ "\t" + cr.toString()
+ "\t" + loocvOut
+ "\t" + eproRHTrain.toStringClassification()
+ "\t" + eproBSTrain.toStringClassification()
+ "\t" + bt632
+ "\t" + testResults
+ "\t" + eproRHTrainTest.toStringClassification()
+ "\t" + eproBSTrainTest.toStringClassification()
+ "\t" + btt632
+ "\t" + stats + "\t" + ao.getRetainedAttributesIdClassInString();
} else {
//statistics
//average CC
List<Double> alCCs = new ArrayList<>();
try {
alCCs.add(Double.valueOf(rr.CC));
} catch (Exception e) {
}
try {
alCCs.add(Double.valueOf(eproRHTrain.meanCCs));
} catch (Exception e) {
}
try {
alCCs.add(Double.valueOf(eproBSTrain.meanCCs));
} catch (Exception e) {
}
try {
alCCs.add(Double.valueOf(rrLoocv.CC));
} catch (Exception e) {
}
try {
alCCs.add(Double.valueOf(tro.CC));
} catch (Exception e) {
}
try {
alCCs.add(Double.valueOf(eproRHTrainTest.meanCCs));
} catch (Exception e) {
}
try {
alCCs.add(Double.valueOf(eproBSTrainTest.meanCCs));
} catch (Exception e) {
}
double CCs[] = alCCs.stream().mapToDouble(Double::doubleValue).toArray();
StandardDeviation std = new StandardDeviation();
std.setData(CCs);
double StdCC = std.evaluate();
Mean m = new Mean();
m.setData(CCs);
double averageCC = m.evaluate();
//average MAE
List<Double> alMAEs = new ArrayList<>();
try {
alMAEs.add(Double.valueOf(rr.MAE));
} catch (Exception e) {
}
try {
alMAEs.add(Double.valueOf(eproRHTrain.meanMAEs));
} catch (Exception e) {
}
try {
alMAEs.add(Double.valueOf(eproBSTrain.meanMAEs));
} catch (Exception e) {
}
try {
alMAEs.add(Double.valueOf(rrLoocv.MAE));
} catch (Exception e) {
}
try {
alMAEs.add(Double.valueOf(tro.MAE));
} catch (Exception e) {
}
try {
alMAEs.add(Double.valueOf(eproRHTrainTest.meanMAEs));
} catch (Exception e) {
}
try {
alMAEs.add(Double.valueOf(eproBSTrainTest.meanMAEs));
} catch (Exception e) {
}
double MAEs[] = alMAEs.stream().mapToDouble(Double::doubleValue).toArray();
std = new StandardDeviation();
std.setData(MAEs);
double StdMAE = std.evaluate();
m = new Mean();
m.setData(MAEs);
double averageMAE = m.evaluate();
//average RMSE
List<Double> alRMSEs = new ArrayList<>();
try {
alRMSEs.add(Double.valueOf(rr.RMSE));
} catch (Exception e) {
}
try {
alRMSEs.add(Double.valueOf(eproRHTrain.meanRMSEs));
} catch (Exception e) {
}
try {
alRMSEs.add(Double.valueOf(eproBSTrain.meanRMSEs));
} catch (Exception e) {
}
try {
alRMSEs.add(Double.valueOf(rrLoocv.RMSE));
} catch (Exception e) {
}
try {
alRMSEs.add(Double.valueOf(tro.RMSE));
} catch (Exception e) {
}
try {
alRMSEs.add(Double.valueOf(eproRHTrainTest.meanRMSEs));
} catch (Exception e) {
}
try {
alRMSEs.add(Double.valueOf(eproBSTrainTest.meanRMSEs));
} catch (Exception e) {
}
double RMSEs[] = alRMSEs.stream().mapToDouble(Double::doubleValue).toArray();
std = new StandardDeviation();
std.setData(RMSEs);
double StdRMSE = std.evaluate();
m = new Mean();
m.setData(RMSEs);
double averageRMSE = m.evaluate();
String stats = df.format(averageCC) + "\t"
+ df.format(StdCC) + "\t"
+ df.format(averageMAE) + "\t"
+ df.format(StdMAE) + "\t"
+ df.format(averageRMSE) + "\t"
+ df.format(StdRMSE);
lastOutput = out
+ "\t" + rr.numberOfFeatures + "\t" + rr.toString()
+ "\t" + loocvOut
+ "\t" + eproRHTrain.toStringRegression()
+ "\t" + eproBSTrain.toStringRegression()
+ "\t" + testResults
+ "\t" + eproRHTrainTest.toStringRegression()
+ "\t" + eproBSTrainTest.toStringRegression()
+ "\t" + stats
+ "\t" + ao.getRetainedAttributesIdClassInString();
}
//CREATE ID
Random r = new Random();
int randomNumber = r.nextInt(10000 - 10) + 10;
out = (lastOutput.split("\t")[0] + "_" + lastOutput.split("\t")[2]
+ "_" + lastOutput.split("\t")[3] + "_" + lastOutput.split("\t")[4] + "_" + lastOutput.split("\t")[5] + "_" + randomNumber);
// temporary models output
// String filename = Main.CVfolder + File.separatorChar + out;
// PrintWriter pw = new PrintWriter(new FileWriter(filename + ".txt"));
// pw.println("#" + classifier + " " + classifier_options + "\n#Value to maximize or minimize:" + valueToMaximizeOrMinimize);
// if (isClassification) {
// pw.print("#ID\tactual\tpredicted\terror\tprobability\n");
// } else {
// pw.print("#ID\tactual\tpredicted\terror\n");
// }
// pw.print(predictions + "\n\n");
// pw.print("#Selected Attributes\t(Total attributes:" + numberOfAttributes + ")\n");
// pw.print(selectedAttributes);
// pw.close();
// //save model
// SerializationHelper.write(filename + ".model", model);
} else {
out = "ERROR";
if (Main.debug) {
System.err.println(o);
}
if (o.toString().equals("null")) {
out += "\t " + classifier + " " + classifier_options + " | " + searchMethod + " | Error probably because of number of features inferior to topX";
}
}
} catch (Exception e) {
out = "ERROR\t" + classifier + " " + classifier_options + " | " + searchMethod + " | " + e.getMessage();
cptFailed++;
if (Main.debug) {
e.printStackTrace();
}
}
Instant finish = Instant.now();
long s = Duration.between(start, finish).toMillis() / 1000;
if (Main.debug) {
System.out.println("model created in [" + s + "s]");
}
return out + "\t" + lastOutput;
}
/**
* get value of interest
*
* @param valueWanted
* @param cr
* @param rr
* @return
*/
private static double getValueToMaximize(String valueWanted, Weka_module.ClassificationResultsObject cr,
Weka_module.RegressionResultsObject rr) {
switch (valueWanted.toLowerCase()) {
//classification
case "auc":
return Double.parseDouble(cr.AUC);
case "pauc":
return Double.parseDouble(cr.pAUC);
case "acc":
return Double.parseDouble(cr.ACC);
case "sen":
return Double.parseDouble(cr.TPR);
case "spe":
return Double.parseDouble(cr.TNR);
case "mcc":
return Double.parseDouble(cr.MCC);
case "kappa":
return Double.parseDouble(cr.kappa);
case "aupcr":
return Double.parseDouble(cr.AUPRC);
case "fscore":
return Double.parseDouble(cr.Fscore);
case "precision":
return Double.parseDouble(cr.precision);
case "recall":
return Double.parseDouble(cr.recall);
case "fdr":
return Double.parseDouble(cr.FDR);//to minimize
case "ber":
return Double.parseDouble(cr.BER);//to minimize
case "tp+fn":
double sumOfTP_TN = Double.parseDouble(cr.TPR) + Double.parseDouble(cr.TNR);
return sumOfTP_TN;
//regression
case "cc":
return Double.parseDouble(rr.CC);
case "mae":
return Double.parseDouble(rr.MAE);//to minimize
case "rmse":
return Double.parseDouble(rr.RMSE);//to minimize
case "rae":
return Double.parseDouble(rr.RAE);//to minimize
case "rrse":
return Double.parseDouble(rr.RRSE);//to minimize
}
return 0;
}
/**
* add to hm weka configurations
* <["misc.VFI", "-B 0.6", "AUC"],"">
* <["CostSensitiveClassifier", "-cost-matrix "[0.0 ratio; 100.0 0.0]" -S 1 -W weka.classifiers.misc.VFI -- -B 0.6", "AUC"],"">
*
* @param classifier
* @param options
* @param costSensitive
*/
private static void addClassificationToQueue(String classifier, String options) {
String optimizers[] = Main.classificationOptimizers.split(",");
String searchmodes[] = Main.searchmodes.split(",");
if (Main.noFeatureSelection) {
searchmodes = new String[]{"all"};
}
// else if(weka.myData.numAttributes()>500){
//
// }
for (String optimizer : optimizers) {//AUC, ACC, SEN, SPE, MCC, TP+FN, kappa
for (String searchmode : searchmodes) {//F, FB, B, BF, topk
alClassifiers.add(new String[]{classifier, options, optimizer.trim(), searchmode.trim()});
}
}
if (Main.metaCostSensitiveClassifier) {
options = "-cost-matrix \"[0.0 ratio; 100.0 0.0]\" -S 1 -W weka.classifiers." + classifier + " -- " + options;
for (String value : optimizers) {
alClassifiers.add(new String[]{"meta.CostSensitiveClassifier", options, value.trim(), "f"});
alClassifiers.add(new String[]{"meta.CostSensitiveClassifier", options, value.trim(), "bf"});
}
}
}
private static void addRegressionToQueue(String classifier, String options) {
//String valuesToMaximizeOrMinimize[] = new String[]{"CC", "MAE", "RMSE", "RAE", "RRSE"};
String optimizers[] = Main.regressionOptimizers.split(",");
String searchmodes[] = Main.searchmodes.split(",");
if (Main.noFeatureSelection) {
searchmodes = new String[]{"all"};
}
for (String optimizer : optimizers) {//only CC is maximized here
for (String searchmode : searchmodes) {
alClassifiers.add(new String[]{classifier, options, optimizer.trim(), searchmode.trim()});
}
}
if (Main.metaAdditiveRegression) {
options = "-S 1.0 -I 10 -W weka.classifiers." + classifier + " -- " + options;
for (String value : optimizers) {
alClassifiers.add(new String[]{"meta.AdditiveRegression", options, value, "F"});
alClassifiers.add(new String[]{"meta.AdditiveRegression", options, value, "FB"});
}
}
}
private static class AttributeOject {
public Integer ID;
public ArrayList<Integer> alAttributes = new ArrayList<>();
public Integer Class;
public ArrayList<Integer> retainedAttributesOnly = new ArrayList<>();
public AttributeOject() {
}
private AttributeOject(ArrayList<Integer> attributes) {
ID = attributes.get(0);
Class = attributes.get(attributes.size() - 1);
attributes.remove(attributes.size() - 1);
attributes.remove(0);
alAttributes = attributes;
}
private void addNewAttributeToRetainedAttributes(int index) {
retainedAttributesOnly.add(alAttributes.get(index));
}
private String getRetainedAttributesIdClassInString() {
String attributes = "";
attributes += (ID) + ",";
for (Integer retainedAttribute : retainedAttributesOnly) {
attributes += (retainedAttribute) + ",";
}
attributes += Class;
return attributes;
}
private ArrayList getRetainedAttributesIdClassInArrayList() {
ArrayList<Integer> al = new ArrayList<>();
al.add(ID);
for (Integer retainedAttribute : retainedAttributesOnly) {
al.add(retainedAttribute);
}
al.add(Class);
return al;
}
private void changeRetainedAttributes(String featuresToTest) {
String features[] = featuresToTest.split(",");
retainedAttributesOnly = new ArrayList<>();
for (int i = 1; i < features.length - 1; i++) {//skip ID and class indexes
retainedAttributesOnly.add(Integer.valueOf(features[i]));
}
}
private String getAttributesIdClassInString() {
String attributes = "";
attributes += (ID) + ",";
for (Integer att : alAttributes) {
attributes += (att) + ",";
}
attributes += Class;
return attributes;
}
}
}
Datasets
Models
