/*
* Weka
*/
package utils;
import biodiscml.Main;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.PrintWriter;
import java.text.DecimalFormat;
import java.text.DecimalFormatSymbols;
import java.time.Duration;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Enumeration;
import java.util.HashMap;
import java.util.List;
import java.util.Random;
import java.util.TreeMap;
import java.util.concurrent.atomic.AtomicIntegerArray;
import org.apache.commons.math3.analysis.UnivariateFunction;
import org.apache.commons.math3.analysis.integration.TrapezoidIntegrator;
import org.apache.commons.math3.analysis.interpolation.SplineInterpolator;
import org.apache.commons.math3.analysis.interpolation.UnivariateInterpolator;
import org.apache.commons.math3.exception.MaxCountExceededException;
import static utils.utils.getMean;
import weka.attributeSelection.AttributeSelection;
import weka.classifiers.Classifier;
import weka.classifiers.Evaluation;
import weka.classifiers.evaluation.ThresholdCurve;
import weka.classifiers.evaluation.output.prediction.PlainText;
import weka.classifiers.meta.FilteredClassifier;
import weka.core.Attribute;
import weka.core.Instance;
import weka.core.Instances;
import weka.core.Range;
import weka.core.UnsupportedAttributeTypeException;
import weka.core.Utils;
import weka.core.converters.ArffLoader;
import weka.core.converters.ArffSaver;
import weka.core.converters.CSVLoader;
import weka.core.converters.CSVSaver;
import weka.core.converters.ConverterUtils;
import weka.filters.Filter;
import weka.filters.supervised.instance.StratifiedRemoveFolds;
import weka.filters.unsupervised.attribute.Remove;
import weka.filters.unsupervised.attribute.Reorder;
import weka.filters.unsupervised.attribute.StringToNominal;
import weka.filters.unsupervised.instance.RemoveRange;
/**
*
* @author Mickael
*/
public class Weka_module {
public DecimalFormat df = new DecimalFormat();
public ArrayList<String> alPrint = new ArrayList<>();
private File CSVFile;
public String ARFFfile;
public boolean debug = true;
public Instances myData;
public Weka_module() {
df.setMaximumFractionDigits(3);
DecimalFormatSymbols dec = new DecimalFormatSymbols();
dec.setDecimalSeparator('.');
df.setGroupingUsed(false);
df.setDecimalFormatSymbols(dec);
// svm.svm_set_print_string_function(new libsvm.svm_print_interface() {
// @Override
// public void print(String s) {
// } // Disables svm output
// });
}
/**
* keyword to search in attribute names to remove them
*
* @param keyword
* @param attributeSelection
*/
public void csvToArffRegressionWithFilter(String keyword, boolean attributeSelection) {
try {
//load csv
if (debug) {
System.out.println("loading csv");
}
CSVLoader csv = new CSVLoader();
csv.setSource(CSVFile);
csv.setMissingValue("?");
//csv.setFieldSeparator("\t");
Instances data = csv.getDataSet();
//remove data having the keyword
if (debug) {
System.out.println("cleaning csv");
}
String[] opts = new String[2];
opts[0] = "-R";
String indexes = "";
for (int i = 0; i < data.numAttributes(); i++) {
if (data.attribute(i).name().contains(keyword)) {
//System.out.println("removing " + data.attribute(i).name());
indexes += (i + 1) + ",";
}
}
opts[1] = indexes;
weka.filters.unsupervised.attribute.Remove r = new weka.filters.unsupervised.attribute.Remove();
r.setOptions(opts);
r.setInputFormat(data);
data = Filter.useFilter(data, r);
if (debug) {
System.out.println("saving full arff");
}
ArffSaver arff = new ArffSaver();
arff.setInstances(data);
ARFFfile = CSVFile.toString().replace(".csv", ".arff");
arff.setFile(new File(ARFFfile));
arff.writeBatch();
//filter out useless data
if (debug) {
System.out.println("attribute selection");
}
ArffLoader arf = new ArffLoader();
arf.setSource(new File(ARFFfile));
data = arf.getDataSet();
if (attributeSelection) {
//weka.filters.supervised.attribute.AttributeSelection -E "weka.attributeSelection.CfsSubsetEval " -S "weka.attributeSelection.BestFirst -D 1 -N 5"
weka.filters.supervised.attribute.AttributeSelection select = new weka.filters.supervised.attribute.AttributeSelection();
String options = "-E "
+ "\"weka.attributeSelection.CfsSubsetEval \" -S \"weka.attributeSelection.BestFirst -D 1 -N 5\"";
select.setOptions(weka.core.Utils.splitOptions((options)));
select.setInputFormat(data);
data = Filter.useFilter(data, select);
}
//save csv to arff
if (debug) {
System.out.println("saving CfsSubsetEval Best Firsts arff");
}
arff = new ArffSaver();
arff.setInstances(data);
ARFFfile = CSVFile.toString().replace(".csv", ".CfsSubset_BestFirst.arff");
arff.setFile(new File(ARFFfile));
arff.writeBatch();
} catch (Exception e) {
e.printStackTrace();
}
}
public void csvToArffRegression() {
if (Main.debug) {
System.out.println("\n\nConvert " + CSVFile + " to arff");
}
File f = new File(CSVFile.toString().replace(".csv", ".arff"));
f.delete();
try {
//load csv
CSVLoader csv = new CSVLoader();
csv.setSource(CSVFile);
csv.setMissingValue("?");
//csv.setFieldSeparator("\t");
Instances data = csv.getDataSet();
//save csv to arff
ArffSaver arff = new ArffSaver();
arff.setInstances(data);
ARFFfile = CSVFile.toString().replace(".csv", ".arff");
arff.setFile(new File(ARFFfile));
arff.writeBatch();
} catch (Exception e) {
e.printStackTrace();
}
}
public void csvToArff(boolean classification) {
if (Main.debug) {
System.out.println("Convert " + CSVFile + " to arff");
}
File f = new File(CSVFile.toString().replace(".csv", ".arff"));
f.delete();
try {
//load csv
CSVLoader csv = new CSVLoader();
csv.setSource(CSVFile);
csv.setNominalAttributes("1"); //change ID to nominal
csv.setMissingValue("?");
String separator = utils.detectSeparator(CSVFile.toString());
csv.setFieldSeparator(separator);
Instances data = csv.getDataSet();
data = convertStringsToNominal(data);
//change numeric to nominal class
if (classification) {
weka.filters.unsupervised.attribute.NumericToNominal n
= new weka.filters.unsupervised.attribute.NumericToNominal();
String[] opts = new String[2];
opts[0] = "-R";
opts[1] = "last";
n.setOptions(opts);
n.setInputFormat(data);
data = Filter.useFilter(data, n);
if (data.classIndex() == -1) {
data.setClassIndex(data.numAttributes() - 1);
}
//class order
if (data.classAttribute().enumerateValues().hasMoreElements()) {
if (data.classAttribute().enumerateValues().nextElement().equals("false")
|| data.classAttribute().enumerateValues().nextElement().equals("0")) {
weka.filters.unsupervised.attribute.SwapValues s = new weka.filters.unsupervised.attribute.SwapValues();
s.setOptions(weka.core.Utils.splitOptions(("-C last -F first -S last")));
s.setInputFormat(data);
data = Filter.useFilter(data, s);
}
}
}
//save csv to arff
ArffSaver arff = new ArffSaver();
arff.setInstances(data);
ARFFfile = CSVFile.toString().replace(".csv", ".arff");
arff.setFile(new File(ARFFfile));
arff.writeBatch();
} catch (Exception e) {
e.printStackTrace();
}
}
public ArrayList<String> trainRegression() {
ArrayList<String> al = new ArrayList<>();
try {
// load data
ConverterUtils.DataSource source = new ConverterUtils.DataSource(ARFFfile);
Instances data = source.getDataSet();
int numberOfFolds = 10;
//reduc numberOfFolds if needed
if (data.numInstances() < numberOfFolds) {
numberOfFolds = data.numInstances();
}
//set last attribute as index
if (data.classIndex() == -1) {
data.setClassIndex(data.numAttributes() - 1);
}
// train model
////weka.classifiers.functions.GaussianProcesses
// weka.classifiers.functions.GaussianProcesses model = new weka.classifiers.functions.GaussianProcesses();
// String options = "-L 1.0 -N 0 -K \"weka.classifiers.functions.supportVector.NormalizedPolyKernel -C 250007 -E 2.0\"";
// model.setOptions(weka.core.Utils.splitOptions((options)));
////weka.classifiers.functions.SMOreg
weka.classifiers.functions.SMOreg model = new weka.classifiers.functions.SMOreg();
String options = "-C 1.0 -N 0 -I \"weka.classifiers.functions.supportVector.RegSMO -L 0.001 -W 1 -P 1.0E-12\" -K \"weka.classifiers.functions.supportVector.NormalizedPolyKernel -C 250007 -E 2.0\"";
model.setOptions(weka.core.Utils.splitOptions((options)));
// cross validation 10 times on the model
Evaluation eval = new Evaluation(data);
eval.crossValidateModel(model, data, numberOfFolds, new Random(1));
//output classification results
//System.out.println(eval.toSummaryString());
al.add(Utils.doubleToString(eval.correlationCoefficient(), 12, 4));
al.add(Utils.doubleToString(eval.meanAbsoluteError(), 12, 4));
al.add(Utils.doubleToString(eval.rootMeanSquaredError(), 12, 4));
al.add(Utils.doubleToString(eval.relativeAbsoluteError(), 12, 4));
al.add(Utils.doubleToString(eval.rootRelativeSquaredError(), 12, 4));
al.add("" + (data.numAttributes() - 1)); // number of genes
} catch (Exception e) {
e.printStackTrace();
}
return al;
}
/**
* short test on 10% of instances
*
* @param attributesToUse
* @return
*/
public Object shortTestTrainClassifier(String classifier,
String classifier_options,
String attributesToUse,
Boolean classification) {
try {
// load data
Instances data = myData;
//get a sample of instances
//// randomize data
data.randomize(new Random());
//// Percent split
int trainSize = (int) Math.round(data.numInstances() * 95 / 100);
int testSize = data.numInstances() - trainSize;
data = new Instances(data, trainSize, testSize);
//set last attribute as index
if (data.classIndex() == -1) {
data.setClassIndex(data.numAttributes() - 1);
}
data.attribute(data.numAttributes() - 1).isNumeric();
/**
* if we are using Vote, it means it already contains the features
* to use for each classifier
*/
String configuration = classifier + " " + classifier_options;
if (!classifier.contains("meta.Vote")) {
//keep only attributesToUse. ID is still here
String options = "";
weka.filters.unsupervised.attribute.Remove r = new weka.filters.unsupervised.attribute.Remove();
options = "-V -R " + attributesToUse;
r.setOptions(weka.core.Utils.splitOptions((options)));
r.setInputFormat(data);
data = Filter.useFilter(data, r);
data.setClassIndex(data.numAttributes() - 1);
//create weka configuration string
//remove ID from classification
String filterID = ""
+ "weka.classifiers.meta.FilteredClassifier "
+ "-F \"weka.filters.unsupervised.attribute.Remove -R 1\" "
+ "-W weka.classifiers.";
//create config
configuration = filterID + "" + classifier + " -- " + classifier_options;
}
//if cost sensitive case
if (classifier.contains("CostSensitiveClassifier")) {
int falseExemples = data.attributeStats(data.classIndex()).nominalCounts[1];
int trueExemples = data.attributeStats(data.classIndex()).nominalCounts[0];
double ratio = (double) trueExemples / (double) falseExemples;
classifier_options = classifier_options.replace("ratio", df.format(ratio));
}
//train
final String config[] = Utils.splitOptions(configuration);
String classname = config[0];
config[0] = "";
Classifier model = null;
model = (Classifier) Utils.forName(Classifier.class, classname, config);
//evaluation
if (Main.debug) {
System.out.println("\tShort test of model on 5% of the data...");
}
//build model to save it
model.buildClassifier(data);
if (Main.debug) {
System.out.println("PASSED");
}
return model;
} catch (Exception e) {
if (Main.debug) {
System.out.println("FAILED");
}
String message = "[model error] " + classifier + " " + classifier_options + " | " + e.getMessage();
if (Main.debug) {
// if (!e.getMessage().contains("handle") && !e.getMessage().contains("supported")) {
//
// }
e.printStackTrace();
System.err.println(message);
}
return message;
}
}
/**
* http://bayesianconspiracy.blogspot.ca/2009/10/usr-bin-env-groovy-import-weka_10.html
*
* @param attributesToUse
* @return
*/
public Object trainClassifier(String classifier,
String classifier_options,
String attributesToUse,
Boolean classification, int numberOfFolds) {
try {
// load data
Instances data = myData;
//reduc numberOfFolds if needed
if (data.numInstances() < numberOfFolds) {
numberOfFolds = data.numInstances();
}
//set last attribute as index
if (data.classIndex() == -1) {
data.setClassIndex(data.numAttributes() - 1);
}
data.attribute(data.numAttributes() - 1).isNumeric();
/**
* if we are using Vote, it means it already contains the features
* to use for each classifier
*/
String configuration = classifier + " " + classifier_options;
if (!classifier.contains("meta.Vote")) {
//keep only attributesToUse. ID is still here
String options = "";
weka.filters.unsupervised.attribute.Remove r = new weka.filters.unsupervised.attribute.Remove();
options = "-V -R " + attributesToUse;
r.setOptions(weka.core.Utils.splitOptions((options)));
r.setInputFormat(data);
data = Filter.useFilter(data, r);
data.setClassIndex(data.numAttributes() - 1);
//create weka configuration string
//remove ID from classification
String filterID = ""
+ "weka.classifiers.meta.FilteredClassifier "
+ "-F \"weka.filters.unsupervised.attribute.Remove -R 1\" "
+ "-W weka.classifiers.";
//create config
configuration = filterID + "" + classifier + " -- " + classifier_options;
}
//if cost sensitive case
if (classifier.contains("CostSensitiveClassifier")) {
int falseExemples = data.attributeStats(data.classIndex()).nominalCounts[1];
int trueExemples = data.attributeStats(data.classIndex()).nominalCounts[0];
double ratio = (double) trueExemples / (double) falseExemples;
classifier_options = classifier_options.replace("ratio", df.format(ratio));
}
//train
final String config[] = Utils.splitOptions(configuration);
String classname = config[0];
config[0] = "";
Classifier model = null;
model = (Classifier) Utils.forName(Classifier.class, classname, config);
//evaluation
Instant start = null;
if (Main.debug2) {
System.out.print("\tEvaluation of model with 10CV...");
start = Instant.now();
}
Evaluation eval = new Evaluation(data);
StringBuffer sb = new StringBuffer();
PlainText pt = new PlainText();
pt.setBuffer(sb);
//10 fold cross validation
eval.crossValidateModel(model, data, numberOfFolds, new Random(1), pt, new Range("first,last"), true);
if (Main.debug2) {
Instant finish = Instant.now();
long s = Duration.between(start, finish).toMillis();
System.out.println(" in " + s + "ms");
}
//build model to save it
model.buildClassifier(data);
// System.out.println(model.toString().split("\n")[0]);
//return
if (classification) {
return new ClassificationResultsObject(eval, sb, data, model);
} else {
return new RegressionResultsObject(eval, sb, data, model);
}
} catch (Exception e) {
String message = "[model error] " + classifier + " " + classifier_options + " | " + e.getMessage();
if (Main.debug) {
// if (!e.getMessage().contains("handle") && !e.getMessage().contains("supported")) {
//
// }
e.printStackTrace();
System.err.println(message);
}
return message;
} catch (Error err) {
String message = "[model error] " + classifier + " " + classifier_options + " | " + err.getMessage();
if (Main.debug) {
// if (!e.getMessage().contains("handle") && !e.getMessage().contains("supported")) {
//
// }
err.printStackTrace();
System.err.println(message);
}
return message;
}
}
/**
* HoldOut
*
* @param classifier
* @param classifier_options
* @param attributesToUse
* @param classification
* @return
*/
public Object trainClassifierHoldOutValidation(String classifier, String classifier_options,
String attributesToUse, Boolean classification, int seed) {
try {
// load data
Instances data = myData;
//set class index
if (data.classIndex() == -1) {
data.setClassIndex(data.numAttributes() - 1);
}
String configuration = classifier + " " + classifier_options;
if (!classifier.contains("meta.Vote") && attributesToUse != null) {
//keep only attributesToUse. ID (if present) is still here
String options = "";
weka.filters.unsupervised.attribute.Remove r = new weka.filters.unsupervised.attribute.Remove();
options = "-V -R " + attributesToUse;
r.setOptions(weka.core.Utils.splitOptions((options)));
r.setInputFormat(data);
data = Filter.useFilter(data, r);
data.setClassIndex(data.numAttributes() - 1);
}
if (!classifier.contains("meta.Vote")) {
//create weka configuration string
String filterID = "weka.classifiers.meta.FilteredClassifier "
+ "-F \"weka.filters.unsupervised.attribute.Remove -R 1\" "
+ "-W weka.classifiers.";
configuration = filterID + "" + classifier + " -- " + classifier_options;
}
// randomize data
data.randomize(new Random(seed));
// Percent split
int trainSize = (int) Math.round(data.numInstances() * 66 / 100);
int testSize = data.numInstances() - trainSize;
Instances train = new Instances(data, 0, trainSize);
Instances test = new Instances(data, trainSize, testSize);
//if cost sensitive case
if (classifier.contains("CostSensitiveClassifier")) {
int falseExemples = train.attributeStats(train.classIndex()).nominalCounts[1];
int trueExemples = train.attributeStats(train.classIndex()).nominalCounts[0];
double ratio = (double) trueExemples / (double) falseExemples;
classifier_options = classifier_options.replace("ratio", df.format(ratio));
}
//train
String config[] = Utils.splitOptions(configuration);
String classname = config[0];
config[0] = "";
Classifier model = (Classifier) Utils.forName(Classifier.class, classname, config);
StringBuffer sb = new StringBuffer();
//build model to save it
try {
model.buildClassifier(train);
} catch (Exception exception) {
//crashing, try to remove ID and run again
train.deleteAttributeAt(0);
test.deleteAttributeAt(0);
model.buildClassifier(train);
}
//evaluation
Instant start = null;
if (Main.debug2) {
System.out.print("\tEvaluation of model [" + seed + "]...");
start = Instant.now();
}
Evaluation eval = new Evaluation(data);
eval.evaluateModel(model, test);
if (Main.debug2) {
Instant finish = Instant.now();
long s = Duration.between(start, finish).toMillis();
System.out.println(" in " + s + "ms");
}
//return
if (classification) {
return new ClassificationResultsObject(eval, sb, data, model);
} else {
return new RegressionResultsObject(eval, sb, data, model);
}
} catch (Exception e) {
if (Main.debug) {
System.err.println(e.getMessage() + " for " + classifier + " " + classifier_options);
e.printStackTrace();
}
return null;
}
}
/**
* Bootstrap
*
* @param classifier
* @param classifier_options
* @param attributesToUse
* @param classification
* @return
*/
public Object trainClassifierBootstrap(String classifier, String classifier_options,
String attributesToUse, Boolean classification, int seed) {
try {
// load data
Instances data = myData;
//set class index
if (data.classIndex() == -1) {
data.setClassIndex(data.numAttributes() - 1);
}
String configuration = classifier + " " + classifier_options;
if (!classifier.contains("meta.Vote") && attributesToUse != null) {
//keep only attributesToUse. ID (if present) is still here
String options = "";
weka.filters.unsupervised.attribute.Remove r = new weka.filters.unsupervised.attribute.Remove();
options = "-V -R " + attributesToUse;
r.setOptions(weka.core.Utils.splitOptions((options)));
r.setInputFormat(data);
data = Filter.useFilter(data, r);
data.setClassIndex(data.numAttributes() - 1);
}
if (!classifier.contains("meta.Vote")) {
//create weka configuration string
String filterID = "weka.classifiers.meta.FilteredClassifier "
+ "-F \"weka.filters.unsupervised.attribute.Remove -R 1\" "
+ "-W weka.classifiers.";
configuration = filterID + "" + classifier + " -- " + classifier_options;
}
Random r = new Random(seed);
//train
String config[] = Utils.splitOptions(configuration);
String classname = config[0];
config[0] = "";
Classifier model = (Classifier) Utils.forName(Classifier.class, classname, config);
StringBuffer sb = new StringBuffer();
//evaluation
Instant start = null;
if (Main.debug2) {
System.out.print("\tEvaluation of model [" + seed + "]...");
start = Instant.now();
}
// Custom sampling (100%, with replacement)
ArrayList<Instance> al_trainSet = new ArrayList<>(data.size()); // Empty list (add one-by-one)
ArrayList<Instance> al_testSet = new ArrayList<>(data); // Full (remove one-by-one)
for (int j = 0; j < data.size(); j++) {
// Random select instance
Instance instance = data.get(r.nextInt(data.size()));
// Add to TRAIN, remove from TEST
al_trainSet.add(instance);
al_testSet.remove(instance);
}
//prepare train and test sets
Instances trainSet = new Instances(data, al_trainSet.size());
trainSet.addAll(al_trainSet);
Instances testSet = new Instances(data, al_testSet.size());
testSet.addAll(al_testSet);
//train the train set
try {
model.buildClassifier(trainSet);
} catch (Exception exception) {
// crashing probably because of presence of ID, removing it
trainSet.deleteAttributeAt(0);
testSet.deleteAttributeAt(0);
model.buildClassifier(trainSet);
}
// Test the test set
Evaluation eval = new Evaluation(data);
eval.evaluateModel(model, testSet);
if (Main.debug2) {
Instant finish = Instant.now();
long s = Duration.between(start, finish).toMillis();
double d = s / 1000;
System.out.println(" in " + s + "ms");
}
//return
if (classification) {
return new ClassificationResultsObject(eval, sb, data, model);
} else {
return new RegressionResultsObject(eval, sb, data, model);
}
} catch (Exception e) {
if (Main.debug) {
e.printStackTrace();
}
//System.err.println(e.getMessage() + " for " + classifier + " " + classifier_options);
return null;
}
}
/**
* Bootstrap .632+
*
* @param classifier
* @param classifier_options
* @param attributesToUse
* @return
*/
public Double trainClassifierBootstrap632plus(String classifier, String classifier_options,
String attributesToUse) {
try {
// load data
Instances data = myData;
//set class index
if (data.classIndex() == -1) {
data.setClassIndex(data.numAttributes() - 1);
}
String configuration = classifier + " " + classifier_options;
if (!classifier.contains("meta.Vote") && attributesToUse != null) {
//keep only attributesToUse. ID (if present) is still here
String options = "";
weka.filters.unsupervised.attribute.Remove r = new weka.filters.unsupervised.attribute.Remove();
options = "-V -R " + attributesToUse;
r.setOptions(weka.core.Utils.splitOptions((options)));
r.setInputFormat(data);
data = Filter.useFilter(data, r);
data.setClassIndex(data.numAttributes() - 1);
//create weka configuration string
String filterID = "weka.classifiers.meta.FilteredClassifier "
+ "-F \"weka.filters.unsupervised.attribute.Remove -R 1\" "
+ "-W weka.classifiers.";
configuration = filterID + "" + classifier + " -- " + classifier_options;
}
//train model
final String config[] = Utils.splitOptions(configuration);
String classname = config[0];
config[0] = "";
Classifier model = null;
model = (Classifier) Utils.forName(Classifier.class, classname, config);
//build model
model.buildClassifier(data);
// evaluate model
Evaluation eval = new Evaluation(data);
eval.evaluateModel(model, data);
// Apparent error rate
Double err = eval.errorRate();
// Calculate Leave One Out (LOO) Bootstrap
AtomicIntegerArray p_l = new AtomicIntegerArray(data.numClasses());
AtomicIntegerArray q_l = new AtomicIntegerArray(data.numClasses());
double sum = 0;
for (int i = 0; i < Main.bootstrapAndRepeatedHoldoutFolds; i++) {
Random r = new Random();
// Custom sampling (100%, with replacement)
ArrayList<Instance> al_trainSet = new ArrayList<>(data.size()); // Empty list (add one-by-one)
ArrayList<Instance> al_testSet = new ArrayList<>(data); // Full (remove one-by-one)
for (int j = 0; j < data.size(); j++) {
// Random select instance
Instance instance = data.get(r.nextInt(data.size()));
// Add to TRAIN, remove from TEST
al_trainSet.add(instance);
al_testSet.remove(instance);
}
//train the train set
Instances trainSet = new Instances(data, al_trainSet.size());
trainSet.addAll(al_trainSet);
model.buildClassifier(trainSet);
// Test the test set
Instances testSet = new Instances(data, al_testSet.size());
testSet.addAll(al_testSet);
Evaluation evaluation = new Evaluation(data);
evaluation.evaluateModel(model, testSet);
// total error rates
sum += evaluation.errorRate();
//GAMMA
double[][] confusionMatrix = evaluation.confusionMatrix();
for (int l = 0; l < data.numClasses(); l++) {
int p_tmp = 0, q_tmp = 0;
for (int n = 0; n < data.numClasses(); n++) {
// Sum for l-th class
p_tmp += confusionMatrix[l][n];
q_tmp += confusionMatrix[n][l];
}
// Add data for l-th class
p_l.addAndGet(l, p_tmp);
q_l.addAndGet(l, q_tmp);
}
}
double Err1 = sum / Main.bootstrapAndRepeatedHoldoutFolds;
// Plain 0.632 bootstrap
Double Err632 = .368 * err + .632 * Err1;
// GAMA
final double observations = data.size() * Main.bootstrapAndRepeatedHoldoutFolds;
double gama = 0;
for (int l = 0; l < data.numClasses(); l++) {
// Normalize numbers -> divide by number of all observations (repeats * dataset size)
gama += ((double) p_l.get(l) / observations) * (1 - ((double) q_l.get(l) / observations));
}
// Relative overfitting rate (R)
double R = (Err1 - err) / (gama - err);
// Modified variables (according to original journal article)
double Err1_ = Double.min(Err1, gama);
double R_ = R;
// R can fall out of [0, 1] -> set it to 0
if (!(Err1 > err && gama > err)) {
R_ = 0;
}
// The 0.632+ bootstrap (as used in original article)
double Err632plus = Err632 + (Err1_ - err) * (.368 * .632 * R_) / (1 - .368 * R_);
return Err632plus;
} catch (Exception e) {
if (Main.debug) {
e.printStackTrace();
}
return -1.0;
}
}
public void attributeSelectionByRelieFFAndSaveToCSV(String outfile) {
// load data
try {
ConverterUtils.DataSource source = new ConverterUtils.DataSource(ARFFfile);
Instances data = source.getDataSet();
if (data.classIndex() == -1) {
data.setClassIndex(data.numAttributes() - 1);
}
//get id attribute
Attribute id = data.attribute(0);
weka.filters.supervised.attribute.AttributeSelection select = new weka.filters.supervised.attribute.AttributeSelection();
//filter based on ranker score threshold
String limit = " -N " + Main.maxNumberOfSelectedFeatures;
String options = "-E \"weka.attributeSelection.ReliefFAttributeEval -M -1 -D 1 -K 10\" -S \"weka.attributeSelection.Ranker -T 0.01" + limit + "\"";
select.setOptions(weka.core.Utils.splitOptions((options)));
select.setInputFormat(data);
Instances filteredData = Filter.useFilter(data, select);
System.out.println("Total attributes: " + (filteredData.numAttributes() - 1));
if (filteredData.numAttributes() == 0) {
System.out.println("Not enough attributes, probably all non-informative. So keeping all and ranking them, but expect low training performance");
options = "-E \"weka.attributeSelection.ReliefFAttributeEval -M -1 -D 1 -K 10\" -S \"weka.attributeSelection.Ranker -T -1 -N -1\"";
select.setOptions(weka.core.Utils.splitOptions((options)));
select.setInputFormat(data);
filteredData = Filter.useFilter(data, select);
}
//add identifier if it has been lost after relieff
if (filteredData.attribute(id.name()) == null) {
filteredData.insertAttributeAt(id, 0);
filteredData.attribute(0).isNominal();
}
//move identifier to index 0
if (filteredData.attribute(id.name()) != null && !filteredData.attribute(0).equals(id)) {
//find index of ID
int idIndex = 0;
for (int i = 0; i < filteredData.numAttributes(); i++) {
if (filteredData.attribute(i).equals(id)) {
idIndex = i;
}
}
//delete ID
filteredData.deleteAttributeAt(idIndex);
//reinsert ID
filteredData.insertAttributeAt(id, 0);
}
//restore IDs
for (int i = 0; i < filteredData.numInstances(); i++) {
filteredData.instance(i).setValue(0, (String) id.value(i));
}
//save data as csv
CSVSaver csv = new CSVSaver();
csv.setInstances(filteredData);
csv.setFile(new File(outfile));
if (new File(outfile.replace(".csv", ".arff")).exists()) {
new File(outfile.replace(".csv", ".arff")).delete();
}
csv.writeBatch();
myData = filteredData;
} catch (Exception e) {
e.printStackTrace();
}
}
public void attributeSelectionByRelieFFAndSaveToCSVbigData(String outfile) {
// load data
try {
ConverterUtils.DataSource source = new ConverterUtils.DataSource(ARFFfile);
Instances data = source.getDataSet();
if (data.classIndex() == -1) {
data.setClassIndex(data.numAttributes() - 1);
}
//get id attribute
Attribute id = data.attribute(0);
weka.filters.supervised.attribute.AttributeSelection select = new weka.filters.supervised.attribute.AttributeSelection();
//go parallel one set of features at a time
//filter based on ranker score threshold
String limit = " -N " + Main.maxNumberOfSelectedFeatures;
String options = "-E \"weka.attributeSelection.ReliefFAttributeEval -M -1 -D 1 -K 10\" -S \"weka.attributeSelection.Ranker -T 0.01" + limit + "\"";
select.setOptions(weka.core.Utils.splitOptions((options)));
select.setInputFormat(data);
Instances filteredData = Filter.useFilter(data, select);
//restoring instances IDs as first attribute
// Attribute attributeInstances = filteredData.attribute("Instance");
// filteredData.deleteAttributeAt(filteredData.attribute("Instance").index());
// filteredData.insertAttributeAt(attributeInstances, 0);
//
// Enumeration e = attributeInstances.enumerateValues();
// int cpt = 0;
// while (e.hasMoreElements()) {
// String s = (String) e.nextElement();
// filteredData.instance(cpt).setValue(0, s);
// cpt++;
// }
// filteredData.attribute(0).isNominal();
System.out.println("Total attributes: " + (filteredData.numAttributes() - 1));
if (filteredData.numAttributes() == 0) {
System.out.println("Not enough attributes, probably all non-informative. So keeping all and ranking them, but expect low training performance");
options = "-E \"weka.attributeSelection.ReliefFAttributeEval -M -1 -D 1 -K 10\" -S \"weka.attributeSelection.Ranker -T -1 -N -1\"";
select.setOptions(weka.core.Utils.splitOptions((options)));
select.setInputFormat(data);
filteredData = Filter.useFilter(data, select);
}
//add identifier if it has been lost after relieff
if (filteredData.attribute(id.name()) == null) {
filteredData.insertAttributeAt(id, 0);
filteredData.attribute(0).isNominal();
}
//move identifier to index 0
if (filteredData.attribute(id.name()) != null && !filteredData.attribute(0).equals(id)) {
//find index of ID
int idIndex = 0;
for (int i = 0; i < filteredData.numAttributes(); i++) {
if (filteredData.attribute(i).equals(id)) {
idIndex = i;
}
}
//delete ID
filteredData.deleteAttributeAt(idIndex);
//reinsert ID
filteredData.insertAttributeAt(id, 0);
}
//restore IDs
for (int i = 0; i < filteredData.numInstances(); i++) {
filteredData.instance(i).setValue(0, (String) id.value(i));
}
//save data as csv
CSVSaver csv = new CSVSaver();
csv.setInstances(filteredData);
csv.setFile(new File(outfile));
if (new File(outfile.replace(".csv", ".arff")).exists()) {
new File(outfile.replace(".csv", ".arff")).delete();
}
csv.writeBatch();
myData = filteredData;
} catch (Exception e) {
e.printStackTrace();
}
}
public void attributeSelectionByInfoGainRankingAndSaveToCSV(String outfile) {
Instances data = myData;
//load data
try {
if (data.classIndex() == -1) {
data.setClassIndex(data.numAttributes() - 1);
}
//get id attribute
Attribute id = data.attribute(0);
// info gain filter on value (remove attributes with 0 merit)
weka.filters.supervised.attribute.AttributeSelection select = new weka.filters.supervised.attribute.AttributeSelection();
String options = "-E \"weka.attributeSelection.InfoGainAttributeEval \" -S \"weka.attributeSelection.Ranker -T 0.001\"";
select.setOptions(weka.core.Utils.splitOptions((options)));
select.setInputFormat(data);
Instances filteredData = Filter.useFilter(data, select);
System.out.println("Total attributes: " + (filteredData.numAttributes() - 1));
// filter on number of maxNumberOfSelectedFeatures
if (filteredData.numAttributes() > Main.maxNumberOfSelectedFeatures) {
System.out.println("Too many attributes, only keep best " + Main.maxNumberOfSelectedFeatures);
String limit = " -N " + Main.maxNumberOfSelectedFeatures;
options = "-E \"weka.attributeSelection.InfoGainAttributeEval \" -S \"weka.attributeSelection.Ranker -T 0.001" + limit + "\"";
select.setOptions(weka.core.Utils.splitOptions((options)));
select.setInputFormat(filteredData);
filteredData = Filter.useFilter(filteredData, select);
}
if (filteredData.numAttributes() <= 10) {
System.out.println("Not enough attributes, probably all non-informative. So keeping all and ranking them, but expect very low performances");
options = "-E \"weka.attributeSelection.InfoGainAttributeEval \" -S \"weka.attributeSelection.Ranker -T -1 -N -1\"";
select.setOptions(weka.core.Utils.splitOptions((options)));
select.setInputFormat(data);
filteredData = Filter.useFilter(data, select);
}
//add identifier if it has been lost after information gain
if (filteredData.attribute(id.name()) == null) {
filteredData.insertAttributeAt(id, 0);
}
//move identifier to index 0
if (filteredData.attribute(id.name()) != null && !filteredData.attribute(0).equals(id)) {
//find index of ID
int idIndex = 0;
for (int i = 0; i < filteredData.numAttributes(); i++) {
if (filteredData.attribute(i).equals(id)) {
idIndex = i;
}
}
Reorder r = new Reorder();
idIndex = idIndex + 1;
if (idIndex != (filteredData.numAttributes() - 1)) {
options = "-R " + idIndex + ",first-" + (idIndex - 1) + "," + (idIndex + 1) + "-last";
} else {
options = "-R " + idIndex + ",first-" + (idIndex - 1) + "," + (idIndex + 1);
}
r.setOptions(weka.core.Utils.splitOptions((options)));
r.setInputFormat(filteredData);
filteredData = Filter.useFilter(filteredData, r);
}
//in case of complete lost ID (replaced by ?)
if (filteredData.instance(0).toString().startsWith("?")) {
for (int i = 0; i < filteredData.numInstances(); i++) {
filteredData.instance(i).setValue(0, data.attribute(0).value((int) data.instance(i).value(0)));
}
}
//save data as csv
CSVSaver csv = new CSVSaver();
csv.setInstances(filteredData);
csv.setFile(new File(outfile));
if (new File(outfile.replace(".csv", ".arff")).exists()) {
new File(outfile.replace(".csv", ".arff")).delete();
}
csv.writeBatch();
myData = filteredData;
//evaluation.crossValidateModel(classifier, data, 10, new Random(1));
} catch (Exception e) {
e.printStackTrace();
System.err.println(e.getMessage() + "\nUnable to execute InfoGain. Trying directly Ranking");
featureRankingForClassification();
}
}
public void attributeSelection(String evaluator, String outfileExtension) {
// load data
try {
Instances data = myData;
if (data.classIndex() == -1) {
data.setClassIndex(data.numAttributes() - 1);
}
// //remove patient ID
// if (myData.attribute(0).name().toLowerCase().contains("patient")) {
// data.deleteAttributeAt(0);
// }
weka.filters.supervised.attribute.AttributeSelection select = new weka.filters.supervised.attribute.AttributeSelection();
select.setOptions(weka.core.Utils.splitOptions((evaluator)));
select.setInputFormat(data);
data = Filter.useFilter(data, select);
//save data as arff
ArffSaver arffsav = new ArffSaver();
arffsav.setInstances(data);
ARFFfile = ARFFfile.replace(".arff", "." + outfileExtension + ".arff");
arffsav.setFile(new File(ARFFfile));
arffsav.writeBatch();
//evaluation.crossValidateModel(classifier, data, 10, new Random(1));
} catch (Exception e) {
e.printStackTrace();
}
}
public AttributeSelection featureRankingForClassification() {
try {
ConverterUtils.DataSource source = new ConverterUtils.DataSource(ARFFfile);
Instances data = source.getDataSet();
data = convertStringsToNominal(data);
data.numAttributes();
AttributeSelection attrsel = new AttributeSelection();
weka.attributeSelection.InfoGainAttributeEval eval = new weka.attributeSelection.InfoGainAttributeEval();
weka.attributeSelection.Ranker rank = new weka.attributeSelection.Ranker();
rank.setOptions(
weka.core.Utils.splitOptions(
"-T 0.001 -N -1"));
attrsel.setEvaluator(eval);
attrsel.setSearch(rank);
attrsel.SelectAttributes(data);
return attrsel;
} catch (Exception e) {
e.printStackTrace();
System.err.println("Check your input files. Probably missing classes");
System.exit(0);
return null;
}
}
/**
*
* @param csvFile
* @return ranking output
*/
public String featureRankingForClassification(String csvFile) {
try {
//csvToArff(true);
ConverterUtils.DataSource source = new ConverterUtils.DataSource(csvFile);
Instances data = source.getDataSet();
AttributeSelection attrsel = new AttributeSelection();
weka.attributeSelection.InfoGainAttributeEval eval = new weka.attributeSelection.InfoGainAttributeEval();
weka.attributeSelection.Ranker rank = new weka.attributeSelection.Ranker();
rank.setOptions(
weka.core.Utils.splitOptions(
"-T -1 -N -1"));
attrsel.setEvaluator(eval);
attrsel.setSearch(rank);
try {
attrsel.SelectAttributes(data);
} catch (UnsupportedAttributeTypeException e) {
//in case of id seen as string instead of nominal
weka.filters.unsupervised.attribute.StringToNominal f = new StringToNominal();
String options = "-R first";
f.setOptions(weka.core.Utils.splitOptions((options)));
f.setInputFormat(data);
data = Filter.useFilter(data, f);
try {
attrsel.SelectAttributes(data);
} catch (Exception ex) {
//in case where one or more features are seen as nominal
for (int i = 1; i < data.numAttributes() - 1; i++) {
if (data.attribute(i).isString()) {
f = new StringToNominal();
options = "-R " + (i + 1);
f.setOptions(weka.core.Utils.splitOptions((options)));
f.setInputFormat(data);
data = Filter.useFilter(data, f);
}
}
attrsel.SelectAttributes(data);
}
}
return attrsel.toResultsString();
} catch (Exception e) {
e.printStackTrace();
return "Can't compute feature ranking";
}
}
public AttributeSelection featureRankingForRegression() {
try {
ConverterUtils.DataSource source = new ConverterUtils.DataSource(ARFFfile);
Instances data = source.getDataSet();
AttributeSelection attrsel = new AttributeSelection();
weka.attributeSelection.CfsSubsetEval eval = new weka.attributeSelection.CfsSubsetEval();
weka.attributeSelection.GreedyStepwise rank = new weka.attributeSelection.GreedyStepwise();
rank.setOptions(
weka.core.Utils.splitOptions(
"-R -T -1.7976931348623157E308 -N -1"));
attrsel.setEvaluator(eval);
attrsel.setSearch(rank);
attrsel.SelectAttributes(data);
return attrsel;
} catch (Exception e) {
e.printStackTrace();
return null;
}
}
public String featureRankingForRegression(String csvFile) {
try {
ConverterUtils.DataSource source = new ConverterUtils.DataSource(csvFile);
Instances data = source.getDataSet();
AttributeSelection attrsel = new AttributeSelection();
weka.attributeSelection.ReliefFAttributeEval eval = new weka.attributeSelection.ReliefFAttributeEval();
weka.attributeSelection.Ranker rank = new weka.attributeSelection.Ranker();
rank.setOptions(
weka.core.Utils.splitOptions(
"-T -1.7976931348623157E308 -N -1"));
attrsel.setEvaluator(eval);
attrsel.setSearch(rank);
attrsel.SelectAttributes(data);
return attrsel.toResultsString();
} catch (Exception e) {
e.printStackTrace();
return null;
}
}
public StringBuffer makePredictions(File testarff, String classifier) {
System.out.println("test file " + testarff.toString() + " on model " + classifier);
StringBuffer sb = new StringBuffer();
StringBuilder sbIds = new StringBuilder();
try {
// load data
ConverterUtils.DataSource source = new ConverterUtils.DataSource(testarff.toString());
Instances data = source.getDataSet();
if (data.classIndex() == -1) {
data.setClassIndex(data.numAttributes() - 1);
}
Classifier model = (Classifier) weka.core.SerializationHelper.read(classifier);
//get class names
HashMap<Integer, String> hmClassCorrespondance = new HashMap<>();
for (String s : model.toString().split("\n")) {
if (s.startsWith("@attribute class")) {
s = s.split("\\{")[1].replace("}", "");
String classes[] = s.split(",");
for (int i = 0; i < classes.length; i++) {
hmClassCorrespondance.put(i, classes[i]);
}
}
}
// label instances
sb.append("instance\tprediction\tprobability\n");
for (int i = 0; i < data.numInstances(); i++) {
double prediction = model.classifyInstance(data.instance(i));
double probability = model.distributionForInstance(data.instance(i))[(int) prediction];
sb.append(data.attribute(0).value(i)).
append("\t").append(hmClassCorrespondance.get((int) prediction)).
append("\t").append(probability).
append("\n");
}
} catch (Exception e) {
e.printStackTrace();
}
return sb;
}
/**
* test a classifier
*
* @param testarff
* @param classifier file
* @param classification
* @return
*/
public Object testClassifierFromFileSource(File testarff, String classifier, boolean classification) {
if (Main.debug) {
System.out.println("test file " + testarff.toString() + " on model " + classifier);
}
StringBuffer sb = new StringBuffer();
try {
// load data
ConverterUtils.DataSource source = new ConverterUtils.DataSource(testarff.toString());
Instances data = source.getDataSet();
if (data.classIndex() == -1) {
data.setClassIndex(data.numAttributes() - 1);
}
//load model
Classifier model = (Classifier) weka.core.SerializationHelper.read(classifier);
Evaluation eval = null;
PlainText pt = null;
//if vote model, we need to simulate features index
if (model.getClass().toString().endsWith("Vote")) {
HashMap<String, String> hmNewDataFeaturesIndex = new HashMap<>();
//get models inside vote
weka.classifiers.meta.Vote m = (weka.classifiers.meta.Vote) model;
Classifier[] modelClassifiers = m.getClassifiers();
//for each classifier, get features indexes of the signature
int classIndex = 0;
for (int i = 0; i < modelClassifiers.length; i++) {
//get model
FilteredClassifier filteredClassifier = (FilteredClassifier) modelClassifiers[i];
//get features of the model in the right order
ArrayList<String> alFeatures = getFeaturesFromClassifier(filteredClassifier);
//get remove filter
Remove rf = (Remove) filteredClassifier.getFilter();
//get index of the remove filter
String indexes[] = rf.getAttributeIndices().split(",");
//set class index
classIndex = Integer.valueOf(indexes[indexes.length - 1]);
//store features index with feature name
for (int j = 0; j < alFeatures.size(); j++) {
hmNewDataFeaturesIndex.put(indexes[j], alFeatures.get(j));
}
}
//create new dataset
Instances newData = new Instances(data);
for (int i = newData.numAttributes() - 2; i >= 0; i--) {
newData.deleteAttributeAt(i);
}
//insert data for each instance
for (int j = 0; j < data.numInstances(); j++) {
//insert attribute "instance" for 1st instance only
if (j == 0) {
newData.insertAttributeAt(data.attribute(0), 0);
}
//insert instance name
try {
newData.instance(j).setValue(0, data.instance(j).stringValue(0));
} catch (Exception e) {
//if not string
newData.instance(j).setValue(0, data.instance(j).value(0));
}
for (int i = 1; i < classIndex - 1; i++) {
//if attribute do not exist in current data
if (hmNewDataFeaturesIndex.containsKey((i + 1) + "")) {
Attribute att = data.attribute(hmNewDataFeaturesIndex.get((i + 1) + ""));
if (j == 0) {
newData.insertAttributeAt(att, i);
}
newData.instance(j).setValue(i, data.instance(j).value(att));
} else {
//create an empty one with missing data
if (j == 0) {
newData.insertAttributeAt(new Attribute("Att_" + i), i);
}
}
}
}
data = newData;
}
// CSVSaver csv = new CSVSaver();
// csv.setInstances(data);
// csv.setFile(new File("E:\\cloud\\Projects\\bacteria\\test\\test.csv"));
// csv.writeBatch();
// ArffSaver arff = new ArffSaver();
// arff.setInstances(data);
// arff.setFile(new File("E:\\cloud\\Projects\\bacteria\\test\\test.arff"));
// arff.writeBatch();
eval = new Evaluation(data);
pt = new PlainText();
pt.setHeader(data);
pt.setBuffer(sb);
pt.setOutputDistribution(true);
pt.setAttributes("first");
eval.evaluateModel(model, data, pt, new Range("first,last"), true);
//return
if (classification) {
return new ClassificationResultsObject(eval, sb, data, model);
} else {
return new RegressionResultsObject(eval, sb, data, model);
}
} catch (Exception e) {
if (Main.debug) {
e.printStackTrace();
}
}
return null;
}
/**
* test a classifier
*
* @param testarff
* @param classifier file
* @param classification
* @return
*/
public Object testClassifierFromModel(Classifier model, String testFile, boolean classification, String out) {
StringBuffer sb = new StringBuffer();
try {
// load data
ConverterUtils.DataSource testSource = new ConverterUtils.DataSource(testFile.replace("csv", "arff"));
Instances testData = testSource.getDataSet();
testData = extractFeaturesFromDatasetBasedOnModel(model, testData);
testData.setClassIndex(testData.numAttributes() - 1);
//remove ID
//testData.deleteAttributeAt(0);
// evaluate test dataset on the model
Evaluation eval = new Evaluation(testData);
PlainText pt = new PlainText();
pt.setHeader(testData);
pt.setBuffer(sb);
eval.evaluateModel(model, testData, pt, new Range("first,last"), true);
// //saving for debug
// SerializationHelper.write(Main.wd + "tmp." + out.replace("\t", "_") + ".model", model);
// ArffSaver arff = new ArffSaver();
// arff.setInstances(testData);
// arff.setFile(new File(Main.wd + "tmp." + out.replace("\t", "_") + ".testdata.arff"));
// arff.writeBatch();
//return
if (classification) {
return new ClassificationResultsObject(eval, sb, testData, model);
} else {
return new RegressionResultsObject(eval, sb, testData, model);
}
} catch (Exception e) {
if (Main.debug) {
System.err.println(e.getMessage());
e.printStackTrace();
}
}
return null;
}
// /**
// * test a classifier
// *
// * @param testarff
// * @param classifier file
// * @param classification
// * @return
// */
// public Object testClassifier(String classifier, String classifier_options, String features, String trainFile, String testFile, boolean classification) {
//
// StringBuffer sb = new StringBuffer();
// try {
// // load data
// ConverterUtils.DataSource trainSource = new ConverterUtils.DataSource(trainFile.replace("csv", "arff"));
// ConverterUtils.DataSource testSource = new ConverterUtils.DataSource(testFile.replace("csv", "arff"));
//
// Instances trainData = trainSource.getDataSet();
// trainData.setClassIndex(trainData.numAttributes() - 1);
// Instances testData = testSource.getDataSet();
// testData.setClassIndex(testData.numAttributes() - 1);
//
// //load model
// String configuration = classifier + " " + classifier_options;
// if (!classifier.contains("meta.Vote")) {
// //keep only attributesToUse. ID is still here
// String options = "";
// weka.filters.unsupervised.attribute.Remove r = new weka.filters.unsupervised.attribute.Remove();
// options = "-V -R " + features;
// r.setOptions(weka.core.Utils.splitOptions((options)));
// r.setInputFormat(trainData);
// trainData = Filter.useFilter(trainData, r);
// trainData.setClassIndex(trainData.numAttributes() - 1);
// testData = Filter.useFilter(testData, r);
// testData.setClassIndex(testData.numAttributes() - 1);
//
// //create weka configuration string
// //remove ID from classification
// String filterID = ""
// + "weka.classifiers.meta.FilteredClassifier "
// + "-F \"weka.filters.unsupervised.attribute.Remove -R 1\" "
// + "-W weka.classifiers.";
//
// //create config
// configuration = filterID + "" + classifier + " -- " + classifier_options;
//
// }
//
// //create model
// final String config[] = Utils.splitOptions(configuration);
// String classname = config[0];
// config[0] = "";
// Classifier model = null;
//
// model = (Classifier) Utils.forName(Classifier.class, classname, config);
//
// // evaluate dataset on the model
// Evaluation eval = new Evaluation(testData);
// PlainText pt = new PlainText();
// pt.setBuffer(sb);
//
// //1 fold cross validation
// eval.crossValidateModel(model, trainData, 10, new Random(1), pt, new Range("first,last"), true);
//
// eval.evaluateModel(model, testData);
// model.buildClassifier(trainData);
//
// //return
// if (classification) {
// return new ClassificationResultsObject(eval, sb, testData, model);
// } else {
// return new RegressionResultsObject(eval, sb, testData, model);
// }
//
// } catch (Exception e) {
// if (Main.debug) {
// e.printStackTrace();
// }
// }
// return null;
//
// }
public void setCSVFile(File file) {
CSVFile = file;
}
public void setARFFfile(String file) {
ARFFfile = file;
}
/**
* set local variable mydata with ARFFfile
*/
public void setDataFromArff() {
// load data
try {
ConverterUtils.DataSource source = new ConverterUtils.DataSource(ARFFfile);
myData = source.getDataSet();
} catch (Exception e) {
e.printStackTrace();
}
}
/**
* set local variable mydata with CSVFfile. VERY DANGEROUS WITHOUT LOADER
* THAT SET THE SEPARATOR Need to load as csvToArff()
*/
public void setDataFromCSV() {
// // load data
// try {
// //implement loader
// ConverterUtils.DataSource source = new ConverterUtils.DataSource(CSVFile.toString());
// myData = source.getDataSet();
// } catch (Exception e) {
// e.printStackTrace();
// }
}
/**
* TODO: find another way to detect classification vs regression. A user
* could use 1 and 0 instead of true false;
*
* @return
*/
public boolean isClassification() {
try {
Double.parseDouble(myData.instance(0).toString(myData.numAttributes() - 1));
//Double.parseDouble(myData.attribute(myData.numAttributes() - 1).enumerateValues().nextElement().toString());
return false;
} catch (Exception e) {
return true;
}
}
public ArrayList<String> getFeaturesFromClassifier(String classifier) {
//HashMap<String, String> hm = new HashMap<>();
ArrayList<String> al = new ArrayList<>();
try {
//load model
Classifier model = (Classifier) weka.core.SerializationHelper.read(classifier);
if (model.getClass().toString().contains("Vote")) {
String tab[] = model.toString().split("\n");
ArrayList<String> alModels = new ArrayList<>();
int i = 1;
while (!tab[i].startsWith("using")) {
alModels.add(tab[i].replaceAll("^\t", ""));
i++;
}
int j = 0;
for (i = i; i < tab.length; i++) {
if (tab[i].startsWith("Filtered Header")) {
al.add("Model: " + alModels.get(j));
j++;
i++;
while (i < tab.length && !tab[i].startsWith("Filtered Header")) {
if (tab[i].startsWith("@attribute")) {
al.add(tab[i].replace("@attribute ", "")
.replaceAll(" \\w+$", "")
.replaceAll(" \\{.*$", ""));
}
i++;
}
i--;
}
}
} else {
for (String s : model.toString().split("\n")) {
if (s.startsWith("@attribute")) {
if (s.split(" ")[1].equals("class")) {
al.add("class");
} else {
al.add(s.replace("@attribute ", "")
.replaceAll(" \\w+$", "")
.replaceAll(" \\{.*$", ""));
}
}
}
}
} catch (Exception e) {
if (Main.debug) {
e.printStackTrace();
}
}
return al;
}
/**
* get classes from classifier
*
* @param classifier
* @param toArrayList
* @return
*/
public ArrayList<String> getClassesFromClassifier(String classifier, boolean toArrayList) {
ArrayList<String> al = new ArrayList<>();
String classes = "";
try {
//load model
Classifier model = (Classifier) weka.core.SerializationHelper.read(classifier);
for (String s : model.toString().split("\n")) {
if (s.startsWith("@attribute class")) {
classes = s;
}
}
} catch (Exception e) {
e.printStackTrace();
}
if (toArrayList) {
classes = classes.replace("@attribute class ", "").replace("{", "").replace("}", "");
for (String s : classes.split(",")) {
al.add(s);
}
return al;
} else {
al.add(classes);
return al;
}
}
public ArrayList<String> getFeaturesFromClassifier(Classifier model) {
//HashMap<String, String> hm = new HashMap<>();
ArrayList<String> al = new ArrayList<>();
try {
//load model
for (String s : model.toString().split("\n")) {
if (s.startsWith("@attribute '") && s.endsWith("numeric")) {
s = s.replace("@attribute '", "").trim();
s = s.substring(0, s.lastIndexOf("'"));
al.add(s);
} else if (s.startsWith("@attribute '")) {
s = s.replace("@attribute '", "").trim();
s = s.substring(0, s.lastIndexOf("' {"));
al.add(s);
} else if (s.startsWith("@attribute")) {
//hm.put(s.split(" ")[1], s.split(" ")[2]); //feature_name, feature_type
s = s.replace("@attribute ", "").trim();
try {
s = s.substring(0, s.lastIndexOf(" {"));
} catch (Exception e) {
s = s.substring(0, s.lastIndexOf(" "));
}
al.add(s);
}
}
} catch (Exception e) {
e.printStackTrace();
}
return al;
}
public HashMap<String, String> getFullFeaturesFromClassifier(String modelFile) {
//HashMap<String, String> hm = new HashMap<>();
HashMap<String, String> hm = new HashMap<>();
try {
Classifier model = (Classifier) weka.core.SerializationHelper.read(modelFile);
//load model
for (String s : model.toString().split("\n")) {
if (s.startsWith("@attribute '")) {
s = s.replace("@attribute '", "").trim();
String attributeName = s.substring(0, s.lastIndexOf("' {")).trim();
String attributeType = s.substring(s.indexOf("' {")).trim();
hm.put("'" + attributeName + "'", attributeType);
} else if (s.startsWith("@attribute")) {
//hm.put(s.split(" ")[1], s.split(" ")[2]); //feature_name, feature_type
s = s.replace("@attribute ", "").trim();
try {
String attributeName = s.substring(0, s.lastIndexOf(" {")).trim();
String attributeType = s.substring(s.indexOf(" {")).trim();
hm.put(attributeName, attributeType);
} catch (Exception e) {
String attributeName = s.substring(0, s.lastIndexOf(" ")).trim();
String attributeType = s.substring(s.indexOf(" ")).trim();
hm.put(attributeName, attributeType);
}
}
}
} catch (Exception e) {
e.printStackTrace();
}
return hm;
}
public void saveFilteredDataToArff(String attributesToUse, boolean classification, String outfile) {
try {
// load data
Instances data = myData;
//set class index
if (data.classIndex() == -1) {
data.setClassIndex(data.numAttributes() - 1);
}
//keep only attributesToUse. ID is still here
String options = "";
weka.filters.unsupervised.attribute.Remove r = new weka.filters.unsupervised.attribute.Remove();
options = "-V -R " + attributesToUse;
r.setOptions(weka.core.Utils.splitOptions((options)));
r.setInputFormat(data);
data = Filter.useFilter(data, r);
data.setClassIndex(data.numAttributes() - 1);
//save csv to arff
ArffSaver arff = new ArffSaver();
arff.setInstances(data);
arff.setFile(new File(outfile));
arff.writeBatch();
} catch (Exception e) {
e.printStackTrace();
}
}
public void saveFilteredDataToCSV(String attributesToUse, boolean classification, String outfile) {
try {
// load data
Instances data = myData;
//set class index
if (data.classIndex() == -1) {
data.setClassIndex(data.numAttributes() - 1);
}
//keep only attributesToUse. ID is still here
String options = "";
weka.filters.unsupervised.attribute.Remove r = new weka.filters.unsupervised.attribute.Remove();
options = "-V -R " + attributesToUse;
r.setOptions(weka.core.Utils.splitOptions((options)));
r.setInputFormat(data);
data = Filter.useFilter(data, r);
data.setClassIndex(data.numAttributes() - 1);
//save csv to arff
CSVSaver csv = new CSVSaver();
csv.setInstances(data);
csv.setFile(new File(outfile));
csv.writeBatch();
} catch (Exception e) {
e.printStackTrace();
}
}
/**
* perform sampling on the dataset
*
* @param fullDatasetFile
* @param dataToTrainFile
* @param dataToTestFile
*/
public void sampling(String fullDatasetFile, String dataToTrainFile, String dataToTestFile, boolean classification, String range) {
System.out.println("# sampling statistics");
boolean useRange = !range.isEmpty();
setCSVFile(new File(fullDatasetFile));
csvToArff(classification);
setDataFromArff();
// load data
Instances data = myData;
//set class index
if (data.classIndex() == -1) {
data.setClassIndex(data.numAttributes() - 1);
}
//calculate seed
Random rand = new Random();
int seed = 1 + rand.nextInt(1000);
try {
Instances TestData = null;
Instances TrainData = null;
//create train set
if (useRange) {
weka.filters.unsupervised.instance.RemoveRange rr = new RemoveRange();
String options = "-V -R " + range;
rr.setOptions(weka.core.Utils.splitOptions((options)));
rr.setInputFormat(data);
TrainData = Filter.useFilter(data, rr);
} else {
weka.filters.supervised.instance.StratifiedRemoveFolds srf = new StratifiedRemoveFolds();
String options = "-S " + seed + " -V -N " + Main.samplingFold + " -F 1"; //split dataset in 3 and inverse the selection
srf.setOptions(weka.core.Utils.splitOptions((options)));
srf.setInputFormat(data);
TrainData = Filter.useFilter(data, srf);
}
System.out.println("Total number of instances in train data file: " + TrainData.numInstances());
if (classification) {
Enumeration e = TrainData.attribute(data.classIndex()).enumerateValues();
int cpt = 0;
while (e.hasMoreElements()) {
String className = (String) e.nextElement();
double n = TrainData.attributeStats(data.classIndex()).nominalWeights[cpt];
System.out.println("class " + className + ": " + n + " (" + df.format(n / (double) TrainData.numInstances() * 100) + "%)");
cpt++;
}
}
//create test set
if (useRange) {
weka.filters.unsupervised.instance.RemoveRange rr = new RemoveRange();
String options = "-R " + range;
rr.setOptions(weka.core.Utils.splitOptions((options)));
rr.setInputFormat(data);
TestData = Filter.useFilter(data, rr);
} else {
weka.filters.supervised.instance.StratifiedRemoveFolds srf = new StratifiedRemoveFolds();
String options = "-S " + seed + " -N " + Main.samplingFold + " -F 1"; //split dataset in 3
srf.setOptions(weka.core.Utils.splitOptions((options)));
srf.setInputFormat(data);
TestData = Filter.useFilter(data, srf);
}
System.out.println("Total number of instances in test data file: " + TestData.numInstances());
if (classification) {
Enumeration e = TestData.attribute(data.classIndex()).enumerateValues();
int cpt = 0;
while (e.hasMoreElements()) {
String className = (String) e.nextElement();
double n = TestData.attributeStats(data.classIndex()).nominalWeights[cpt];
System.out.println("class " + className + ": " + n + " (" + df.format(n / (double) TestData.numInstances() * 100) + "%)");
cpt++;
}
}
//convert strings to nominal
TestData = convertStringsToNominal(TestData);
TrainData = convertStringsToNominal(TrainData);
//save to csv and arff
if (!useRange) {
///test set to csv
CSVSaver csv = new CSVSaver();
csv.setInstances(TestData);
csv.setFile(new File(dataToTestFile));
csv.writeBatch();
///train set to csv
csv = new CSVSaver();
csv.setInstances(TrainData);
csv.setFile(new File(dataToTrainFile));
csv.writeBatch();
}
//test set to arff
ArffSaver arff = new ArffSaver();
arff.setInstances(TestData);
arff.setFile(new File(dataToTestFile.replace("csv", "arff")));
arff.writeBatch();
//train set to arff
arff = new ArffSaver();
arff.setInstances(TrainData);
arff.setFile(new File(dataToTrainFile.replace("csv", "arff")));
arff.writeBatch();
} catch (Exception e) {
e.printStackTrace();
}
}
/**
* in case of combined models using filters for selecting specific filters
* based on indexes
*
* @param featureSelectedData
* @param arffFile
* @param outfile
*/
public void extractFeaturesFromArffFileBasedOnSelectedFeatures(Instances featureSelectedData, Instances arffFile,
String outfile) {
//get attribute names of feature selection data
ArrayList<String> aFeaturesSelected = new ArrayList<>();
Enumeration<Attribute> enumerate = featureSelectedData.enumerateAttributes();
while (enumerate.hasMoreElements()) {
Attribute a = enumerate.nextElement();
aFeaturesSelected.add(a.name());
}
HashMap<String, String> hmModelFeatures = new HashMap<>();//indexed hashed features
for (String f : aFeaturesSelected) {
hmModelFeatures.put(f, f);
}
Instances data = arffFile;
//get index of model features in the arff file
String indexesToKeep = "";
Enumeration e = data.enumerateAttributes();
while (e.hasMoreElements()) {
Attribute a = (Attribute) e.nextElement();
if (hmModelFeatures.containsKey(a.name())) {
indexesToKeep += (a.index() + 1) + ",";
}
}
indexesToKeep += myData.numAttributes();
try {
//apply remove filter
String[] opts = new String[3];
opts[0] = "-V";
opts[1] = "-R";
opts[2] = indexesToKeep;
weka.filters.unsupervised.attribute.Remove r = new weka.filters.unsupervised.attribute.Remove();
r.setOptions(opts);
r.setInputFormat(data);
data = Filter.useFilter(data, r);
//reorder
String order = ",";
for (String featureOrdered : aFeaturesSelected) {
int featureNotOrdered = data.attribute(featureOrdered).index() + 1;
order += featureNotOrdered + ",";
}
order += data.attribute("class").index() + 1;
Reorder reorder = new Reorder();
reorder.setAttributeIndices(order);
reorder.setInputFormat(data);
data = Filter.useFilter(data, reorder);
//save
ArffSaver arff = new ArffSaver();
arff.setInstances(data);
arff.setFile(new File(outfile));
arff.writeBatch();
} catch (Exception ex) {
ex.printStackTrace();
}
}
/**
* extract features from test file while keeping arff compatibility
*
* @param modelFile
* @param arffTestFile
* @param outfile
*/
public void extractFeaturesFromTestFileBasedOnModel(String modelFile, Instances arffTestFile, String outfile) {
Instances data = arffTestFile;
//get model features
ArrayList<String> alModelFeatures = getFeaturesFromClassifier(modelFile); //features in the right order
HashMap<String, String> hmModelFeatures = new HashMap<>();//indexed hashed features
for (String f : alModelFeatures) {
hmModelFeatures.put(f, f);
}
//get index of model features in the arff test file
String indexesToKeep = "1"; //keep ID
Enumeration e = myData.enumerateAttributes();
while (e.hasMoreElements()) {
Attribute a = (Attribute) e.nextElement();
if (hmModelFeatures.containsKey(a.name())) {
indexesToKeep += "," + (a.index() + 1);
}
}
if (!hmModelFeatures.containsKey("class")) {
indexesToKeep += "," + myData.numAttributes();
}
try {
//apply remove filter
String[] opts = new String[3];
opts[0] = "-V";
opts[1] = "-R";
opts[2] = indexesToKeep;
weka.filters.unsupervised.attribute.Remove r = new weka.filters.unsupervised.attribute.Remove();
r.setOptions(opts);
r.setInputFormat(data);
data = Filter.useFilter(data, r);
//reorder
String order = "1,";
for (String featureOrdered : alModelFeatures) {
int featureNotOrdered = data.attribute(featureOrdered).index() + 1;
order += featureNotOrdered + ",";
}
Reorder reorder = new Reorder();
reorder.setAttributeIndices(order);
reorder.setInputFormat(data);
data = Filter.useFilter(data, reorder);
//save to arff
ArffSaver arff = new ArffSaver();
arff.setInstances(data);
arff.setFile(new File(outfile + ".tmp"));
arff.writeBatch();
//change non-numeric attributes to ensure compatibility
String classString = "";
Classifier model = (Classifier) weka.core.SerializationHelper.read(modelFile);
HashMap<String, String> hm = new HashMap<>();
for (String s : model.toString().split("\n")) {
if (s.startsWith("@attribute") && !s.endsWith(" numeric")) {
String attribute = s.replace("@attribute ", "");
attribute = attribute.substring(0, attribute.indexOf("{"));
hm.put(attribute, s);
}
}
try {
BufferedReader br = new BufferedReader(new FileReader(outfile + ".tmp"));
PrintWriter pw = new PrintWriter(new FileWriter(outfile));
while (br.ready()) {
String line = br.readLine();
if (line.startsWith("@attribute") && !line.endsWith(" numeric")
&& !line.contains("@attribute Instance")
&& !line.contains("@attribute " + Main.mergingID)) {
String attribute = line.replace("@attribute ", "").trim();
attribute = attribute.substring(0, attribute.indexOf("{"));
if (hm.get(attribute) != null) {
pw.println(hm.get(attribute));
} else {
pw.println(line);
}
} else {
pw.println(line);
}
}
pw.close();
br.close();
new File(outfile + ".tmp").delete();
} catch (Exception ex) {
ex.printStackTrace();
}
} catch (Exception ex) {
ex.printStackTrace();
}
}
/**
* extract features from dataset
*
* @param modelFile
* @param dataset
* @param outfile
*/
public Instances extractFeaturesFromDatasetBasedOnModel(Classifier model, Instances dataset) {
//get model features
ArrayList<String> alModelFeatures = getFeaturesFromClassifier(model); //features in the right order
HashMap<String, String> hmModelFeatures = new HashMap<>();//indexed hashed features
for (String f : alModelFeatures) {
hmModelFeatures.put(f, f);
}
//get index of model features in the arff test file
String indexesToKeep = "1"; //keep ID
Enumeration e = dataset.enumerateAttributes();
while (e.hasMoreElements()) {
Attribute a = (Attribute) e.nextElement();
if (hmModelFeatures.containsKey(a.name())) {
indexesToKeep += "," + (a.index() + 1);
}
}
if (!hmModelFeatures.containsKey("class")) {
indexesToKeep += "," + dataset.numAttributes();
}
Instances filteredDataset = null;
try {
//apply remove filter
String[] opts = new String[3];
opts[0] = "-V";
opts[1] = "-R";
opts[2] = indexesToKeep;
weka.filters.unsupervised.attribute.Remove r = new weka.filters.unsupervised.attribute.Remove();
r.setOptions(opts);
r.setInputFormat(dataset);
filteredDataset = Filter.useFilter(dataset, r);
//reorder
String order = "1,";
for (String featureOrdered : alModelFeatures) {
int featureNotOrdered = filteredDataset.attribute(featureOrdered).index() + 1;
order += featureNotOrdered + ",";
}
Reorder reorder = new Reorder();
reorder.setAttributeIndices(order);
reorder.setInputFormat(filteredDataset);
filteredDataset = Filter.useFilter(filteredDataset, reorder);
} catch (Exception ex) {
ex.printStackTrace();
}
return filteredDataset;
}
/**
* ensure compatibility of non-numeric attributes between 2 arff files
*
* @param ArffFileWithReferenceHeader contains the header you want
* @param ArffFileToModify file fo modify header
*/
public void makeCompatibleARFFheaders(String ArffFileWithReferenceHeader, String ArffFileToModify) {
HashMap<String, String> hm = new HashMap<>();
try {
// read source. Get non-numeric attributes
BufferedReader br = new BufferedReader(new FileReader(ArffFileWithReferenceHeader));
String line = "";
while (br.ready()) {
line = br.readLine();
if (line.startsWith("@attribute") && !line.trim().endsWith("numeric")) {
String attribute = line.replace("@attribute ", "");
attribute = attribute.substring(0, attribute.indexOf("{"));
hm.put(attribute, line);
}
}
br.close();
//read destination
br = new BufferedReader(new FileReader(ArffFileToModify));
PrintWriter pw = new PrintWriter(new FileWriter(ArffFileToModify + "_tmp"));
while (br.ready()) {
line = br.readLine();
if (line.startsWith("@attribute") && !line.endsWith(" numeric")) {
String attribute = line.replace("@attribute ", "").trim();
attribute = attribute.substring(0, attribute.indexOf("{"));
pw.println(hm.get(attribute));
} else {
pw.println(line);
}
}
pw.close();
br.close();
new File(ArffFileToModify).delete();
new File(ArffFileToModify + "_tmp").renameTo(new File(ArffFileToModify));
} catch (Exception ex) {
ex.printStackTrace();
}
}
public Instances convertStringsToNominal(Instances data) {
try {
Enumeration en = data.enumerateAttributes();
while (en.hasMoreElements()) {
Attribute a = (Attribute) en.nextElement();
if (a.isString()) {
StringToNominal stn = new StringToNominal();
stn.setOptions(new String[]{"-R", (a.index() + 1) + ""});
stn.setInputFormat(data);
data = Filter.useFilter(data, stn);
}
}
} catch (Exception e) {
e.printStackTrace();
}
return data;
}
public static class RegressionResultsObject {
public String CC;//correlation coefficient
public String MAE;//Mean absolute error
public String RMSE;//Root mean squared error
public String RAE;//Relative absolute error
public String RRSE;//Root relative squared error
public StringBuilder predictions = new StringBuilder();
public StringBuilder features = new StringBuilder();
public int numberOfFeatures = 0; //number of features
public Classifier model;
public String featuresRankingResults;
public Evaluation evaluation;
public Instances dataset;
public RegressionResultsObject() {
}
private RegressionResultsObject(Evaluation eval, StringBuffer sb, Instances data, Classifier classifier) {
dataset = data;
evaluation = eval;
if (!sb.toString().isEmpty()) {
try {
parsePredictions(sb);
} catch (Exception e) {
}
}
//getFeatures(data);
getFeatures(classifier);
model = classifier;
//measures
try {
CC = Utils.doubleToString(eval.correlationCoefficient(), 12, 4).trim();
MAE = Utils.doubleToString(eval.meanAbsoluteError(), 12, 4).trim();
RMSE = Utils.doubleToString(eval.rootMeanSquaredError(), 12, 4).trim();
RAE = Utils.doubleToString(eval.relativeAbsoluteError() / 100, 12, 4).trim();
RRSE = Utils.doubleToString(eval.rootRelativeSquaredError() / 100, 12, 4).trim();
} catch (Exception exception) {
exception.printStackTrace();
}
}
@Override
public String toString() {
return (CC + "\t"
+ MAE + "\t"
+ RMSE + "\t"
+ RAE + "\t"
+ RRSE);
}
public String toStringDetails() {
return ("[score_training] CC: " + CC + "\n"
+ "[score_training] MAE: " + MAE + "\n"
+ "[score_training] RMSE: " + RMSE + "\n"
+ "[score_training] RAE: " + RAE + "\n"
+ "[score_training] RRSE: " + RRSE);
}
private void parsePredictions(StringBuffer sb) {
String lines[] = sb.toString().split("\n");
for (int i = 1; i < lines.length; i++) {
String p = lines[i].replaceAll(" +", "\t");
String tab[] = p.split("\t");
String inst = tab[1];
String actual = tab[2];
String predicted = tab[3];
String error = tab[4];
String ID = tab[5];
predictions.append(ID.replace("(", "").replace(")", "") + "\t" + actual + "\t" + predicted + "\t" + error + "\n");
}
}
private void getFeatures(Instances data) {
for (int i = 1; i < data.numAttributes() - 1; i++) {
numberOfFeatures++;
features.append(data.attribute(i).name() + "\n");
}
}
private void getFeatures(Classifier model) {
boolean voteClassifier = model.toString().split("\n")[0].contains("Vote");
TreeMap<String, Integer> tm = new TreeMap<>();
for (String s : model.toString().split("\n")) {
if (s.startsWith("@attribute")) {
//hm.put(s.split(" ")[1], s.split(" ")[2]); //feature_name, feature_type
String name = s.split(" ")[1];
if (tm.containsKey(name)) {
int i = tm.get(name);
i++;
tm.put(name, i);
} else {
tm.put(s.split(" ")[1], 1);
}
}
}
for (String f : tm.keySet()) {
if (!f.equals("class")) {
numberOfFeatures++;
if (voteClassifier) {
features.append(f + "\t" + tm.get(f) + "\n");
} else {
features.append(f + "\n");
}
}
}
}
public String getFeatureRankingResults() {
String[] s = featuresRankingResults.split("\n");
StringBuilder toreturn = new StringBuilder();
int index = 0;
while (!s[index].equals("Ranked attributes:")) {
index++;
}
index++;
while (!s[index].startsWith("Selected")) {
if (!s[index].contains(Main.mergingID) && !s[index].isEmpty()) {
String out = s[index].substring(1); //remove first space
out = out.replaceAll(" +", "\t");
out = out.split("\t")[0] + "\t" + out.split("\t")[2];
toreturn.append(out + "\n");
}
index++;
}
return toreturn.toString();
}
}
public static class ClassificationResultsObject {
public String ACC;
public String MCC;
public String TNR;//specificity
public String AUC;
public String TPR;//sensitivity
public String Fscore;
public double TP;
public double FN;
public double TN;
public double FP;
public String kappa;
public String MAE;
public StringBuilder predictions = new StringBuilder();
public StringBuilder features = new StringBuilder();
public int numberOfFeatures = 0; //number of features
public Classifier model;
public String featuresRankingResults;
public Evaluation evaluation;
public Instances dataset;
public String FPR;
public String recall;
public String FNR;
public String AUPRC;
public String precision;
public String FDR;
public String matrix;
public String pAUC;
public String BCR;
public String BER;
public String classes = "";
private ClassificationResultsObject() {
}
public void getPredictions() {
System.out.println("instance\tactual\tpredicted\terror\tprobability");
System.out.println(predictions);
}
private ClassificationResultsObject(Evaluation eval, StringBuffer sb, Instances data, Classifier classifier) {
dataset = data;
evaluation = eval;
model = classifier;
getFeaturesAndClasses(classifier);
if (!sb.toString().isEmpty()) {
try {
parsePredictions(sb);
} catch (Exception e) {
try {
parsePredictions2(sb);
} catch (Exception e2) {
if (Main.debug) {
e.printStackTrace();
e2.printStackTrace();
}
}
}
}
//measures
ACC = Utils.doubleToString(eval.pctCorrect() / 100, 12, 4).trim();
MCC = Utils.doubleToString(eval.weightedMatthewsCorrelation(), 12, 4).trim();
TPR = Utils.doubleToString(eval.weightedTruePositiveRate(), 12, 4).trim();//sensitivity
TNR = Utils.doubleToString(eval.weightedTrueNegativeRate(), 12, 4).trim();//specificity
FPR = Utils.doubleToString(eval.weightedFalsePositiveRate(), 12, 4).trim();
FNR = Utils.doubleToString(eval.weightedFalseNegativeRate(), 12, 4).trim();
AUC = Utils.doubleToString(eval.weightedAreaUnderROC(), 12, 4).trim();
AUPRC = Utils.doubleToString(eval.weightedAreaUnderPRC(), 12, 4).trim();
Fscore = Utils.doubleToString(eval.weightedFMeasure(), 12, 4).trim();
kappa = Utils.doubleToString(eval.kappa(), 12, 4).trim();
MAE = Utils.doubleToString(eval.meanAbsoluteError(), 12, 4).trim();
precision = Utils.doubleToString(eval.weightedPrecision(), 12, 4).trim();
FDR = Utils.doubleToString(1 - eval.weightedPrecision(), 12, 4).trim();
recall = Utils.doubleToString(eval.weightedRecall(), 12, 4).trim();
double[][] m = eval.confusionMatrix();
matrix = "";
for (int i = 0; i < m.length; i++) {
matrix += "[";
for (int j = 0; j < m.length; j++) {
matrix += (int) m[i][j] + " ";
}
matrix = matrix.substring(0, matrix.length() - 1) + "] ";
}
try {
//pAUC
int numberOfClasses = m.length;
double pAUCs = 0;
for (int i = 0; i < numberOfClasses; i++) {
Instances rocCurve = new ThresholdCurve().getCurve(eval.predictions(), i);
pAUCs += getPartialROCArea(rocCurve, Main.pAUC_lower, Main.pAUC_upper);
}
pAUC = Utils.doubleToString((pAUCs / (double) numberOfClasses), 12, 4).trim();
} catch (Exception e) {
pAUC = "NaN";
}
//balanced classification rate
BCR = Utils.doubleToString(
((eval.weightedTruePositiveRate() + eval.weightedTrueNegativeRate()) / m.length), 12, 4).trim();
//BER (Balanced error rate)
//https://arxiv.org/pdf/1207.3809.pdf
BER = Utils.doubleToString(0.5 * ((eval.weightedFalsePositiveRate()) + (eval.weightedFalseNegativeRate())), 12, 4).trim();
// BER = Utils.doubleToString(
// (1 - (eval.weightedTruePositiveRate() + eval.weightedTrueNegativeRate()) / m.length), 12, 4).trim();
}
@Override
public String toString() {
return (ACC + "\t"
+ AUC + "\t"
+ AUPRC + "\t"
+ TPR + "\t" //SEN == recall
+ TNR + "\t" //SPE
+ MCC + "\t"
+ MAE + "\t"
+ BER + "\t"
+ FPR + "\t"
+ FNR + "\t"
+ precision + "\t" //= PPV (positive predictive value)
+ FDR + "\t"
+ Fscore + "\t"
+ kappa + "\t"
+ matrix);
}
public String toStringShort() {
return (ACC + "\t"
+ AUC + "\t"
+ AUPRC + "\t"
+ TPR + "\t" //SEN == recall
+ TNR + "\t" //SPE
+ MCC + "\t"
+ MAE + "\t"
+ BER);
}
public String toStringDetails() {
return ("[score_training] ACC: " + ACC + "\n"
+ "[score_training] AUC: " + AUC + "\n"
+ "[score_training] AUPRC: " + AUPRC + "\n"
+ "[score_training] TPR: " + TPR + "\n"
+ "[score_training] TNR: " + TNR + "\n"
+ "[score_training] MCC: " + MCC + "\n"
+ "[score_training] MAE: " + MAE + "\n"
+ "[score_training] BER: " + BER + "\n"
+ "[score_training] FPR: " + FPR + "\n"
+ "[score_training] FNR: " + FNR + "\n"
+ "[score_training] PPV: " + precision + "\n"
+ "[score_training] FDR: " + FDR + "\n"
+ "[score_training] recall: " + recall + "\n"
+ "[score_training] Fscore: " + Fscore + "\n"
+ "[score_training] kappa: " + kappa + "\n"
+ "[score_training] matrix: " + matrix // + "pAUC[" + Main.pAUC_lower + "-" + Main.pAUC_upper + pAUC + "]: \n"
);
}
public String toStringDetailsTesting() {
return ("[score_testing] ACC: " + ACC + "\n"
+ "[score_testing] AUC: " + AUC + "\n"
+ "[score_testing] AUPRC: " + AUPRC + "\n"
+ "[score_testing] TPR: " + TPR + "\n"
+ "[score_testing] TNR: " + TNR + "\n"
+ "[score_testing] MCC: " + MCC + "\n"
+ "[score_testing] MAE: " + MAE + "\n"
+ "[score_testing] BER: " + BER + "\n"
+ "[score_testing] FPR: " + FPR + "\n"
+ "[score_testing] FNR: " + FNR + "\n"
+ "[score_testing] PPV: " + precision + "\n"
+ "[score_testing] FDR: " + FDR + "\n"
+ "[score_testing] recall: " + recall + "\n"
+ "[score_testing] Fscore: " + Fscore + "\n"
+ "[score_testing] kappa: " + kappa + "\n"
+ "[score_testing] matrix: " + matrix // + "pAUC[" + Main.pAUC_lower + "-" + Main.pAUC_upper + pAUC + "]: \n"
);
}
/**
* first attempt to retrieve predictions
*
* @param sb
*/
private void parsePredictions(StringBuffer sb) {
String lines[] = sb.toString().split("\n");
//get predictions
for (int i = 0; i < lines.length; i++) {
String p = lines[i].replaceAll(" +", "\t");
if (!p.trim().startsWith("inst#")) {
String tab[] = p.split("\t");
String inst = tab[1];
try {
String actual = tab[2].split(":")[1];
String predicted = "";
try {
predicted = tab[3].split(":")[1];
} catch (Exception e) {
predicted = tab[3];
}
String error;
String prob;
String ID;
if (tab.length == 6) {
if (actual.contains("?")) {
error = "?";
} else {
error = "No";
}
prob = tab[4];
ID = tab[5];
if (prob.contains("?")) {
error = "?";
}
} else {
error = "Yes";
prob = tab[5];
ID = tab[6];
}
predictions.append(ID.replace("(", "").replace(")", "") + "\t" + actual + "\t" + predicted + "\t" + error + "\t" + prob.replace(",", "\t") + "\n");
} catch (Exception e) {
//e.printStackTrace();
}
}
}
}
// in case of different format
private void parsePredictions2(StringBuffer sb) {
String lines[] = sb.toString().split("\n");
//skip header
int a = 0;
if (lines[0].trim().startsWith("inst#")) {
a = 1;
}
//get predictions
for (int i = a; i < lines.length; i++) {
String p = lines[i].trim().replaceAll(" +", "\t");
String tab[] = p.split("\t");
try {
String inst = tab[0];
String actual = tab[1].split(":")[1];
String predicted = tab[2].split(":")[1];
String error;
String prob;
String ID;
if (tab.length == 4) {
if (actual.contains("?")) {
error = "?";
} else {
error = "No";
}
prob = tab[3];
try {
ID = tab[4];
} catch (Exception e) {
ID = "";
}
} else {
error = "Yes";
prob = tab[4];
try {
ID = tab[5];
} catch (Exception e) {
ID = "";
}
}
predictions.append(ID.replace("(", "").replace(")", "") + "\t"
+ actual + "\t" + predicted + "\t" + error + "\t" + prob.replace(",", "\t") + "\n");
} catch (Exception e) {
//e.printStackTrace();
}
}
}
private void getFeaturesAndClasses(Classifier model) {
boolean voteClassifier = model.toString().split("\n")[0].contains("Vote");
TreeMap<String, Integer> tm = new TreeMap<>();
for (String s : model.toString().split("\n")) {
if (s.startsWith("@attribute")) {
//hm.put(s.split(" ")[1], s.split(" ")[2]); //feature_name, feature_type
String name = s.split(" ")[1];
if (tm.containsKey(name)) {
int i = tm.get(name);
i++;
tm.put(name, i);
} else {
tm.put(s.split(" ")[1], 1);
}
}
if (s.startsWith("@attribute class ")) {
classes = s.replace("@attribute class {", "").replace("}", "").replace(",", "\t");
}
}
for (String f : tm.keySet()) {
if (!f.equals("class")) {
numberOfFeatures++;
if (voteClassifier) {
features.append(f + "\t" + tm.get(f) + "\n");
} else {
features.append(f + "\n");
}
}
}
}
public String getFeatureRankingResults() {
String[] s = featuresRankingResults.split("\n");
StringBuilder toreturn = new StringBuilder();
int index = 0;
while (!s[index].equals("Ranked attributes:")) {
index++;
}
index++;
while (!s[index].startsWith("Selected")) {
if (!s[index].contains(Main.mergingID) && !s[index].isEmpty()) {
//String out = s[index].substring(1); //remove first space
String out = s[index].trim(); //remove first space
out = out.replaceAll(" +", "\t");
String output = out.split("\t")[0] + "\t";
for (int i = 2; i < out.split("\t").length; i++) {
output += out.split("\t")[i] + " ";
}
toreturn.append(output + "\n");
}
index++;
}
return toreturn.toString();
}
}
/**
* compute partial AUC using trapezoid function
*/
public static double getPartialROCArea(Instances tcurve, double limit_inf, double limit_sup) {
int tpr = tcurve.attribute("True Positive Rate").index();
int fpr = tcurve.attribute("False Positive Rate").index();
double[] tprVals = tcurve.attributeToDoubleArray(tpr);
double[] fprVals = tcurve.attributeToDoubleArray(fpr);
//retrive coordinates
//transform list type in arraylist to revert them after
ArrayList<Double> alx = new ArrayList<>();
ArrayList<Double> aly = new ArrayList<>();
for (int i = 0; i < fprVals.length; i++) {
alx.add(fprVals[i]);
aly.add(tprVals[i]);
}
Collections.reverse(alx);
Collections.reverse(aly);
//create function points
double[] x = new double[alx.size()];
double[] y = new double[aly.size()];
for (int i = 0; i < alx.size(); i++) {
x[i] = alx.get(i);
y[i] = aly.get(i);
}
//correct to avoid multiple x values
try {
for (int i = 1; i < x.length; i++) {
if (x[i] <= x[i - 1]) {
x[i] = x[i - 1] + 0.0001;
int j = i + 1;
while (x[j] == x[i - 1] && j < x.length - 1) {
x[j] += 0.0001;
j++;
}
}
}
} catch (Exception e) {
//e.printStackTrace();
}
//show curve
// System.out.println("x\ty");
// for (int i = 0; i < y.length; i++) {
// System.out.println(x[i]+"\t"+y[i]);
//
// }
//https://stackoverflow.com/questions/16896961/how-to-compute-integration-of-a-function-in-apache-commons-math3-library
//estimate function
UnivariateInterpolator interpolator = new SplineInterpolator();
UnivariateFunction function = interpolator.interpolate(x, y);
//compute integrale
TrapezoidIntegrator trapezoid = new TrapezoidIntegrator();
double pauc = 0;
try {
pauc = trapezoid.integrate(10000000, function, limit_inf, 1);
//because of the spline curve, some area can be counter as 0
if (pauc < 0) {
return 0;
}
} catch (MaxCountExceededException maxCountExceededException) {
//pauc = trapezoid.integrate(10000000, function, limit_inf, limit_sup);
}
return pauc;
}
/**
* this object is compatible with bootstrap and repeated holdout method to
* store repeated iterations performances in arrays
*/
public static class evaluationPerformancesResultsObject {
public ArrayList<Double> alAUCs = new ArrayList<>();
public ArrayList<Double> alpAUCs = new ArrayList<>();
public ArrayList<Double> alAUPRCs = new ArrayList<>();
public ArrayList<Double> alACCs = new ArrayList<>();
public ArrayList<Double> alSEs = new ArrayList<>();
public ArrayList<Double> alSPs = new ArrayList<>();
public ArrayList<Double> alMCCs = new ArrayList<>();
public ArrayList<Double> alBERs = new ArrayList<>();
public String meanACCs;
public String meanAUCs;
public String meanAUPRCs;
public String meanSENs;
public String meanSPEs;
public String meanMCCs;
public String meanBERs;
public ArrayList<Double> alCCs = new ArrayList<>();
public ArrayList<Double> alRMSEs = new ArrayList<>();
public ArrayList<Double> alMAEs = new ArrayList<>();
public ArrayList<Double> alRAEs = new ArrayList<>();
public ArrayList<Double> alRRSEs = new ArrayList<>();
public String meanCCs;
public String meanMAEs;
public String meanRMSEs;
public String meanRAEs;
public String meanRRSEs;
public evaluationPerformancesResultsObject() {
}
public String toStringClassification() {
if (meanACCs == null) {
return "\t\t\t\t\t\t\t";
} else {
return meanACCs + "\t"
+ meanAUCs + "\t"
+ meanAUPRCs + "\t"
+ meanSENs + "\t"
+ meanSPEs + "\t"
+ meanMCCs + "\t"
+ meanMAEs + "\t"
+ meanBERs;
}
}
public String toStringClassificationDetails() {
return "[score_training] Average weighted ACC: " + meanACCs
+ "\t(" + utils.getStandardDeviation(alACCs) + ")\n"
+ "[score_training] Average weighted AUC: " + meanAUCs
+ "\t(" + utils.getStandardDeviation(alAUCs) + ")\n"
+ "[score_training] Average weighted AUPRC: " + meanAUPRCs
+ "\t(" + utils.getStandardDeviation(alAUPRCs) + ")\n"
+ "[score_training] Average weighted SEN: " + meanSENs
+ "\t(" + utils.getStandardDeviation(alSEs) + ")\n"
+ "[score_training] Average weighted SPE: " + meanSPEs
+ "\t(" + utils.getStandardDeviation(alSPs) + ")\n"
+ "[score_training] Average weighted MCC: " + meanMCCs
+ "\t(" + utils.getStandardDeviation(alMCCs) + ")\n"
+ "[score_training] Average weighted MAE: " + meanMAEs
+ "\t(" + utils.getStandardDeviation(alMAEs) + ")\n"
+ "[score_training] Average weighted BER: " + meanBERs
+ "\t(" + utils.getStandardDeviation(alBERs) + ")";
}
public String toStringRegression() {
if (meanCCs == null) {
return "\t\t\t\t";
} else {
return meanCCs + "\t"
+ meanMAEs + "\t"
+ meanRMSEs + "\t"
+ meanRAEs + "\t"
+ meanRRSEs;
}
}
public String toStringRegressionDetails() {
return "[score_training] Average weighted CC: " + meanCCs
+ "\t(" + utils.getStandardDeviation(alCCs) + ")\n"
+ "[score_training] Average weighted MAE: " + meanMAEs
+ "\t(" + utils.getStandardDeviation(alMAEs) + ")\n"
+ "[score_training] Average weighted RMSE: " + meanRMSEs
+ "\t(" + utils.getStandardDeviation(alRMSEs) + ")\n"
+ "[score_training] Average weighted RAE: " + meanRAEs
+ "\t(" + utils.getStandardDeviation(alRAEs) + ")\n"
+ "[score_training] Average weighted RRSE: " + meanRRSEs
+ "\t(" + utils.getStandardDeviation(alRRSEs) + ")";
}
public void computeMeans() {
meanACCs = getMean(alACCs);
meanAUCs = getMean(alAUCs);
meanAUPRCs = getMean(alAUPRCs);
meanSENs = getMean(alSEs);
meanSPEs = getMean(alSPs);
meanMCCs = getMean(alMCCs);
meanBERs = getMean(alBERs);
meanCCs = getMean(alCCs);
meanRMSEs = getMean(alRMSEs);
meanRAEs = getMean(alRAEs);
meanRRSEs = getMean(alRRSEs);
meanMAEs = getMean(alMAEs);
}
}
public static class testResultsObject {
public String ACC;
public String AUC;
public String AUPRC;
public String SP; //TNR
public String SE; //TPR
public String MCC;
public String BER;
public String CC;
public String MAE;
public String RMSE;
public String RAE;
public String RRSE;
public testResultsObject() {
}
public String toStringClassification() {
return ACC + "\t"
+ AUC + "\t"
+ AUPRC + "\t"
+ SE + "\t"
+ SP + "\t"
+ MCC + "\t"
+ MAE + "\t"
+ BER;
}
public String toStringRegression() {
return CC + "\t"
+ MAE + "\t"
+ RMSE + "\t"
+ RAE + "\t"
+ RRSE;
}
}
}
Datasets
Models
