/*
 * Get clinical and genes expression
 * Make some feature extraction
 */
package biodiscml;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.PrintWriter;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.TreeMap;
import utils.Weka_module;
import utils.utils.TableObject;
import static utils.utils.*;

/**
 *
 * @author Mickael
 */
public class AdaptDatasetToTraining {

    public static boolean debug = Main.debug;
    public static HashMap<String, String> removedFeatures = new HashMap<>();

    public AdaptDatasetToTraining() {
    }

    /**
     * create dataset with a determined class
     *
     * @param trainFile
     */
    public AdaptDatasetToTraining(String trainFile) {
        //remove files of previous run if exist
        if (!Main.restoreRun || !Main.resumeTraining) {
            System.out.println("Check if files exist and should be deleted...");
            String allDataFile = trainFile.replace("data_to_train", "all_data");
            if (new File(allDataFile).exists()) {
                System.out.println("\t" + allDataFile + " exist... deleting...");
                new File(allDataFile).delete();
            }
            if (new File(trainFile).exists()) {
                System.out.println("\t" + trainFile + " exist... deleting...");
                new File(trainFile).delete();
                new File(trainFile.replace(".csv", ".arff")).delete();
            }
        }

        //create the adapted training file
        System.out.println("# Training file(s)");
        if (Main.doClassification) {
            createFileCompatibleForWeka(Main.classificationClassName, Main.hmTrainFiles, trainFile, Main.separator, true);
        } else {
            createFileCompatibleForWeka(Main.regressionClassName, Main.hmTrainFiles, trainFile, Main.separator, true);
        }
        //create the adapted tested file
        if (Main.doSampling) {
            System.out.println("## Apply sampling configuration");
            //if a test file is provided
            String trainAndTestFile = trainFile.replace("data_to_train", "all_data");
            String testFile = trainFile.replace("data_to_train.csv", "data_to_test.csv");
            Weka_module weka = new Weka_module();
            String trainSetRange = "";
            if (!Main.hmNewDataFiles.isEmpty()) {
                System.out.println("# Testing file(s)");
                if (Main.doClassification) {
                    createFileCompatibleForWeka(Main.classificationClassName, Main.hmNewDataFiles, testFile, Main.separator, false);
                } else {
                    createFileCompatibleForWeka(Main.regressionClassName, Main.hmNewDataFiles, testFile, Main.separator, false);
                }
                //if a test file is provided, we need to merge it to the train file and
                // split it again to preserve a compatible arff format between train and test sets
                trainSetRange = mergeTrainAndTestFiles(trainFile, testFile, trainAndTestFile);

            } else {// else we split
                // just rename the train file to a file that contains all train and test data
                new File(trainFile).renameTo(new File(trainAndTestFile));
            }

            //perform sampling
            weka.sampling(trainAndTestFile, trainFile, testFile, Main.isClassification, trainSetRange);
        }
        System.out.println("");

    }

    /**
     * get common ids between infiles
     *
     * @param al_tables
     * @return
     */
    public HashMap<String, String> getCommonIds(ArrayList<TableObject> al_tables) {
        HashMap<String, String> hm_ids = new HashMap<>();
        //if many infiles
        if (al_tables.size() > 0) {
            HashMap<String, Integer> hm_counts = new HashMap<>();
            //get all ids, count how many times each one is seen
            for (TableObject table : al_tables) {
                for (String s : table.hmIDsList.keySet()) {
                    s = s.toLowerCase();
                    if (hm_counts.containsKey(s)) {
                        int tmp = hm_counts.get(s);
                        tmp++;
                        hm_counts.put(s, tmp);
                    } else {
                        hm_counts.put(s, 1);
                    }
                }
            }
            //check number of times ids have been seen
            for (String s : hm_counts.keySet()) {
                if (hm_counts.get(s) == al_tables.size()) {
                    hm_ids.put(s, "");
                }
            }
        } else {//for one infile
            for (String s : al_tables.get(0).hmIDsList.keySet()) {
                hm_ids.put(s, "");
            }
        }

        return hm_ids;
    }

    private void createFileCompatibleForWeka(String theClass, HashMap<String, String> infiles, String outfile, String separator, Boolean trainingFile) {
        //convert hashmap to list
        String[] files = new String[infiles.size()];
        String[] prefixes = new String[infiles.size()];
        int cpt = 0;
        for (String f : infiles.keySet()) {
            files[cpt] = f;
            prefixes[cpt] = infiles.get(f);
            cpt++;
        }

        //load datasets of features
        if (debug) {
            System.out.println("loading files");
        }

        ArrayList<TableObject> al_tables = new ArrayList<>();
        int classIndex = -1;

        for (int i = 0; i < files.length; i++) {
            String file = files[i];
            if (debug) {
                System.out.println(file);
            }

            TableObject tbo = new TableObject(readTable(file, separator));
            //locate class
            if (tbo.containsClass(theClass)) {
                classIndex = i;
            }
            al_tables.add(tbo);
        }

        //extract class
        ArrayList<String> myClass = new ArrayList<>();
        try {
            myClass = al_tables.get(classIndex).getTheClass(theClass);
        } catch (Exception e) {
            System.err.println("[error] Class " + theClass + " not found. Error in the input file.");
            if (Main.debug) {
                e.printStackTrace();
            }
            System.exit(0);
        }
        //remove useless features having 100% the same value
        if (trainingFile) {
            try {
                for (TableObject tbo : al_tables) {
                    for (String s : tbo.getSortedHmDataKeyset()) {
                        HashMap<String, String> hm = new HashMap<>();
                        for (String value : tbo.hmData.get(s)) {
                            hm.put(value, value);
                        }
                        if (hm.size() == 1) {
                            tbo.hmData.remove(s);
                            if (hm.keySet().toArray()[0].equals("?")) {
                                System.out.println("Removing feature " + s + " "
                                        + "because 100% of values are missing");
                            } else {
                                System.out.println("Removing feature " + s + " "
                                        + "because 100% of values are identical "
                                        + "{" + hm.keySet().toArray()[0] + "}");
                            }
                            removedFeatures.put(s, s);
                        }
                    }
                    cpt++;
                }
            } catch (Exception e) {
                e.printStackTrace();
            }
        } else {
            try {
                for (TableObject tbo : al_tables) {
                    for (String s : tbo.getSortedHmDataKeyset()) {
                        if (removedFeatures.containsKey(s)) {
                            tbo.hmData.remove(s);
                        }
                    }
                    cpt++;
                }
            } catch (Exception e) {
                e.printStackTrace();
            }
        }

        //create outfile
        if (debug) {
            System.out.println("create outfile " + outfile);
        }
        try {
            PrintWriter pw = new PrintWriter(new FileWriter(outfile));
            ///////// PRINT HEADER
            int featuresCpt = 0;
            pw.print(Main.mergingID);
            cpt = 0;
            for (TableObject tbo : al_tables) {
                for (String s : tbo.getSortedHmDataKeyset()) {
                    if (!Main.hmExcludedFeatures.containsKey(s)) {
                        if (!prefixes[cpt].isEmpty()) {
                            pw.print("\t" + prefixes[cpt] + "__" + s);
                        } else {
                            pw.print("\t" + s);
                        }
                        featuresCpt++;
                    }
                }
                cpt++;
            }
            pw.println("\tclass");
            pw.flush();

            //search for ids present in all datasets
            HashMap<String, String> hm_ids = getCommonIds(al_tables);
            if (debug && al_tables.size() > 1) {
                System.out.println("Total number of common instances between files: " + hm_ids.size());
            } else {
                System.out.println("Total number of instances between files: " + hm_ids.size());
            }

            System.out.println("Total number of features: " + featuresCpt);

            ///////PRINT CONTENT
            TreeMap<String, Integer> tm = new TreeMap<>();
            tm.putAll(al_tables.get(0).hmIDsList);
            int existing_spaces = 0;
            for (String id : tm.keySet()) {
                if (hm_ids.containsKey(id.toLowerCase()) && !id.equals(Main.mergingID.toLowerCase())) {
                    // if (hm_ids.containsKey(id) && !id.equals(Main.mergingID)) {
                    pw.print(id);
                    for (TableObject tbo : al_tables) {
                        int idIndex = tbo.hmIDsList.get(id);
                        for (String s : tbo.getSortedHmDataKeyset()) {
                            if (!Main.hmExcludedFeatures.containsKey(s)) { //if it is not a rejected feature
                                // print values and replace , by .
                                String out = tbo.hmData.get(s).get(idIndex).replace(",", ".").trim();
                                if (out.isEmpty() || out.equals("NA") || out.equals("na")
                                        || out.equals("N/A") || out.equals("n/a")) {
                                    out = "?";
                                }
                                pw.print("\t" + out);
                            }
                        }
                    }
                    String classe = myClass.get(al_tables.get(classIndex).hmIDsList.get(id));
                    if (classe.contains(" ")) {
                        existing_spaces++;
                    }
                    classe = classe.replace(" ", "_");
                    pw.print("\t" + classe);
                    //pw.print("\t" + myClass.get(idIndex).replace("1", "true").replace("0", "false"));
                    pw.println();
                }
            }
            if (existing_spaces > 0) {
                System.out.println("Spaces detected in class label. They were replaced by _");
            }
            pw.flush();

            if (debug) {
                System.out.println("closing outfile " + outfile);
            }
            pw.close();

        } catch (Exception e) {
            e.printStackTrace();
        }
    }

    /**
     *
     * @param outfile
     * @param replace
     * @return range of the train set (ex: 1-100)
     */
    private String mergeTrainAndTestFiles(String trainFile, String testFile, String trainAndTestFile) {
        int cpt = -1;
        try {
            //create train+set file
            PrintWriter pw = new PrintWriter(new FileWriter(trainAndTestFile));

            //read train
            BufferedReader br = new BufferedReader(new FileReader(trainFile));

            while (br.ready()) {
                pw.println(br.readLine());
                cpt++;
            }
            br.close();
            pw.flush();

            //read test
            br = new BufferedReader(new FileReader(testFile));
            br.readLine(); // skip header
            while (br.ready()) {
                pw.println(br.readLine());
            }
            br.close();
            pw.close();
        } catch (Exception e) {
            e.printStackTrace();
        }
        return "1-" + cpt;
    }

}