package dataprep;

import java.io.BufferedReader;

import java.io.BufferedWriter;

import java.io.File;

import java.io.FileNotFoundException;

import java.io.FileReader;

import java.io.FileWriter;

import java.io.IOException;

import java.util.ArrayList;

import java.util.HashMap;

import java.util.HashSet;

import java.util.List;

import java.util.Map;

import java.util.Set;

/**  

 * Provides methods for processing UMLS mapped FAERS data

 * @author zhengc

 *

 */

public class ProcessUMLS {

    public static void main(String[] args) throws IOException {

        String input = "/Users/zhengc/workspace/FARES/data/FARES/UMLS_map_data/ID_indications_all_clean_width_umls_id_diso_AD";

        String output = "/Users/zhengc/workspace/FARES/data/FARES/UMLS_map_data/ID_indications_all_clean_width_umls_id_diso_AD_filtered";

        String idnummapfile = "/Users/zhengc/workspace/FARES/data/FARES/UMLS_map_data/ID_indications_all_clean_width_umls_id_diso_AD_idnummap";

        // Compute total terms

        Set<String> terms = getTermSet(input);

        System.out.println(terms.size());

        // Create numbers of each termID appeared in the data

        Map<String, Integer> idnummap = getIdNumMap(input);

        BufferedWriter bw = new BufferedWriter(new FileWriter(new File(idnummapfile)));

        bw.write("UMLS_id" + "," + "Number" + "\n");

        for (String id : idnummap.keySet()) {

            bw.write(id + "," + idnummap.get(id) + "\n");

        }

        bw.close();

        System.out.println("Number of total diseases: " + idnummap.size());

        // Set threshold and get processed data

        Set<String> freqset = getFreqTermSet(idnummap, 1, 500000);

        System.out.printf("%d disease will be filtered", freqset.size());

//      for (String term : freqset) {

//          System.out.println(term);

//      }

        procUMLS(input, freqset, output);

    }

    /**

     * Get all disease terms

     * @param filename raw FDA labeling data file

     * @return disease term set

     * @throws IOException

     */

    private static Set<String> getTermSet(String filename) throws IOException {

        Set<String> termSet = new HashSet<String>();

        BufferedReader br = new BufferedReader(new FileReader(new File(filename)));

        String line = null;

        while ((line=br.readLine()) != null) {

            String[] terms = line.split(",");

            for (int i=0; i<terms.length; i++) {

                termSet.add(terms[i]);

            }

        }

        br.close();

        return termSet;

    }

    /**

     * Compute the number of occurrence of each disease term in FAERS data 

     * @param filename cleaned, UMLS mapped FAERS file

     * @return id_number map that key is disease UMLS term and value is number of occurrence

     * @throws IOException

     */

    private static Map<String, Integer> getIdNumMap(String filename) throws IOException {

        Map<String, Integer> idNumMap = new HashMap<String, Integer>();

        BufferedReader br = new BufferedReader(new FileReader(new File(filename)));

        String line = null;

        while ((line=br.readLine()) != null) {

            String[] terms = line.split(",");

            // get unique terms

            Set<String> termset = new HashSet<String>();

            for (int i=0; i<terms.length; i++) {

                termset.add(terms[i]);

            }

            for (String term : termset) {

                if (!idNumMap.containsKey(term)) {

                    idNumMap.put(term, 1);

                } else {

                    idNumMap.put(term, idNumMap.get(term) + 1);

                }

            }

        }

        br.close();

        return idNumMap;

    }

    /**

     * Remove rare and too many occurrence disease term in FAERS

     * @param idNumMap id_number map

     * @param minfreq low bound of threshold

     * @param maxfreq up bound threshold

     * @return filtered disease term set 

     */

    private static Set<String> getFreqTermSet(Map<String, Integer> idNumMap, int minfreq, int maxfreq) {

        Set<String> freqSet = new HashSet<String>();

        int value = 0;

        for (String key : idNumMap.keySet()) {

            value = idNumMap.get(key);

            if (value <= minfreq | value >= maxfreq) {

                freqSet.add(key);

            }

        }

        return freqSet;

    }

    /**

     * Gets filtered FAERS data

     * @param input cleaned, UMLS mapped FARES data

     * @param freqset filtered disease term set 

     * @param output filter FAERS data

     * @throws IOException

     */

    private static void procUMLS(String input, Set<String> freqset,  String output) throws IOException {

        BufferedReader br = new BufferedReader(new FileReader(new File(input)));

        BufferedWriter bw = new BufferedWriter(new FileWriter(new File(output)));

        String line = null;

        while ((line=br.readLine()) != null) {

            String[] terms = line.split(",");

            // get unique terms

            Set<String> termset = new HashSet<String>();

            for (int i=0; i<terms.length; i++) {

                termset.add(terms[i]);

            }

            List<String> termlist = new ArrayList<String>(termset);

            StringBuilder sb = new StringBuilder();

            for (String term : termlist) {

                if (!freqset.contains(term)) {

                    sb.append(term + ",");

                }

            }

            String sbline = sb.toString();

//          System.out.println(sbline);

            if (!sbline.isEmpty()) {

                sbline = sbline.replaceFirst(",$", "");

                bw.write(sbline + "\n");

            }

        }

        bw.close();

        br.close();

    }

}