--- a
+++ b/bin/ann_script_val_kfold.lua
@@ -0,0 +1,779 @@
+
+
+print('\n\n @ @ @ @ @ @ START @ @ @ @ @ @ @ ');
+print('file: script_start.lua');
+print('author Davide Chicco <davide.chicco@gmail.com>');
+print(os.date("%c", os.time()));
+require './metrics_ROC_AUC_computer.lua'
+
+MAX_MSE = 4
+
+local timeStart = os.time()
+
+-- Create k fold subset
+function kfold_subset(k, k_fold, training_set_and_validation_set)
+  
+     print("< < < k_fold cross validation: "..k.." / "..k_fold.." > > >")
+  
+     local fold_perc = round((k * 100 / k_fold), 0)
+     local fold_size = math.ceil(#training_set_and_validation_set/k_fold)
+     io.write("#training_set_and_validation_set = "..#training_set_and_validation_set.."\n")
+     io.write("fold_perc = "..fold_perc.."%\n")
+     io.write("fold_size = "..fold_size.."\n")
+     local training_set = {}
+     local validation_set = {}
+     
+     local validation_set_start_index = -1
+     local validation_set_end_index = -1
+     
+
+     if k==k_fold then
+       validation_set_start_index = (fold_size*(k-1))+1
+       validation_set_end_index = #training_set_and_validation_set
+     else
+       validation_set_start_index = (fold_size*(k-1))+1
+       validation_set_end_index = (fold_size*(k-1)+fold_size)       
+     end     
+     
+     print("validation_set_start_index = "..validation_set_start_index)
+     print("validation_set_end_index = "..validation_set_end_index)
+     
+     for i=1,#training_set_and_validation_set do       
+       if i>=validation_set_start_index and i<=validation_set_end_index then
+	 validation_set[#validation_set+1] = training_set_and_validation_set[i]	 
+       else
+	 training_set[#training_set+1] = training_set_and_validation_set[i]
+       end       
+     end     
+  
+     return {training_set, validation_set, validation_set_start_index, validation_set_end_index}
+     
+end
+
+-- Training function
+function trainingNN(perceptron, train_patient_profile)
+  
+  
+    local criterion = nn.MSECriterion()  
+    local lossSum = 0
+    local positiveLossSum = 0
+    local error_progress = 0
+    local numberOfOnes = 0
+    local positiveErrorProgress = 0
+  
+    if OPTIM_PACKAGE == false then
+
+      myTrainer = nn.StochasticGradient(perceptron, criterion)
+      myTrainer.learningRate = LEARN_RATE
+      myTrainer.maxIteration = ITERATIONS
+      myTrainer:train(train_patient_profile)
+      
+    else
+      
+      require 'optim'
+      local params, gradParams = perceptron:getParameters()     
+      local optimState = nil
+	
+      if MOMENTUM==true then 
+	optimState = {learningRate = LEARN_RATE}
+      else 
+	optimState = {learningRate = LEARN_RATE,
+			  momentum = MOMENTUM_ALPHA }
+      end
+      
+      local total_runs = ITERATIONS*#train_patient_profile
+      local loopIterations = 1
+      for epoch=1,ITERATIONS do
+	for k=1,#train_patient_profile do
+	  
+	    -- Function feval 
+	    local function feval(params)
+		gradParams:zero()
+		
+		local thisProfile = train_patient_profile[k][1]
+		local thisLabel = train_patient_profile[k][2]
+
+		local thisPrediction = perceptron:forward(thisProfile)
+		local loss = criterion:forward(thisPrediction, thisLabel)
+		
+		-- [-1,+1] -> [0,1]
+		thisPrediction = (thisPrediction+1)/2
+		
+		-- print("thisPrediction = "..round(thisPrediction[1],2).." thisLabel = "..thisLabel[1])
+		
+		lossSum = lossSum + loss
+		error_progress = lossSum*100 / (loopIterations*MAX_MSE)
+		
+		--print("thisLabel[1] = "..thisLabel[1].." positiveLossSum = "..positiveLossSum.." numberOfOnes = "..numberOfOnes);
+		
+		if thisLabel[1]==1 then
+		  positiveLossSum = positiveLossSum + loss
+		  numberOfOnes = numberOfOnes + 1		  
+		end
+		if (numberOfOnes > 0 ) then 
+		  positiveErrorProgress = positiveLossSum*100 / (numberOfOnes*MAX_MSE) 
+		end
+		
+		if ((loopIterations*100/total_runs)*5)%100==0 then
+		  io.write("completion: ", round((loopIterations*100/total_runs),2).."%" )
+		  io.write(" (epoch="..epoch..")(element="..k..") loss = "..round(loss,3).." ")      
+		  io.write("\terror progress = "..round(error_progress,5).."%\n")
+		end
+		if printError== true then
+		  filePointer:write(loopIterations..","..positiveErrorProgress.."\n")
+		end
+
+		local dloss_doutput = criterion:backward(thisPrediction, thisLabel)
+		
+		perceptron:backward(thisProfile, dloss_doutput)
+
+		return loss,gradParams
+	    end
+	  optim.sgd(feval, params, optimState)
+	  loopIterations = loopIterations+1
+	end     
+      end
+
+    end
+    
+    return perceptron;
+end
+
+-- Create training set, validation set, test set
+function create_subsets(patients_vett, patient_outcome, permutedIndexVect, training_set_size, validation_set_size, test_set_size)
+
+
+	local train_patient_profile = {}
+	local validation_patient_profile = {}
+	local test_patient_profile = {}
+
+	local original_validation_indexes = {}
+	
+	local training_set_and_validation_set = {}
+
+	for i=1,#patients_vett do
+	    
+	  if i>=1 and i<=(training_set_size) then
+	    
+	    train_patient_profile[#train_patient_profile+1] = {torch.Tensor(patients_vett[permutedIndexVect[i]]), torch.Tensor{patient_outcome[permutedIndexVect[i]]}}
+	    
+	    --print("training outcome["..#train_patient_profile.."] = "..train_patient_profile[#train_patient_profile][2][1]);
+	    
+	    training_set_and_validation_set[#training_set_and_validation_set+1] = {torch.Tensor(patients_vett[permutedIndexVect[i]]), torch.Tensor{patient_outcome[permutedIndexVect[i]]}}
+	  
+	  elseif i>= (training_set_size+1) and i <= (training_set_size+validation_set_size) then
+	      
+	    original_validation_indexes[#original_validation_indexes+1] = permutedIndexVect[i];
+	    -- print("original_validation_indexes =".. permutedIndexVect[i]);
+	    
+	    validation_patient_profile[#validation_patient_profile+1] = {torch.Tensor(patients_vett[permutedIndexVect[i]]), torch.Tensor{patient_outcome[permutedIndexVect[i]]}}
+	    --print("validation outcome["..#validation_patient_profile.."] = "..validation_patient_profile[#validation_patient_profile][2][1]);
+	    
+	    training_set_and_validation_set[#training_set_and_validation_set+1] = {torch.Tensor(patients_vett[permutedIndexVect[i]]), torch.Tensor{patient_outcome[permutedIndexVect[i]]}}
+	   
+	  else
+	    
+	    test_patient_profile[#test_patient_profile+1] = {torch.Tensor(patients_vett[permutedIndexVect[i]]), torch.Tensor{patient_outcome[permutedIndexVect[i]]}}
+	    
+	  end
+	end
+
+	return {train_patient_profile, validation_patient_profile, test_patient_profile, training_set_and_validation_set}
+end
+
+-- createPerceptron
+function createPerceptron(this_input_number, this_hidden_units, this_hidden_layers, this_output_number)
+
+
+  perceptron = nn.Sequential()  
+
+  perceptron:add(nn.Linear(this_input_number, this_hidden_units))
+  -- perceptron:add(nn.Sigmoid())
+  perceptron:add(nn.ReLU())
+  if DROPOUT_FLAG==true then perceptron:add(nn.Dropout()) end
+
+  for w=1,this_hidden_layers do
+    perceptron:add(nn.Linear(this_hidden_units, this_hidden_units))
+    -- perceptron:add(nn.Sigmoid())
+    perceptron:add(nn.ReLU())
+    if DROPOUT_FLAG==true then perceptron:add(nn.Dropout()) end
+  end
+  perceptron:add(nn.Linear(this_hidden_units, this_output_number))
+ 
+  if XAVIER_INITIALIZATION==true then 
+    print("XAVIER_INITIALIZATION = "..tostring(XAVIER_INITIALIZATION))
+--     perceptron = require("./weight-init.lua")(perceptron,  'xavier') -- XAVIER
+  end
+
+  return perceptron;
+end
+
+
+-- function executeTest
+function executeTest(testPerceptron, dataset_patient_profile, computeConfusionMatrix)
+
+    local atleastOneTrue = false
+    local atleastOneFalse = false
+    local predictionTestVect = {}
+    local truthVect = {}
+
+    for i=1,#dataset_patient_profile do
+      local current_label = dataset_patient_profile[i][2][1]
+      local original_prediction = testPerceptron:forward(dataset_patient_profile[i][1])[1]
+      
+      -- io.write("original_prediction = ".. original_prediction)
+      
+      prediction = original_prediction --(original_prediction+1)/2
+      predictionTestVect[i] = prediction
+      truthVect[i] = current_label      
+
+--       io.write(" prediction = ".. round(prediction,2))
+--       io.write(" current_label = ".. current_label.."\n")
+--       io.flush()
+      
+      local labelResult = false      
+      if current_label >= THRESHOLD and prediction >= THRESHOLD  then
+	labelResult = true
+      elseif current_label < THRESHOLD and prediction < THRESHOLD  then
+	labelResult = true
+      end
+            
+      if prediction>=THRESHOLD then
+	atleastOneTrue = true
+      else
+	atleastOneFalse = true
+      end
+    end
+
+    if atleastOneTrue==false then print("ATTENTION: all the predictions are FALSE") end
+    if atleastOneFalse==false then print("ATTENTION: all the predictions are TRUE") end
+
+   if computeConfusionMatrix == false then
+     
+     return {predictionTestVect, truthVect}
+     
+   else
+      
+      local output_AUC_computer = metrics_ROC_AUC_computer(predictionTestVect, truthVect)
+      local auroc = output_AUC_computer[1]
+      local aupr = output_AUC_computer[2]
+
+      local printValues = false
+      local output_confusion_matrix = confusion_matrix(predictionTestVect, truthVect, THRESHOLD, printValues)
+
+      return {output_confusion_matrix[4], output_confusion_matrix[1], output_confusion_matrix[5], auroc, aupr}; 
+	-- MCC, accuracy, f1_score, AUROC, AUPR
+   end
+end
+
+
+-- Function sleep
+function sleep(n) os.execute("sleep " .. tonumber(n)); end
+
+  -- Function table.contains
+function table.contains(table, element)  
+  local count = 1
+  for _, value in pairs(table) do
+   -- print("value: "..tostring(value).." element: "..tostring(element));
+    if tostring(value) == tostring(element) or value==element then
+      return {true,count}
+    end
+    count = count + 1
+  end
+  return {false,-1}
+end
+
+-- Function that prints 
+function printTime(timeStart, stringToPrint)
+	timeEnd = os.time();
+	duration = timeEnd - timeStart;
+	print('\nduration '..stringToPrint.. ': '.. comma_value(tonumber(duration)).. ' seconds');
+	io.flush();
+	print('duration '..stringToPrint.. ': '..string.format("%.2d days, %.2d hours, %.2d minutes, %.2d seconds", (duration/(60*60))/24, duration/(60*60)%24, duration/60%60, duration%60)) 
+	io.flush();
+	
+    return duration;
+end
+
+
+
+-- Function that reads a value and returns the string of the signed value
+function signedValueFunction(value)
+  
+      local value = tonumber(value);
+      --print("value = "..value);
+      local charPlus = ""
+      if tonumber(value) >= 0 then charPlus = "+"; end
+      local outputString = charPlus..""..tostring(round(value,2));
+      --print("outputString = "..outputString);
+      
+      return tostring(outputString);
+end
+       
+
+-- from sam_lie
+-- Compatible with Lua 5.0 and 5.1.
+-- Disclaimer : use at own risk especially for hedge fund reports :-)
+
+---============================================================
+-- add comma to separate thousands
+-- 
+function comma_value(amount)
+  local formatted = amount
+  while true do  
+    formatted, k = string.gsub(formatted, "^(-?%d+)(%d%d%d)", '%1,%2')
+    if (k==0) then
+      break
+    end
+  end
+  return formatted
+end
+
+-- function that computes the confusion matrix
+function confusion_matrix(predictionTestVect, truthVect, threshold, printValues)
+
+  local tp = 0
+  local tn = 0
+  local fp = 0
+  local fn = 0
+  local f1_score = -2
+  local MatthewsCC = -2
+  local accuracy = -2
+  local arrayFPindices = {}
+  local arrayFPvalues = {}
+  local arrayTPvalues = {}
+  local areaRoc = 0
+  
+  
+  local fpRateVett = {}
+  local tpRateVett = {}
+  local precisionVett = {}
+  local recallVett = {}
+    
+  for i=1,#predictionTestVect do
+
+    if printValues == true then
+      io.write("predictionTestVect["..i.."] = ".. round(predictionTestVect[i],4).."\ttruthVect["..i.."] = "..truthVect[i].." ");
+      io.flush();
+    end
+
+    if predictionTestVect[i] >= threshold and truthVect[i] >= threshold then
+      tp = tp + 1
+      arrayTPvalues[#arrayTPvalues+1] = predictionTestVect[i]
+      if printValues == true then print(" TP ") end
+    elseif  predictionTestVect[i] < threshold and truthVect[i] >= threshold then
+      fn = fn + 1
+      if printValues == true then print(" FN ") end
+    elseif  predictionTestVect[i] >= threshold and truthVect[i] < threshold then
+      fp = fp + 1
+      if printValues == true then print(" FP ") end
+      arrayFPindices[#arrayFPindices+1] = i;
+      arrayFPvalues[#arrayFPvalues+1] = predictionTestVect[i]  
+    elseif  predictionTestVect[i] < threshold and truthVect[i] < threshold then
+      tn = tn + 1
+      if printValues == true then print(" TN ") end
+    end
+     
+  end
+  
+
+  print("TOTAL:")
+    print(" FN = "..comma_value(fn).." / "..comma_value(tonumber(fn+tp)).."\t (truth == 1) & (prediction < threshold)");
+    print(" TP = "..comma_value(tp).." / "..comma_value(tonumber(fn+tp)).."\t (truth == 1) & (prediction >= threshold)\n");
+	
+
+    print(" FP = "..comma_value(fp).." / "..comma_value(tonumber(fp+tn)).."\t (truth == 0) & (prediction >= threshold)");
+    print(" TN = "..comma_value(tn).." / "..comma_value(tonumber(fp+tn)).."\t (truth == 0) & (prediction < threshold)\n");
+
+  local continueLabel = true
+
+    
+    if continueLabel then
+      upperMCC = (tp*tn) - (fp*fn)
+      innerSquare = (tp+fp)*(tp+fn)*(tn+fp)*(tn+fn)
+      lowerMCC = math.sqrt(innerSquare)
+      
+      MatthewsCC = -2
+      if lowerMCC>0 then MatthewsCC = upperMCC/lowerMCC end
+      local signedMCC = signedValueFunction(MatthewsCC);
+      -- print("signedMCC = "..signedMCC);
+      
+      if MatthewsCC > -2 then print("\n::::\tMatthews correlation coefficient = "..signedMCC.."\t::::\n");
+      else print("Matthews correlation coefficient = NOT computable");	end
+      
+      accuracy = (tp + tn)/(tp + tn +fn + fp)
+      print("accuracy = "..round(accuracy,2).. " = (tp + tn) / (tp + tn +fn + fp) \t  \t [worst = -1, best =  +1]");
+      
+      f1_score = -2
+      if (tp+fp+fn)>0 then   
+	f1_score = (2*tp) / (2*tp+fp+fn)
+	print("f1_score = "..round(f1_score,2).." = (2*tp) / (2*tp+fp+fn) \t [worst = 0, best = 1]");
+      else
+	print("f1_score CANNOT be computed because (tp+fp+fn)==0")    
+      end
+	
+      
+      local totalRate = 0
+      if MatthewsCC > -2 and f1_score > -2 then 
+	totalRate = MatthewsCC + accuracy + f1_score 
+	-- print("total rate = "..round(totalRate,2).." in [-1, +3] that is "..round((totalRate+1)*100/4,2).."% of possible correctness");
+      end
+      
+--       local numberOfPredictedOnes = tp + fp;
+--       print("numberOfPredictedOnes = (TP + FP) = "..comma_value(numberOfPredictedOnes).." = "..round(numberOfPredictedOnes*100/(tp + tn + fn + fp),2).."%");
+--       
+--       io.write("\nDiagnosis: ");
+--       if (fn >= tp and (fn+tp)>0) then print("too many FN false negatives"); end
+--       if (fp >= tn and (fp+tn)>0) then print("too many FP false positives"); end
+--       
+--       
+--       if (tn > (10*fp) and tp > (10*fn)) then print("Excellent ! ! !");
+--       elseif (tn > (5*fp) and tp > (5*fn)) then print("Very good ! !"); 
+--       elseif (tn > (2*fp) and tp > (2*fn)) then print("Good !"); 
+--       elseif (tn >= fp and tp >= fn) then print("Alright"); 
+--       else print("Baaaad"); end
+    end
+    
+    return {accuracy, arrayFPindices, arrayFPvalues, MatthewsCC, f1_score};
+end
+
+
+
+-- Permutations
+-- tab = {1,2,3,4,5,6,7,8,9,10}
+-- permute(tab, 10, 10)
+function permute(tab, n, count)
+      n = n or #tab
+      for i = 1, count or n do
+	math.randomseed(os.time())
+        local j = math.random(i, n)
+        tab[i], tab[j] = tab[j], tab[i]
+      end
+      return tab
+end
+
+-- round a real value
+function round(num, idp)
+  local mult = 10^(idp or 0)
+  return math.floor(num * mult + 0.5) / mult
+end
+
+
+
+-- ##############################3
+
+local profile_vett = {}
+
+
+local csv = require("csv")
+-- local fileName = tostring(arg[1])
+local fileName = "../data/LungCancerDataset_AllRecords_NORM_reduced_features.csv"
+-- cervical_arranged_NORM.csv
+-- cervical_arranged_NORM_ONLY_BIOPSY_TARGET.csv
+
+
+print("Readin' "..tostring(fileName));
+
+local f = csv.open(fileName)
+local column_names = {}
+
+local j = 0
+for fields in f:lines() do
+  
+  if j>0 then
+    profile_vett[j] = {}
+      for i, v in ipairs(fields) do 
+	profile_vett[j][i] = tonumber(v);
+      end
+    j = j + 1
+  else
+    for i, v in ipairs(fields) do 
+	column_names[i] = v
+     end
+    j = j + 1
+  end
+end
+
+OPTIM_PACKAGE = true
+MAX_VALUE = 1
+local output_number = 1
+THRESHOLD = 0.5 -- ORIGINAL
+-- THRESHOLD = 0.1529
+XAVIER_INITIALIZATION = true
+DROPOUT_FLAG = true
+MOMENTUM_ALPHA = 0.5
+
+MOMENTUM = true
+LEARN_RATE = 0.01 -- default was 0.01
+ITERATIONS = 200 -- default was 200 -- I'M ANALYZING THIS PARAMETER IN THIS ANALYSIS
+local hidden_units = 50 -- default was 50
+
+local mcc = "mcc"
+local aupr = "aupr"
+OPTIMIZE_SCORE = aupr
+
+
+print("\nOPTIM_PACKAGE  = ".. tostring(OPTIM_PACKAGE));
+print("XAVIER_INITIALIZATION = ".. tostring(XAVIER_INITIALIZATION));
+print("DROPOUT_FLAG = ".. tostring(DROPOUT_FLAG));
+print("MOMENTUM_ALPHA = ".. tostring(MOMENTUM_ALPHA));
+
+print("MOMENTUM = ".. tostring(MOMENTUM));
+print("LEARN_RATE = ".. tostring(LEARN_RATE)); 
+print("ITERATIONS = ".. tostring(ITERATIONS)); 
+
+
+
+
+
+-- local hidden_layers = 1 -- best is 1
+-- local hiddenUnitVect = {25,50,75,100,125,150,175,200,225,250,275,300,350,400}
+local hiddenUnitVect = {10,20,30,40,50,60,70}
+local hiddenLayerVect = {1,2,3,4}
+-- local hiddenLayerVect = {1}
+
+local profile_vett_data = {}
+local label_vett = {}
+
+for i=1,#profile_vett do
+  profile_vett_data[i] = {}
+--   io.write("#"..i.."# ")
+--   io.flush()
+  for j=1,#(profile_vett[1]) do    
+ 
+    if j<#(profile_vett[1]) then
+      profile_vett_data[i][j] = profile_vett[i][j]
+      -- io.write("profile_vett_data["..i.."]["..j.."] = "..profile_vett_data[i][j].." ")
+      -- filePointer:write(round(profile_vett_data[i][j],2)..",")
+      -- io.flush()
+    else
+      label_vett[i] = profile_vett[i][j]
+      -- filePointer:write(round(label_vett[i],2)..",")
+      -- io.flush()
+    end    
+  end
+   -- filePointer:write("\n")
+--   io.flush()
+end
+
+print("Number of value profiles (rows) = "..#profile_vett_data);
+print("Number features (columns) = "..#(profile_vett_data[1]));
+print("Number of targets (rows) = "..#label_vett);
+
+
+local patient_outcome = label_vett
+local patients_vett = profile_vett_data
+
+
+-- START
+
+local timeStart = os.time();
+
+local indexVect = {}; 
+for i=1, #patients_vett do indexVect[i] = i;  end
+local permutedIndexVect = permute(indexVect, #indexVect, #indexVect);
+
+print("#patients_vett = "..#patients_vett);
+
+-- SET TRAINING SET SIZE, VALDIATION SET SIZE, TEST_SET SIZE
+TRAINING_SET_PERC = 60
+VALIDATION_SET_PERC = 20
+TEST_SET_PERC = 20
+
+local training_set_size = round((TRAINING_SET_PERC*(#patients_vett))/100)
+local validation_set_size = round((VALIDATION_SET_PERC*(#patients_vett))/100)
+local test_set_size = #patients_vett - validation_set_size - training_set_size
+
+print("\ntraining_set_size = "..training_set_size);
+print("validation_set_size = "..validation_set_size);
+print("test_set_size = "..test_set_size.."\n");
+local modelFileVect = {}
+
+
+
+-- CREATE TRAINING SET, VALDIATION SET, TEST_SET
+local output_create_subsets = create_subsets(patients_vett, patient_outcome, permutedIndexVect, training_set_size, validation_set_size, test_set_size)
+local train_patient_profile = output_create_subsets[1]
+local validation_patient_profile = output_create_subsets[2]
+local test_patient_profile = output_create_subsets[3]
+local training_set_and_validation_set = output_create_subsets[4] 
+
+
+require 'nn'
+local input_number = (#(train_patient_profile[1][1]))[1]
+print("input_number = "..(#(train_patient_profile[1][1]))[1])
+
+
+function train_patient_profile:size() return #train_patient_profile end 
+function validation_patient_profile:size() return #validation_patient_profile end 
+
+local printError = false  
+local fileName = nil
+local filePointer = nil
+if printError == true then 
+  fileName = "./mse_log/positive_error_progress"..tostring(os.time())..".csv" 
+  filePointer = io.open(fileName, "w")  
+end
+
+ 
+
+  
+-- OPTIMIZATION LOOPS  
+local MCC_vect = {}  
+local f1score_vect = {}  
+local auroc_vett = {}
+local aupr_vett = {}
+
+local hus_vect = {}
+local hl_vect = {}
+local general_result_vect = {}
+local general_truth_vect = {}
+
+local k_fold = 5
+
+for b=1,#hiddenLayerVect do
+   for a=1,#hiddenUnitVect do
+    
+   local hidden_units = hiddenUnitVect[a]
+   local hidden_layers = hiddenLayerVect[b]
+   print("$$$ hidden_units = "..hidden_units.."\t hidden_layers = "..hidden_layers.." $$$")
+
+   for k=1,k_fold do
+     
+     local output_kfold_subset = kfold_subset(k, k_fold, training_set_and_validation_set)
+     local training_set = output_kfold_subset[1]
+     local validation_set = output_kfold_subset[2]
+     local validation_set_start_index = output_kfold_subset[3]
+     local validation_set_end_index = output_kfold_subset[4]
+      
+      -- CREATE THE NEURAL NETWORK
+      local perceptron = createPerceptron(input_number, hidden_units, hidden_layers, output_number)
+      
+      -- TRAIN THE NEURAL NETWORK
+      perceptron = trainingNN(perceptron, training_set)
+	
+      -- APPLY THE NEURAL NETWORK TO THE VALIDATION SET
+      local computeConfusionMatrix = false 
+      print("\n\n### executeTest(perceptron, validation_set)")     
+      local testOutput = executeTest(perceptron, validation_set, computeConfusionMatrix)
+      
+      local thisPredictionTestVect = nil
+      local thisTruthVect = nil
+      if (computeConfusionMatrix == true) then
+	  MCC_vect[#MCC_vect+1] = testOutput[1]
+	  f1score_vect[#f1score_vect+1] = testOutput[3]
+	  auroc_vett[#auroc_vett+1] = testOutput[4]
+	  aupr_vett[#aupr_vett+1] = testOutput[5]
+      else
+	  thisPredictionTestVect = testOutput[1]
+	  thisTruthVect = testOutput[2]
+	  
+	  local h = 1
+	  for p=validation_set_start_index,validation_set_end_index do
+	      general_result_vect[p]=thisPredictionTestVect[h]
+	      general_truth_vect[p]=thisTruthVect[h]
+	      h = h+1
+	  end	  
+      end   
+   end
+   
+--    print("general_result_vect")
+--    print(general_result_vect)
+--    
+--    print("general_truth_vect")
+--    print(general_truth_vect)
+   
+   local output_AUC_computer = metrics_ROC_AUC_computer(general_result_vect, general_truth_vect)
+   local auroc = output_AUC_computer[1]
+   local aupr = output_AUC_computer[2]
+
+   local printValues = false
+   local output_confusion_matrix = confusion_matrix(general_result_vect, general_truth_vect, THRESHOLD, printValues)
+   
+   MCC_vect[#MCC_vect+1] = output_confusion_matrix[4]
+   f1score_vect[#f1score_vect+1] = output_confusion_matrix[5]
+   auroc_vett[#auroc_vett+1] = auroc
+   aupr_vett[#aupr_vett+1] = aupr
+   
+   hus_vect[#hus_vect+1] = hidden_units
+   hl_vect[#hl_vect+1] = hidden_layers
+      
+  end
+end
+
+local maxMCC = -1
+local maxMCCpos = -1
+
+local max_aupr = -1
+local max_aupr_pos = -1
+local optHiddenLayers = -1
+local optHiddenUnits = -1
+
+for k=1,#MCC_vect do
+      io.write("@ @ ["..k.."] ")
+      io.write("\tAUPR = "..round(aupr_vett[k],2).."% ")
+      io.write("\tMCC = "..round(MCC_vect[k],2))
+      io.write("\tF1_score  = "..round(f1score_vect[k],2))
+      io.write("\tAUROC  = "..round(auroc_vett[k],2).."% ")
+      io.write("\thidden units = "..hus_vect[k].." ")
+      io.write("\thidden layers = "..hl_vect[k].." ")      
+      io.write(" @ @ \n")
+      io.flush()
+      
+      if (OPTIMIZE_SCORE == mcc and MCC_vect[k]>=maxMCC) then 
+	    maxMCC = MCC_vect[k]
+	    maxMCCpos = k
+	    optHiddenLayers = hl_vect[k]
+	    optHiddenUnits = hus_vect[k]
+      elseif (OPTIMIZE_SCORE == aupr and aupr_vett[k]>=max_aupr) then 
+	    max_aupr = aupr_vett[k]
+	    max_aupr_pos = k
+	    optHiddenLayers = hl_vect[k]
+	    optHiddenUnits = hus_vect[k]
+      end
+end
+
+-- CHOOSING THE MODEL BY OPTIMZING THE MCC OR AUPR
+local modelFileToLoad = nil
+if OPTIMIZE_SCORE == mcc then
+
+  print("\nmodelFileVect["..maxMCCpos.."]")
+  print("optHiddenLayers = "..optHiddenLayers)
+  print("optHiddenUnits = "..optHiddenUnits)
+  
+elseif OPTIMIZE_SCORE == aupr then
+
+  print("\nmodelFileVect["..max_aupr_pos.."]")
+  print("optHiddenLayers = "..optHiddenLayers)
+  print("optHiddenUnits = "..optHiddenUnits)     
+  
+end
+
+      -- CREATE THE NEURAL NETWORK
+local optPerceptron = createPerceptron(input_number, optHiddenUnits, optHiddenLayers, output_number)
+      
+-- TRAIN THE NEURAL NETWORK
+optPerceptron = trainingNN(optPerceptron, train_patient_profile)
+
+local computeConfusionMatrix = true
+print("\n\n### executeTest(optPerceptron, test_patient_profile)")
+local executeTestOutput = executeTest(optPerceptron, test_patient_profile, computeConfusionMatrix)
+
+local lastMCC = executeTestOutput[1]
+local lastAccuracy = executeTestOutput[2]
+local lastF1score = executeTestOutput[3]
+
+print("':':':':' lastMCC = "..round(lastMCC,2).."  lastF1score = "..round(lastF1score,2).." ':':':':'")
+
+for i=1,#modelFileVect do
+  local command = "rm "..tostring(modelFileVect[i])
+  io.write("command: "..command.." \n")
+  local res = sys.execute(command)
+  -- print("command response: "..res)
+end
+
+if printError == true then 
+  filePointer:close()
+end
+
+printTime(timeStart, " complete execution")