Diff of /experiment_runner.py [000000] .. [8d2107]

Switch to side-by-side view

--- a
+++ b/experiment_runner.py
@@ -0,0 +1,98 @@
+import logging
+import sys
+import time
+import csv
+from daemon import runner
+import pandas as pd
+from slack_log_handler import SlackLogHandler
+
+from model_tester import test_model, execute_test
+
+
+class ExperimentRunner():
+
+    def __init__(self):
+        #base = '../experiments/'
+        base = '/home/ubuntu/experiments/'
+        self.experiments_file = base + 'experiments.csv'
+        self.results_file = base + 'results.csv'
+        self.pidfile_path = base + 'runner.pid'
+        self.pidfile_timeout = 5
+        self.stdin_path = '/dev/null'
+        self.stdout_path = '/dev/null'
+        self.stderr_path = '/dev/null'
+
+    def do_next_experiment(self):
+        experiments = pd.read_csv(self.experiments_file)
+        idx = experiments.loc[experiments.loc[:,'Run?'] < 1].index.tolist()
+        if len(idx) > 0:
+            idx = idx[0]
+            next_experiment = experiments.iloc[idx]
+            self.run_experiment(next_experiment, idx)
+            return True
+        else:
+            return False
+        
+    def run_experiment(self, experiment, idx):
+        logger.info("Running experiment: " + str(experiment['Id']))
+        experiments = pd.read_csv(self.experiments_file)
+        experiments.loc[idx, 'Run?'] = 1 
+        # Mark experiment as run
+        experiments.to_csv(self.experiments_file, index=False)
+        try:
+            exec(experiment['Dependencies'])
+            pipeline = eval(experiment['Model'])
+
+            logger.info(str(pipeline))
+            results = execute_test(pipeline, int(experiment['Patients']), experiment['CV'])
+        except Exception as e:
+            logger.error("Error on the following experiment:\n" + str(experiment))
+            logger.exception("Here was the stack trace:")
+
+
+        #post results
+        try:
+            with open(self.results_file, "a") as results_file:
+                writer = csv.writer(results_file)
+                writer.writerow([experiment[key] for key in ['Id', 'Purpose', 'CV', 'Patients', 'Model']] + 
+                                [results[key] for key in ['mode', 'precision_mean', 'precision_std', 'recall_mean',
+                                                          'recall_std', 'f1_mean', 'f1_std', 'accuracy_mean',
+                                                          'accuracy_std', 'train_precision_mean', 'train_precision_std', 
+                                                          'train_recall_mean', 'train_recall_std', 'train_f1_mean', 
+                                                          'train_f1_std', 'train_accuracy_mean', 'train_accuracy_std', 
+                                                          'important_features']
+                                ])
+
+        except Exception as e:
+            logger.error(e)
+
+    def run(self):
+        waiting = False
+        while True:
+            success = self.do_next_experiment()
+            if not success:
+                if not waiting:
+                    logger.info("No experiments to run, waiting for more")
+                waiting = True
+                time.sleep(5)
+            else:
+                waiting = False
+
+if __name__ == "__main__":
+    # Configure Logger
+    logger = logging.getLogger("DaemonLog")
+    logger.setLevel(logging.INFO)
+    formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
+    handler = logging.FileHandler("/home/ubuntu/experiments/runner.log")
+    handler.setFormatter(formatter)
+    logger.addHandler(handler)
+
+    slack_handler = SlackLogHandler("https://hooks.slack.com/services/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")
+    slack_handler.setLevel(logging.INFO)
+    logger.addHandler(slack_handler)
+
+    # Configure app to run as daemon
+    exp_runner = ExperimentRunner()
+    daemon_runner = runner.DaemonRunner(exp_runner)
+    daemon_runner.daemon_context.files_preserve=[handler.stream]
+    daemon_runner.do_action()