1 lines (1 with data), 158.3 kB

{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"MLP_models.ipynb","provenance":[],"collapsed_sections":["pMOATtfckQrD","-5k5NxKfkW7p","zyvdoJ0Vmuog","WA8cHUO8pMlb","qWjp7FlTrTWg"]},"kernelspec":{"name":"python3","display_name":"Python 3"},"language_info":{"name":"python"}},"cells":[{"cell_type":"code","execution_count":1,"metadata":{"id":"84nJFHFrK4Tp","executionInfo":{"status":"ok","timestamp":1651886496422,"user_tz":240,"elapsed":11193,"user":{"displayName":"Benedikt Geiger","userId":"17925887631246406508"}}},"outputs":[],"source":["import tensorflow as tf\n","import tensorflow.keras.layers as layers\n","import torch\n","import numpy as np\n","import pandas as pd\n","import os\n","import pickle\n","import matplotlib.pyplot as plt\n","from sklearn import metrics"]},{"cell_type":"code","source":["from google.colab import drive\n","drive.mount(\"/content/gdrive\")"],"metadata":{"id":"NwTz__WVK_sU","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1651886514076,"user_tz":240,"elapsed":17660,"user":{"displayName":"Benedikt Geiger","userId":"17925887631246406508"}},"outputId":"07a021e3-bd4a-4664-f783-7281dc89b820"},"execution_count":2,"outputs":[{"output_type":"stream","name":"stdout","text":["Mounted at /content/gdrive\n"]}]},{"cell_type":"markdown","source":["Loading the datasets"],"metadata":{"id":"kMU4jE5qK6-l"}},{"cell_type":"code","source":["# project folder\n","deep_learning_dir = \"/content/gdrive/My Drive/BMI 707 Project\""],"metadata":{"id":"lifWknGULF_d","executionInfo":{"status":"ok","timestamp":1651886514078,"user_tz":240,"elapsed":15,"user":{"displayName":"Benedikt Geiger","userId":"17925887631246406508"}}},"execution_count":3,"outputs":[]},{"cell_type":"code","source":["# get nctid's and labels\n","train = pd.read_csv(deep_learning_dir + \"/data_formatting/training_data.tsv\", sep=\"\\t\")\n","val = pd.read_csv(deep_learning_dir + \"/data_formatting/validation_data.tsv\", sep=\"\\t\")\n","test = pd.read_csv(deep_learning_dir + \"/data_formatting/testing_data.tsv\", sep=\"\\t\")\n","\n","train_nctids = train[\"nctid\"].to_numpy()\n","train_labels = train[\"label\"].to_numpy()\n","\n","val_nctids = val[\"nctid\"].to_numpy()\n","val_labels = val[\"label\"].to_numpy()\n","\n","test_nctids = test[\"nctid\"].to_numpy()\n","test_labels = test[\"label\"].to_numpy()"],"metadata":{"id":"7-XMLJW8LH6x","executionInfo":{"status":"ok","timestamp":1651886516727,"user_tz":240,"elapsed":2660,"user":{"displayName":"Benedikt Geiger","userId":"17925887631246406508"}}},"execution_count":4,"outputs":[]},{"cell_type":"markdown","source":["Loading the embeddings"],"metadata":{"id":"LHlFydAxK8SM"}},{"cell_type":"code","source":["## diseases\n","with open(deep_learning_dir + \"/embeddings/nctid2diseases.pkl\", \"rb\") as handle:\n","  nctid2diseases = pickle.load(handle)\n","\n","\n","## eligibility criteria\n","with open(deep_learning_dir + \"/embeddings/nctid2incl_criteria.pkl\", \"rb\") as handle:\n","  nctid2incl_criteria = pickle.load(handle)\n","\n","with open(deep_learning_dir + \"/embeddings/nctid2excl_criteria.pkl\", \"rb\") as handle:\n","  nctid2excl_criteria = pickle.load(handle)\n","\n","\n","## drugs\n","with open(deep_learning_dir + \"/embeddings/nctid2drugs.pkl\", \"rb\") as handle:\n","  nctid2drugs = pickle.load(handle)\n","\n","\n","## targets\n","with open(deep_learning_dir + \"/embeddings/nctid2drug_targets.pkl\", \"rb\") as handle:\n","  nctid2target = pickle.load(handle)"],"metadata":{"id":"KrPTAj29LABG","executionInfo":{"status":"ok","timestamp":1651886541109,"user_tz":240,"elapsed":24390,"user":{"displayName":"Benedikt Geiger","userId":"17925887631246406508"}}},"execution_count":5,"outputs":[]},{"cell_type":"code","source":["# convert to np array\n","for nctid in nctid2diseases:\n","  nctid2incl_criteria[nctid] = nctid2incl_criteria[nctid].numpy()\n","  nctid2excl_criteria[nctid] = nctid2excl_criteria[nctid].numpy()\n","  \n","  if nctid in nctid2target:\n","    nctid2target[nctid] = nctid2target[nctid].numpy()\n","  else:\n","    nctid2target[nctid] = np.zeros(1024, dtype=np.float32)"],"metadata":{"id":"RqpIFddbZ4MI","executionInfo":{"status":"ok","timestamp":1651886541113,"user_tz":240,"elapsed":19,"user":{"displayName":"Benedikt Geiger","userId":"17925887631246406508"}}},"execution_count":6,"outputs":[]},{"cell_type":"code","source":["## meta data\n","with open(deep_learning_dir + \"/embeddings/nctid2n_diseases.pkl\", \"rb\") as handle:\n","  nctid2n_diseases = pickle.load(handle)\n","\n","\n","with open(deep_learning_dir + \"/embeddings/length_nctid2incl_criteria.pkl\", \"rb\") as handle:\n","  nctid2n_incl = pickle.load(handle)\n","\n","with open(deep_learning_dir + \"/embeddings/length_nctid2excl_criteria.pkl\", \"rb\") as handle:\n","  nctid2n_excl = pickle.load(handle)\n","\n","\n","with open(deep_learning_dir + \"/embeddings/nctid2npart_success.pkl\", \"rb\") as handle:\n","  nctid2npart_success = pickle.load(handle)"],"metadata":{"id":"oOPY5HPIj4ot","executionInfo":{"status":"ok","timestamp":1651886542289,"user_tz":240,"elapsed":1188,"user":{"displayName":"Benedikt Geiger","userId":"17925887631246406508"}}},"execution_count":7,"outputs":[]},{"cell_type":"code","source":["# concatenate meta info\n","nctid2meta = {}\n","\n","for nctid in nctid2diseases:\n","  nctid2meta[nctid] = np.array([nctid2n_diseases[nctid], nctid2n_incl[nctid], nctid2n_excl[nctid], \n","                                nctid2npart_success[nctid][0], nctid2npart_success[nctid][1]])"],"metadata":{"id":"NHQ96_4dxEDj","executionInfo":{"status":"ok","timestamp":1651886542291,"user_tz":240,"elapsed":14,"user":{"displayName":"Benedikt Geiger","userId":"17925887631246406508"}}},"execution_count":8,"outputs":[]},{"cell_type":"markdown","source":["Build input data for the neural network."],"metadata":{"id":"LCV1mhpans5A"}},{"cell_type":"code","source":["def build_data(nctids, nctid_dict):\n","  \"\"\"\n","  Build NN input matrix for given list of nctid's and a dictionary,\n","  which maps an nctid to an embedding.\n","  \"\"\"\n","  n_rows = len(nctids)\n","  n_cols = len(list(nctid_dict.values())[0])\n","\n","  data = np.zeros((n_rows, n_cols), dtype=float)\n","\n","  for i,nctid in enumerate(nctids):\n","    \n","    data[i,:] = nctid_dict[nctid]\n","\n","  return data"],"metadata":{"id":"wvWOoXLdnSd7","executionInfo":{"status":"ok","timestamp":1651886542292,"user_tz":240,"elapsed":12,"user":{"displayName":"Benedikt Geiger","userId":"17925887631246406508"}}},"execution_count":9,"outputs":[]},{"cell_type":"code","source":["# train inputs\n","train_data_diseases = build_data(train_nctids, nctid2diseases)\n","train_data_incl = build_data(train_nctids, nctid2incl_criteria)\n","train_data_excl = build_data(train_nctids, nctid2excl_criteria)\n","train_data_drug = build_data(train_nctids, nctid2drugs)\n","train_data_target = build_data(train_nctids, nctid2target)\n","train_data_meta = build_data(train_nctids, nctid2meta)\n","\n","# validation inputs\n","val_data_diseases = build_data(val_nctids, nctid2diseases)\n","val_data_incl = build_data(val_nctids, nctid2incl_criteria)\n","val_data_excl = build_data(val_nctids, nctid2excl_criteria)\n","val_data_drug = build_data(val_nctids, nctid2drugs)\n","val_data_target = build_data(val_nctids, nctid2target)\n","val_data_meta = build_data(val_nctids, nctid2meta)\n","\n","# test inputs\n","test_data_diseases = build_data(test_nctids, nctid2diseases)\n","test_data_incl = build_data(test_nctids, nctid2incl_criteria)\n","test_data_excl = build_data(test_nctids, nctid2excl_criteria)\n","test_data_drug = build_data(test_nctids, nctid2drugs)\n","test_data_target = build_data(test_nctids, nctid2target)\n","test_data_meta = build_data(test_nctids, nctid2meta)"],"metadata":{"id":"kaPmiRbXqilL","executionInfo":{"status":"ok","timestamp":1651886544063,"user_tz":240,"elapsed":1780,"user":{"displayName":"Benedikt Geiger","userId":"17925887631246406508"}}},"execution_count":10,"outputs":[]},{"cell_type":"code","source":["# example\n","train_data_target.shape"],"metadata":{"id":"ay8w9686rFwa","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1651886555520,"user_tz":240,"elapsed":179,"user":{"displayName":"Benedikt Geiger","userId":"17925887631246406508"}},"outputId":"2efe023d-4536-463d-d84f-00dcfd57f479"},"execution_count":13,"outputs":[{"output_type":"execute_result","data":{"text/plain":["(3094, 1024)"]},"metadata":{},"execution_count":13}]},{"cell_type":"code","source":["# input dims of the model, i.e. the embedding dimensions\n","diseases_dim = train_data_diseases.shape[1]\n","\n","incl_criteria_dim = train_data_incl.shape[1]\n","excl_criteria_dim = train_data_excl.shape[1]\n","\n","drug_dim = train_data_drug.shape[1]\n","\n","targets_dim = train_data_target.shape[1]\n","\n","meta_dim = train_data_meta.shape[1]"],"metadata":{"id":"-l_kLfPsbe0A","executionInfo":{"status":"ok","timestamp":1651886544067,"user_tz":240,"elapsed":10,"user":{"displayName":"Benedikt Geiger","userId":"17925887631246406508"}}},"execution_count":12,"outputs":[]},{"cell_type":"markdown","source":["# Final Model"],"metadata":{"id":"s9LU1NeKE1pV"}},{"cell_type":"markdown","source":["# Automatic model evaluation\n","\n","Includes: ROC-plot, Precision-Recall curve, accuracy, ROC-AUC, PR-AUC, F1-score"],"metadata":{"id":"_HUYggIvVSeS"}},{"cell_type":"code","source":["def mean_and_sd(l):\n","  return np.mean(l), np.sqrt(np.var(l))"],"metadata":{"id":"LEmf4c6ZlR2d"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["def plot_learning_curve(history, save_path=None):\n","  \"\"\"\n","  Plot training and validation loss per epoch.\n","  \"\"\"\n","  plt.figure(figsize=(8,5))\n","  plt.title(\"Model accuracy\")\n","\n","  plt.plot(history.history[\"loss\"], label=\"Train\", color=\"black\")\n","  plt.plot(history.history[\"val_loss\"], label=\"Validation\", color=\"#990000\")\n","  \n","  plt.xlabel(\"epoch\")\n","  plt.ylabel(\"loss\")\n","  \n","  plt.legend(loc=\"best\")\n","\n","  if save_path: plt.savefig(save_path)\n","  \n","  plt.show()\n","\n","  \n","\n","\n","\n","def model_performance(model, input, y_true, save_path=None):\n","  \"\"\"\n","  Compute performance of 'model' applied to 'input' data with true labels 'y_true'\n","  Saves the ROC plot and precision-recall curve to 'save_path'\n","\n","  Return: dict\n","    accuracy, ROC-AUC, PR-AUC, F1-score\n","  \"\"\"\n","\n","  pred_prob = model.predict(input).squeeze()\n","  y_pred = (pred_prob > 0.5).astype(\"int32\")\n","\n","  #fig, axes = plt.subplots(1, 2, figsize=(10, 5))\n","\n","  fpr, tpr, _ = metrics.roc_curve(y_true, pred_prob)\n","  #metrics.RocCurveDisplay(fpr=fpr, tpr=tpr).plot(ax=axes[0], color=\"black\")\n","\n","  prec, recall, _ = metrics.precision_recall_curve(y_true, pred_prob)\n","  #metrics.PrecisionRecallDisplay(precision=prec, recall=recall).plot(ax=axes[1], color=\"black\")\n","\n","  #axes[0].set(xlabel=\"1 - specificity\", ylabel=\"sensitivity\", title=\"ROC plot\")\n","  #axes[1].set(xlabel=\"recall\", ylabel=\"precision\", title=\"Precision-Recall Curve\")\n","\n","  if save_path: plt.savefig(save_path)\n","  \n","  roc_auc = metrics.auc(fpr, tpr)\n","  pr_auc = metrics.auc(recall, prec)\n","\n","  ac = metrics.accuracy_score(y_true, y_pred)\n","  f1 = metrics.precision_recall_fscore_support(y_true, y_pred, average=\"binary\")[2]\n","\n","  return {\"accuracy\": np.round(ac, 3), \"roc_auc\": np.round(roc_auc, 3), \n","          \"pr_auc\": np.round(pr_auc, 3), \"F1\": np.round(f1, 3)}"],"metadata":{"id":"4ZuUb5tX1FlO"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# number of successful test trials\n","sum(test_labels) / len(test_labels)"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"9zyIEbrLGv2T","executionInfo":{"status":"ok","timestamp":1651874955515,"user_tz":240,"elapsed":248,"user":{"displayName":"Benedikt Geiger","userId":"17925887631246406508"}},"outputId":"38d2f868-09cb-49d0-ad41-77b21e6fab15"},"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["0.7495636998254799"]},"metadata":{},"execution_count":15}]},{"cell_type":"code","source":["min_val_loss = np.inf\n","best_dropout = 1.\n","best_lr = np.inf"],"metadata":{"id":"rtnEdJoGmjXP"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["dropouts = np.linspace(0.2, 0.4, 6)\n","lrs = [0.01, 0.005, 0.001, 0.0005, 0.0001]\n","n_runs = 5"],"metadata":{"id":"Os3SK3xpm3Ct"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["for do in dropouts:\n","\n","  for lr in lrs:\n","    \n","    print(f\"Current dropout: {do}, current learning rate:{lr}\")\n","    val_losses = np.zeros(n_runs, dtype=float)\n","\n","    for i in range(n_runs):\n","\n","      if i%5 == 0: print(f\"Run {i} for dropout {do} and learning rate {lr}\")\n","\n","      # inclusion and exclusion criteria\n","      inclusion_input = layers.Input(shape=(incl_criteria_dim,), name=\"inclusion\")\n","      inclusion_emb = layers.Dense(256, activation=\"relu\")(inclusion_input)\n","\n","      exclusion_input = layers.Input(shape=(excl_criteria_dim,), name=\"exclusion\")\n","      exclusion_emb = layers.Dense(256, activation=\"relu\")(exclusion_input)\n","\n","      inclusion_exclusion_raw = layers.Concatenate()([inclusion_emb, exclusion_emb])\n","      ie_dropout = layers.Dropout(rate=do)(inclusion_exclusion_raw)\n","      inclusion_exclusion_emb = layers.Dense(256, activation=\"relu\")(ie_dropout)\n","\n","      # diseases\n","      diseases_input = layers.Input(shape=(diseases_dim,), name=\"diseases\")\n","      diseases_emb = layers.Dense(128, activation=\"relu\")(diseases_input)\n","\n","\n","      # drug\n","      drug_input = layers.Input(shape=(drug_dim,), name=\"drug\")\n","      drug_emb = layers.Dense(64, activation=\"relu\")(drug_input)\n","\n","      # targets\n","      targets_input = layers.Input(shape=(targets_dim,), name=\"targets\")\n","      targets_emb = layers.Dense(64, activation=\"relu\")(targets_input)\n","\n","      # drug-target interaction\n","      drug_target_raw = layers.Concatenate()([drug_emb, targets_emb])\n","      dt = layers.Dropout(rate=do)(drug_target_raw)\n","      drug_target_emb = layers.Dense(64, activation=\"relu\")(dt)\n","\n","\n","      all_emb = layers.Concatenate()([inclusion_exclusion_emb, diseases_emb, drug_target_emb])\n","      ae = layers.Dropout(rate=do)(all_emb)\n","      trial_embedding1 = layers.Dense(128, activation=\"relu\")(ae)\n","\n","      meta_input = layers.Input(shape=(meta_dim,), name=\"meta\")\n","      emb_and_meta = layers.Concatenate()([trial_embedding1, meta_input])\n","\n","      trial_embedding2 = layers.Dense(64, activation=\"relu\")(emb_and_meta)\n","      trial_embedding3 = layers.Dense(32, activation=\"relu\")(trial_embedding2)\n","\n","      o = layers.Dense(1, activation=\"sigmoid\", name=\"trial_success\")(trial_embedding3)\n","\n","      model = tf.keras.Model(inputs=[inclusion_input, exclusion_input, diseases_input, drug_input, targets_input, meta_input], outputs=[o])\n","      # model.summary()\n","\n","\n","      # compile and train\n","\n","      model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=lr), loss=\"binary_crossentropy\", metrics=[\"accuracy\", \"AUC\"])\n","      callback = tf.keras.callbacks.EarlyStopping(monitor=\"val_loss\", patience=5, restore_best_weights=True)\n","\n","      history = model.fit(\n","        x={\"inclusion\": train_data_incl, \"exclusion\": train_data_excl, \n","          \"diseases\": train_data_diseases, \"drug\": train_data_drug,\n","          \"targets\": train_data_target, \"meta\": train_data_meta},\n","        y={\"trial_success\": train_labels},\n","        validation_data=(\n","            {\"inclusion\": val_data_incl, \"exclusion\": val_data_excl,\n","            \"diseases\": val_data_diseases, \"drug\": val_data_drug,\n","            \"targets\": val_data_target, \"meta\": val_data_meta},\n","            {\"trial_success\": val_labels}\n","        ), \n","        epochs=10,\n","        batch_size=128,\n","        callbacks=[callback]\n","      )\n","\n","      val_losses[i] = min(history.history[\"val_loss\"])\n","    \n","    if np.mean(val_losses) < min_val_loss:\n","        \n","        best_dropout = do\n","        best_lr = lr\n","        min_val_loss = np.mean(val_losses)\n","        print(f\"Current best dropout: {best_dropout}, current best learning rate:{best_lr}, current best average loss: {np.mean(val_losses)}\")"],"metadata":{"id":"1XUeqe8AE3aL"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["print(best_dropout, best_lr)\n","\n","# 0.32, 0.01"],"metadata":{"id":"Ls0cC6WzzJgO"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# RUN BEST MODEL AFTER GRID SEARCH 30 TIMES\n","n_runs = 30"],"metadata":{"id":"5-aBXz_fiL-3"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["val_F1 = np.zeros(n_runs, dtype=float)\n","val_acc = np.zeros(n_runs, dtype=float)\n","val_pr_auc = np.zeros(n_runs, dtype=float)\n","val_roc_auc = np.zeros(n_runs, dtype=float)\n","\n","test_F1 = np.zeros(n_runs, dtype=float)\n","test_acc = np.zeros(n_runs, dtype=float)\n","test_pr_auc = np.zeros(n_runs, dtype=float)\n","test_roc_auc = np.zeros(n_runs, dtype=float)"],"metadata":{"id":"RVuo6IPlh4HP"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["for i in range(n_runs):\n","\n","  # inclusion and exclusion criteria\n","  inclusion_input = layers.Input(shape=(incl_criteria_dim,), name=\"inclusion\")\n","  inclusion_emb = layers.Dense(256, activation=\"relu\", name=\"lower_dim_inclusion\")(inclusion_input)\n","\n","  exclusion_input = layers.Input(shape=(excl_criteria_dim,), name=\"exclusion\")\n","  exclusion_emb = layers.Dense(256, activation=\"relu\", name=\"lower_dim_exclusion\")(exclusion_input)\n","\n","  inclusion_exclusion_raw = layers.Concatenate(name=\"criteria\")([inclusion_emb, exclusion_emb])\n","  ie_dropout = layers.Dropout(rate=0.32)(inclusion_exclusion_raw)\n","  inclusion_exclusion_emb = layers.Dense(256, activation=\"relu\", name=\"lower_dim_criteria\")(ie_dropout)\n","\n","  # diseases\n","  diseases_input = layers.Input(shape=(diseases_dim,), name=\"diseases\")\n","  diseases_emb = layers.Dense(128, activation=\"relu\", name=\"lower_dim_diseases\")(diseases_input)\n","\n","\n","  # drug\n","  drug_input = layers.Input(shape=(drug_dim,), name=\"drug\")\n","  drug_emb = layers.Dense(64, activation=\"relu\", name=\"lower_dim_drug\")(drug_input)\n","\n","  # targets\n","  targets_input = layers.Input(shape=(targets_dim,), name=\"targets\")\n","  targets_emb = layers.Dense(64, activation=\"relu\", name=\"lower_dim_targets\")(targets_input)\n","\n","  # drug-target interaction\n","  drug_target_raw = layers.Concatenate(name=\"drug-targets\")([drug_emb, targets_emb])\n","  dt = layers.Dropout(rate=0.32)(drug_target_raw)\n","  drug_target_emb = layers.Dense(64, activation=\"relu\", name=\"lower_dim_drug-targets\")(dt)\n","\n","\n","  all_emb = layers.Concatenate(name=\"all_embeddings\")([inclusion_exclusion_emb, diseases_emb, drug_target_emb])\n","  ae = layers.Dropout(rate=0.32)(all_emb)\n","  trial_embedding1 = layers.Dense(128, activation=\"relu\", name=\"trial_embedding_1\")(ae)\n","\n","  meta_input = layers.Input(shape=(meta_dim,), name=\"meta\")\n","  emb_and_meta = layers.Concatenate()([trial_embedding1, meta_input])\n","\n","  trial_embedding2 = layers.Dense(64, activation=\"relu\", name=\"trial_embedding_2\")(emb_and_meta)\n","  trial_embedding3 = layers.Dense(32, activation=\"relu\", name=\"trial_embedding_3\")(trial_embedding2)\n","\n","  o = layers.Dense(1, activation=\"sigmoid\", name=\"trial_success\")(trial_embedding3)\n","\n","  model = tf.keras.Model(inputs=[inclusion_input, exclusion_input, diseases_input, drug_input, targets_input, meta_input], outputs=[o])\n","  model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=0.01), loss=\"binary_crossentropy\", metrics=[\"accuracy\", \"AUC\"])\n","  callback = tf.keras.callbacks.EarlyStopping(monitor=\"val_loss\", patience=5, restore_best_weights=True)\n","\n","  history = model.fit(\n","    x={\"inclusion\": train_data_incl, \"exclusion\": train_data_excl, \n","      \"diseases\": train_data_diseases, \"drug\": train_data_drug,\n","      \"targets\": train_data_target, \"meta\": train_data_meta},\n","    y={\"trial_success\": train_labels},\n","    validation_data=(\n","        {\"inclusion\": val_data_incl, \"exclusion\": val_data_excl,\n","        \"diseases\": val_data_diseases, \"drug\": val_data_drug,\n","        \"targets\": val_data_target, \"meta\": val_data_meta},\n","        {\"trial_success\": val_labels}\n","    ), \n","    epochs=10,\n","    batch_size=128,\n","    callbacks=[callback]\n","  )\n","\n","\n","  val_metrics = model_performance(\n","    model,\n","    {\"inclusion\": val_data_incl, \"exclusion\": val_data_excl, \n","    \"diseases\": val_data_diseases, \"drug\": val_data_drug,\n","    \"targets\": val_data_target, \"meta\": val_data_meta},\n","    val_labels,\n","    save_path=None\n","  )\n","\n","  test_metrics = model_performance(\n","    model,\n","    {\"inclusion\": test_data_incl, \"exclusion\": test_data_excl, \n","    \"diseases\": test_data_diseases, \"drug\": test_data_drug,\n","    \"targets\": test_data_target, \"meta\": test_data_meta},\n","    test_labels,\n","    save_path=None\n","  )\n","\n","  val_F1[i] = val_metrics[\"F1\"]\n","  val_acc[i] = val_metrics[\"accuracy\"]\n","  val_pr_auc[i] = val_metrics[\"pr_auc\"]\n","  val_roc_auc[i] = val_metrics[\"roc_auc\"]\n","\n","  test_F1[i] = test_metrics[\"F1\"]\n","  test_acc[i] = test_metrics[\"accuracy\"]\n","  test_pr_auc[i] = test_metrics[\"pr_auc\"]\n","  test_roc_auc[i] = test_metrics[\"roc_auc\"]"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":1000},"id":"Abp5DL43f7ic","executionInfo":{"status":"ok","timestamp":1651518004414,"user_tz":240,"elapsed":88484,"user":{"displayName":"Benedikt Geiger","userId":"17925887631246406508"}},"outputId":"856e52cb-9425-40e7-d0a0-20a17b7ac8ed"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["Epoch 1/10\n","25/25 [==============================] - 11s 327ms/step - loss: 5.1901 - accuracy: 0.5837 - auc: 0.4969 - val_loss: 0.6553 - val_accuracy: 0.6831 - val_auc: 0.5050\n","Epoch 2/10\n","25/25 [==============================] - 7s 287ms/step - loss: 0.6480 - accuracy: 0.6658 - auc: 0.5857 - val_loss: 0.6123 - val_accuracy: 0.6802 - val_auc: 0.6481\n","Epoch 3/10\n","25/25 [==============================] - 7s 284ms/step - loss: 0.6103 - accuracy: 0.6813 - auc: 0.6581 - val_loss: 0.6064 - val_accuracy: 0.6831 - val_auc: 0.6698\n","Epoch 4/10\n","25/25 [==============================] - 5s 193ms/step - loss: 0.5893 - accuracy: 0.6894 - auc: 0.6982 - val_loss: 0.5899 - val_accuracy: 0.6744 - val_auc: 0.6827\n","Epoch 5/10\n","25/25 [==============================] - 5s 199ms/step - loss: 0.5504 - accuracy: 0.7143 - auc: 0.7569 - val_loss: 0.5811 - val_accuracy: 0.7122 - val_auc: 0.7064\n","Epoch 6/10\n","25/25 [==============================] - 5s 194ms/step - loss: 0.5093 - accuracy: 0.7524 - auc: 0.8060 - val_loss: 0.6642 - val_accuracy: 0.6831 - val_auc: 0.6762\n","Epoch 7/10\n","25/25 [==============================] - 5s 198ms/step - loss: 0.4902 - accuracy: 0.7592 - auc: 0.8183 - val_loss: 0.6573 - val_accuracy: 0.6948 - val_auc: 0.6892\n","Epoch 8/10\n","25/25 [==============================] - 5s 197ms/step - loss: 0.4225 - accuracy: 0.8009 - auc: 0.8712 - val_loss: 0.6614 - val_accuracy: 0.6221 - val_auc: 0.6569\n","Epoch 9/10\n","25/25 [==============================] - 5s 191ms/step - loss: 0.3865 - accuracy: 0.8332 - auc: 0.8969 - val_loss: 0.7221 - val_accuracy: 0.7006 - val_auc: 0.6865\n","Epoch 10/10\n","25/25 [==============================] - 5s 193ms/step - loss: 0.3507 - accuracy: 0.8516 - auc: 0.9149 - val_loss: 0.7224 - val_accuracy: 0.6802 - val_auc: 0.7006\n"]},{"output_type":"display_data","data":{"text/plain":["<Figure size 720x360 with 2 Axes>"],"image/png":"iVBORw0KGgoAAAANSUhEUgAAAmEAAAFNCAYAAABIc7ibAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dfZxcdXn//9fboFINIEJqlSQEESrxpotdwdvu9ucCkSp4b4j8Chbd1YoV71qsFmjUije1aMV2R6QgdUFAa/PVIC5KhqqgCbKoxC8So0girZG7iigGuL5/nHM2Zyezu2d3Z+bMzfv5eOwjM+dm5polOVxzfa7z+SgiMDMzM7PWeljZAZiZmZn1IidhZmZmZiVwEmZmZmZWAidhZmZmZiVwEmZmZmZWAidhZmZmZiVwEmZdSdLJkr5Rdhxm1hqSXiPpqwWO+1dJf9eKmFpB0k8lDaWPz5L072XHZMU5CbMFSy8Cv5F0r6T/lnSBpMU1xzxH0tcl/UrSPZL+j6SVNcfsLekcST9LX+vH6fP9mxz/Bkmva+Z7mPWymmvE/9S7RixURHw2Io4ucNwbIuK9jXzvjKSQ9Ov0c26X9FFJi5rxXvNR1jXWpuckzBrlxRGxGOgDDgfele2Q9Gzgq8B/Ak8ADgJuBL4p6YnpMY8AvgY8BVgF7A08G7gDOKJ1H8PMmiS7RjwD6AfeU3uApD1aHlXj/VH6OQeAVwN/UXI8QOOvsV3y36p0TsKsoSLiv4ErSZKxzIeAz0TExyLiVxFxZ0S8B7gOOCs95s+B5cBLI2JzRDwUEb+IiPdGxPp675V+6/wrSVsl/VLShyXV/TudVuI2plW4jZKek25/P/B84BPpN8NPNOL3YGb1RcR24ArgqTD57/hNkm4Bbkm3vUjShKS7JX1L0tOz8yUtk/QFSTsk3ZH9m823ICjxT5J+Iel/JX1fUvZ+F0h6X+71Xi9pi6Q7Ja2T9ITcvpD0Bkm3pLGcK0kFP+cW4JvkroXz/FwHp6MId6TXuc9Kesycf/EFrrHp531SLqbJ35WkQUnbJP2NpP8G/k3SDyW9KHf8Hmn8z0ifPyv9nHdLulHS4Dzi7mpOwqyhJC0FXghsSZ8/CngOcFmdwy8FjkofDwFfiYh75/iWLyX5Vv0M4HjqfOuU9Fjgy8DHgf2AjwJflrRfRLwb+C/g1IhYHBGnzvH9zWwOJC0DjgVuyG1+CXAksFLS4cD5wAjJv9dRYJ2kR6ZDe18CbgVWAAcAl9R5m6OBPwEOBfYBXkVS8amN5f8DPpDuf3z6urWv9yLgmcDT0+OOKfg5n0zyBS+7Fs73cymN8QnAYcAydn15nYv5XmPz/gB4LHAgMAxcDJyQ238M8MuI+K6kA0iuu+9Lz3kH8HlJSxbw/l3HSZg1yhcl/Qq4DfgFcGa6/bEkf89ur3PO7UDWi7DfNMfM5oNpZe1nwDlMvSBk/gy4JSIuiogHIuJi4P8CL57H+5nZ/HxR0t3AN4Aq8A+5fR9I/x3/huR/7qMR8e2IeDAiLgTuB55FMmz2BOCdEfHriPhtRNS7AWcnsBfwZEAR8cOIqHd9eQ1wfkR8NyLuJ2mjeLakFbljzo6Iu9NrzNVMrfLX811JvwZ+CGwAPplun9fniogtETEeEfdHxA6SL5EDs8RQz3yvsXkPAWemsfwGGAOOS79sA6whScwATgTWR8T6tOo2DmwiScAt5STMGuUlEbEXMEhy4cuSq7tI/uE+vs45jwd+mT6+Y5pjZnNb7vGtJBeyWk9I91Fz7AHzeD8zm5+XRMRjIuLAiPjL9H/imfy/4wOBt6dDWHenidsykn/Hy4BbI+KBmd4oIr4OfAI4F/iFpIqkvescOuXakFaJ7mDqteG/c4/vAxYDSLopbWG4V9Lzc8c8Iz3m1STVvUcv5HNJepykS5Q0+v8v8O/sur7OxXyvsXk7IuK32ZN0yPWHwIvTROw4ksQMks/7yprP+7wGxNBVnIRZQ0VEFbgA+Ej6/NfAtcAr6xz+KpJGUYCrgGMkPbrOcTNZlnu8HPh5nWN+TnJBoObY7VnYc3xPM2us/L/B24D3pwlb9vOotIJ9G7C8SFN4RHw8Iv4YWEkyLPnOOodNuTak15/92HVtmOn1n5K2MCyOiP+q2RcRcSnJte+MBX6ufyD5/TwtIvYmqTAV6kurUeQaex/wqNzzP6jZX+9amQ1JHg9sThMzSD7TRTWf99ERcfY8Yu9aTsKsGc4BjpL0R+nz04GTlDTR7yVp37TZ89nA36fHXETyj/bzkp4s6WGS9pP0t5JmKl+/M329ZcBbgM/VOWY9cKikNWnj6KtJLsxfSvf/D/DEBX1iM2uUTwFvkHRk2mD/aEl/Jmkv4DskQ2pnp9v3lPTc2heQ9Mz0/IcDvwZ+S1KRr3Ux8FpJfZIeSZLwfDsiftqgz3I28HpJf7CAz7UXcC9wT9pnVS+ZLKLINXYCWCNpkaRVFBv2vISkB++N7KqCQVKxe7GkY9LX2zNt7l86z/i7kpMwa7i0b+EzpN8A096GY4CXkVxobiWZxuJ5EXFLesz9JI2j/xcYB/6X5MK0P/DtGd7uP4HrSS4eXwY+XSeeO0iaa99OUpL/a+BFEZENhX4MeIWkuyR9fN4f3MwWLCI2Aa8nGU68i6Sx/eR034MkvZxPAn4GbCMZ9qu1N0nScxfJ9eYO4MN13usq4O+Az5Ncmw4GVjfws3wfuIak12u+n+vvSYY47yG5xn1hnrEUuca+JY3jbpJ+uS8WeN3bSSp+zyH3JTgibiOpjv0tsIMkAXwnzjumUIRHYqwzSQrgkFz528zMrGM4IzUzMzMrgZMwMzMzsxJ4ONLMzMysBK6EmZmZmZXASZiZmZlZCTpuFfT9998/VqxYUXYYZtZC119//S8jouPXnPP1y6z3zHT96rgkbMWKFWzatKnsMMyshSTVLjvVkXz9Mus9M12/PBxpZmZmVgInYWZmZmYlcBJmZmZmVgInYWZmZmYlcBJmZmZmVgInYWZmZmYlcBJmZmZmVoKmJWGSzpf0C0k/mGa/JH1c0hZJ35P0jGbFYmZWz0KuU5JOknRL+nNS66I2s27RzErYBcCqGfa/EDgk/RkG/qWJsZiZ1XMB87hOSXoscCZwJHAEcKakfZsaqZl1nabNmB8R10haMcMhxwOfiYgArpP0GEmPj4jbmxWTmTVepVJhbGxsTuf09fVxzjnnNCmi4uZ7nQIGgfGIuBNA0jhJMndxo2I77bTTmJiYmHb/mjVrGB4ebtTbmVkJyuwJOwC4Lfd8W7ptN5KGJW2StGnHjh0tCc7MZlapVBgcHGRkZIRqtVp2OM0y3XWq1OvXxMTEnBNfM2s/HbF2ZERUgApAf39/lByOmQFjY2NMTEwwMDDgqswM5nv9mqlSODg4uOC4zKx8ZSZh24FluedL021m1say4ceJiQn6+vrYsGFD2SE103TXqe0kQ5L57RtaFpWZdYUyhyPXAX+e3n30LOAe94OZtb98ArZmzZqyw2m26a5TVwJHS9o3bcg/Ot1mZlZY0yphki4m+aa4v6RtJHcSPRwgIv4VWA8cC2wB7gNe26xYzKyxuqUCNt/rVETcKem9wMb0pdZmTfpmZkU18+7IE2bZH8CbmvX+ZtZYtcOQ3WAh16mIOB84vxlxmVlv6IjGfDNrnemmnMjugMwa8c3MbGGchJnZFNNVu3wXpJlZYzkJM7PddEvPl5lZO3MSZtajpht27KaeLzOzduYkzKzLFenxyuuRqSfMzErnJMysy7nHy8ysPTkJM+sB7vEyM2s/TsLMOtx0w40Z93iZmbUnJ2FmHWKuvV0Z93iZmbUnJ2FmHcK9XWZm3cVJmFkHcW+XmVn3eFjZAZiZmZn1IidhZmZmZiVwEmbW5iqVCoODg0xMTJQdipmZNZB7wszaVHY3ZP7uR9/laGbWPZyEmbXIbPN51apNvnz3o5lZd3ESZtYi000xMR0nX2Zm3c1JmFkLeYoJMzPLuDHfrMncWG9mZvW4EmbWIEWWFXJjvZmZZZyEmTWIlxUyM7O5cBJmNo253s2YJWDu+TIzsyLcE2Y2jayyVVRfX5+HG83MrDBXwsxm4MqWmZk1iythZmZmZiVwJcwsVdsDNpeJVc3MzObKSZj1vHprNIJ7vMzMrLmchFnPyxrwPZWEmZm1kpMwM9yAb2ZmrefGfDPraZJWSbpZ0hZJp9fZf6Ckr0n6nqQNkpbm9j0oaSL9WdfayM2s0zkJs57lNR1N0iLgXOCFwErgBEkraw77CPCZiHg6sBb4QG7fbyKiL/05riVBm1nXcBJmPSu/zJAb8HvWEcCWiNgaEb8DLgGOrzlmJfD19PHVdfabmc2Le8Ks52R3Q3qZIQMOAG7LPd8GHFlzzI3Ay4CPAS8F9pK0X0TcAewpaRPwAHB2RHyxBTGbWZdwEmY9o95UFK6AWQHvAD4h6WTgGmA78GC678CI2C7picDXJX0/In6cP1nSMDAMsHz58tZFbWZtz0mYdb3pki9PRWEkCdWy3POl6bZJEfFzkkoYkhYDL4+Iu9N929M/t0raABwO/Ljm/ApQAejv74+mfAoz60hOwqzreR4wm8FG4BBJB5EkX6uBKeVRSfsDd0bEQ8C7gPPT7fsC90XE/ekxzwU+1MrgzayzOQmznuDeL6snIh6QdCpwJbAIOD8ibpK0FtgUEeuAQeADkoJkOPJN6emHAaOSHiK5yensiNjc8g9hZh3LSZh1jdq1HzNeA9JmEhHrgfU1287IPb4cuLzOed8Cntb0AM2sa3mKCusa2bBjLU9BYWZm7ciVMOsqHnY0M7NO0dRKWIHlQJZLulrSDemSIMc2Mx4zMzOzdtG0JKzgciDvAS6NiMNJ7kr6ZLPiMTMzM2snzayEFVkOJIC908f7AD9vYjzWpbwGpJmZdaJm9oQVWQ7kLOCrkt4MPBoYamI81qW8BqSZmXWishvzTwAuiIh/lPRs4CJJT00nRZzkZT9sNm7INzOzTtPM4chZlwMBTgEuBYiIa4E9gf1rXygiKhHRHxH9S5YsaVK4ZmZmZq3TzErYrMuBAD8DXgBcIOkwkiRsRxNjsg4w3aSr0/FkrGZm1omaloQVXA7k7cCnJL2VpEn/5IjwArc9pjbpyi+0XYR7wczMrBM1tSeswHIgm0kWvbUelm+sB7zQtpmZ9YSyG/Oth2UVsCwBc2O9mZn1Eq8daaXx1BJmZtbLXAmzlnMFzMzMzJUwK4ErYGZmZq6EWUlcATMzs17nSpiZmZlZCZyEWct4oW0zM7NdnIRZy7gXzMzMbBcnYdZSWS+YJ2I1a72sGl2pVMoOxcxwEmZm1jPGxsaoVqtzWpvVzJrHSZiZmZlZCZyEmZn1gEqlQrVaLTsMM8txEmZm1gMaPQTp/jKzhXMSZk3nqSnM2sPAwAADAwMNeS33l5ktnJMwa5os+RoZGaFarXpqCrMGK1qNavRQpIc2zRrDSZg1TTYv2MDAAKOjo56awqzBilajsv2N+BJUqVQYGRlZ8OvkX8/DmtarvHakNZXXiDRrDwMDAwwPDy94+DA7f3R0dMGvVZvQ+Uua9RpXwszMulgjhw6z18oSuoW+ViMramadyEmYmfU0Sask3Sxpi6TT6+w/UNLXJH1P0gZJS3P7TpJ0S/pzUmsjL6aRQ5GNeq18AjY6OtqwmwXMOo2TMGs43w1pnULSIuBc4IXASuAESStrDvsI8JmIeDqwFvhAeu5jgTOBI4EjgDMl7duq2OeiUZWrRlTBahMwD0FaL3NPmC1YpVKZ0huSDX0MDAz4bkhrd0cAWyJiK4CkS4Djgc25Y1YCb0sfXw18MX18DDAeEXem544Dq4CLWxB3yzWqCpbvKXMCZr3OlTBbsOwuyIzvhrQOcgBwW+75tnRb3o3Ay9LHLwX2krRfwXObplqtztrr1ah+sCJVsCJ3OTayp8ysG7gSZg3huyCti70D+ISkk4FrgO3Ag0VPljQMDAMsX768GfFNq9HVq+lep8hdjvljXCE3S7gSZma9bDuwLPd8abptUkT8PCJeFhGHA+9Ot91d5Nz02EpE9EdE/5IlSxoafJEZ8BvRwzVb9Wq2qSqK9IFVq1XPF2Y9x5UwM+tlG4FDJB1EkkCtBqaUaSTtD9wZEQ8B7wLOT3ddCfxDrhn/6HR/V5mtCpbvAZ3tNWZKwPJ/eqjSeoUrYTZvvgvSOl1EPACcSpJQ/RC4NCJukrRW0nHpYYPAzZJ+BDwOeH967p3Ae0kSuY3A2qxJvx00oh9sLj1cMw1Vug/MrD5XwmzOsrshfRekdYOIWA+sr9l2Ru7x5cDl05x7PrsqY22lEf1gc6mC1ZuNv2gf2NDQEAA7d+6cd6xmnchJmM1Zfk3INWvW+NutWZtaSPWpEVWwotNRjI+PAzA4ODivWM06lZMwmxffDWnW3YpU0rIK1nR3Q3oY0mxm7gkzM+swC50fbLY7EYsmUOPj45NVrNrzPR2F2eychJmZdajZhgFnSoCq1eq0U0sstJ9sIbPie6oK6yVOwszMOtRMCc5ChwHne342k/9C3n+mBNGsmzgJMzOzSY1a6mg+VbRGvK9ZJ3Fjvs2qdoHuiYkJ+vr6SozIzOZrtkRnIUORtVNWmNnMnITZrLIpKbLEq6+vz822Zm0q31S/kHMXkkTN9/rg+cKs1zgJs0I8JYVZZyhayaqXpC20IX+mKSuK8Hxh1mvcE2Zm1mUWUslayLnTTVkxX9nSaL5T0rqVK2G2G/eAmXWv6Yb8FjKM2QzZXZYZ95hZN3IlzHaT9YBl3ANm1j2mq1Y1Yq1JM5sbV8KsLveAmfWedrmrMX+XZa2sUu91a60bNLUSJmmVpJslbZF0+jTHvErSZkk3SfLsfCXK+i/yVTAz6xzzneOrUXODNVptVS5bDsmTuVq3aFolTNIi4FzgKGAbsFHSuojYnDvmEOBdwHMj4i5Jv9+seGx2+akoPCRh1r6yvq5a8x1SbLehyPxdllls+fUozbpFM4cjjwC2RMRWAEmXAMcDm3PHvB44NyLuAoiIXzQxHivAw5Bm7W+mOxDnO6TYLkORsPvnyzfpj46OugpmXaOZw5EHALflnm9Lt+UdChwq6ZuSrpO0qonxmJlZjXYdiqyndkFwT2Fhna7sxvw9gEOAQWApcI2kp0XE3fmDJA0DwwDLly9vdYxdy1NRmFm7DUXWmm4pJE9hYd2gmZWw7cCy3POl6ba8bcC6iNgZET8BfkSSlE0REZWI6I+I/iVLljQt4F7jqSjMDNprKHI6vjZZNypUCZP0BeDTwBUR8VDB194IHCLpIJLkazVQ+6/oi8AJwL9J2p9keHJrwde3ecoqYFnlyz1gZp1vPpOtzjQVRLuotxRSJ8RtVkTRStgnSRKoWySdLekPZzshIh4ATgWuBH4IXBoRN0laK+m49LArgTskbQauBt4ZEXfM+VNYIVn/RHaLtytfZt1jIcOK7XwdmGkppHaO26yIQpWwiLgKuErSPiSVq6sk3QZ8Cvj3iKi75H1ErAfW12w7I/c4gLelP9YkWeUr/+3REx2adZ/5DCt2wlBkrXpTWJh1osKN+ZL2A04E/n/gBuCzwPOAk0ga661NZUOPTr6s20k6ADiQ3LUtIq4pLyJrhkYuEm5WpqI9Yf8B/CFwEfDiiLg93fU5SZuaFZw1jnu/rNtJ+iDwapK5CB9MNwfgJKyOTpmWwqybFa2EfSodWpwk6ZERcX9E9DchLmuA2gZ8sy73EuAPI+L+sgPpJO6rMitP0cb899XZdm0jA7HG8zJE1mO2Ag8vO4hO4/YEs/LMWAmT9Acks9z/nqTDAaW79gYe1eTYrAE8DGk95D5gQtLXgMlqWET8VXkhtYanbDDrTLMNRx4DnEwy0epHc9t/Bfxtk2KygmpnvK/lYUjrMevSn541l4r3dIuAm1nrzJiERcSFwIWSXh4Rn29RTFbQbP1eHoa0XhIRF0p6BMmkzwA3Tzd9Tjea61QT3XaHYfal1HeAWyeZbTjyxIj4d2CFpN3m8oqIj9Y5zVrIw41mCUmDwIXAT0laJ5ZJOmm2KSokrQI+BiwCzouIs2v2L09f9zHpMadHxHpJK0gmor45PfS6iHhDoz6PFed1JK1TzTYc+ej0z8XNDsTMbIH+ETg6Im4GkHQocDHwx9OdIGkRcC5wFMlathslrYuIzbnD3kOy4se/SFpJMgH1inTfjyPCY/4l8lQb1slmG44cTR9+MiJ2tCAeM7P5eniWgAFExI8kzXa35BHAlojYCiDpEuB4krnGJl+K5GYkgH2AnzcuZGuU0dHRBc2e7+FMK0PRKSq+Kemrkk6RtG9TIzIzm59Nks6TNJj+fAqYbTLpA4Dbcs+3pdvyzgJOlLSNpAr25ty+gyTdIKkq6fkLjN8WYCGJU6VSmVxT18sgWSsVSsIi4lCSkvxTgOslfUnSiU2NzMxsbt5IUsH6q/Rnc7ptoU4ALoiIpcCxwEWSHgbcDiyPiMNJ1r8dk7R37cmShiVtkrRpxw4PKDTa0NBQoTs9K5UKg4ODVCqV3baPjIw0KzyzGRWthBER34mIt5GU7+8kaVS1EmQXk4mJibJDMWsb6QoeH42Il6U//1Rg9vztwLLc86XptrxTgEvT97gW2BPYP32/O9Lt1wM/Ztedmfm4KhHRHxH9S5Ysmd+Hs2mNj49P3umZNejXJlvTVbryCdjo6CgDAwN1zzdrlkJJmKS9JZ0k6QrgWyTfAI9oamQ2Lc+Eb7aLpEvTP78v6Xu1P7OcvhE4RNJB6fQWq9l9rrGfAS9I3+MwkiRsh6QlaWM/kp4IHEIya39LuTF9dxMTE5PJ1nSVrtoEbHh4ePJ36WFJa5Wia0feCHwRWJt+E7SSeWoKs0lvSf980VxPjIgHJJ0KXEky/cT5EXGTpLXApohYB7wd+JSkt5I06Z8cESHpT4C1knYCDwFviIg7G/GB5sNfyHZNQLtz567p4bJkKt+4Xy8BMytD0STsiRERTY3EzGweIuL29OEvgd9ExEPp9BRPBq4ocP56kob7/LYzco83A8+tc97ngbaZxNqJxK4JaAcHB4Ek2apWq5MT2Y6NjU2ZU2y6BMzLP1mrzDZZ6zkRcRqwTtJuSVhEHNe0yMzM5uYa4PnpHdxfJRlqfDXwmlKjslLkk62sSpgfuq2XgNWrpJk102yVsIvSPz/S7EDMzBZIEXGfpFNI5jb8kCTfvdKDZku2pquA1VbSzJptxsb89I4fgL6IqOZ/AM8SbWbtRJKeTVL5+nK6bVGJ8VjJpku2PHRr7aLoFBUn1dl2cgPjsAI8NYXZjE4D3gX8R9pc/0Tg6pJjshLVG24sMqeYWavM1hN2ArCGZFbo/G3be5HMFWYt5KkpzKaXVuiruedbSSZttR4zXaKVDTeatYvZesKyOcH2J1kcN/MrYLb5d6wJPDWF2VTZDUSS/g/JFBJT+Aai3uNkyzrFbAt43wrcCjy7NeFYPdnCslkVzMym8A1EZtaRZhuO/EZEPE/Sr5j6DVNARMRu66RZ43kY0mx6uRuINpHOEwaQzmb/yNICMzObxWyVsOelf+7VmnAsr7YC5mFIsxl9DRgC7k2f/x7JfGHPKS2iFnCjuVnnKjRjvqSDgW0Rcb+kQeDpwGci4u5mBtfrXAEzm5M9IyJLwIiIeyU9qsyAWsH9T2adq+iyRZ8H+iU9CagA/wmMAcc2KzBLuAJmVtivJT0jIr4LIOmPgd+UHJOZ2bSKzhP2UEQ8ALwU+OeIeCfw+OaF1ds8H5jZvJwGXCbpvyR9A/gccGrJMZmZTatoJWxnOmfYScCL020Pb05I5mFIs7mLiI2Sngz8Ybrp5ojwIoA2Z9myR5VKxbPrW1MVrYS9lmSaivdHxE8kHcSu28KtCbJhSF8AzIpJ+7/+BnhLRPwAWCHpRSWHZR1sbGxs2n3ZiEWlUmlhRNZtCiVhEbE5Iv4qIi5On/8kIj7Y3NB6j4chzRbk34DfsWtew+3A+8oLxzrVbHecVioVRkZGqFarMyZqZrMplIRJeq6kcUk/krRV0k8kbW12cL3Gw5BmC3JwRHwI2AkQEfeRzGloNifj4+MMDAzU3ZclYLXbXBWz+SjaE/Zp4K3A9cCDzQvHfDek2bz9TtLvkU4snU6tc3+5IVmnqtcXlk/ARkdHGRsbo1qtTh6bGRsbY82aNW4nsVkV7Qm7JyKuiIhfRMQd2U9TIzMzm5szga8AyyR9lmTy1r8uNyTrdNlwY20CNjw8vFvyVa1WGzJM6cpa7yhaCbta0oeBL5D7ZpnNx2NmViZJDwP2BV4GPItkGPItEfHLUgOzrlAvAcsbHR3dbYiyEe8FuJrW5YpWwo4E+oF/AP4x/fFiuWbWFtL1Iv86rdJ/OSK+5ATMFiJrzs+qW7B7AjY0NMTQ0NCUbaOjowwMDFCtVmetZtVWvOr1m1l3K1QJi4g/bXYgZmYLdJWkd5BM0vrrbGNE3FleSNapxsfHGRwcnBxyrFcByy8ZlSVtw8PDk4lUdm69ala9hKu238y6X9G7Ix8n6dOSrkifr5R0SnNDMzObk1cDfwlUgU25H7N5yZKogYGBWYcFx8fH667jWa8iVpuAzVRts+5WdDjyAuBK4Anp8x+RLBFiZtYuVgLnAjcCE8A/A08pNSLrCnOdMigbpszkG/Vr+8vynID1nqJJ2P4RcSnwEEC6jqSnqmgQT9Jq1hAXAocBHydJwFam28zmpV7PVxHTVcVmavB3Atabit4d+WtJ+7Fr/p1nAfc0LaoeUalUJueZgaTk7UlazebtqRGxMvf8akmbS4vGOl69RGousmrYzp07p8wnlk+48r1k1nuKJmFvA9YBB0v6JrAEeMVsJ0laBXwMWAScFxFnT3Pcy4HLgWdGRNf1cGTJVq3a5Mv/CM0W5LuSnhUR1wFIOhL3hFmJsiRO2rVwQ23Fa6GJnnW2oknYwcALgWXAy0mmrJjxXEmLSPozjgK2ARslrYuIzTXH7QW8Bfj23ELvHPnliPKcfJk11B8D35L0s/T5cuBmSd8HIiKeXl5oZh5ytN0VTcL+LiIuk7Qv8Kckc4T9C0kyNp0jgC0RsRVA0iXA8UDt8MB7gQ8C75xL4J0gq4BlCZiXIzJrqlVlB2BWj4ccbTpFk7CsCS2cqKYAABaCSURBVP/PgE9FxJclvW+Wcw4Abss930ZN0ibpGcCy9PW6LgnzgtxmrRMRt5Ydg1k9HnK06RS9O3K7pFGSeXjWS3rkHM6tK11m5KPA2wscOyxpk6RNO3bsWMjbtlxWAfM3ILP2JGmVpJslbZF0ep39yyVdLekGSd+TdGxu37vS826WdExrIzezTlc0kXoVyTxhx0TE3cBjmX34cDtJD1lmabotsxfwVGCDpJ+SrPe2TlJ/7QtFRCUi+iOif8mSJQVDNjObWa539YUkU1qcIGllzWHvAS6NiMOB1cAn03NXps+fQjIU+sn09cwaxot5d7eiyxbdR7J4d/b8duD2WU7bCBwi6SCS5Gs1MDkmFxH3APtnzyVtAN7RjXdHmlnbKtK7GsDe6eN9gJ+nj48HLomI+4GfSNqSvt61rQjcul9+WovM2NiYb+jqIkV7wuYsIh6QdCpJBW0RcH5E3CRpLbApItY1673NzAqatXcVOAv4qqQ3A48GsqnQDwCuqzn3gOaEab2mNvmqTcichHWHBfV1zSYi1kfEoRFxcES8P912Rr0ELCIGXQUzszZ0AnBBRCwFjgUuSntaC+nknlYrX+3SRtZdmpqEmZm1udl6VwFOAS4FiIhrgT1JWimKnOueVpuXeksmZQlZtVp1j1iXcBJmZr1ssndV0iNIeldrK/U/A14AIOkwkiRsR3rcakmPTHtfDwG+07LIravl15+sl5DVW4XFOo+TsCbwgtxmnSEiHgCy3tUfktwFeZOktZKOSw97O/B6STcCFwMnR+ImkgrZZuArwJsi4sHd38VsYWoTMuseTWvM72WepNWsc0TEemB9zbYzco83A8+d5tz3A+9vaoBmOePj40iaHJJ0g35ncyWsSTxJq5mZNdPIyIjnEOtwTsLMzMw6SH5Islqtuj+sgzkJMzMz6yBei7J7uCfMzMysw2TVsJ07d5YciS2EK2EN5LsizcysFfJ3TILXmOxUroQ1QKVSYWxsbHJJiYGBAd8VaWZmLeE1JjuXk7AGyKakyJIv/6U3M7NW8BqTnc3DkQ3iKSnMzKwsXmOyMzkJMzMz61DTrTE5MDAw+dz9Yu3Lw5HzkPWAZbLZ8c3MzFop35yf3TE5PDw8+f+oSqXCyMjI5DH5ZC37f5nbaMrjStg8ZD1gGS9PZGZmZau9Y7JarU5JwPKy5KzeZK+unLWOK2HzlPWAmZmZtZt8c/7o6OiURKu2OpZfh3Kmypk1nithc+B5wMzMrJOMjo5OJlLVapXBwcHJJCvfzJ+tQzld5cyaw0nYHGTDkB5+NDOzdlbbsJ9VxqrVKgMDA5PJWe06lNm+fGN/EbVDmB7SLMbDkXPkYUgzM2t3060vma+MZcdJ2m1fvT6x6Zr4a4cwAQ9pFuQkzMzMrMvl75wssi+rnGWVrJnusKztL8v3o+X7zWx3Ho40MzPrcrV3ThbdNzIyUreJH6YmYPn+smxIM/8aHpqsz0lYAW7INzOzXpLvFYOZm/jz/WVDQ0OTq8fU9pvVDnGak7BC3JBvZma9JF8Zm62JPxtqrK2ojY+Pz6vJv5e4J6wgN+SbmVkvqe0Vm66JfybDw8MMDw8zODjYtDg7mZOwGWR3g3hZIjMz6zX1+sRmavCfSb7R3036u3g4cgYehjQzM9tlpib+ItwXNpUrYbPwMKSZmdnCDA0NcdVVV5UdRttxJawO3w1pZmbWOOPj427Qr8NJWB0ehjQzM7Nm83BkTm0jvochzczMrFlcCctxBczMzMxaxZWwGq6AmZmZWSu4EmZmZmZWAidhZmZmZiVwEmZmPU3SKkk3S9oi6fQ6+/9J0kT68yNJd+f2PZjbt661kZt1l2x6qEqlUnYoLeOeMDPrWZIWAecCRwHbgI2S1kXE5uyYiHhr7vg3A4fnXuI3EeE1zcwWqFKpMDIyMvm8V5Y2chJmZr3sCGBLRGwFkHQJcDyweZrjTwDObFFsZl0pmw4K4N5772Xx4sWTa0v2GidhZtbLDgBuyz3fBhxZ70BJBwIHAV/Pbd5T0ibgAeDsiPhinfOGgWGA5cuXNyhss85UrVbrJlwDAwOsWbOm59aWdE+YmVkxq4HLI+LB3LYDI6IfWAOcI+ng2pMiohIR/RHRv2TJklbFatZ28snX6OjolMcbNmzomSHIPCdhZtbLtgPLcs+XptvqWQ1cnN8QEdvTP7cCG5jaL2ZmdYyOjjI8PMzQ0BBDQ0O7JV/VarVnGvSbOhwpaRXwMWARcF5EnF2z/23A60hK+TuAv4iIW5sZk5lZzkbgEEkHkSRfq0mqWlNIejKwL3Btbtu+wH0Rcb+k/YHnAh9qSdRmHWhoaAjY1XQ/Pj6+2zFZtSz7s9urY02rhOXuOnohsBI4QdLKmsNuAPoj4unA5fgCZmYtFBEPAKcCVwI/BC6NiJskrZV0XO7Q1cAlERG5bYcBmyTdCFxN0hM2XUO/Wc8bHx+vm3hNp1qtdn01rJmVsFnvOoqIq3PHXwec2MR4zMx2ExHrgfU1286oeX5WnfO+BTytqcGZ9ZisWnbVVVcBTE5bka+IZXdXrlmzpuMrZc1MwgrfdZQ6BbiiifGYmZlZG8sqZUcdddSURCxLurLn0B1Dlm3RmC/pRKAf+PA0+4clbZK0aceOHQ1//2yW3omJiYa/tpmZmc3N+Pj4ZFUMkoRrZGRkyoSuQMdPadHMJKzQXUeShoB3A8dFxP31XqjZt3iPjY0xMTFBX1/fZKZtZmZm5RkfH2d0dJSBgYHJbQMDA4yOjk5J0Dp5uSNN7TNt4AtLewA/Al5AknxtBNZExE25Yw4nachfFRG3FHnd/v7+2LRpU0NjHRwcBGDDhg0NfV0zawxJ16fzcXW0Zly/zHrBUUcdBUy9o3JwcHDK3GMDAwNt+f/xma5fTesJi4gHJGV3HS0Czs/uOgI2RcQ6kuHHxcBlkgB+FhHHTfuiZmZm1nNmms6ikzV1nrDZ7jqKiKHdTmqh7A6LbCjSzMzMOsvo6GjH9oa1RWN+WdwLZmZm1pmmm3G/k/T8At59fX1tOYZsZmZm05vLxK/tqqcrYWZmZmZlcRJmZmZmVgInYWZmZmYlcBJmZmZmVgInYWZmZmYlcBJmZmZmHa9arXbc8kVOwszMzKwrVKvVjpq41UmYmZmZdbQ1a9ZMWei7UzgJMzMzs442PDzMhg0bOi4RcxJmZmZmVoKeX7bIzMzMukO1WgWgv7+fxYsXA8lQZbuuL+kkzMzMzLrK9ddfD8A+++wD0LZJmIcjzczMrCsMDQ1NPh4dHaWvr6/EaGbnJMzMzMy6wvj4OENDQwwNDbVt9SvPw5FmZmbWNcbHx8sOoTBXwszMzMxK0JNJWKVSYXBwkImJibJDMTMzsx7Vk0nY2NgYExMT9PX1sWbNmrLDMTMzsx7UUz1hlUplSgK2YcOGskMyMzOzHtVTlTBXwMzMzKxd9FQSBkxWwDrh1lUzMzObv2q1SrVapVKpTG7L+sLz28rSU8ORZma1JK0CPgYsAs6LiLNr9v8T8Kfp00cBvx8Rj0n3nQS8J933voi4sDVRm9lcjIyMUKlUWLx48eTSRlD+TPo9VwkzM8tIWgScC7wQWAmcIGll/piIeGtE9EVEH/DPwBfScx8LnAkcCRwBnClp31bGb2Yzy8+gf/3111OtVhkYGCgxoqmchJlZLzsC2BIRWyPid8AlwPEzHH8CcHH6+BhgPCLujIi7gHFgVVOjNbM5yWbQz4yOjrJhw4a2ScQ8HGlmvewA4Lbc820kla3dSDoQOAj4+gznHtCEGM1sAcbHxznqqKOAXcOP2ZBkpVIpdUjSlTAzs2JWA5dHxINzOUnSsKRNkjbt2LGjSaGZ2UzGx8frLmc0NjZWQjS7OAkzs162HViWe7403VbPanYNRRY+NyIqEdEfEf1LlixZYLhm1gj5IcoyOQkzs162EThE0kGSHkGSaK2rPUjSk4F9gWtzm68Ejpa0b9qQf3S6zcza3Pj4eFv0hbknzMx6VkQ8IOlUkuRpEXB+RNwkaS2wKSKyhGw1cElERO7cOyW9lySRA1gbEXe2Mn4z62xOwsysp0XEemB9zbYzap6fNc255wPnNy04M2u6bEnDNWvWtLxJ30mYmZmZ9ZzsDskyJ2/tiZ6wbImCiYmJskMxMzOzNlJmb1hPVMK8cLeZmZnlZXdIjo+PMzg4WEoMPZGEwa6Fu83MzMzy84aVNXlrVw9HehjSzMzMimr15K1dnYR5GNLMzMxmU9bkrV0/HOlhSDMzM5tJ1hdWrVan9Ic1e9qKrq6EmZmZmRVRO2XFxMRE04cnnYSZmZmZpUZHR9mwYQN9fX1Nf6+mJmGSVkm6WdIWSafX2f9ISZ9L939b0opmxmNmZmZWz9DQEENDQ3WHH7Mb/SqVSkPfs2k9YZIWAecCRwHbgI2S1kXE5txhpwB3RcSTJK0GPgi8ulkxmZmZmdWTn7IiU61W6e/v5/rrr598Do2bWb+ZlbAjgC0RsTUifgdcAhxfc8zxwIXp48uBF0hSE2MyMzMzm1U2q0KWgGVGRkY47bTTGvIezbw78gDgttzzbcCR0x0TEQ9IugfYD/hl/iBJw8AwwPLlywsH0IrxXDMzM+s+WbUrW9z7sssu46qrrmroMkcdMUVFRFSACkB/f38UPe+cc85pWkxmZmbW3YaHhyeTsWZMVdHM4cjtwLLc86XptrrHSNoD2Ae4o4kxmZmZmbWFZiZhG4FDJB0k6RHAamBdzTHrgJPSx68Avh4RhStdZmZmZp2qacORaY/XqcCVwCLg/Ii4SdJaYFNErAM+DVwkaQtwJ0miZmZmZtb1mtoTFhHrgfU1287IPf4t8MpmxmBmZmbWjjxjvpmZmVkJnISZmZmZlcBJmJmZmVkJnISZmZmZlcBJmJmZmVkJnISZmZmZlcBJmJmZmVkJ1GkT1EvaAdw6h1P2p2ZB8A7huFvLcbfWXOM+MCKWNCuYVvH1q+057tbq1LhhbrFPe/3quCRsriRtioj+suOYK8fdWo67tTo17lbr1N+T424tx916jYrdw5FmZmZmJXASZmZmZlaCXkjCKmUHME+Ou7Ucd2t1atyt1qm/J8fdWo679RoSe9f3hJmZmZm1o16ohJmZmZm1na5IwiStknSzpC2STq+z/5GSPpfu/7akFa2PcncF4n6bpM2Svifpa5IOLCPOemaLPXfcyyWFpLa4A6ZI3JJelf7eb5I01uoY6ynwd2W5pKsl3ZD+fTm2jDhrYjpf0i8k/WCa/ZL08fQzfU/SM1odY7vwNay1fP1qrU68fkGLrmER0dE/wCLgx8ATgUcANwIra475S+Bf08ergc91SNx/CjwqffzGdoi7aOzpcXsB1wDXAf2dEDdwCHADsG/6/Pc7JO4K8Mb08Urgp20Q958AzwB+MM3+Y4ErAAHPAr5ddsxt/N/X17AWxp0e5+tX6+Juu+tXGkvTr2HdUAk7AtgSEVsj4nfAJcDxNcccD1yYPr4ceIEktTDGemaNOyKujoj70qfXAUtbHON0ivzOAd4LfBD4bSuDm0GRuF8PnBsRdwFExC9aHGM9ReIOYO/08T7Az1sYX10RcQ1w5wyHHA98JhLXAY+R9PjWRNdWfA1rLV+/Wqsjr1/QmmtYNyRhBwC35Z5vS7fVPSYiHgDuAfZrSXTTKxJ33ikkGXc7mDX2tCy7LCK+3MrAZlHkd34ocKikb0q6TtKqlkU3vSJxnwWcKGkbsB54c2tCW5C5/hvoVr6GtZavX63VrdcvaMA1bI+GhmNNIelEoB8YKDuWIiQ9DPgocHLJoczHHiQl/UGSb+3XSHpaRNxdalSzOwG4ICL+UdKzgYskPTUiHio7MLNOuob5+lWKnr1+dUMlbDuwLPd8abqt7jGS9iApd97RkuimVyRuJA0B7waOi4j7WxTbbGaLfS/gqcAGST8lGStf1wbNrUV+59uAdRGxMyJ+AvyI5KJWpiJxnwJcChAR1wJ7kqxt1s4K/RvoAb6GtZavX63VrdcvaMQ1rOzGtwY0zu0BbAUOYlfT31NqjnkTU5taL+2QuA8naWg8pOx45xp7zfEbaI/G1iK/81XAhenj/UlKzft1QNxXACenjw8j6alQG/zOVzB9U+ufMbWp9Ttlx9vG/319DWth3DXH+/rV/Ljb8vqVxtPUa1jpH7BBv6RjSTL+HwPvTretJfnmBUlWfRmwBfgO8MSyYy4Y91XA/wAT6c+6smMuGnvNsW1xESv4OxfJUMRm4PvA6rJjLhj3SuCb6QVuAji6DWK+GLgd2EnyDf0U4A3AG3K/63PTz/T9dvk70qb/fX0Na2HcNcf6+tX8uNvu+pXG1fRrmGfMNzMzMytBN/SEmZmZmXUcJ2FmZmZmJXASZmZmZlYCJ2FmZmZmJXASZmZmZlYCJ2FWyGyryZdJ0tp0QkgkPV/STZImJB0g6fJZzj1P0sr08d+2Il4zs1qSTpb0ifTxWZLeUXZM1nxOwqyoC0gmAmw7EXFGRFyVPn0N8IGI6IuI7RHxilnOfV1EbE6fOgkzszlRwv8vtXnxXxwrJGZfTX5Gks6WtFnS9yR9JN12gaR/lbRJ0o8kvSjdvkjShyVtTI8fyb3O30j6vqQbJZ2de51XSHod8CrgvZI+K2lFVrlLX/Mjkn6Qvuab0+0bJPWnr/V7aQXts2l17bTc+75f0lvm+/nNrHuk15abJX0G+AHwd7nr1d/njvvzdNuNki5Kt71Y0rcl3SDpKkmPK+tzWPm8gLc1naT9gJcCT46IkPSY3O4VwBHAwcDVkp4E/DlwT0Q8U9IjgW9K+irwZOB44MiIuE/SY/PvExHnSXoe8KWIuFzSitzu4fS9+iLigTrnni7p1IjoS2NeAXwBOCf9lrs6jdPMDJI1GU8C9gZeQXJ9EMk6k39Csrbne4DnRMQvc9ecbwDPSq+FrwP+Gnh7y6O3tuAkzFrhHuC3wKclfQn4Um7fpRHxEHCLpK0kidbRwNMlZUOJ+5Bc8IaAf4uI+wAiYi6VuSGStfceKHJuRPxU0h2SDgceB9wQEWUvmGxm7ePWiLgurewfDdyQbl9Mcr36I+CyiPglTLnmLAU+J+nxJGsp/qS1YVs78XCkNUQ63DeR/qzN70sTnyOAy4EXAV/J7655qSD5NvnmtK+rLyIOioivNjP+aZwHnAy8Fji/hPc3s/b16/RPsasPtS8inhQRn57hvH8GPhERTwNGSNYFtR7lJMwaIiIezF2Ezsjvk7QY2Cci1gNvJfmGmHmlpIdJOhh4InAzcCXwRkkPT88/VNKjgXHgtZIelW6fMqQ4i3FgRNIeM5y7M3vP1H+Q3IzwzDQmM7NaVwJ/kV7nSO/K/n3g6yTXt/3S7dk1Zx9ge/r4pFYHa+3Fw5FWiKSLgUFgf0nbgDNn+baXtxfwn5L2JPnW+Lbcvp8B3yHpq3hDRPxW0nkk/VvflSRgB/CSiPiKpD5gk6TfAespfkfjecChwPck7QQ+BXyi5phKuv+7EfGaiPidpKuBuyPiwYLvY2Y9JCK+Kukw4NrkcsW9wIkRcZOk9wNVSQ+SDFeeDJwFXCbpLpJE7aBSAre2oIja0SCz1pB0AWkTfdmx1JM25H8XeGVE3FJ2PGZm1l08HGlWRzqB6xbga07AzMysGVwJMzMzMyuBK2FmZmZmJXASZmZmZlYCJ2FmZmZmJXASZmZmZlYCJ2FmZmZmJXASZmZmZlaC/wcztrhCyVlwtAAAAABJRU5ErkJggg==\n"},"metadata":{"needs_background":"light"}},{"output_type":"display_data","data":{"text/plain":["<Figure size 720x360 with 2 Axes>"],"image/png":"iVBORw0KGgoAAAANSUhEUgAAAmEAAAFNCAYAAABIc7ibAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdeZhU1bX38e8SRMRmEhCUUVAkSEyr7YxpjBMa0cRZ1AtO6E301WgcEr3qxRiHRMTkaqQjRuUGEYdrUEEJCogDCiio4AA4MogIggLKuN4/zqmiuqimq5uqOjX8Ps9TT59hnzqrWnq7au999jZ3R0RERERya7uoAxAREREpRUrCRERERCKgJExEREQkAkrCRERERCKgJExEREQkAkrCRERERCKgJEyKkpkNNLNXoo5DRHLDzM42s/FplLvfzP4rFzHlgpl9amZHhds3m9n/Rh2TpE9JmGyzsBL43sxWmdmXZvaQmZUllTnUzF4ys+/MbKWZPWNmPZPKNDOzoWb2efhe88P91lmOf5KZXZjNe4iUsqQ6YkmqOmJbufs/3f2YNMpd4u63ZPLeMWbmZrY6/JwLzWyImTXIxr3qI6o6VmqmJEwypZ+7lwHlwL7A72InzOwQYDzwL2A3YHdgFvCqmXUNyzQCXgT2BvoCzYBDgGXAgbn7GCKSJbE6Yj+gArghuYCZNcx5VJn3k/BzVgJnAOdHHA+Q+Tq2SP5bRU5JmGSUu38JvECQjMXcCTzi7ve4+3fuvtzdbwCmAjeHZf4D6AT80t3nuPsmd//K3W9x97Gp7hV+6/x/ZvaxmX1tZn8ys5T/psOWuGlhK9w0Mzs0PH4rcDjwP+E3w//JxO9BRFJz94XAOKAXxP+Of21mc4G54bETzGymma0ws9fMbJ/Y9WbW0cyeMrOlZrYs9jebOATBAneb2Vdm9q2ZvWtmsfs9ZGZ/SHi/i8xsnpktN7MxZrZbwjk3s0vMbG4Yy71mZml+znnAqyTUhfX8XN3CXoRlYT33TzNrUedffBp1bPh590iIKf67MrM+ZrbAzK41sy+Bf5jZ+2Z2QkL5hmH8+4X7B4efc4WZzTKzPvWIu6gpCZOMMrMOwHHAvHC/CXAo8HiK4qOBo8Pto4Dn3X1VHW/5S4Jv1fsBJ5HiW6eZ7Qw8B/wFaAUMAZ4zs1bufj0wBbjU3cvc/dI63l9E6sDMOgLHA28nHP4FcBDQ08z2BR4ELib4ex0GjDGzHcKuvWeBz4AuQHtgVIrbHAP8FOgONAdOJ2jxSY7lZ8Bt4fldw/dNfr8TgAOAfcJyx6b5OXsQfMGL1YX1/VwWxrgb8COgI5u/vNZFfevYRO2AnYHOwCDgUeCshPPHAl+7+1tm1p6g3v1DeM1vgSfNrM023L/oKAmTTHnazL4DvgC+Am4Kj+9M8O9scYprFgOxsQitaihTmzvClrXPgaFUrxBifg7MdfcR7r7B3R8FPgD61eN+IlI/T5vZCuAVYDLwx4Rzt4V/x98T/M99mLu/4e4b3f1hYC1wMEG32W7A1e6+2t1/cPdUD+CsB5oCPQBz9/fdPVX9cjbwoLu/5e5rCYZRHGJmXRLK3O7uK8I6ZiLVW/lTecvMVgPvA5OA+8Lj9fpc7j7P3f/t7mvdfSnBl8jKWmJIpb51bKJNwE1hLN8DI4ETwy/bAP0JEjOAc4Cx7j42bHX7NzCdIAGXkJIwyZRfuHtToA9BxRdLrr4h+MPdNcU1uwJfh9vLaihTmy8Stj8jqMiS7RaeI6ls+3rcT0Tq5xfu3sLdO7v7r8L/icck/h13Bq4Ku7BWhIlbR4K/447AZ+6+YWs3cveXgP8B7gW+MrMqM2uWomi1uiFsJVpG9brhy4TtNUAZgJnNDocwrDKzwxPK7BeWOYOgdW+nbflcZtbWzEZZMND/W+B/2Vy/1kV969hES939h9hO2OX6PtAvTMROJEjMIPi8pyV93t4ZiKGoKAmTjHL3ycBDwJ/D/dXA68BpKYqfTjBQFGACcKyZ7ZSi3NZ0TNjuBCxKUWYRQYVAUtmFsbDreE8RyazEv8EvgFvDhC32ahK2YH8BdEpnULi7/8Xd9wd6EnRLXp2iWLW6Iax/WrG5btja++8dDmEoc/cpSefc3UcT1H03buPn+iPB7+fH7t6MoIUprXFpSdKpY9cATRL22yWdT1VXxrokTwLmhIkZBJ9pRNLn3cndb69H7EVLSZhkw1DgaDP7Sbh/HTDAgkH0Tc2sZTjY8xDgv8MyIwj+aJ80sx5mtp2ZtTKz35vZ1pqvrw7fryNwOfBYijJjge5m1j8cOHoGQcX8bHh+CdB1mz6xiGTK34FLzOygcID9Tmb2czNrCrxJ0KV2e3i8sZkdlvwGZnZAeP32wGrgB4IW+WSPAueZWbmZ7UCQ8Lzh7p9m6LPcDlxkZu224XM1BVYBK8NxVqmSyXSkU8fOBPqbWQMz60t63Z6jCMbg/SebW8EgaLHrZ2bHhu/XOBzc36Ge8RclJWGSceG4hUcIvwGGYxuOBU4mqGg+I5jGore7zw3LrCUYOPoB8G/gW4KKqTXwxlZu9y9gBkHl8RwwPEU8ywgG115F0CR/DXCCu8e6Qu8BTjWzb8zsL/X+4CKyzdx9OnARQXfiNwQD2weG5zYSjOXcA/gcWEDQ7ZesGUHS8w1BfbMM+FOKe00A/gt4kqBu6gacmcHP8i7wMsFYr/p+rv8m6OJcSVDHPVXPWNKpYy8P41hBMF7u6TTedzFBi9+hJHwJdvcvCFrHfg8sJUgAr0Z5RzXmrp4YKUxm5sCeCc3fIiIiBUMZqYiIiEgElISJiIiIREDdkSIiIiIRUEuYiIiISASUhImIiIhEoOBWQW/durV36dIl6jBEJIdmzJjxtbsX/Jpzqr9ESs/W6q+CS8K6dOnC9OnTow5DRHLIzJKXnSpIqr9ESs/W6i91R4qIiIhEQEmYiIiISASUhImIiIhEQEmYiIiISASUhImIiIhEQEmYiIiISASUhImIiIhEIGtJmJk9aGZfmdl7NZw3M/uLmc0zs3fMbL9sxSIiksq21FNmNsDM5oavAbmLWkSKRTZbwh4C+m7l/HHAnuFrEPC3LMYiIpLKQ9SjnjKznYGbgIOAA4GbzKxlViMVkaKTtRnz3f1lM+uylSInAY+4uwNTzayFme3q7ouzFZOIRO+KK64AYOjQoRFHUv96CugD/NvdlwOY2b8JkrlHMxXbv/71L7766isAGjVqxCmnnEJZWVmm3l5E8kCUyxa1B75I2F8QHtsiCTOzQQTfQunUqVNOghORzKqqqmLkyJFMnjyZysrKqMNJV031VE3Ht1Df+uu2227jjTfeiO9v2rSJ8847L+3rRST/FcTake5eBVQBVFRUeMThiEgdVVVVcfHFFwNQWVlJ//79I44od+pbf40ZM4b169ezZMkS9t9/f9atW5e1GEUkGlEmYQuBjgn7HcJjIlIkElu/AIYNG8agQYMijqpOaqqnFhJ0SSYen5TJG++yyy4AbLedHmIXKVZR/nWPAf4jfProYGClxoOJFI9Y61es+7EAEzCouZ56ATjGzFqGA/KPCY+JiKQtay1hZvYowTfF1ma2gOBJou0B3P1+YCxwPDAPWANosINIgYu1fAEF0fpV33rK3Zeb2S3AtPCtBscG6YuIpCubT0eeVct5B36drfuLSG4lj/uKjf3K1wQMtq2ecvcHgQezEZeIlIaCGJgvIvmrCMZ9iYhEQkmYiNRZqm7HQmj5EhHJJ0rCRCRtya1ehdLtKCKSj5SEiUitUiVfSrxERLaNkjAR2UJidyOoy1FEJBuUhImUuOSEC6onXbGfSr5ERDJLSZhIiUrVxRijpEtEJPuUhImUED3VKCKSP5SEiZSQkSNHMnPmTMrLy5V8iYhETEmYSBFLHu8VS8AmTZoUXVAiIgIoCRMpKlt7qhGgvLyc/v37RxKbiIhUpyRMpEgkr90Y+6kuRxGR/KQkTKTAae1GEZHCpCRMpIAlt36p1UtEpHAoCRMpMKmmmVDrl4hI4VESJlJAklu+1PolIlK4lISJFIjEBEwtXyIihU9JmEieqmm6CSVgIiLFYbuoAxCR1GKz28dUVlYqARMRKSJqCRPJY5rdXkSkeKklTCTPVFVV0adPn2qtYCIiUnzUEiaSR1LN+yUiIsVJSZhIHtCs9yIipUdJmEhEUk26qnm/RERKh5IwkRypacoJTboqIlKalISJZFlyV2NlZWX8pxIvEZHSpSRMJMti830p6RIRkURKwkSyqKqqismTJ1NZWan5vkREpBolYSJZkNwFqakmREQkmZIwkQxKNf5LXZAiIpKKkjCRDEk10aqSLxERqYmSMJEMSEzANNGqiIikQ0mYSD3UNOeXEjAREUmXFvAWqYfYtBMxlZWVSsBERKRO1BImUovkVi+AmTNnUl5ermknRESk3pSEiaRQ07qOMeXl5Zp2QkREtomSMJEEqaaY0JOOIiKSDUrCRBJoiaHSY2Z9gXuABsAD7n570vnOwINAG2A5cI67LwjPbQTeDYt+7u4n5ixwESl4SsJE2NwCprFepcXMGgD3AkcDC4BpZjbG3eckFPsz8Ii7P2xmPwNuA84Nz33v7uU5DVpEioaejhSBagmYxnqVlAOBee7+sbuvA0YBJyWV6Qm8FG5PTHFeRKRespqEmVlfM/vQzOaZ2XUpzncys4lm9raZvWNmx2czHpGtibWAqQuypLQHvkjYXxAeSzQLODnc/iXQ1MxahfuNzWy6mU01s19kN1QRKTZZS8ISmvmPI/gmeZaZ9UwqdgMw2t33Bc4E7stWPCKpVFVV0adPn2pzfokk+S1QaWZvA5XAQmBjeK6zu1cA/YGhZtYt+WIzGxQmatOXLl2as6BFJP9lsyUsnWZ+B5qF282BRVmMR2QL6oYseQuBjgn7HcJjce6+yN1PDr8sXh8eWxH+XBj+/BiYBOybfAN3r3L3CnevaNOmTVY+hIgUpmwOzE/VzH9QUpmbgfFmdhmwE3BUFuOREqdJVyWFacCeZrY7QfJ1JkGrVpyZtQaWu/sm4HcET0piZi2BNe6+NixzGHBnLoMXkcIW9cD8s4CH3L0DcDwwwsy2iEnN+bKtYgtsx+b/ilELWGlz9w3ApcALwPsEwyNmm9lgM4tNN9EH+NDMPgLaAreGx38ETDezWQQD9m9PeqpSRGSrstkSVmszP3AB0BfA3V83s8ZAa+CrxELuXgVUAVRUVHi2ApbiFEvAQAtsy5bcfSwwNunYjQnbTwBPpLjuNeDHWQ9QRIpWNpOwWpv5gc+BI4GHzOxHQGNATV2yTZK7HWOtX0rAREQkn2QtCXP3DWYWa+ZvADwYa+YHprv7GOAq4O9m9huCQfoD3V0tXVJvia1esbUeNfu9iIjko6zOmJ9GM/8cgsGsIttM3Y4iIlJIoh6YL5IxsS5IJWAiIlIIlIRJUamsrFQCJiIiBUFJmIiIiEgElIRJwdPSQyIiUoiyOjBfJJNSzXgPm6egiD0FKSIiUgiUhEnBSFznMZGmoBARkUKkJEwKQlVVFZMnT6ayslLrPIqISFHQmDApCLFuSHU3iohIsVASJnkvsRVMXY4iIlIs1B0peSs2ED828F6tYCIiUkyUhEleSl4DUgPvRUSk2CgJk7yjNSBFRKQUaEyY5B2tASkiIqVASZjkFQ3CFxGRUqEkTPJGYjekBuGLiEix05gwiVzyU5DqhhQRkVKgJEwiF1uOSE9BiohIKVESJpGJtYDF1oPUckQiueXumFnUYYiULCVhklOxxAuIdz/GWsBEJHdWr15NWVkZV155JXfddVfU4YiUJA3Ml5yoqqqiT58+XHzxxdWSr2HDhjFp0iR1QYrk2G233QbAkCFDIo5EpHSpJUyyTrPfi+SfoUOHAsHfpIhEQ0mYZJ0mXxXJL8uWLWP16tUA7LzzzhFHI1K61B0pWaXJV0Xyzy233BJ1CCKCkjDJIk2+KpKf/vrXvwLQpUuXaAMRKXFKwiRr1A0pkn++/fZbNm3aRIsWLfj000/5v//7v/jfqojklpIwySp1Q4rkl6qqKgB+85vfxI89/fTTUYUjUtKUhEnGxaajmDlzZtShiEiSWBJ22WWXxY/16tUrqnBESpqSMMm4xFnwNRZMJH9s2LCBuXPnAtCyZcv4cc2aLxINTVEhGaNliETy25QpUwA47bTTgCApa9iwIYsXL2bjxo00aNAgyvBESo5awiRj1AImkt9is+NfeeWV1Y7/7W9/46qrrooiJJGSppYwyYjE+cDUAiaSn5599lkADjrooC3OvfHGG7kOR6TkqSVMtpnmAxPJf8uXLwdgzz33jI8Bc/f4+QMPPDCSuERKmZIw2WaaD0wk/z311FMAnH/++fFjiUnYdtvpfwciuaa/OtkmWpZIpDCMGDECgLPPPjt+bPvtt+ecc86JKiSRkqckTLZJrBVM3ZBSqMysr5l9aGbzzOy6FOc7m9mLZvaOmU0ysw4J5waY2dzwNSC3kdfNyy+/DEDHjh2rHR8xYgTNmjXL6L1mzZrF7373O3744YeMvq9IsdHAfKk3tYJJoTOzBsC9wNHAAmCamY1x9zkJxf4MPOLuD5vZz4DbgHPNbGfgJqACcGBGeO03uf0UtVuyZAmQekA+BEsZ1TS5sruzceNGGjZM738X7k55eTkA/fr149BDD61HxCKlQS1hUm9qBZMicCAwz90/dvd1wCjgpKQyPYGXwu2JCeePBf7t7svDxOvfQN8cxFxnsWWJTj755BrL1PRU88EHH0zz5s3TbtUaOHBgfPujjz6qNu5MRKpTEibbRK1gUuDaA18k7C8IjyWaBcSyl18CTc2sVZrX5oUnnngCgDPPPLNO161YsYI333yTNWvWsGjRIjZu3Jiy3Ndff82iRYtYs2YNjzzySPz4eeedx8SJE+sfuEiRUxImdaa1IaXE/BaoNLO3gUpgIZA6G0nBzAaZ2XQzm7506dJsxbhVsUQoeTxYzCmnnMLee++9xfHEL1jdunWjf//+PPzww9UW/N64cSNt2rShY8eO9O7de4v3+O6777Y1fJGipSRM6kwz40sRWQgkZiYdwmNx7r7I3U92932B68NjK9K5Nixb5e4V7l7Rpk2bTMdfq40bN7Jx40a6d+9epzUi3Z3HH3+82rHRo0czcOBAfvnLX8aPXXrppQBs2rSJt99+G4CHHnoofn5b1qXcsGEDY8aMYeXKlfV+D5F8ltWB+WbWF7gHaAA84O63pyhzOnAzwcDWWe6u/6vnmdiakDFaG1KKyDRgTzPbnSCBOhOoVgeZWWtgubtvAn4HPBieegH4o5nFVsI+JjyfV15//XUATjzxxDpdN2HChFrLbNy4kfvvv7/asauvvpoBAwbQrFmzrY5BS0dZWRlr166lb9++XHjhhZxyyinb9H4i+SZrLWEJTx0dRzCw9Swz65lUZk+CSuswd98buCJb8Uj9xVq+YtQCJsXC3TcAlxIkVO8Do919tpkNNrNY1tIH+NDMPgLaAreG1y4HbiFI5KYBg8NjeWX06NHA5kW70xXrikzVktWjRw+AlOtN3n578F27c+fOdbpfsuuuu461a9cC8Pzzz3PqqafSrFmzOieTIvksmy1h8aeOAMws9tRR4qPfFwH3xh7pdvevshiP1IPWhJRi5+5jgbFJx25M2H4CeKKGax9kc8tYXho3bhwAFRUVtZZdsGABzz33HGeffTaffvopAEceeeQWrWJmhrtzzz33VDt+yy23ZGTm/Y8++og77rhji+PfffcdzzzzzDa/v0i+yOaYsHSeHOoOdDezV81sath9KRGLDbzv06eP1oQUKXDz5s1ju+22Sys5+slPfsIll1wSn0V/wIABPP300/zrX/+Kl/n5z38OwL333gvAXnvtFe92vP766+sd59KlSxk8eDBLly5lr732AuCyyy6r9/uJFIKoB+Y3BPYkaO4/C/i7mbVILpQPTxcVu+TEa/LkyUAwBYXWhBQpTCtWrADgsMMOq7Xsxo0b44t8x5KuW2+9lZ122qlaF2CTJk2AzQnSiy++yJNPPom7V+u6jE1ncfrpp9d6b3dnl1124aabbmKXXXaJH//LX/7CwoULufbaawE44IADaNSoUa3vJ1IospmEpfPk0AJgjLuvd/dPgI8IkrJqon66qBQkjvuKJV6TJk1i0qRJSsBEClRsCEE646g++OCDavsNGzakffvNnRfuHp949f333wegcePG1cokmjdvHgBr165l11133ep8YRdccMEWx9asWQPAbrvtxu233467c+SRR9b4Hhs2bKBfv37VHiISyXdpJWFm9pSZ/dzM6pK0xZ86MrNGBE8djUkq8zRBK1jsCaTuwMd1uIdkQGzcV+yJRyVeIsVh7NhgqNtRRx1V52uvueaaWsv8+9//rvHcGWecEd/+8ssvueaaa3jvvffix1566SWOOOIIJk+ezD/+8Y9q1z722GPsuOOOdYp3++2359lnn+Xss89m1qxZdbpWJCrpJlX3ETy2PdfMbjezvWq7IM2njl4AlpnZHILlQK5292V1/hSyTbT8kEhxmjFjBkDKiVgTpVpa6Le//W3Ksp999ll8O9XkrDHJY9CmT5/O/vvvDwRdlUceeSSTJk2iT58+ANXmHkunCxOCNS8BDjnkkGrHY2tXiuS7tJIwd5/g7mcD+wGfAhPM7DUzO8/Mtt/KdWPdvbu7d3P32GPdN7r7mHDb3f1Kd+/p7j9291Hb/pGkPrT8kEjxWbRoERC0Em3N9OnTtzjWsmXLFCXhzTffBOBPf/pTneNZt24dACeccMIW55588kmmTJnC+vXr03qv0047jebNm9OlSxemTp1a51hE8kHa3YvhWmkDgQuBtwkmYd2PYNFaEZHImVl7MzvUzH4ae0UdU1RWrlzJl19+mVar0ueffw5snsZia2OvYq688spaywwfPjz+pGXMhRdeyPPPP1/t2Ny5czEzevfuTcOGtc+c9PTTT8fXw4y1zL300ktMnDiRsrKyWlv+RPJFumPC/g+YAjQB+rn7ie7+mLtfBpRlM0DJrth4MJFCZ2Z3AK8CNwBXh6/UfWolIDZ4vlu3bmlfE0ts/vnPf9ZYZtasWUyYMCGtKS/OP/98RowYUS0RHD58OLB5qosmTZqwxx57pB3junXrqnVdQjBL/xFHHEGfPn049thjmT179hZzmInko3Rbwv4edhne5u6LAcxsBwB3r30GQMlbGg8mReQXwF7ufry79wtfJTu9+rRp04D0pqc444wz6Nq1K507d8bdadu2bY1l99lnn7RayhI99thjWySDzzzzDDNmzGDVqlVpv0/i2LUDDzwwvn3nnXfGt5988kkArrhCC7BI/ks3CftDimOvZzIQyb3E2fA1HkyKwMfA1gc/lZAxY4KH0ROTlZqMGjWK+fPnZzWe2JQVAOPHj8fM2G+//eq0wPeQIUPi22+88Ua1aTNSeeWVV+oXrEiObLXz3czaEcxyv6OZ7QvE/lqaEXRNSoFJXIw71g2pVjApEmuAmWb2IrA2dtDd/190IUUntuxQPs6tePTRR9frumOOOYbnnnuOr7/+usYy69ati0/o+vzzz2/1CU6RqNU2AvJYgsH4HYAhCce/A36fpZgki2KTspaXl1NZWUn//v3VCibFYgxbzkVYsubPn1/jE45RWb16da1Pam7Ns88+W2uZ7bffnnfeeYd99tmHcePG8Yc/pOrIEckPW03C3P1h4GEzO8Xdn8xRTJIFsRawWAKmxbil2Lj7w+HE0N3DQx+6e3rzHRSZlStX4u6cddZZUYdSTWzJo2zbfffdAXjrrbdycj+R+qqtO/Icd/9foIuZbfE8srsPSXGZ5Jmqqqr4Qtyx1i+RYmNmfYCHCeYyNKCjmQ1w95ejjCsKsSWIWrduHXEk0SgrK6OiooJddtmFgw8+mB9++IHXXnstZ0mgSLpq647cKfypaSgKWGwMmBbiliJ3F3CMu38IYGbdgUeB/SONKgIvvPACsPUZ7UtBbNkmCOYT+9GPfhRhNCJbqq07cli4eZ+7L81BPJIlegJSSsD2sQQMwN0/2tqKHsUsNoHpoYceGnEk0UleBeDUU09l9uzZEUUjklq6U1S8ambjzewCM8uvkZ6yVZqMVUrIdDN7wMz6hK+/A1uux1MCYgtYN27cOOJIote3b18A5syZw4QJEyKORqS62teHANy9u5kdCJwJXB8uuD0qHC8meUyTsUoJ+U/g10BsSoopwH3RhROdGTNmUF5eToMGDaIOJTLXX389GzZs4Pbbb4/PRXb00UczZ84cysrK6NixY8QRiqSZhAG4+5vAm2b2R4LpKh4GlITlqcSnIdUVKaXA3dcS1E0l/cBQbAHsrU1iWgoSp6Y4/fTTGT16NAA9e/akUaNGrF27dotrVq5cSfPmzXMWo0i6a0c2M7MBZjYOeA1YDNQ+DbNEJnE6CrWCSTEzs9Hhz3fN7J3kV9Tx5dqXX34JwGmnnRZxJPkjeS3MdevW8eGHwfDBJUuWsH79esyMFi1aaPoeyal0W8JmAU8Dg91dyxUVCM0HJiXi8vDnCZFGkSfeeSfIO1u1ahVxJPmjYcOGjBs3juOOOy5+7O9//zsNGzbkjjvuqFZ2a7Pxi2RauklYVy/1tm0RyUvuvjjc/Br43t03hdNT9ADGRRdZNKZMmQLAfvvtF3Ek+aVv3758+umnTJ06lTPPPJO77rorZbm6rGUpsq222h1pZkPDzTFmtsUrB/GJiKTrZaCxmbUHxgPnAg9FGlEE1q1bB0CvXr0ijiT/dO7cmeOPPz7luUceeQSA4cOH5zIkKXG1tYSNCH/+OduBSObEpqWorKyMOhSRXDJ3X2NmFxDMbXinmc2MOqhcGz16NI0aNdLs8DVo2rRpfPukk06ib9++dOrUiYYNg/8djhs3junTp1NRURFViFJCapusdUa4We7u9ySeM7PLAU1AlYc0LYWUKDOzQ4CzgQvCYyU3R0OLFi1YsWJF1GHktVdeeYVhw4bFW7+g+tOkBxxwgFYYkZxId7LWASmODcxgHJJhmpZCStAVwO+A/3P32WbWFZgYcUw5tW7dOmbPns0xxxwTdSh57bDDDquWgEEwFiyx9yC23q5INtW2gPdZQH9g9+pLoV0AACAASURBVKQxYE2B5dkMTOpHXZFSqtx9Mgmt8+7+MZsnbi0Jy5cH1bK6Iutn0qRJ8YH5vXv3ZtOmTWy3XbptFSJ1V9uYsNicYK0JFseN+Q4oufl3CoG6IqXUmNlQd7/CzJ4BtniK291PjCCsSMyfPx8IWnqkftydFi1a8Morr3D00Ufz4osvRh2SFLHaxoR9BnwGHJKbcCQT1BUpJUYPEIU++eQTAC3Js41WrlwJwEsvvcTcuXPZc889I45IilVtU1S8Ev78zsy+TXh9Z2bf5iZESUdVVRV9+vRh5sySexhMSlzCA0TTgSnuPjnsmnwFmBZdZLk3Y0bwq+jRo0fEkRS28847L77dvXv3+JJHIpm21STM3XuHP5u6e7OEV1N3b5abECUdWqZIhBeBxMFQOwITIoolEkuXLgWgXbt2EUdS2B588EGGDNm8BOkZZ5wRYTRSzNJdO7Kbme0Qbvcxs/9nZi2yG5qkI7EFLLZMkboipUQ1dvdVsZ1wu6RGqM+dO5dOnTpRVlYWdSgF79e//nW1/S+++CKiSKSYpfvYx5PARjPbA6gCOgIjsxaVpE0tYCJxq80svlaPme0PfB9hPDk3Z86c+ALesm0aNWpUbe6wTp06MWrUqAgjkmKUbhK2yd03AL8E/uruVwO7Zi8sqQu1gIkAwTxhj5vZlHA862PApRHHlFONGzfmlFNOiTqMovLkk0/Gt7WkkWRaugt4rw/nDBsA9AuPbZ+dkERE6s7dp5lZD2Cv8NCH7r4+yphyaenSpXz99dc0atQo6lCKysknnxzfTnzg4auvvqJVq1Y0aFByizJIBqXbEnYewTQVt7r7J2a2O5sfC5cI6GlIkerMrAlwLXC5u78HdDGzEyIOK2e++uorAPbaa69aSkpduTs777xzfP+8886jbdu2/OpXv4owKikGabWEufscEmaedvdPgDuyFZRsXVVVVXxJjcrKSo0FEwn8A5jB5nkNFwKPA89GFlEOxeYI22OPPSKOpLjdddddPPTQQwAsXLgw2mCk4KWVhJnZYcDNQOfwGgPc3btmLzRJVlVVxciRI5k8OViZRQvMilTTzd3PCIdO4O5rLLYGTQmYO3cuAB06dIg4kuI1bty4+KoEAN98802E0UgxSLc7cjgwBOgNHABUhD8lh2JPQlZWVioBE9nSOjPbkXDpIjPrBqyt7SIz62tmH5rZPDO7LsX5TmY20czeNrN3zOz48HgXM/vezGaGr/sz/YHqYtOmTYAmas2W5cuXxxOwm2++GYDXXnuNRx99NMKopNClm4StdPdx7v6Vuy+LvbIamaSkJyFFanQT8DzQ0cz+STB56zVbu8DMGgD3AscBPYGzzKxnUrEbgNHuvi9wJnBfwrn57l4evi7J0Oeol9j0Cdtvr2emsqlfv37cdNNN8f3+/fuzZs2aer3Xww8/zH333Vd7QSla6SZhE83sT2Z2iJntF3tlNTIRkTSZ2XZAS+BkYCDwKFDh7pNqufRAYJ67f+zu64BRwElJZRyIrRDSHFiUobAzqm3btmy//faaqDXLxowZAwTDQ2JuvfVWAN555x3MjIEDB9b6PqeffjoDBw7k17/+Na+99lr8+MMPP8zNN98cb9mU4pbuFBUHhT8rEo458LPMhiPJYuPAgPikrCJSnbtvMrNr3H008FwdLm0PJE6FvoDN9V3MzcB4M7sM2Ak4KuHc7mb2NvAtcIO7T0m+gZkNAgZBMOFntkyePJkf//jHWXv/Upc4cSvARRddxJw5cxg6dCh//OMfmT59OuPHjweCRCo2eD+Vk046KZ7MARx22GGMGDGC8ePHM2JEMPFAy5YtufzyyzP/QSSvpNUS5u5HpHgpAcuB2DgwQLPii2zdBDP7rZl1NLOdY68MvO9ZwEPu3gE4HhgRtrwtBjqF3ZRXAiPNbIs1dd29yt0r3L2iTZs2GQinusWLF/PUU0+xatWq+NqRkht33XVXfDuWgMXEnmCPWb58OW3btsXM4gnYY489Fj9/7rnnxhMwgCuuuAIz4+23385G6JIn0l07sq2ZDTezceF+TzO7ILuhlbZUa0JqLJjIVp0B/AqYDExPeG3NQoJl2GI6hMcSXQCMBnD314HGQGt3XxsbG+vuM4D5QPdt/Ax1dsEFF8RnyT/77LNzffuStt12Nf8vtKqqioYNG9KhQwc6d+5Mq1at4nO5AUydOpXTTz893pUZ89JLL1XbTxx/JsUn3TFhDwEvALuF+x8RLBEiWRCbB2zy5Mlq/RJJX0+CQfazgJnAX4G9a7lmGrCnme1uZo0IBt6PSSrzOXAkgJn9iCAJW2pmbcKB/ZhZV2BP4OMMfZa0zZo1K76d3GUm2ff+++8zceJEBgwYgLtz5JFHxs9t3LiRhQsX8vnnn1e75vXXX+egg4Je79///ve8//77AEybNo0jjjiCDz74IF62a1fNBFXM0h0T1trdR5vZ7wDcfYOZbcxiXCUrcSJWTUMhUicPE4zN+ku43z88dnpNF4R12aUEXzIbAA+6+2wzGwxMd/cxwFXA383sNwRjYQe6u5vZT4HBZrYe2ARc4u7Ls/XharJo0ebnBNq2bZvr25e8Hj160KNHD/r06QPAhAkTmDZtGgceeOAWZT/77DNgy7GBPXr0qJZA77XXXrg7zZo147333mPTpk1st912uDuXX3459913H9dccw0VFRW0adOGww8/PHsfULIq3SRstZm1YvP8OwcDK7MWVQmLDcJXAiZSZ73cPXF6iYlmNqe2i9x9LDA26diNCdtzgMNSXPck8GTy8Sjts88+UYcgwAEHHMDUqVNZsWIFDRo0oGXLluy3337Ude7g7777jhdffJGmTZtuMQ3GbbfdFt9WC2jhSjcJu5Kgib6bmb0KtAFOre0iM+sL3EPwDfMBd7+9hnKnAE8AB7h7bWM4ilLsKcjYZKxKwETq7C0zO9jdpwKY2UHUPiasqDRsmG6VLtkW627MhPrOQyb5L90xYd0IJjM8lKDZfi61JHBpToKImTUFLgfeSD/s4qIxYCIZsT/wmpl9amafAq8DB5jZu2b2TrSh5cYhhxxSeyEpGBs2bNji2OzZs1m/fn21sWLl5eUsX57znnDJgHS/Nv2Xuz9uZi2BI4A/A39jy/l0EsUnQQQws9gkiMndA7cQLAZ+dV0CLwZaC1Iko/pGHUCurVq1qtp+o0aNIopEsqFBgwZs2rSJV199ld69e1c7N3/+/Hj35qxZs/jggw849NBDowhTtkG6LWGxQfg/B/7u7s8Btf21p5oEsX1igXDW/Y7h+5UcrQUpkjnu/tnWXlHHlw1q/Sh+ZrZFAhbz7rvvxrcTH9CQwpFuErbQzIYRzMMz1sx2qMO1KYWTHQ4hePKotrKDzGy6mU0vtskItRakiIjUR69evbj//mDd+OHDh0ccjdRHuonU6QRjwY519xXAztTefVjbJIhNgV7ApHD8xsHAGDNLXBoJyP6M01GoqqqKd0OKiNTHkiVL4tu77LJLhJFIVGJf4J9//nn2339/Pv74Yz0tWUDSXbZojbs/5e5zw/3F7j6+lsu2Ogmiu69099bu3sXduwBTgRNL5enI2FQUGoQvIvWVuKTN6tWrI4xEopKYcL311lt069aNv/3tbxFGJHWxTV2KW+PuG4DYJIjvA6NjkyCa2YnZum8h0VQUIrItGjRoEN/W4t2labvttqNfv37VjiU/sCH5K2tJGASTILp7d3fv5u63hsduDGehTi7bp1RawdQVKSKZ1q1bt6hDkIiMGTMGd+fuu+8GNF9cIclqEiZbSlyWSF2RIrItnn766fh2qjmlpLScf/75NZ7buHEjq1ev5o033tCYsTyiJCzHtCyRiGTKTjvtFN/u1atXhJFIPrnqqqto3749H3/8MePHj+eMM86gYcOGlJWVcfDBB/POO++watUqvv3226hDLXlqs8yhWDekxoKJSKZ16NAh6hAkYolrUy5atKjGLury8nIAWrZsyTPPPMO+++5LkyZNchKjVKeWsBxRN6SIZNrXX38ddQiSR5o2bcrYsWNTnmvduvUWx7755ht69+7NTjvtxMqVKzEzGjdurCdtc0hJWJZVVVXRp0+feAKmbkgR2VZr1qxh8ODBvPjii/Fjxx13XIQRSb447rjjmDlzZnz/r3/9K59//jlLly5l9erVNS5t1aJFCwDWrl1LWVkZZoaZ8Y9//CMncZcqJWFZpqWJRCTTpk6dyk033QTACSecgLvTtm3biKOSfPGTn/yEVatW4e5ceumldOwYzJvepEkT1q5di7szYsSItP7NbG2wv2w7JWE5oKWJRCSTNm3aFN/u0qVLdIFI3kp8aCOVc845hy+//JIvv/wSCFrQ3J3HH398i7LTp5fE7FGRUBKWJbFuyMRmYRERkXzStm1b3D0+luzUU0/F3atNY7FgwYKowit6SsKyIDYIf/LkyZSXl2sgvohkjaYZkGx56623gODf2PLlyyOOpjhpiooMS3wKUmPARCQbJk2aFN8+9thjowtESsKAAQNo0qSJnprMArWEZZASMBHJhfXr18e3axv7I1JfixYtim+vWbOGvffeOz6GTDJDLWEZpNnwRSTbTj31VJ588smow5AScMQRR1TbnzNnDrvuuisrV66kWbNmEUVVXNQSlmGaDV9Esik5AWvQoEFEkUixa9KkCe7Oe++9V+148+bNOeecc/jTn/6klrFtpCRMRKSAHXbYYVGHIEVu7733xt3p2rVr/Ng///lPrrnmGnbddVcuvPDCCKMrbErCMiS2LqSISC7tsMMOUYcgJWL+/Pl88MEHWxwfPnw4zz//PD/88AMffPABo0aNqjbFhdRMY8IyJDYeTNNRiIhIsdprr73YsGEDa9euZcOGDTRv3hzYctmsXr160atXryhCLChKwjJI48FERKTYNWjQgCZNmgCwatUqysrKtiizbt26XIdVkNQdKSJSIL7//vstju24444RRCIS2Gmnnfjhhx/o3LkzQ4YM4eyzzwZg2rRpEUdWGJSEbYPY0kRankhEciHV8jFmFkEkIpvtsMMOfPrpp/zmN7/h5JNPBrTeZLqUhG2DkSNHxpMvLU8kIiKlLraCwwMPPMCGDRsAcHfGjx/Pa6+9xqpVq6IML+9oTFg9xZ6GrKysrLaEiIiISKlKXMHhZz/7Gf/617/Yeeedq5XRk5ObqSWsnvQ0pIiISM2mTJmyRQIm1SkJq4fEVjA9DSkiuTBmzBi6d+8edRgitZo9e3a1/W7durFp06b4/uLFi3MdUt5SElYPagUTkVwbO3Zs1CGIpKVnz57x7RtvvJF58+ZVe4Bk8ODBUYSVlzQmrJ7UCiYiIpLasmXLWL9+PW3bto0fW7hwIe3bt+f+++/n2Wef5dNPPy35tU/VEiYiIiIZtfPOO1dLwABatGgR316wYAF33HFHrsPKO0rCREQKVJcuXaIOQSRtTZo0YcaMGfH9oUOHRhhNflASVkdaqFtE8sGwYcN4++23ow5DpE7222+/+BQVp5xySsTRRE9JWB1pUL5IcTGzvmb2oZnNM7PrUpzvZGYTzextM3vHzI5POPe78LoPzezYXMZ9+OGHV+veESk0999/P5988knUYURKSVg9aFC+SHEwswbAvcBxQE/gLDPrmVTsBmC0u+8LnAncF17bM9zfG+gL3Be+n4ikqWvXrhx66KFMnDgRM6NLly60a9eOO++8M+rQckJJmIiUsgOBee7+sbuvA0YBJyWVcaBZuN0cWBRunwSMcve17v4JMC98PxGpxbnnnhvffv311/nZz34GwGeffcaSJUu49tprowotp5SEiUgpaw98kbC/IDyW6GbgHDNbAIwFLqvDtVmz/fbb5+pWIhn3yCOP0LVr16jDiJySsDrQoHyRknQW8JC7dwCOB0aYWdp1p5kNMrPpZjZ96dKlGQuqW7duGXsvkSjMnz8fd+fOO+9kypQpfPfdd8ydOzd+fsmSJRFGlxtKwupAg/JFis5CoGPCfofwWKILgNEA7v460Bhonea1uHuVu1e4e0WbNm0yFnjiDOQihezqq6+md+/elJWVsccee3DTTTcBwcD9YqckrI40KF+kqEwD9jSz3c2sEcFA+zFJZT4HjgQwsx8RJGFLw3JnmtkOZrY7sCfwZs4iFylSr7zyCgA333xztIHkgJIwESlZ7r4BuBR4AXif4CnI2WY22MxODItdBVxkZrOAR4GBHphN0EI2B3ge+LW7b8z9pxApLg888AAAzZs3jziS7NPakSJS0tx9LMGA+8RjNyZszwEOq+HaW4FbsxqgSInp0qULhx9+OFOmTGHJkiXstNNOlJWVRR1WVqglLE0alC8iIpIbU6ZMAaBdu3a0a9eOdevWRRxRdigJS0NVVRUXX3wxoEH5IiIi2XbJJZfEt1evXl20Y7GVhKUh9lTksGHDivYfgogUjtjaeyLFKvaEZIxawuohjTXZrjSzOeF6bC+aWedsxlNXVVVV9OnTh5kzZ+qpSBHJC7vvvnvUIYhkXbt27XD3+BeORx99lFtvvZW1a9dGHFlmZS0JS3NNtreBCnffB3gCyKvFokaOHMnMmTMpLy9XN6SI5IUdd9wx6hBEInHDDTfQuHFjNm4snoeQs9kSVuuabO4+0d3XhLtTCSY7zAuxgfjl5eVMmjRJrWAiIiIR6NGjR7X9a665hnXr1vHII4/wzTffRBRVZmQzCavrumoXAOOyGE+daHZ8ERGR6L3//vvVxkEOGTKEHXbYgQEDBjB48OAII9t2eTFPmJmdA1QAlTWcHwQMAujUqVPO4tI4MBGJWrEOSBapK3ffYrmuoUOH4u7cfffdBbmUVzZbwtJaV83MjgKuB05095Qj7rK19pqISL5btmxZ1CGI5I3nnnuO0047jWHDhsWP3XPPPSxatCjCqOovm0lYrWuymdm+wDCCBOyrLMZSJ5qYVUREJP8cf/zxjB49mkGDBrFw4eZ2nUKdtiVrSViaa7L9CSgDHjezmWaWvHBuJDQeTETy1YYNG6IOQSQv7Lbbblx11VUAfPjhhxFHUz9ZHROWxppsR2Xz/ttC48FEJB9169Yt6hBE8kZsHNgNN9zAQQcdVHBrTGrGfBGRAtK9e/eoQxDJG7HljaZOnVqQX1CUhImIiEhBatq0aXz7q6++YrfdduPdd9/FzGjVqlXed1MqCRMREZGCtMsuu1QblL948WL22WcfAJYvX878+fOjCi0tSsISJK4VKSIiIoWhppnzzz333Lxeb1JJWAKtFSkiIlJ4WrRowYoVKxg+fDjuztdffw0ErWGNGzfmz3/+c8QRpqYkLInWihQRESk8zZs35/zzzwegVatW1c5dffXVUYRUKyVhIiIiUnS+//57xo8fH983M1599dUII9qSkrCQZskXEREpHo0bN+boo4/myCOPjB/r3bs3J554ImvWrIkwss2UhIU0S76IFILf/va3UYcgUlAmTJhQba3JZ555hiFDhkQY0WZKwhJolnwRyWdvvvkmHTp0iDoMkYIzaNAgjjvuuPh+o0aNIoxmMyVhIiIiUvTGjh3LqlWrALj22mt59tlnI45ISZiISF7bY489og5BpGhs3Lgxvt2vX78IIwkoCUOD8kUkfzVr1izqEESKRrNmzWjfvn1838y46aabIotHSRgalC8iIlIqFixYUG1/8ODB1ZY+yqWSTsISlynSoHwREZHSYGbV9t96661I4ijpJEzLFIlIIUmeBVxE6mfTpk24OxdddBEAFRUVTJo0KedxlGwSFhsHpmWKRKRQdO3aNeoQRIrKPffcE9++++67c37/kkzCqqqquPjiiwGNAxMRESlVO+64Iz/88AMQzZeckkzCYgPxhw0bphYwERGRErbDDjsAMHToUNatW5fTe5dkEgaaHV9ERESqW7ZsWU7vV7JJmIiIiAjAHXfcAQTTVXz//fc5u6+SMBERESlpjz/+OAD3338/TZo04eWXX87JfZWEiYiISEmbOHFitf1f/OIXObmvkjAREREpaWVlZbh7fOb8b775hiVLlmT9vkrCRERERJK0a9eOuXPnZvUeSsJEpKSZWV8z+9DM5pnZdSnO321mM8PXR2a2IuHcxoRzY3IbuYhkw/r16+Pb3bt358EHH8zavZSEiUjJMrMGwL3AcUBP4Cwz65lYxt1/4+7l7l4O/BV4KuH097Fz7n5izgIXkaxp2LAh11xzTXz/ggsuwMyYOnVqxu+lJExEStmBwDx3/9jd1wGjgJO2Uv4s4NGcRCYikbnjjjvYtGlTtWOHHHIIZsZPf/rTjN1HSZiIlLL2wBcJ+wvCY1sws87A7sBLCYcbm9l0M5tqZrl5nEpEcsLMOP7447c4PmXKFD777LOM3ENJmIhIes4EnnD3jQnHOrt7BdAfGGpm3ZIvMrNBYaI2fenSpbmKVUQy4LnnnsPdWb58OX/4wx/ix2fMmJGR91cSJiKlbCHQMWG/Q3gslTNJ6op094Xhz4+BScC+yRe5e5W7V7h7RZs2bTIRs4jkWMuWLbn++uv58ssvufXWWznmmGMy8r5KwkSklE0D9jSz3c2sEUGitcVTjmbWA2gJvJ5wrKWZ7RButwYOA+bkJGoRiUTbtm35/e9/T1lZWUber+SSsKqqKiZPnhx1GCKSB9x9A3Ap8ALwPjDa3Web2WAzS3za8UxglMdmcgz8CJhuZrOAicDt7q4kTETS1jDqAHJt5MiRAPTv3z/iSEQkH7j7WGBs0rEbk/ZvTnHda8CPsxqciBS1kmsJA6isrGTQoEFRhyEiIiIlrKSSMHVFioiISL4oqSRMXZEiIiKSL0oqCQN1RYqIiEh+KLkkTERERCQflEwSpvFgIiIikk9KIgmrqqri4osvBjQeTERERPJDVpMwM+trZh+a2Twzuy7F+R3M7LHw/Btm1iWT96+qqqJPnz7xBGzYsGEaDyYiIiJ5IWtJmJk1AO4FjgN6AmeZWc+kYhcA37j7HsDdwB2ZjGHkyJHMnDmTyspKJWAiIiKSV7I5Y/6BwLxwYVvMbBRwEtXXVjsJuDncfgL4HzOzpKVBtkl5eTmTJk3K1NuJiIiIZEQ2k7D2wBcJ+wuAg2oq4+4bzGwl0Ar4OrGQmQ0CBgF06tQp7QDKy8vrHLSIiIhILhTE2pHuXgVUAVRUVKTdSjZ06NCsxSQikiuLFy+mcePGUYchIhmWzSRsIdAxYb9DeCxVmQVm1hBoDizLYkwiIgWnXbt2UYcgIlmQzacjpwF7mtnuZtYIOBMYk1RmDDAg3D4VeCmT48FERERE8lXWWsLCMV6XAi8ADYAH3X22mQ0Gprv7GGA4MMLM5gHLCRI1ERERkaKX1TFh7j4WGJt07MaE7R+A07IZg4iIiEg+KokZ80VERETyjZIwERERkQgoCRMRERGJgJIwERERkQgoCRMRERGJgJIwERERkQgoCRMRERGJgBXaBPVmthT4rA6XtCZpQfACobhzS3HnVl3j7uzubbIVTK6o/sp7iju3CjVuqFvsNdZfBZeE1ZWZTXf3iqjjqCvFnVuKO7cKNe5cK9Tfk+LOLcWde5mKXd2RIiIiIhFQEiYiIiISgVJIwqqiDqCeFHduKe7cKtS4c61Qf0+KO7cUd+5lJPaiHxMmIiIiko9KoSVMREREJO8URRJmZn3N7EMzm2dm16U4v4OZPRaef8PMuuQ+yi2lEfeVZjbHzN4xsxfNrHMUcaZSW+wJ5U4xMzezvHgCJp24zez08Pc+28xG5jrGVNL4t9LJzCaa2dvhv5fjo4gzKaYHzewrM3uvhvNmZn8JP9M7ZrZfrmPMF6rDckv1V24VYv0FOarD3L2gX0ADYD7QFWgEzAJ6JpX5FXB/uH0m8FiBxH0E0CTc/s98iDvd2MNyTYGXgalARSHEDewJvA20DPd3KZC4q4D/DLd7Ap/mQdw/BfYD3qvh/PHAOMCAg4E3oo45j//7qg7LYdxhOdVfuYs77+qvMJas12HF0BJ2IDDP3T9293XAKOCkpDInAQ+H208AR5qZ5TDGVGqN290nuvuacHcq0CHHMdYknd85wC3AHcAPuQxuK9KJ+yLgXnf/BsDdv8pxjKmkE7cDzcLt5sCiHMaXkru/DCzfSpGTgEc8MBVoYWa75ia6vKI6LLdUf+VWQdZfkJs6rBiSsPbAFwn7C8JjKcu4+wZgJdAqJ9HVLJ24E11AkHHng1pjD5tlO7r7c7kMrBbp/M67A93N7FUzm2pmfXMWXc3Siftm4BwzWwCMBS7LTWjbpK5/A8VKdVhuqf7KrWKtvyADdVjDjIYjWWFm5wAVQGXUsaTDzLYDhgADIw6lPhoSNOn3IfjW/rKZ/djdV0QaVe3OAh5y97vM7BBghJn1cvdNUQcmUkh1mOqvSJRs/VUMLWELgY4J+x3CYynLmFlDgubOZTmJrmbpxI2ZHQVcD5zo7mtzFFttaou9KdALmGRmnxL0lY/Jg8Gt6fzOFwBj3H29u38CfERQqUUpnbgvAEYDuPvrQGOCtc3yWVp/AyVAdVhuqf7KrWKtvyATdVjUA98yMHCuIfAxsDubB/3tnVTm11Qf1Dq6QOLel2BA455Rx1vX2JPKTyI/Bram8zvvCzwcbrcmaGpuVQBxjwMGhts/IhhTYXnwO+9CzYNaf071Qa1vRh1vHv/3VR2Ww7iTyqv+yn7ceVl/hfFktQ6L/ANm6Jd0PEHGPx+4Pjw2mOCbFwRZ9ePAPOBNoGvUMacZ9wRgCTAzfI2JOuZ0Y08qmxeVWJq/cyPoipgDvAucGXXMacbdE3g1rOBmAsfkQcyPAouB9QTf0C8ALgEuSfhd3xt+pnfz5d9Inv73VR2Ww7iTyqr+yn7ceVd/hXFlvQ7TjPkiIiIiESiGMWEiIiIiBUdJmIiIiEgElISJiIiIREBJmIiIiEgElISJiIiIREBJmKSlttXko2Rmg8MJITGzw81stpnNNLP2ZvZELdc+YGY9w+3f5yJeRJEm+gAAA/BJREFUEZFkZjbQzP4n3L7ZzH4bdUySfUrCJF0PEUwEmHfc/UZ3nxDung3c5u7l7r7Q3U+t5doL3X1OuKskTETqxAL6f6nUi/7hSFq89tXkt8rMbjezOWb2jpn9OTz2kJndb2bTzewjMzshPN7AzP5kZtPC8hcnvM+1Zvaumc0ys9sT3udUM7sQOB24xcz+aWZdYi134Xv+2czeC9/zsvD4JDOrCN9rx7AF7Z9h69oVCfe91cwur+/nF5HiEdYtH5rZI8B7wH8l1Ff/nVDuP8Jjs8xsRHisn5m9YWZvm9kEM2sb1eeQ6GkBb8k6M2sF/BLo4e5uZi0STncBDgS6ARPNbA/gP4CV7n6Ame0AvGpm44EewEnAQe6+xsx2TryPuz9gZr2BZ939CTPrknB6UHivcnffkOLa68zsUncvD2PuAjwFDA2/5Z4ZxikiAsGajAOAZsCpBPWDEawz+VOCtT1vAA51968T6pxXgIPDuvBC4BrgqpxHL3lBSZjkwkrgB2C4mT0LPJtwbrS7bwLmmtnHBInWMcA+ZhbrSmxOUOEdBfzD3dcAuHtdWuaOIlh7b0M617r7p2a2zMz2BdoCb7t71Asmi0j++Mzdp4Yt+8cAb4fHywjqq58Aj7v711CtzukAPGZmuxKspfhJbsOWfKLuSMmIsLtvZvganHguTHwOBJ4ATgCeTzyd9FZO8G3ysnBcV7m77+7u4/9/e3fPWkUQxWH8+YuC+EIKxd4XAhZCGns/gk0qBRObaKkfQG38BKbSKHYWFmITjEHFSgiSQrAQrUQrCy0UJCEci53gNcQkhckG8/yau+zsMKcazuyc2buZ8f/FFDAGjAP3ehhf0vb1o/2G33WoI1V1oqrurtHvFjBZVaeACbr/BdUOZRKmf6KqlgYmoWuDbUkOAENVNQ1coVshLhtNsivJceAY8A6YAS4n2dP6DyfZD8wC40n2tft/bCmuYxaYSLJ7jb6Ly2M2j+gOI5xuMUnSSjPAxTbP0U5lHwGe081vh9r95TlnCPjcri9sdbDaXtyO1IYkeQCcAQ4n+QRcX2e1N+gg8DjJXrpV49WBto/AHF1dxaWq+plkiq5+az5JgC/A2ap6kmQEeJ1kAZhm4ycap4Bh4E2SReAOMLnimdutfb6qzlXVQpIXwLeqWtrgOJJ2kKp6muQk8KqbrvgOnK+qt0luAi+TLNFtV44BN4CHSb7SJWpHewlc20KqVu4GSVsjyX1aEX3fsaymFeTPA6NV9b7veCRJ/xe3I6VVtA+4fgCemYBJkjaDb8IkSZJ64JswSZKkHpiESZIk9cAkTJIkqQcmYZIkST0wCZMkSeqBSZgkSVIPfgEQA9OUZdCubgAAAABJRU5ErkJggg==\n"},"metadata":{"needs_background":"light"}}]},{"cell_type":"code","source":["val_F1_mean, val_F1_sd = mean_and_sd(val_F1)\n","val_acc_mean, val_acc_sd = mean_and_sd(val_acc)\n","val_pr_auc_mean, val_pr_auc_sd = mean_and_sd(val_pr_auc)\n","val_roc_auc_mean, val_roc_auc_sd = mean_and_sd(val_roc_auc)\n","\n","test_F1_mean, test_F1_sd = mean_and_sd(test_F1)\n","test_acc_mean, test_acc_sd = mean_and_sd(test_acc)\n","test_pr_auc_mean, test_pr_auc_sd = mean_and_sd(test_pr_auc)\n","test_roc_auc_mean, test_roc_auc_sd = mean_and_sd(test_roc_auc)"],"metadata":{"id":"OcYnEyGKnn5n"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["print(f\"validation F1: {round(val_F1_mean, 3)} +/- {round(val_F1_sd, 3)}\")\n","print(f\"test F1: {round(test_F1_mean, 3)} +/- {round(test_F1_sd, 3)}\")\n","\n","print(f\"validation accuracy: {round(val_acc_mean, 3)} +/- {round(val_acc_sd, 3)}\")\n","print(f\"test accuracy: {round(test_acc_mean, 3)} +/- {round(test_acc_sd, 3)}\")\n","\n","print(f\"validation PR-AUC: {round(val_pr_auc_mean, 3)} +/- {round(val_pr_auc_sd, 3)}\")\n","print(f\"test PR-AUC: {round(test_pr_auc_mean, 3)} +/- {round(test_pr_auc_sd, 3)}\")\n","\n","print(f\"validation ROC-AUC: {round(val_roc_auc_mean, 3)} +/- {round(val_roc_auc_sd, 3)}\")\n","print(f\"test ROC-AUC: {round(test_roc_auc_mean, 3)} +/- {round(test_roc_auc_sd, 3)}\")"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"-liHd50Ypx4b","executionInfo":{"status":"ok","timestamp":1651514835741,"user_tz":240,"elapsed":25,"user":{"displayName":"Benedikt Geiger","userId":"17925887631246406508"}},"outputId":"b4c679af-24eb-49c1-8846-2f83d9dad6e9"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["validation F1: 0.787 +/- 0.016\n","test F1: 0.831 +/- 0.027\n","validation accuracy: 0.679 +/- 0.012\n","test accuracy: 0.728 +/- 0.028\n","validation PR-AUC: 0.799 +/- 0.013\n","test PR-AUC: 0.841 +/- 0.009\n","validation ROC-AUC: 0.665 +/- 0.026\n","test ROC-AUC: 0.65 +/- 0.032\n"]}]},{"cell_type":"code","source":["model.save(deep_learning_dir + \"/MLP_models/model_weights_plot\")\n","\n","# model = tf.keras.models.load_model(deep_learning_dir + \"clinical_trial_model_weights\")"],"metadata":{"id":"t5boyGKM0sg7","executionInfo":{"status":"ok","timestamp":1651518148927,"user_tz":240,"elapsed":6838,"user":{"displayName":"Benedikt Geiger","userId":"17925887631246406508"}},"colab":{"base_uri":"https://localhost:8080/"},"outputId":"c7ee6b89-47d8-4e0a-a6c2-af162ae10323"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["INFO:tensorflow:Assets written to: /content/gdrive/My Drive/BMI 707 Project/MLP_models/model_weights_plot/assets\n"]}]},{"cell_type":"markdown","source":["Model structure"],"metadata":{"id":"Wg5dCkNvK-Bu"}},{"cell_type":"code","source":["# print model structure\n","tf.keras.utils.plot_model(model, \"clinical_trial_model.pdf\", show_shapes=True)"],"metadata":{"id":"Ei7D7SQmlq3E"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["!mv clinical_trial_model.pdf \"/content/gdrive/My Drive/BMI 707 Project/MLP_models/\""],"metadata":{"id":"dh87OEEm-QZI"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["plot_learning_curve(history, save_path=\"/content/gdrive/My Drive/BMI 707 Project/MLP_models/loss_history.pdf\")"],"metadata":{"id":"eAUGpVh2KLc5"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["model_metrics = model_performance(\n","  model,\n","  {\"inclusion\": test_data_incl, \"exclusion\": test_data_excl, \n","   \"diseases\": test_data_diseases, \"drug\": test_data_drug,\n","   \"targets\": test_data_target, \"meta\": test_data_meta},\n","  test_labels,\n","  save_path=deep_learning_dir+\"/MLP_models/model_performance.pdf\"\n",")"],"metadata":{"id":"_w6DnF131LWv"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["model_metrics"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"X1fZY5urQunF","executionInfo":{"status":"ok","timestamp":1651518476978,"user_tz":240,"elapsed":143,"user":{"displayName":"Benedikt Geiger","userId":"17925887631246406508"}},"outputId":"c24e7bf0-3599-4532-f895-b8403f57b2dd"},"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["{'F1': 0.83, 'accuracy': 0.727, 'pr_auc': 0.844, 'roc_auc': 0.67}"]},"metadata":{},"execution_count":67}]},{"cell_type":"markdown","source":["# Leave one out models"],"metadata":{"id":"GCVkz9nCiezw"}},{"cell_type":"markdown","source":["### Without eligibility criteria"],"metadata":{"id":"pMOATtfckQrD"}},{"cell_type":"code","source":["n_runs = 30"],"metadata":{"id":"41-pz1IXigzs"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["val_F1 = np.zeros(n_runs, dtype=float)\n","val_acc = np.zeros(n_runs, dtype=float)\n","val_pr_auc = np.zeros(n_runs, dtype=float)\n","val_roc_auc = np.zeros(n_runs, dtype=float)\n","\n","test_F1 = np.zeros(n_runs, dtype=float)\n","test_acc = np.zeros(n_runs, dtype=float)\n","test_pr_auc = np.zeros(n_runs, dtype=float)\n","test_roc_auc = np.zeros(n_runs, dtype=float)"],"metadata":{"id":"BPpr81y9ijnC"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["for i in range(n_runs):\n","\n","  # diseases\n","  diseases_input = layers.Input(shape=(diseases_dim,), name=\"diseases\")\n","  diseases_emb = layers.Dense(128, activation=\"relu\", name=\"lower_dim_diseases\")(diseases_input)\n","\n","\n","  # drug\n","  drug_input = layers.Input(shape=(drug_dim,), name=\"drug\")\n","  drug_emb = layers.Dense(64, activation=\"relu\", name=\"lower_dim_drug\")(drug_input)\n","\n","  # targets\n","  targets_input = layers.Input(shape=(targets_dim,), name=\"targets\")\n","  targets_emb = layers.Dense(64, activation=\"relu\", name=\"lower_dim_targets\")(targets_input)\n","\n","  # drug-target interaction\n","  drug_target_raw = layers.Concatenate(name=\"drug-targets\")([drug_emb, targets_emb])\n","  dt = layers.Dropout(rate=0.32)(drug_target_raw)\n","  drug_target_emb = layers.Dense(64, activation=\"relu\", name=\"lower_dim_drug-targets\")(dt)\n","\n","\n","  all_emb = layers.Concatenate(name=\"all_embeddings\")([diseases_emb, drug_target_emb])\n","  ae = layers.Dropout(rate=0.32)(all_emb)\n","  trial_embedding1 = layers.Dense(128, activation=\"relu\", name=\"trial_embedding_1\")(ae)\n","\n","  meta_input = layers.Input(shape=(meta_dim,), name=\"meta\")\n","  emb_and_meta = layers.Concatenate()([trial_embedding1, meta_input])\n","\n","  trial_embedding2 = layers.Dense(64, activation=\"relu\", name=\"trial_embedding_2\")(emb_and_meta)\n","  trial_embedding3 = layers.Dense(32, activation=\"relu\", name=\"trial_embedding_3\")(trial_embedding2)\n","\n","  o = layers.Dense(1, activation=\"sigmoid\", name=\"trial_success\")(trial_embedding3)\n","\n","  model = tf.keras.Model(inputs=[diseases_input, drug_input, targets_input, meta_input], outputs=[o])\n","  model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=0.01), loss=\"binary_crossentropy\", metrics=[\"accuracy\", \"AUC\"])\n","\n","  history = model.fit(\n","    x={\"diseases\": train_data_diseases, \"drug\": train_data_drug,\n","      \"targets\": train_data_target, \"meta\": train_data_meta},\n","    y={\"trial_success\": train_labels},\n","    validation_data=(\n","        {\"diseases\": val_data_diseases, \"drug\": val_data_drug,\n","        \"targets\": val_data_target, \"meta\": val_data_meta},\n","        {\"trial_success\": val_labels}\n","    ), \n","    epochs=5,\n","    batch_size=128\n","  )\n","\n","\n","  val_metrics = model_performance(\n","    model,\n","    {\"diseases\": val_data_diseases, \"drug\": val_data_drug,\n","    \"targets\": val_data_target, \"meta\": val_data_meta},\n","    val_labels,\n","    save_path=None\n","  )\n","\n","  test_metrics = model_performance(\n","    model,\n","    {\"diseases\": test_data_diseases, \"drug\": test_data_drug,\n","    \"targets\": test_data_target, \"meta\": test_data_meta},\n","    test_labels,\n","    save_path=None\n","  )\n","\n","  val_F1[i] = val_metrics[\"F1\"]\n","  val_acc[i] = val_metrics[\"accuracy\"]\n","  val_pr_auc[i] = val_metrics[\"pr_auc\"]\n","  val_roc_auc[i] = val_metrics[\"roc_auc\"]\n","\n","  test_F1[i] = test_metrics[\"F1\"]\n","  test_acc[i] = test_metrics[\"accuracy\"]\n","  test_pr_auc[i] = test_metrics[\"pr_auc\"]\n","  test_roc_auc[i] = test_metrics[\"roc_auc\"]"],"metadata":{"id":"ND3BoknyisOR"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["val_F1_mean, val_F1_sd = mean_and_sd(val_F1)\n","val_acc_mean, val_acc_sd = mean_and_sd(val_acc)\n","val_pr_auc_mean, val_pr_auc_sd = mean_and_sd(val_pr_auc)\n","val_roc_auc_mean, val_roc_auc_sd = mean_and_sd(val_roc_auc)\n","\n","test_F1_mean, test_F1_sd = mean_and_sd(test_F1)\n","test_acc_mean, test_acc_sd = mean_and_sd(test_acc)\n","test_pr_auc_mean, test_pr_auc_sd = mean_and_sd(test_pr_auc)\n","test_roc_auc_mean, test_roc_auc_sd = mean_and_sd(test_roc_auc)"],"metadata":{"id":"05s6938Oi7LY"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["print(f\"validation F1: {round(val_F1_mean, 3)} +/- {round(val_F1_sd, 3)}\")\n","print(f\"test F1: {round(test_F1_mean, 3)} +/- {round(test_F1_sd, 3)}\")\n","\n","print(f\"validation accuracy: {round(val_acc_mean, 3)} +/- {round(val_acc_sd, 3)}\")\n","print(f\"test accuracy: {round(test_acc_mean, 3)} +/- {round(test_acc_sd, 3)}\")\n","\n","print(f\"validation PR-AUC: {round(val_pr_auc_mean, 3)} +/- {round(val_pr_auc_sd, 3)}\")\n","print(f\"test PR-AUC: {round(test_pr_auc_mean, 3)} +/- {round(test_pr_auc_sd, 3)}\")\n","\n","print(f\"validation ROC-AUC: {round(val_roc_auc_mean, 3)} +/- {round(val_roc_auc_sd, 3)}\")\n","print(f\"test ROC-AUC: {round(test_roc_auc_mean, 3)} +/- {round(test_roc_auc_sd, 3)}\")"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"nsTA6lp8i8R5","executionInfo":{"status":"ok","timestamp":1651525179337,"user_tz":240,"elapsed":12,"user":{"displayName":"Benedikt Geiger","userId":"17925887631246406508"}},"outputId":"aca0e6dc-3f33-480f-8828-2f518587b3ef"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["validation F1: 0.781 +/- 0.012\n","test F1: 0.817 +/- 0.02\n","validation accuracy: 0.667 +/- 0.012\n","test accuracy: 0.71 +/- 0.021\n","validation PR-AUC: 0.795 +/- 0.013\n","test PR-AUC: 0.83 +/- 0.011\n","validation ROC-AUC: 0.659 +/- 0.02\n","test ROC-AUC: 0.631 +/- 0.022\n"]}]},{"cell_type":"markdown","source":["### Without target data"],"metadata":{"id":"-5k5NxKfkW7p"}},{"cell_type":"code","source":["n_runs = 30"],"metadata":{"id":"TxIzT1AUkM0m"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["val_F1 = np.zeros(n_runs, dtype=float)\n","val_acc = np.zeros(n_runs, dtype=float)\n","val_pr_auc = np.zeros(n_runs, dtype=float)\n","val_roc_auc = np.zeros(n_runs, dtype=float)\n","\n","test_F1 = np.zeros(n_runs, dtype=float)\n","test_acc = np.zeros(n_runs, dtype=float)\n","test_pr_auc = np.zeros(n_runs, dtype=float)\n","test_roc_auc = np.zeros(n_runs, dtype=float)"],"metadata":{"id":"j2HytVzakfMk"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["for i in range(n_runs):\n","\n","  # inclusion and exclusion criteria\n","  inclusion_input = layers.Input(shape=(incl_criteria_dim,), name=\"inclusion\")\n","  inclusion_emb = layers.Dense(256, activation=\"relu\", name=\"lower_dim_inclusion\")(inclusion_input)\n","\n","  exclusion_input = layers.Input(shape=(excl_criteria_dim,), name=\"exclusion\")\n","  exclusion_emb = layers.Dense(256, activation=\"relu\", name=\"lower_dim_exclusion\")(exclusion_input)\n","\n","  inclusion_exclusion_raw = layers.Concatenate(name=\"criteria\")([inclusion_emb, exclusion_emb])\n","  ie_dropout = layers.Dropout(rate=0.32)(inclusion_exclusion_raw)\n","  inclusion_exclusion_emb = layers.Dense(256, activation=\"relu\", name=\"lower_dim_criteria\")(ie_dropout)\n","\n","  # diseases\n","  diseases_input = layers.Input(shape=(diseases_dim,), name=\"diseases\")\n","  diseases_emb = layers.Dense(128, activation=\"relu\", name=\"lower_dim_diseases\")(diseases_input)\n","\n","\n","  # drug\n","  drug_input = layers.Input(shape=(drug_dim,), name=\"drug\")\n","  drug_emb = layers.Dense(64, activation=\"relu\", name=\"lower_dim_drug\")(drug_input)\n","\n","\n","  all_emb = layers.Concatenate(name=\"all_embeddings\")([inclusion_exclusion_emb, diseases_emb, drug_emb])\n","  ae = layers.Dropout(rate=0.32)(all_emb)\n","  trial_embedding1 = layers.Dense(128, activation=\"relu\", name=\"trial_embedding_1\")(ae)\n","\n","  meta_input = layers.Input(shape=(meta_dim,), name=\"meta\")\n","  emb_and_meta = layers.Concatenate()([trial_embedding1, meta_input])\n","\n","  trial_embedding2 = layers.Dense(64, activation=\"relu\", name=\"trial_embedding_2\")(emb_and_meta)\n","  trial_embedding3 = layers.Dense(32, activation=\"relu\", name=\"trial_embedding_3\")(trial_embedding2)\n","\n","  o = layers.Dense(1, activation=\"sigmoid\", name=\"trial_success\")(trial_embedding3)\n","\n","  model = tf.keras.Model(inputs=[inclusion_input, exclusion_input, diseases_input, drug_input, meta_input], outputs=[o])\n","  model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=0.01), loss=\"binary_crossentropy\", metrics=[\"accuracy\", \"AUC\"])\n","\n","  history = model.fit(\n","    x={\"inclusion\": train_data_incl, \"exclusion\": train_data_excl, \n","      \"diseases\": train_data_diseases, \"drug\": train_data_drug,\n","      \"meta\": train_data_meta},\n","    y={\"trial_success\": train_labels},\n","    validation_data=(\n","        {\"inclusion\": val_data_incl, \"exclusion\": val_data_excl,\n","        \"diseases\": val_data_diseases, \"drug\": val_data_drug,\n","        \"meta\": val_data_meta},\n","        {\"trial_success\": val_labels}\n","    ), \n","    epochs=5,\n","    batch_size=128\n","  )\n","\n","\n","  val_metrics = model_performance(\n","    model,\n","    {\"inclusion\": val_data_incl, \"exclusion\": val_data_excl, \n","    \"diseases\": val_data_diseases, \"drug\": val_data_drug,\n","    \"meta\": val_data_meta},\n","    val_labels,\n","    save_path=None\n","  )\n","\n","  test_metrics = model_performance(\n","    model,\n","    {\"inclusion\": test_data_incl, \"exclusion\": test_data_excl, \n","    \"diseases\": test_data_diseases, \"drug\": test_data_drug, \n","    \"meta\": test_data_meta},\n","    test_labels,\n","    save_path=None\n","  )\n","\n","  val_F1[i] = val_metrics[\"F1\"]\n","  val_acc[i] = val_metrics[\"accuracy\"]\n","  val_pr_auc[i] = val_metrics[\"pr_auc\"]\n","  val_roc_auc[i] = val_metrics[\"roc_auc\"]\n","\n","  test_F1[i] = test_metrics[\"F1\"]\n","  test_acc[i] = test_metrics[\"accuracy\"]\n","  test_pr_auc[i] = test_metrics[\"pr_auc\"]\n","  test_roc_auc[i] = test_metrics[\"roc_auc\"]"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"QPWTu9XYkkjD","executionInfo":{"status":"ok","timestamp":1651526772411,"user_tz":240,"elapsed":1286826,"user":{"displayName":"Benedikt Geiger","userId":"17925887631246406508"}},"outputId":"f1f9e993-c82c-4d6c-e230-bea851a7f242"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["Epoch 1/5\n","25/25 [==============================] - 7s 210ms/step - loss: 3.0516 - accuracy: 0.5915 - auc: 0.5278 - val_loss: 0.7818 - val_accuracy: 0.6628 - val_auc: 0.5612\n","Epoch 2/5\n","25/25 [==============================] - 5s 195ms/step - loss: 0.7825 - accuracy: 0.6574 - auc: 0.6027 - val_loss: 0.6396 - val_accuracy: 0.6744 - val_auc: 0.6260\n","Epoch 3/5\n","25/25 [==============================] - 5s 192ms/step - loss: 0.7362 - accuracy: 0.6642 - auc: 0.6320 - val_loss: 0.6168 - val_accuracy: 0.6773 - val_auc: 0.6550\n","Epoch 4/5\n","25/25 [==============================] - 5s 197ms/step - loss: 0.6388 - accuracy: 0.6800 - auc: 0.6745 - val_loss: 0.6311 - val_accuracy: 0.6773 - val_auc: 0.6396\n","Epoch 5/5\n","25/25 [==============================] - 5s 200ms/step - loss: 0.6032 - accuracy: 0.6949 - auc: 0.7047 - val_loss: 0.6036 - val_accuracy: 0.6890 - val_auc: 0.6658\n","Epoch 1/5\n","25/25 [==============================] - 8s 223ms/step - loss: 3.2503 - accuracy: 0.5695 - auc: 0.5043 - val_loss: 0.6779 - val_accuracy: 0.6541 - val_auc: 0.6130\n","Epoch 2/5\n","25/25 [==============================] - 5s 193ms/step - loss: 0.6538 - accuracy: 0.6542 - auc: 0.5508 - val_loss: 0.6417 - val_accuracy: 0.6657 - val_auc: 0.5747\n","Epoch 3/5\n","25/25 [==============================] - 5s 193ms/step - loss: 0.6393 - accuracy: 0.6532 - auc: 0.5856 - val_loss: 0.6247 - val_accuracy: 0.6657 - val_auc: 0.6029\n","Epoch 4/5\n","25/25 [==============================] - 5s 207ms/step - loss: 0.6362 - accuracy: 0.6532 - auc: 0.6055 - val_loss: 0.6184 - val_accuracy: 0.6657 - val_auc: 0.6306\n","Epoch 5/5\n","25/25 [==============================] - 5s 190ms/step - loss: 0.6073 - accuracy: 0.6532 - auc: 0.6814 - val_loss: 0.6163 - val_accuracy: 0.6657 - val_auc: 0.6367\n","Epoch 1/5\n","25/25 [==============================] - 7s 225ms/step - loss: 2.7811 - accuracy: 0.5944 - auc: 0.5325 - val_loss: 0.6888 - val_accuracy: 0.6715 - val_auc: 0.5158\n","Epoch 2/5\n","25/25 [==============================] - 5s 200ms/step - loss: 0.6513 - accuracy: 0.6668 - auc: 0.5851 - val_loss: 0.6220 - val_accuracy: 0.6831 - val_auc: 0.6167\n","Epoch 3/5\n","25/25 [==============================] - 5s 201ms/step - loss: 0.6226 - accuracy: 0.6797 - auc: 0.6495 - val_loss: 0.6090 - val_accuracy: 0.6773 - val_auc: 0.6392\n","Epoch 4/5\n","25/25 [==============================] - 5s 198ms/step - loss: 0.5957 - accuracy: 0.6894 - auc: 0.6904 - val_loss: 0.6031 - val_accuracy: 0.6657 - val_auc: 0.6670\n","Epoch 5/5\n","25/25 [==============================] - 5s 197ms/step - loss: 0.5666 - accuracy: 0.6981 - auc: 0.7306 - val_loss: 0.6022 - val_accuracy: 0.6860 - val_auc: 0.6755\n","Epoch 1/5\n","25/25 [==============================] - 7s 216ms/step - loss: 2.7659 - accuracy: 0.5895 - auc: 0.5276 - val_loss: 0.6746 - val_accuracy: 0.6599 - val_auc: 0.5285\n","Epoch 2/5\n","25/25 [==============================] - 5s 198ms/step - loss: 0.6566 - accuracy: 0.6642 - auc: 0.6104 - val_loss: 0.6700 - val_accuracy: 0.6744 - val_auc: 0.6584\n","Epoch 3/5\n","25/25 [==============================] - 5s 208ms/step - loss: 0.6450 - accuracy: 0.6726 - auc: 0.6325 - val_loss: 0.6229 - val_accuracy: 0.6715 - val_auc: 0.6678\n","Epoch 4/5\n","25/25 [==============================] - 5s 220ms/step - loss: 0.6456 - accuracy: 0.6800 - auc: 0.6793 - val_loss: 0.6611 - val_accuracy: 0.6744 - val_auc: 0.6819\n","Epoch 5/5\n","25/25 [==============================] - 5s 204ms/step - loss: 0.7195 - accuracy: 0.6923 - auc: 0.7027 - val_loss: 0.6543 - val_accuracy: 0.6337 - val_auc: 0.6687\n","Epoch 1/5\n","25/25 [==============================] - 8s 271ms/step - loss: 2.0996 - accuracy: 0.5750 - auc: 0.5257 - val_loss: 1.0610 - val_accuracy: 0.6453 - val_auc: 0.5251\n","Epoch 2/5\n","25/25 [==============================] - 8s 322ms/step - loss: 0.7630 - accuracy: 0.6542 - auc: 0.5899 - val_loss: 0.6516 - val_accuracy: 0.6715 - val_auc: 0.6333\n","Epoch 3/5\n","25/25 [==============================] - 7s 279ms/step - loss: 0.6190 - accuracy: 0.6761 - auc: 0.6478 - val_loss: 0.6308 - val_accuracy: 0.6773 - val_auc: 0.6626\n","Epoch 4/5\n","25/25 [==============================] - 5s 196ms/step - loss: 0.7346 - accuracy: 0.6694 - auc: 0.6555 - val_loss: 0.6943 - val_accuracy: 0.6017 - val_auc: 0.6423\n","Epoch 5/5\n","25/25 [==============================] - 5s 196ms/step - loss: 0.6278 - accuracy: 0.6858 - auc: 0.7035 - val_loss: 0.6227 - val_accuracy: 0.6860 - val_auc: 0.6876\n","Epoch 1/5\n","25/25 [==============================] - 9s 265ms/step - loss: 3.1197 - accuracy: 0.5782 - auc: 0.4986 - val_loss: 0.7682 - val_accuracy: 0.6512 - val_auc: 0.6309\n","Epoch 2/5\n","25/25 [==============================] - 5s 204ms/step - loss: 0.7477 - accuracy: 0.6571 - auc: 0.5851 - val_loss: 0.7635 - val_accuracy: 0.6715 - val_auc: 0.5649\n","Epoch 3/5\n","25/25 [==============================] - 5s 201ms/step - loss: 0.6401 - accuracy: 0.6791 - auc: 0.6595 - val_loss: 0.6444 - val_accuracy: 0.6802 - val_auc: 0.6620\n","Epoch 4/5\n","25/25 [==============================] - 10s 398ms/step - loss: 0.5945 - accuracy: 0.6942 - auc: 0.6967 - val_loss: 0.5965 - val_accuracy: 0.6831 - val_auc: 0.6819\n","Epoch 5/5\n","25/25 [==============================] - 8s 316ms/step - loss: 0.5739 - accuracy: 0.7065 - auc: 0.7269 - val_loss: 0.5841 - val_accuracy: 0.6890 - val_auc: 0.6943\n","Epoch 1/5\n","25/25 [==============================] - 10s 317ms/step - loss: 4.2090 - accuracy: 0.5792 - auc: 0.5125 - val_loss: 0.7939 - val_accuracy: 0.5116 - val_auc: 0.6493\n","Epoch 2/5\n","25/25 [==============================] - 6s 238ms/step - loss: 0.7171 - accuracy: 0.6354 - auc: 0.5632 - val_loss: 0.7715 - val_accuracy: 0.6831 - val_auc: 0.6399\n","Epoch 3/5\n","25/25 [==============================] - 7s 283ms/step - loss: 0.6826 - accuracy: 0.6829 - auc: 0.6556 - val_loss: 1.5883 - val_accuracy: 0.6831 - val_auc: 0.6741\n","Epoch 4/5\n","25/25 [==============================] - 5s 219ms/step - loss: 0.6835 - accuracy: 0.6761 - auc: 0.6738 - val_loss: 0.8813 - val_accuracy: 0.6948 - val_auc: 0.6968\n","Epoch 5/5\n","25/25 [==============================] - 10s 385ms/step - loss: 0.6380 - accuracy: 0.6955 - auc: 0.7093 - val_loss: 0.7119 - val_accuracy: 0.6628 - val_auc: 0.6652\n","Epoch 1/5\n","25/25 [==============================] - 7s 212ms/step - loss: 2.6217 - accuracy: 0.5798 - auc: 0.5231 - val_loss: 0.6823 - val_accuracy: 0.6628 - val_auc: 0.5858\n","Epoch 2/5\n","25/25 [==============================] - 6s 244ms/step - loss: 0.7247 - accuracy: 0.6500 - auc: 0.6172 - val_loss: 0.6562 - val_accuracy: 0.6628 - val_auc: 0.6441\n","Epoch 3/5\n","25/25 [==============================] - 7s 292ms/step - loss: 0.7213 - accuracy: 0.6629 - auc: 0.6381 - val_loss: 0.7133 - val_accuracy: 0.6192 - val_auc: 0.6104\n","Epoch 4/5\n","25/25 [==============================] - 5s 211ms/step - loss: 0.5931 - accuracy: 0.6959 - auc: 0.7046 - val_loss: 0.6356 - val_accuracy: 0.6686 - val_auc: 0.6598\n","Epoch 5/5\n","25/25 [==============================] - 6s 253ms/step - loss: 0.5955 - accuracy: 0.6952 - auc: 0.7158 - val_loss: 0.5910 - val_accuracy: 0.6860 - val_auc: 0.6851\n","Epoch 1/5\n","25/25 [==============================] - 9s 216ms/step - loss: 2.7464 - accuracy: 0.5999 - auc: 0.4886 - val_loss: 0.6688 - val_accuracy: 0.6686 - val_auc: 0.5023\n","Epoch 2/5\n","25/25 [==============================] - 5s 200ms/step - loss: 0.6545 - accuracy: 0.6587 - auc: 0.5608 - val_loss: 0.6301 - val_accuracy: 0.6657 - val_auc: 0.5938\n","Epoch 3/5\n","25/25 [==============================] - 5s 194ms/step - loss: 0.6286 - accuracy: 0.6532 - auc: 0.6185 - val_loss: 0.6206 - val_accuracy: 0.6657 - val_auc: 0.6217\n","Epoch 4/5\n","25/25 [==============================] - 7s 277ms/step - loss: 0.6137 - accuracy: 0.6532 - auc: 0.6570 - val_loss: 0.6051 - val_accuracy: 0.6657 - val_auc: 0.6605\n","Epoch 5/5\n","25/25 [==============================] - 9s 364ms/step - loss: 0.5848 - accuracy: 0.6532 - auc: 0.7185 - val_loss: 0.6058 - val_accuracy: 0.6657 - val_auc: 0.6611\n","Epoch 1/5\n","25/25 [==============================] - 10s 328ms/step - loss: 2.4270 - accuracy: 0.5856 - auc: 0.5219 - val_loss: 0.7504 - val_accuracy: 0.6657 - val_auc: 0.4887\n","Epoch 2/5\n","25/25 [==============================] - 7s 287ms/step - loss: 0.6812 - accuracy: 0.6422 - auc: 0.5762 - val_loss: 0.6269 - val_accuracy: 0.6802 - val_auc: 0.6338\n","Epoch 3/5\n","25/25 [==============================] - 8s 321ms/step - loss: 0.6170 - accuracy: 0.6781 - auc: 0.6529 - val_loss: 0.6180 - val_accuracy: 0.6773 - val_auc: 0.6544\n","Epoch 4/5\n","25/25 [==============================] - 7s 289ms/step - loss: 0.5969 - accuracy: 0.6813 - auc: 0.6915 - val_loss: 0.6128 - val_accuracy: 0.6744 - val_auc: 0.6599\n","Epoch 5/5\n","25/25 [==============================] - 5s 203ms/step - loss: 0.5770 - accuracy: 0.6849 - auc: 0.7179 - val_loss: 0.5948 - val_accuracy: 0.6860 - val_auc: 0.6854\n","Epoch 1/5\n","25/25 [==============================] - 7s 219ms/step - loss: 2.5684 - accuracy: 0.6021 - auc: 0.5325 - val_loss: 0.6450 - val_accuracy: 0.6744 - val_auc: 0.5836\n","Epoch 2/5\n","25/25 [==============================] - 7s 296ms/step - loss: 0.6439 - accuracy: 0.6580 - auc: 0.5895 - val_loss: 0.6288 - val_accuracy: 0.6831 - val_auc: 0.6062\n","Epoch 3/5\n","25/25 [==============================] - 5s 198ms/step - loss: 0.6153 - accuracy: 0.6846 - auc: 0.6428 - val_loss: 0.6061 - val_accuracy: 0.6802 - val_auc: 0.6580\n","Epoch 4/5\n","25/25 [==============================] - 8s 326ms/step - loss: 0.5905 - accuracy: 0.6949 - auc: 0.7025 - val_loss: 0.6118 - val_accuracy: 0.6860 - val_auc: 0.6483\n","Epoch 5/5\n","25/25 [==============================] - 7s 298ms/step - loss: 0.5737 - accuracy: 0.7078 - auc: 0.7322 - val_loss: 0.5962 - val_accuracy: 0.6802 - val_auc: 0.6767\n","Epoch 1/5\n","25/25 [==============================] - 10s 322ms/step - loss: 2.5810 - accuracy: 0.5659 - auc: 0.4943 - val_loss: 0.8021 - val_accuracy: 0.6802 - val_auc: 0.5091\n","Epoch 2/5\n","25/25 [==============================] - 6s 247ms/step - loss: 0.6780 - accuracy: 0.6438 - auc: 0.5843 - val_loss: 0.6288 - val_accuracy: 0.6715 - val_auc: 0.6130\n","Epoch 3/5\n","25/25 [==============================] - 7s 280ms/step - loss: 0.6062 - accuracy: 0.6878 - auc: 0.6680 - val_loss: 0.6049 - val_accuracy: 0.6919 - val_auc: 0.6681\n","Epoch 4/5\n","25/25 [==============================] - 7s 279ms/step - loss: 0.5883 - accuracy: 0.6810 - auc: 0.6909 - val_loss: 0.6058 - val_accuracy: 0.6802 - val_auc: 0.6630\n","Epoch 5/5\n","25/25 [==============================] - 7s 279ms/step - loss: 0.5733 - accuracy: 0.6984 - auc: 0.7280 - val_loss: 0.6048 - val_accuracy: 0.6831 - val_auc: 0.6794\n","Epoch 1/5\n","25/25 [==============================] - 10s 257ms/step - loss: 3.4692 - accuracy: 0.6138 - auc: 0.5288 - val_loss: 0.6400 - val_accuracy: 0.6802 - val_auc: 0.5729\n","Epoch 2/5\n","25/25 [==============================] - 6s 255ms/step - loss: 0.6350 - accuracy: 0.6723 - auc: 0.5813 - val_loss: 0.6633 - val_accuracy: 0.6657 - val_auc: 0.6267\n","Epoch 3/5\n","25/25 [==============================] - 7s 288ms/step - loss: 0.6235 - accuracy: 0.6736 - auc: 0.6505 - val_loss: 0.6079 - val_accuracy: 0.6831 - val_auc: 0.6738\n","Epoch 4/5\n","25/25 [==============================] - 7s 276ms/step - loss: 0.5702 - accuracy: 0.6981 - auc: 0.7286 - val_loss: 0.6071 - val_accuracy: 0.6948 - val_auc: 0.6758\n","Epoch 5/5\n","25/25 [==============================] - 9s 351ms/step - loss: 0.5300 - accuracy: 0.7356 - auc: 0.7828 - val_loss: 0.6174 - val_accuracy: 0.6744 - val_auc: 0.6771\n","Epoch 1/5\n","25/25 [==============================] - 11s 353ms/step - loss: 3.8252 - accuracy: 0.5633 - auc: 0.4954 - val_loss: 0.7803 - val_accuracy: 0.5610 - val_auc: 0.5570\n","Epoch 2/5\n","25/25 [==============================] - 10s 376ms/step - loss: 0.8035 - accuracy: 0.6367 - auc: 0.5596 - val_loss: 1.2881 - val_accuracy: 0.6017 - val_auc: 0.6355\n","Epoch 3/5\n","25/25 [==============================] - 8s 303ms/step - loss: 0.7916 - accuracy: 0.6645 - auc: 0.6412 - val_loss: 0.6234 - val_accuracy: 0.6657 - val_auc: 0.6305\n","Epoch 4/5\n","25/25 [==============================] - 6s 222ms/step - loss: 0.6503 - accuracy: 0.6745 - auc: 0.6759 - val_loss: 0.6906 - val_accuracy: 0.6657 - val_auc: 0.6468\n","Epoch 5/5\n","25/25 [==============================] - 5s 194ms/step - loss: 0.6085 - accuracy: 0.6836 - auc: 0.7120 - val_loss: 0.5927 - val_accuracy: 0.6599 - val_auc: 0.6736\n","Epoch 1/5\n","25/25 [==============================] - 7s 214ms/step - loss: 2.4624 - accuracy: 0.5944 - auc: 0.5385 - val_loss: 0.6212 - val_accuracy: 0.6890 - val_auc: 0.5988\n","Epoch 2/5\n","25/25 [==============================] - 5s 200ms/step - loss: 0.6638 - accuracy: 0.6677 - auc: 0.6151 - val_loss: 0.6547 - val_accuracy: 0.6802 - val_auc: 0.6844\n","Epoch 3/5\n","25/25 [==============================] - 5s 195ms/step - loss: 0.6306 - accuracy: 0.6858 - auc: 0.6718 - val_loss: 0.6037 - val_accuracy: 0.6860 - val_auc: 0.6720\n","Epoch 4/5\n","25/25 [==============================] - 5s 196ms/step - loss: 0.5911 - accuracy: 0.6975 - auc: 0.7089 - val_loss: 0.6075 - val_accuracy: 0.6541 - val_auc: 0.6816\n","Epoch 5/5\n","25/25 [==============================] - 5s 205ms/step - loss: 0.6108 - accuracy: 0.6936 - auc: 0.7174 - val_loss: 0.5936 - val_accuracy: 0.6860 - val_auc: 0.6746\n","Epoch 1/5\n","25/25 [==============================] - 7s 224ms/step - loss: 3.3112 - accuracy: 0.6047 - auc: 0.5420 - val_loss: 0.6967 - val_accuracy: 0.6686 - val_auc: 0.6080\n","Epoch 2/5\n","25/25 [==============================] - 5s 201ms/step - loss: 0.6366 - accuracy: 0.6826 - auc: 0.6482 - val_loss: 0.6372 - val_accuracy: 0.6424 - val_auc: 0.6573\n","Epoch 3/5\n","25/25 [==============================] - 5s 200ms/step - loss: 0.5902 - accuracy: 0.6813 - auc: 0.6974 - val_loss: 0.5977 - val_accuracy: 0.6860 - val_auc: 0.6912\n","Epoch 4/5\n","25/25 [==============================] - 5s 196ms/step - loss: 0.5932 - accuracy: 0.6962 - auc: 0.7236 - val_loss: 0.5954 - val_accuracy: 0.6860 - val_auc: 0.6813\n","Epoch 5/5\n","25/25 [==============================] - 5s 196ms/step - loss: 0.5564 - accuracy: 0.7185 - auc: 0.7500 - val_loss: 0.5941 - val_accuracy: 0.6831 - val_auc: 0.6968\n","Epoch 1/5\n","25/25 [==============================] - 7s 220ms/step - loss: 3.0030 - accuracy: 0.5963 - auc: 0.5402 - val_loss: 0.7164 - val_accuracy: 0.6628 - val_auc: 0.5879\n","Epoch 2/5\n","25/25 [==============================] - 5s 199ms/step - loss: 0.7052 - accuracy: 0.6516 - auc: 0.5897 - val_loss: 0.6358 - val_accuracy: 0.6860 - val_auc: 0.6255\n","Epoch 3/5\n","25/25 [==============================] - 5s 200ms/step - loss: 0.6307 - accuracy: 0.6778 - auc: 0.6575 - val_loss: 0.8902 - val_accuracy: 0.6773 - val_auc: 0.6436\n","Epoch 4/5\n","25/25 [==============================] - 5s 204ms/step - loss: 0.6705 - accuracy: 0.6710 - auc: 0.6700 - val_loss: 0.5994 - val_accuracy: 0.6802 - val_auc: 0.6770\n","Epoch 5/5\n","25/25 [==============================] - 5s 217ms/step - loss: 0.5874 - accuracy: 0.6942 - auc: 0.7184 - val_loss: 0.6224 - val_accuracy: 0.6802 - val_auc: 0.6846\n","Epoch 1/5\n","25/25 [==============================] - 7s 217ms/step - loss: 2.0739 - accuracy: 0.5989 - auc: 0.4932 - val_loss: 0.6955 - val_accuracy: 0.5233 - val_auc: 0.5777\n","Epoch 2/5\n","25/25 [==============================] - 5s 201ms/step - loss: 0.6504 - accuracy: 0.6626 - auc: 0.5616 - val_loss: 0.6218 - val_accuracy: 0.6802 - val_auc: 0.5804\n","Epoch 3/5\n","25/25 [==============================] - 5s 220ms/step - loss: 0.6237 - accuracy: 0.6781 - auc: 0.6261 - val_loss: 0.6372 - val_accuracy: 0.6657 - val_auc: 0.6127\n","Epoch 4/5\n","25/25 [==============================] - 5s 196ms/step - loss: 0.6153 - accuracy: 0.6791 - auc: 0.6415 - val_loss: 0.6023 - val_accuracy: 0.6802 - val_auc: 0.6617\n","Epoch 5/5\n","25/25 [==============================] - 5s 196ms/step - loss: 0.5883 - accuracy: 0.6955 - auc: 0.6956 - val_loss: 0.5989 - val_accuracy: 0.6860 - val_auc: 0.6759\n","Epoch 1/5\n","25/25 [==============================] - 7s 213ms/step - loss: 5.2044 - accuracy: 0.5721 - auc: 0.5154 - val_loss: 0.7226 - val_accuracy: 0.6134 - val_auc: 0.5367\n","Epoch 2/5\n","25/25 [==============================] - 6s 244ms/step - loss: 0.6862 - accuracy: 0.6545 - auc: 0.5759 - val_loss: 0.6411 - val_accuracy: 0.6890 - val_auc: 0.6032\n","Epoch 3/5\n","25/25 [==============================] - 5s 196ms/step - loss: 0.6571 - accuracy: 0.6690 - auc: 0.6047 - val_loss: 0.6166 - val_accuracy: 0.6773 - val_auc: 0.6466\n","Epoch 4/5\n","25/25 [==============================] - 5s 201ms/step - loss: 0.5997 - accuracy: 0.6816 - auc: 0.6851 - val_loss: 0.6233 - val_accuracy: 0.6831 - val_auc: 0.6883\n","Epoch 5/5\n","25/25 [==============================] - 5s 195ms/step - loss: 0.5794 - accuracy: 0.6933 - auc: 0.7242 - val_loss: 0.6075 - val_accuracy: 0.6686 - val_auc: 0.6663\n","Epoch 1/5\n","25/25 [==============================] - 7s 223ms/step - loss: 3.5444 - accuracy: 0.5866 - auc: 0.5215 - val_loss: 0.8044 - val_accuracy: 0.6366 - val_auc: 0.5591\n","Epoch 2/5\n","25/25 [==============================] - 5s 201ms/step - loss: 0.6924 - accuracy: 0.6658 - auc: 0.6150 - val_loss: 0.9770 - val_accuracy: 0.6715 - val_auc: 0.5864\n","Epoch 3/5\n","25/25 [==============================] - 5s 192ms/step - loss: 0.9201 - accuracy: 0.6564 - auc: 0.6389 - val_loss: 0.8999 - val_accuracy: 0.6628 - val_auc: 0.6413\n","Epoch 4/5\n","25/25 [==============================] - 6s 248ms/step - loss: 0.8226 - accuracy: 0.6677 - auc: 0.6724 - val_loss: 0.8026 - val_accuracy: 0.6599 - val_auc: 0.6561\n","Epoch 5/5\n","25/25 [==============================] - 11s 430ms/step - loss: 0.7156 - accuracy: 0.6900 - auc: 0.7079 - val_loss: 0.6913 - val_accuracy: 0.6453 - val_auc: 0.6535\n","Epoch 1/5\n","25/25 [==============================] - 9s 310ms/step - loss: 4.0803 - accuracy: 0.6021 - auc: 0.5279 - val_loss: 1.1291 - val_accuracy: 0.6570 - val_auc: 0.6175\n","Epoch 2/5\n","25/25 [==============================] - 11s 432ms/step - loss: 0.7530 - accuracy: 0.6632 - auc: 0.5903 - val_loss: 0.6945 - val_accuracy: 0.6831 - val_auc: 0.6541\n","Epoch 3/5\n","25/25 [==============================] - 8s 298ms/step - loss: 0.6382 - accuracy: 0.6745 - auc: 0.6458 - val_loss: 0.6029 - val_accuracy: 0.6744 - val_auc: 0.6458\n","Epoch 4/5\n","25/25 [==============================] - 7s 278ms/step - loss: 0.6262 - accuracy: 0.6813 - auc: 0.6768 - val_loss: 0.7143 - val_accuracy: 0.6657 - val_auc: 0.6664\n","Epoch 5/5\n","25/25 [==============================] - 5s 199ms/step - loss: 0.5877 - accuracy: 0.6991 - auc: 0.7234 - val_loss: 0.6996 - val_accuracy: 0.6773 - val_auc: 0.6711\n","Epoch 1/5\n","25/25 [==============================] - 7s 215ms/step - loss: 3.6606 - accuracy: 0.5769 - auc: 0.5110 - val_loss: 0.7526 - val_accuracy: 0.6424 - val_auc: 0.6245\n","Epoch 2/5\n","25/25 [==============================] - 5s 203ms/step - loss: 0.8067 - accuracy: 0.6341 - auc: 0.5831 - val_loss: 0.6776 - val_accuracy: 0.6744 - val_auc: 0.6348\n","Epoch 3/5\n","25/25 [==============================] - 5s 196ms/step - loss: 0.6468 - accuracy: 0.6816 - auc: 0.6608 - val_loss: 0.6210 - val_accuracy: 0.6744 - val_auc: 0.6522\n","Epoch 4/5\n","25/25 [==============================] - 5s 197ms/step - loss: 0.6583 - accuracy: 0.6752 - auc: 0.6697 - val_loss: 0.8086 - val_accuracy: 0.6308 - val_auc: 0.6648\n","Epoch 5/5\n","25/25 [==============================] - 5s 196ms/step - loss: 0.6531 - accuracy: 0.6849 - auc: 0.6914 - val_loss: 0.6765 - val_accuracy: 0.6657 - val_auc: 0.6181\n","Epoch 1/5\n","25/25 [==============================] - 7s 217ms/step - loss: 2.1563 - accuracy: 0.6044 - auc: 0.5169 - val_loss: 0.6735 - val_accuracy: 0.6802 - val_auc: 0.5441\n","Epoch 2/5\n","25/25 [==============================] - 5s 199ms/step - loss: 0.6321 - accuracy: 0.6774 - auc: 0.6040 - val_loss: 0.6285 - val_accuracy: 0.6657 - val_auc: 0.6545\n","Epoch 3/5\n","25/25 [==============================] - 5s 199ms/step - loss: 0.6422 - accuracy: 0.6668 - auc: 0.6571 - val_loss: 0.6944 - val_accuracy: 0.6860 - val_auc: 0.6636\n","Epoch 4/5\n","25/25 [==============================] - 5s 199ms/step - loss: 0.6196 - accuracy: 0.6878 - auc: 0.6764 - val_loss: 0.6032 - val_accuracy: 0.7035 - val_auc: 0.6629\n","Epoch 5/5\n","25/25 [==============================] - 6s 239ms/step - loss: 0.5737 - accuracy: 0.7033 - auc: 0.7331 - val_loss: 0.6076 - val_accuracy: 0.6831 - val_auc: 0.6676\n","Epoch 1/5\n","25/25 [==============================] - 7s 221ms/step - loss: 3.1150 - accuracy: 0.5750 - auc: 0.5051 - val_loss: 0.7842 - val_accuracy: 0.5698 - val_auc: 0.6041\n","Epoch 2/5\n","25/25 [==============================] - 5s 201ms/step - loss: 0.7039 - accuracy: 0.6493 - auc: 0.5539 - val_loss: 0.6307 - val_accuracy: 0.6860 - val_auc: 0.5702\n","Epoch 3/5\n","25/25 [==============================] - 5s 200ms/step - loss: 0.6311 - accuracy: 0.6765 - auc: 0.6229 - val_loss: 0.5930 - val_accuracy: 0.6860 - val_auc: 0.6865\n","Epoch 4/5\n","25/25 [==============================] - 5s 194ms/step - loss: 0.6057 - accuracy: 0.6765 - auc: 0.6671 - val_loss: 0.6420 - val_accuracy: 0.6831 - val_auc: 0.6627\n","Epoch 5/5\n","25/25 [==============================] - 5s 198ms/step - loss: 0.5961 - accuracy: 0.6871 - auc: 0.7037 - val_loss: 0.5829 - val_accuracy: 0.6977 - val_auc: 0.6990\n","Epoch 1/5\n","25/25 [==============================] - 8s 278ms/step - loss: 3.5574 - accuracy: 0.5792 - auc: 0.5238 - val_loss: 0.6546 - val_accuracy: 0.6512 - val_auc: 0.5775\n","Epoch 2/5\n","25/25 [==============================] - 7s 296ms/step - loss: 0.6565 - accuracy: 0.6736 - auc: 0.6220 - val_loss: 1.0334 - val_accuracy: 0.6657 - val_auc: 0.5666\n","Epoch 3/5\n","25/25 [==============================] - 7s 284ms/step - loss: 0.7975 - accuracy: 0.6312 - auc: 0.6064 - val_loss: 0.6256 - val_accuracy: 0.6715 - val_auc: 0.6544\n","Epoch 4/5\n","25/25 [==============================] - 9s 351ms/step - loss: 0.6417 - accuracy: 0.6729 - auc: 0.6732 - val_loss: 0.6585 - val_accuracy: 0.6802 - val_auc: 0.6541\n","Epoch 5/5\n","25/25 [==============================] - 9s 380ms/step - loss: 0.5990 - accuracy: 0.6826 - auc: 0.7089 - val_loss: 0.6161 - val_accuracy: 0.6831 - val_auc: 0.6796\n","Epoch 1/5\n","25/25 [==============================] - 12s 369ms/step - loss: 2.9459 - accuracy: 0.5837 - auc: 0.5284 - val_loss: 0.7329 - val_accuracy: 0.6483 - val_auc: 0.5562\n","Epoch 2/5\n","25/25 [==============================] - 8s 310ms/step - loss: 0.8338 - accuracy: 0.6490 - auc: 0.5790 - val_loss: 0.7063 - val_accuracy: 0.6715 - val_auc: 0.6165\n","Epoch 3/5\n","25/25 [==============================] - 5s 211ms/step - loss: 0.6300 - accuracy: 0.6668 - auc: 0.6546 - val_loss: 0.6363 - val_accuracy: 0.6715 - val_auc: 0.6586\n","Epoch 4/5\n","25/25 [==============================] - 5s 206ms/step - loss: 0.7162 - accuracy: 0.6622 - auc: 0.6708 - val_loss: 0.7727 - val_accuracy: 0.6977 - val_auc: 0.6665\n","Epoch 5/5\n","25/25 [==============================] - 8s 333ms/step - loss: 0.7299 - accuracy: 0.6907 - auc: 0.7066 - val_loss: 0.6007 - val_accuracy: 0.6860 - val_auc: 0.6948\n","Epoch 1/5\n","25/25 [==============================] - 10s 287ms/step - loss: 2.9779 - accuracy: 0.5708 - auc: 0.5143 - val_loss: 0.6720 - val_accuracy: 0.5785 - val_auc: 0.5434\n","Epoch 2/5\n","25/25 [==============================] - 7s 288ms/step - loss: 0.6610 - accuracy: 0.6506 - auc: 0.5826 - val_loss: 0.6947 - val_accuracy: 0.6657 - val_auc: 0.5686\n","Epoch 3/5\n","25/25 [==============================] - 7s 291ms/step - loss: 0.6333 - accuracy: 0.6784 - auc: 0.6283 - val_loss: 0.5968 - val_accuracy: 0.6831 - val_auc: 0.6740\n","Epoch 4/5\n","25/25 [==============================] - 8s 309ms/step - loss: 0.6014 - accuracy: 0.6807 - auc: 0.6991 - val_loss: 0.6325 - val_accuracy: 0.6744 - val_auc: 0.6826\n","Epoch 5/5\n","25/25 [==============================] - 7s 292ms/step - loss: 0.5949 - accuracy: 0.6842 - auc: 0.7130 - val_loss: 0.5897 - val_accuracy: 0.6890 - val_auc: 0.6807\n","Epoch 1/5\n","25/25 [==============================] - 11s 361ms/step - loss: 2.9335 - accuracy: 0.5850 - auc: 0.5299 - val_loss: 0.9148 - val_accuracy: 0.6657 - val_auc: 0.4665\n","Epoch 2/5\n","25/25 [==============================] - 8s 301ms/step - loss: 0.7654 - accuracy: 0.6461 - auc: 0.5910 - val_loss: 0.6279 - val_accuracy: 0.6919 - val_auc: 0.6306\n","Epoch 3/5\n","25/25 [==============================] - 7s 284ms/step - loss: 0.6131 - accuracy: 0.6816 - auc: 0.6600 - val_loss: 0.6027 - val_accuracy: 0.6919 - val_auc: 0.6693\n","Epoch 4/5\n","25/25 [==============================] - 7s 289ms/step - loss: 0.6090 - accuracy: 0.6778 - auc: 0.6803 - val_loss: 0.6420 - val_accuracy: 0.6715 - val_auc: 0.6623\n","Epoch 5/5\n","25/25 [==============================] - 7s 286ms/step - loss: 0.6012 - accuracy: 0.6849 - auc: 0.7031 - val_loss: 0.6013 - val_accuracy: 0.6628 - val_auc: 0.6686\n","Epoch 1/5\n","25/25 [==============================] - 11s 334ms/step - loss: 3.6025 - accuracy: 0.5837 - auc: 0.5193 - val_loss: 0.6376 - val_accuracy: 0.6628 - val_auc: 0.5889\n","Epoch 2/5\n","25/25 [==============================] - 7s 289ms/step - loss: 0.6586 - accuracy: 0.6538 - auc: 0.5961 - val_loss: 0.6665 - val_accuracy: 0.6570 - val_auc: 0.6319\n","Epoch 3/5\n","25/25 [==============================] - 7s 288ms/step - loss: 0.6030 - accuracy: 0.6723 - auc: 0.6746 - val_loss: 0.6088 - val_accuracy: 0.6512 - val_auc: 0.6527\n","Epoch 4/5\n","25/25 [==============================] - 8s 331ms/step - loss: 0.5913 - accuracy: 0.6823 - auc: 0.7041 - val_loss: 0.6493 - val_accuracy: 0.6744 - val_auc: 0.6662\n","Epoch 5/5\n","25/25 [==============================] - 7s 293ms/step - loss: 0.5881 - accuracy: 0.6913 - auc: 0.7131 - val_loss: 0.6047 - val_accuracy: 0.6919 - val_auc: 0.6995\n","Epoch 1/5\n","25/25 [==============================] - 10s 303ms/step - loss: 4.2498 - accuracy: 0.5928 - auc: 0.5234 - val_loss: 0.6393 - val_accuracy: 0.6802 - val_auc: 0.5735\n","Epoch 2/5\n","25/25 [==============================] - 7s 275ms/step - loss: 0.6363 - accuracy: 0.6784 - auc: 0.6186 - val_loss: 0.6330 - val_accuracy: 0.6715 - val_auc: 0.6473\n","Epoch 3/5\n","25/25 [==============================] - 5s 199ms/step - loss: 0.6446 - accuracy: 0.6774 - auc: 0.6666 - val_loss: 0.7163 - val_accuracy: 0.6017 - val_auc: 0.6487\n","Epoch 4/5\n","25/25 [==============================] - 5s 195ms/step - loss: 0.6313 - accuracy: 0.6600 - auc: 0.6718 - val_loss: 0.6187 - val_accuracy: 0.6773 - val_auc: 0.6567\n","Epoch 5/5\n","25/25 [==============================] - 5s 202ms/step - loss: 0.5806 - accuracy: 0.6962 - auc: 0.7185 - val_loss: 0.5942 - val_accuracy: 0.6890 - val_auc: 0.6696\n"]}]},{"cell_type":"code","source":["val_F1_mean, val_F1_sd = mean_and_sd(val_F1)\n","val_acc_mean, val_acc_sd = mean_and_sd(val_acc)\n","val_pr_auc_mean, val_pr_auc_sd = mean_and_sd(val_pr_auc)\n","val_roc_auc_mean, val_roc_auc_sd = mean_and_sd(val_roc_auc)\n","\n","test_F1_mean, test_F1_sd = mean_and_sd(test_F1)\n","test_acc_mean, test_acc_sd = mean_and_sd(test_acc)\n","test_pr_auc_mean, test_pr_auc_sd = mean_and_sd(test_pr_auc)\n","test_roc_auc_mean, test_roc_auc_sd = mean_and_sd(test_roc_auc)"],"metadata":{"id":"9WTovGNylc-Q"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["print(f\"validation F1: {round(val_F1_mean, 3)} +/- {round(val_F1_sd, 3)}\")\n","print(f\"test F1: {round(test_F1_mean, 3)} +/- {round(test_F1_sd, 3)}\")\n","\n","print(f\"validation accuracy: {round(val_acc_mean, 3)} +/- {round(val_acc_sd, 3)}\")\n","print(f\"test accuracy: {round(test_acc_mean, 3)} +/- {round(test_acc_sd, 3)}\")\n","\n","print(f\"validation PR-AUC: {round(val_pr_auc_mean, 3)} +/- {round(val_pr_auc_sd, 3)}\")\n","print(f\"test PR-AUC: {round(test_pr_auc_mean, 3)} +/- {round(test_pr_auc_sd, 3)}\")\n","\n","print(f\"validation ROC-AUC: {round(val_roc_auc_mean, 3)} +/- {round(val_roc_auc_sd, 3)}\")\n","print(f\"test ROC-AUC: {round(test_roc_auc_mean, 3)} +/- {round(test_roc_auc_sd, 3)}\")"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"E7zHHIs6ls4f","executionInfo":{"status":"ok","timestamp":1651526772416,"user_tz":240,"elapsed":25,"user":{"displayName":"Benedikt Geiger","userId":"17925887631246406508"}},"outputId":"a89c59de-e86e-42ae-93dd-bdd4df2d2a86"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["validation F1: 0.784 +/- 0.02\n","test F1: 0.822 +/- 0.042\n","validation accuracy: 0.677 +/- 0.014\n","test accuracy: 0.719 +/- 0.04\n","validation PR-AUC: 0.798 +/- 0.018\n","test PR-AUC: 0.839 +/- 0.008\n","validation ROC-AUC: 0.674 +/- 0.017\n","test ROC-AUC: 0.649 +/- 0.015\n"]}]},{"cell_type":"markdown","source":["### Without disease data"],"metadata":{"id":"zyvdoJ0Vmuog"}},{"cell_type":"code","source":["n_runs = 30"],"metadata":{"id":"-m1MCcOcm5vi"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["val_F1 = np.zeros(n_runs, dtype=float)\n","val_acc = np.zeros(n_runs, dtype=float)\n","val_pr_auc = np.zeros(n_runs, dtype=float)\n","val_roc_auc = np.zeros(n_runs, dtype=float)\n","\n","test_F1 = np.zeros(n_runs, dtype=float)\n","test_acc = np.zeros(n_runs, dtype=float)\n","test_pr_auc = np.zeros(n_runs, dtype=float)\n","test_roc_auc = np.zeros(n_runs, dtype=float)"],"metadata":{"id":"tS9OXzy2m8Nz"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["for i in range(n_runs):\n","\n","  # inclusion and exclusion criteria\n","  inclusion_input = layers.Input(shape=(incl_criteria_dim,), name=\"inclusion\")\n","  inclusion_emb = layers.Dense(256, activation=\"relu\", name=\"lower_dim_inclusion\")(inclusion_input)\n","\n","  exclusion_input = layers.Input(shape=(excl_criteria_dim,), name=\"exclusion\")\n","  exclusion_emb = layers.Dense(256, activation=\"relu\", name=\"lower_dim_exclusion\")(exclusion_input)\n","\n","  inclusion_exclusion_raw = layers.Concatenate(name=\"criteria\")([inclusion_emb, exclusion_emb])\n","  ie_dropout = layers.Dropout(rate=0.32)(inclusion_exclusion_raw)\n","  inclusion_exclusion_emb = layers.Dense(256, activation=\"relu\", name=\"lower_dim_criteria\")(ie_dropout)\n","\n","\n","  # drug\n","  drug_input = layers.Input(shape=(drug_dim,), name=\"drug\")\n","  drug_emb = layers.Dense(64, activation=\"relu\", name=\"lower_dim_drug\")(drug_input)\n","\n","  # targets\n","  targets_input = layers.Input(shape=(targets_dim,), name=\"targets\")\n","  targets_emb = layers.Dense(64, activation=\"relu\", name=\"lower_dim_targets\")(targets_input)\n","\n","  # drug-target interaction\n","  drug_target_raw = layers.Concatenate(name=\"drug-targets\")([drug_emb, targets_emb])\n","  dt = layers.Dropout(rate=0.32)(drug_target_raw)\n","  drug_target_emb = layers.Dense(64, activation=\"relu\", name=\"lower_dim_drug-targets\")(dt)\n","\n","\n","  all_emb = layers.Concatenate(name=\"all_embeddings\")([inclusion_exclusion_emb, drug_target_emb])\n","  ae = layers.Dropout(rate=0.32)(all_emb)\n","  trial_embedding1 = layers.Dense(128, activation=\"relu\", name=\"trial_embedding_1\")(ae)\n","\n","  meta_input = layers.Input(shape=(meta_dim,), name=\"meta\")\n","  emb_and_meta = layers.Concatenate()([trial_embedding1, meta_input])\n","\n","  trial_embedding2 = layers.Dense(64, activation=\"relu\", name=\"trial_embedding_2\")(emb_and_meta)\n","  trial_embedding3 = layers.Dense(32, activation=\"relu\", name=\"trial_embedding_3\")(trial_embedding2)\n","\n","  o = layers.Dense(1, activation=\"sigmoid\", name=\"trial_success\")(trial_embedding3)\n","\n","  model = tf.keras.Model(inputs=[inclusion_input, exclusion_input, drug_input, targets_input, meta_input], outputs=[o])\n","  model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=0.01), loss=\"binary_crossentropy\", metrics=[\"accuracy\", \"AUC\"])\n","\n","  history = model.fit(\n","    x={\"inclusion\": train_data_incl, \"exclusion\": train_data_excl, \n","      \"drug\": train_data_drug, \"targets\": train_data_target,\n","      \"meta\": train_data_meta},\n","    y={\"trial_success\": train_labels},\n","    validation_data=(\n","        {\"inclusion\": val_data_incl, \"exclusion\": val_data_excl,\n","        \"drug\": val_data_drug, \"targets\": val_data_target,\n","        \"meta\": val_data_meta},\n","        {\"trial_success\": val_labels}\n","    ), \n","    epochs=5,\n","    batch_size=128\n","  )\n","\n","\n","  val_metrics = model_performance(\n","    model,\n","    {\"inclusion\": val_data_incl, \"exclusion\": val_data_excl, \n","    \"drug\": val_data_drug, \"targets\": val_data_target,\n","    \"meta\": val_data_meta},\n","    val_labels,\n","    save_path=None\n","  )\n","\n","  test_metrics = model_performance(\n","    model,\n","    {\"inclusion\": test_data_incl, \"exclusion\": test_data_excl, \n","    \"drug\": test_data_drug, \"targets\": test_data_target,\n","    \"meta\": test_data_meta},\n","    test_labels,\n","    save_path=None\n","  )\n","\n","  val_F1[i] = val_metrics[\"F1\"]\n","  val_acc[i] = val_metrics[\"accuracy\"]\n","  val_pr_auc[i] = val_metrics[\"pr_auc\"]\n","  val_roc_auc[i] = val_metrics[\"roc_auc\"]\n","\n","  test_F1[i] = test_metrics[\"F1\"]\n","  test_acc[i] = test_metrics[\"accuracy\"]\n","  test_pr_auc[i] = test_metrics[\"pr_auc\"]\n","  test_roc_auc[i] = test_metrics[\"roc_auc\"]"],"metadata":{"id":"KIfFSkeenCmF"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["val_F1_mean, val_F1_sd = mean_and_sd(val_F1)\n","val_acc_mean, val_acc_sd = mean_and_sd(val_acc)\n","val_pr_auc_mean, val_pr_auc_sd = mean_and_sd(val_pr_auc)\n","val_roc_auc_mean, val_roc_auc_sd = mean_and_sd(val_roc_auc)\n","\n","test_F1_mean, test_F1_sd = mean_and_sd(test_F1)\n","test_acc_mean, test_acc_sd = mean_and_sd(test_acc)\n","test_pr_auc_mean, test_pr_auc_sd = mean_and_sd(test_pr_auc)\n","test_roc_auc_mean, test_roc_auc_sd = mean_and_sd(test_roc_auc)"],"metadata":{"id":"RggtOU2noIDa"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["print(f\"validation F1: {round(val_F1_mean, 3)} +/- {round(val_F1_sd, 3)}\")\n","print(f\"test F1: {round(test_F1_mean, 3)} +/- {round(test_F1_sd, 3)}\")\n","\n","print(f\"validation accuracy: {round(val_acc_mean, 3)} +/- {round(val_acc_sd, 3)}\")\n","print(f\"test accuracy: {round(test_acc_mean, 3)} +/- {round(test_acc_sd, 3)}\")\n","\n","print(f\"validation PR-AUC: {round(val_pr_auc_mean, 3)} +/- {round(val_pr_auc_sd, 3)}\")\n","print(f\"test PR-AUC: {round(test_pr_auc_mean, 3)} +/- {round(test_pr_auc_sd, 3)}\")\n","\n","print(f\"validation ROC-AUC: {round(val_roc_auc_mean, 3)} +/- {round(val_roc_auc_sd, 3)}\")\n","print(f\"test ROC-AUC: {round(test_roc_auc_mean, 3)} +/- {round(test_roc_auc_sd, 3)}\")"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"kHliRYWLoKND","executionInfo":{"status":"ok","timestamp":1651527968428,"user_tz":240,"elapsed":14,"user":{"displayName":"Benedikt Geiger","userId":"17925887631246406508"}},"outputId":"063c2103-e15e-422a-c19c-b6199d2dcd20"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["validation F1: 0.798 +/- 0.007\n","test F1: 0.838 +/- 0.019\n","validation accuracy: 0.685 +/- 0.011\n","test accuracy: 0.729 +/- 0.023\n","validation PR-AUC: 0.754 +/- 0.012\n","test PR-AUC: 0.814 +/- 0.012\n","validation ROC-AUC: 0.639 +/- 0.013\n","test ROC-AUC: 0.604 +/- 0.024\n"]}]},{"cell_type":"markdown","source":["### Without drug data"],"metadata":{"id":"WA8cHUO8pMlb"}},{"cell_type":"code","source":["n_runs = 30"],"metadata":{"id":"qRZwNXrQpMle"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["val_F1 = np.zeros(n_runs, dtype=float)\n","val_acc = np.zeros(n_runs, dtype=float)\n","val_pr_auc = np.zeros(n_runs, dtype=float)\n","val_roc_auc = np.zeros(n_runs, dtype=float)\n","\n","test_F1 = np.zeros(n_runs, dtype=float)\n","test_acc = np.zeros(n_runs, dtype=float)\n","test_pr_auc = np.zeros(n_runs, dtype=float)\n","test_roc_auc = np.zeros(n_runs, dtype=float)"],"metadata":{"id":"jL8YVYJ3pMlg"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["for i in range(n_runs):\n","\n","  # inclusion and exclusion criteria\n","  inclusion_input = layers.Input(shape=(incl_criteria_dim,), name=\"inclusion\")\n","  inclusion_emb = layers.Dense(256, activation=\"relu\", name=\"lower_dim_inclusion\")(inclusion_input)\n","\n","  exclusion_input = layers.Input(shape=(excl_criteria_dim,), name=\"exclusion\")\n","  exclusion_emb = layers.Dense(256, activation=\"relu\", name=\"lower_dim_exclusion\")(exclusion_input)\n","\n","  inclusion_exclusion_raw = layers.Concatenate(name=\"criteria\")([inclusion_emb, exclusion_emb])\n","  ie_dropout = layers.Dropout(rate=0.32)(inclusion_exclusion_raw)\n","  inclusion_exclusion_emb = layers.Dense(256, activation=\"relu\", name=\"lower_dim_criteria\")(ie_dropout)\n","\n","  # diseases\n","  diseases_input = layers.Input(shape=(diseases_dim,), name=\"diseases\")\n","  diseases_emb = layers.Dense(128, activation=\"relu\", name=\"lower_dim_diseases\")(diseases_input)\n","\n","  # targets\n","  targets_input = layers.Input(shape=(targets_dim,), name=\"targets\")\n","  targets_emb = layers.Dense(64, activation=\"relu\", name=\"lower_dim_targets\")(targets_input)\n","\n","\n","  all_emb = layers.Concatenate(name=\"all_embeddings\")([inclusion_exclusion_emb, diseases_emb, targets_emb])\n","  ae = layers.Dropout(rate=0.32)(all_emb)\n","  trial_embedding1 = layers.Dense(128, activation=\"relu\", name=\"trial_embedding_1\")(ae)\n","\n","  meta_input = layers.Input(shape=(meta_dim,), name=\"meta\")\n","  emb_and_meta = layers.Concatenate()([trial_embedding1, meta_input])\n","\n","  trial_embedding2 = layers.Dense(64, activation=\"relu\", name=\"trial_embedding_2\")(emb_and_meta)\n","  trial_embedding3 = layers.Dense(32, activation=\"relu\", name=\"trial_embedding_3\")(trial_embedding2)\n","\n","  o = layers.Dense(1, activation=\"sigmoid\", name=\"trial_success\")(trial_embedding3)\n","\n","  model = tf.keras.Model(inputs=[inclusion_input, exclusion_input, diseases_input, targets_input, meta_input], outputs=[o])\n","  model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=0.01), loss=\"binary_crossentropy\", metrics=[\"accuracy\", \"AUC\"])\n","\n","  history = model.fit(\n","    x={\"inclusion\": train_data_incl, \"exclusion\": train_data_excl,\n","       \"diseases\": train_data_diseases, \"targets\": train_data_target,\n","      \"meta\": train_data_meta},\n","    y={\"trial_success\": train_labels},\n","    validation_data=(\n","        {\"inclusion\": val_data_incl, \"exclusion\": val_data_excl,\n","        \"diseases\": val_data_diseases, \"targets\": val_data_target,\n","        \"meta\": val_data_meta},\n","        {\"trial_success\": val_labels}\n","    ), \n","    epochs=5,\n","    batch_size=128\n","  )\n","\n","\n","  val_metrics = model_performance(\n","    model,\n","    {\"inclusion\": val_data_incl, \"exclusion\": val_data_excl, \n","    \"diseases\": val_data_diseases, \"targets\": val_data_target,\n","    \"meta\": val_data_meta},\n","    val_labels,\n","    save_path=None\n","  )\n","\n","  test_metrics = model_performance(\n","    model,\n","    {\"inclusion\": test_data_incl, \"exclusion\": test_data_excl, \n","    \"diseases\": test_data_diseases, \"targets\": test_data_target,\n","    \"meta\": test_data_meta},\n","    test_labels,\n","    save_path=None\n","  )\n","\n","  val_F1[i] = val_metrics[\"F1\"]\n","  val_acc[i] = val_metrics[\"accuracy\"]\n","  val_pr_auc[i] = val_metrics[\"pr_auc\"]\n","  val_roc_auc[i] = val_metrics[\"roc_auc\"]\n","\n","  test_F1[i] = test_metrics[\"F1\"]\n","  test_acc[i] = test_metrics[\"accuracy\"]\n","  test_pr_auc[i] = test_metrics[\"pr_auc\"]\n","  test_roc_auc[i] = test_metrics[\"roc_auc\"]"],"metadata":{"id":"jzs9CyMZpMlh"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["val_F1_mean, val_F1_sd = mean_and_sd(val_F1)\n","val_acc_mean, val_acc_sd = mean_and_sd(val_acc)\n","val_pr_auc_mean, val_pr_auc_sd = mean_and_sd(val_pr_auc)\n","val_roc_auc_mean, val_roc_auc_sd = mean_and_sd(val_roc_auc)\n","\n","test_F1_mean, test_F1_sd = mean_and_sd(test_F1)\n","test_acc_mean, test_acc_sd = mean_and_sd(test_acc)\n","test_pr_auc_mean, test_pr_auc_sd = mean_and_sd(test_pr_auc)\n","test_roc_auc_mean, test_roc_auc_sd = mean_and_sd(test_roc_auc)"],"metadata":{"id":"CNHR5bcJpMll"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["print(f\"validation F1: {round(val_F1_mean, 3)} +/- {round(val_F1_sd, 3)}\")\n","print(f\"test F1: {round(test_F1_mean, 3)} +/- {round(test_F1_sd, 3)}\")\n","\n","print(f\"validation accuracy: {round(val_acc_mean, 3)} +/- {round(val_acc_sd, 3)}\")\n","print(f\"test accuracy: {round(test_acc_mean, 3)} +/- {round(test_acc_sd, 3)}\")\n","\n","print(f\"validation PR-AUC: {round(val_pr_auc_mean, 3)} +/- {round(val_pr_auc_sd, 3)}\")\n","print(f\"test PR-AUC: {round(test_pr_auc_mean, 3)} +/- {round(test_pr_auc_sd, 3)}\")\n","\n","print(f\"validation ROC-AUC: {round(val_roc_auc_mean, 3)} +/- {round(val_roc_auc_sd, 3)}\")\n","print(f\"test ROC-AUC: {round(test_roc_auc_mean, 3)} +/- {round(test_roc_auc_sd, 3)}\")"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"oJx1NwE3pMlo","executionInfo":{"status":"ok","timestamp":1651529173009,"user_tz":240,"elapsed":17,"user":{"displayName":"Benedikt Geiger","userId":"17925887631246406508"}},"outputId":"3dcdd25f-42a1-4e50-957c-d7897af4ca4e"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["validation F1: 0.787 +/- 0.014\n","test F1: 0.83 +/- 0.027\n","validation accuracy: 0.683 +/- 0.01\n","test accuracy: 0.729 +/- 0.028\n","validation PR-AUC: 0.802 +/- 0.012\n","test PR-AUC: 0.842 +/- 0.007\n","validation ROC-AUC: 0.679 +/- 0.011\n","test ROC-AUC: 0.667 +/- 0.015\n"]}]},{"cell_type":"markdown","source":["### Without meta data"],"metadata":{"id":"qWjp7FlTrTWg"}},{"cell_type":"code","source":["n_runs = 30"],"metadata":{"id":"7V4NvDTUrXvz"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["val_F1 = np.zeros(n_runs, dtype=float)\n","val_acc = np.zeros(n_runs, dtype=float)\n","val_pr_auc = np.zeros(n_runs, dtype=float)\n","val_roc_auc = np.zeros(n_runs, dtype=float)\n","\n","test_F1 = np.zeros(n_runs, dtype=float)\n","test_acc = np.zeros(n_runs, dtype=float)\n","test_pr_auc = np.zeros(n_runs, dtype=float)\n","test_roc_auc = np.zeros(n_runs, dtype=float)"],"metadata":{"id":"Uk51fO2graji"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["for i in range(n_runs):\n","\n","  # inclusion and exclusion criteria\n","  inclusion_input = layers.Input(shape=(incl_criteria_dim,), name=\"inclusion\")\n","  inclusion_emb = layers.Dense(256, activation=\"relu\", name=\"lower_dim_inclusion\")(inclusion_input)\n","\n","  exclusion_input = layers.Input(shape=(excl_criteria_dim,), name=\"exclusion\")\n","  exclusion_emb = layers.Dense(256, activation=\"relu\", name=\"lower_dim_exclusion\")(exclusion_input)\n","\n","  inclusion_exclusion_raw = layers.Concatenate(name=\"criteria\")([inclusion_emb, exclusion_emb])\n","  ie_dropout = layers.Dropout(rate=0.32)(inclusion_exclusion_raw)\n","  inclusion_exclusion_emb = layers.Dense(256, activation=\"relu\", name=\"lower_dim_criteria\")(ie_dropout)\n","\n","\n","  # diseases\n","  diseases_input = layers.Input(shape=(diseases_dim,), name=\"diseases\")\n","  diseases_emb = layers.Dense(128, activation=\"relu\", name=\"lower_dim_diseases\")(diseases_input)\n","\n","\n","  # drug\n","  drug_input = layers.Input(shape=(drug_dim,), name=\"drug\")\n","  drug_emb = layers.Dense(64, activation=\"relu\", name=\"lower_dim_drug\")(drug_input)\n","\n","  # targets\n","  targets_input = layers.Input(shape=(targets_dim,), name=\"targets\")\n","  targets_emb = layers.Dense(64, activation=\"relu\", name=\"lower_dim_targets\")(targets_input)\n","\n","  # drug-target interaction\n","  drug_target_raw = layers.Concatenate(name=\"drug-targets\")([drug_emb, targets_emb])\n","  dt = layers.Dropout(rate=0.32)(drug_target_raw)\n","  drug_target_emb = layers.Dense(64, activation=\"relu\", name=\"lower_dim_drug-targets\")(dt)\n","\n","\n","  all_emb = layers.Concatenate(name=\"all_embeddings\")([inclusion_exclusion_emb, diseases_emb, drug_target_emb])\n","  ae = layers.Dropout(rate=0.32)(all_emb)\n","  trial_embedding1 = layers.Dense(128, activation=\"relu\", name=\"trial_embedding_1\")(ae)\n","  trial_embedding2 = layers.Dense(64, activation=\"relu\", name=\"trial_embedding_2\")(trial_embedding1)\n","  trial_embedding3 = layers.Dense(32, activation=\"relu\", name=\"trial_embedding_3\")(trial_embedding2)\n","\n","  o = layers.Dense(1, activation=\"sigmoid\", name=\"trial_success\")(trial_embedding3)\n","\n","  model = tf.keras.Model(inputs=[inclusion_input, exclusion_input, diseases_input, drug_input, targets_input], outputs=[o])\n","  model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=0.01), loss=\"binary_crossentropy\", metrics=[\"accuracy\", \"AUC\"])\n","\n","  history = model.fit(\n","    x={\"inclusion\": train_data_incl, \"exclusion\": train_data_excl,\n","       \"drug\": train_data_drug, \"diseases\": train_data_diseases, \n","       \"targets\": train_data_target},\n","    y={\"trial_success\": train_labels},\n","    validation_data=(\n","        {\"inclusion\": val_data_incl, \"exclusion\": val_data_excl,\n","         \"drug\": val_data_drug, \"diseases\": val_data_diseases, \n","         \"targets\": val_data_target\n","        },\n","        {\"trial_success\": val_labels}\n","    ), \n","    epochs=5,\n","    batch_size=128\n","  )\n","\n","\n","  val_metrics = model_performance(\n","    model,\n","    {\"inclusion\": val_data_incl, \"exclusion\": val_data_excl,\n","     \"drug\": val_data_drug, \"diseases\": val_data_diseases, \n","     \"targets\": val_data_target},\n","    val_labels,\n","    save_path=None\n","  )\n","\n","  test_metrics = model_performance(\n","    model,\n","    {\"inclusion\": test_data_incl, \"exclusion\": test_data_excl,\n","     \"drug\": test_data_drug, \"diseases\": test_data_diseases, \n","     \"targets\": test_data_target},\n","    test_labels,\n","    save_path=None\n","  )\n","\n","  val_F1[i] = val_metrics[\"F1\"]\n","  val_acc[i] = val_metrics[\"accuracy\"]\n","  val_pr_auc[i] = val_metrics[\"pr_auc\"]\n","  val_roc_auc[i] = val_metrics[\"roc_auc\"]\n","\n","  test_F1[i] = test_metrics[\"F1\"]\n","  test_acc[i] = test_metrics[\"accuracy\"]\n","  test_pr_auc[i] = test_metrics[\"pr_auc\"]\n","  test_roc_auc[i] = test_metrics[\"roc_auc\"]"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"eTOicaYtrfD5","executionInfo":{"status":"ok","timestamp":1651530561600,"user_tz":240,"elapsed":1191457,"user":{"displayName":"Benedikt Geiger","userId":"17925887631246406508"}},"outputId":"52aa55d1-02e5-46cb-853a-e3f9dbe38ad3"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["Epoch 1/5\n","25/25 [==============================] - 7s 221ms/step - loss: 2.8720 - accuracy: 0.5957 - auc: 0.5178 - val_loss: 0.6364 - val_accuracy: 0.6802 - val_auc: 0.5521\n","Epoch 2/5\n","25/25 [==============================] - 5s 201ms/step - loss: 0.6398 - accuracy: 0.6739 - auc: 0.5995 - val_loss: 0.6445 - val_accuracy: 0.6831 - val_auc: 0.6133\n","Epoch 3/5\n","25/25 [==============================] - 5s 202ms/step - loss: 0.6252 - accuracy: 0.6723 - auc: 0.6217 - val_loss: 0.6202 - val_accuracy: 0.6628 - val_auc: 0.6541\n","Epoch 4/5\n","25/25 [==============================] - 5s 198ms/step - loss: 0.6048 - accuracy: 0.6732 - auc: 0.6838 - val_loss: 0.5979 - val_accuracy: 0.6773 - val_auc: 0.6830\n","Epoch 5/5\n","25/25 [==============================] - 5s 200ms/step - loss: 0.5752 - accuracy: 0.6871 - auc: 0.7254 - val_loss: 0.6089 - val_accuracy: 0.6802 - val_auc: 0.6766\n","Epoch 1/5\n","25/25 [==============================] - 7s 228ms/step - loss: 2.2530 - accuracy: 0.5798 - auc: 0.5297 - val_loss: 0.6348 - val_accuracy: 0.6570 - val_auc: 0.5560\n","Epoch 2/5\n","25/25 [==============================] - 5s 201ms/step - loss: 0.6528 - accuracy: 0.6690 - auc: 0.6302 - val_loss: 0.6147 - val_accuracy: 0.6802 - val_auc: 0.6401\n","Epoch 3/5\n","25/25 [==============================] - 6s 223ms/step - loss: 0.5987 - accuracy: 0.6787 - auc: 0.6730 - val_loss: 0.6055 - val_accuracy: 0.6686 - val_auc: 0.6477\n","Epoch 4/5\n","25/25 [==============================] - 5s 205ms/step - loss: 0.5866 - accuracy: 0.6862 - auc: 0.6983 - val_loss: 0.6094 - val_accuracy: 0.6744 - val_auc: 0.6683\n","Epoch 5/5\n","25/25 [==============================] - 5s 202ms/step - loss: 0.5718 - accuracy: 0.7004 - auc: 0.7258 - val_loss: 0.6061 - val_accuracy: 0.6890 - val_auc: 0.6649\n","Epoch 1/5\n","25/25 [==============================] - 7s 213ms/step - loss: 4.1644 - accuracy: 0.5960 - auc: 0.5108 - val_loss: 0.6416 - val_accuracy: 0.6686 - val_auc: 0.5058\n","Epoch 2/5\n","25/25 [==============================] - 5s 202ms/step - loss: 0.6529 - accuracy: 0.6622 - auc: 0.5624 - val_loss: 0.6288 - val_accuracy: 0.6686 - val_auc: 0.6495\n","Epoch 3/5\n","25/25 [==============================] - 5s 196ms/step - loss: 0.6168 - accuracy: 0.6758 - auc: 0.6300 - val_loss: 0.5982 - val_accuracy: 0.6831 - val_auc: 0.6803\n","Epoch 4/5\n","25/25 [==============================] - 5s 200ms/step - loss: 0.5979 - accuracy: 0.6810 - auc: 0.6803 - val_loss: 0.5886 - val_accuracy: 0.6773 - val_auc: 0.6971\n","Epoch 5/5\n","25/25 [==============================] - 5s 197ms/step - loss: 0.5944 - accuracy: 0.6823 - auc: 0.6990 - val_loss: 0.6495 - val_accuracy: 0.6628 - val_auc: 0.6731\n","Epoch 1/5\n","25/25 [==============================] - 7s 222ms/step - loss: 3.5438 - accuracy: 0.5727 - auc: 0.5115 - val_loss: 0.6754 - val_accuracy: 0.6570 - val_auc: 0.5285\n","Epoch 2/5\n","25/25 [==============================] - 5s 209ms/step - loss: 0.6461 - accuracy: 0.6652 - auc: 0.6196 - val_loss: 0.5988 - val_accuracy: 0.6773 - val_auc: 0.6633\n","Epoch 3/5\n","25/25 [==============================] - 5s 207ms/step - loss: 0.6084 - accuracy: 0.6794 - auc: 0.6943 - val_loss: 0.5917 - val_accuracy: 0.6744 - val_auc: 0.6721\n","Epoch 4/5\n","25/25 [==============================] - 5s 208ms/step - loss: 0.5733 - accuracy: 0.6997 - auc: 0.7280 - val_loss: 0.5934 - val_accuracy: 0.6831 - val_auc: 0.6758\n","Epoch 5/5\n","25/25 [==============================] - 5s 212ms/step - loss: 0.5467 - accuracy: 0.7127 - auc: 0.7653 - val_loss: 0.5903 - val_accuracy: 0.7093 - val_auc: 0.6985\n","Epoch 1/5\n","25/25 [==============================] - 7s 223ms/step - loss: 2.9898 - accuracy: 0.5886 - auc: 0.5237 - val_loss: 0.6816 - val_accuracy: 0.6628 - val_auc: 0.5093\n","Epoch 2/5\n","25/25 [==============================] - 5s 200ms/step - loss: 0.6733 - accuracy: 0.6522 - auc: 0.5819 - val_loss: 0.6484 - val_accuracy: 0.6657 - val_auc: 0.5529\n","Epoch 3/5\n","25/25 [==============================] - 5s 201ms/step - loss: 0.6358 - accuracy: 0.6677 - auc: 0.6220 - val_loss: 0.6174 - val_accuracy: 0.6715 - val_auc: 0.6534\n","Epoch 4/5\n","25/25 [==============================] - 5s 197ms/step - loss: 0.6154 - accuracy: 0.6687 - auc: 0.6667 - val_loss: 0.5978 - val_accuracy: 0.6773 - val_auc: 0.6751\n","Epoch 5/5\n","25/25 [==============================] - 5s 198ms/step - loss: 0.6034 - accuracy: 0.6745 - auc: 0.6911 - val_loss: 0.6256 - val_accuracy: 0.6715 - val_auc: 0.7009\n","Epoch 1/5\n","25/25 [==============================] - 33s 221ms/step - loss: 2.8836 - accuracy: 0.5866 - auc: 0.5262 - val_loss: 0.6612 - val_accuracy: 0.6424 - val_auc: 0.6132\n","Epoch 2/5\n","25/25 [==============================] - 5s 196ms/step - loss: 0.6504 - accuracy: 0.6412 - auc: 0.5913 - val_loss: 0.6649 - val_accuracy: 0.6657 - val_auc: 0.5088\n","Epoch 3/5\n","25/25 [==============================] - 5s 199ms/step - loss: 0.6178 - accuracy: 0.6723 - auc: 0.6478 - val_loss: 0.5954 - val_accuracy: 0.6831 - val_auc: 0.6776\n","Epoch 4/5\n","25/25 [==============================] - 5s 198ms/step - loss: 0.5925 - accuracy: 0.6823 - auc: 0.6983 - val_loss: 0.6142 - val_accuracy: 0.6831 - val_auc: 0.6552\n","Epoch 5/5\n","25/25 [==============================] - 5s 197ms/step - loss: 0.5782 - accuracy: 0.6871 - auc: 0.7194 - val_loss: 0.5957 - val_accuracy: 0.6773 - val_auc: 0.6703\n","Epoch 1/5\n","25/25 [==============================] - 7s 224ms/step - loss: 2.4289 - accuracy: 0.5902 - auc: 0.5076 - val_loss: 0.7202 - val_accuracy: 0.5552 - val_auc: 0.5643\n","Epoch 2/5\n","25/25 [==============================] - 5s 199ms/step - loss: 0.6485 - accuracy: 0.6487 - auc: 0.6008 - val_loss: 0.6181 - val_accuracy: 0.6657 - val_auc: 0.6296\n","Epoch 3/5\n","25/25 [==============================] - 5s 202ms/step - loss: 0.6157 - accuracy: 0.6755 - auc: 0.6414 - val_loss: 0.6223 - val_accuracy: 0.6773 - val_auc: 0.6396\n","Epoch 4/5\n","25/25 [==============================] - 5s 200ms/step - loss: 0.5965 - accuracy: 0.6836 - auc: 0.6816 - val_loss: 0.5999 - val_accuracy: 0.6802 - val_auc: 0.6619\n","Epoch 5/5\n","25/25 [==============================] - 5s 198ms/step - loss: 0.5775 - accuracy: 0.6878 - auc: 0.7153 - val_loss: 0.5888 - val_accuracy: 0.6831 - val_auc: 0.6833\n","Epoch 1/5\n","25/25 [==============================] - 7s 212ms/step - loss: 4.5040 - accuracy: 0.5960 - auc: 0.5215 - val_loss: 0.6671 - val_accuracy: 0.6657 - val_auc: 0.4863\n","Epoch 2/5\n","25/25 [==============================] - 5s 202ms/step - loss: 0.6476 - accuracy: 0.6661 - auc: 0.5737 - val_loss: 0.6202 - val_accuracy: 0.6831 - val_auc: 0.6112\n","Epoch 3/5\n","25/25 [==============================] - 5s 201ms/step - loss: 0.6220 - accuracy: 0.6813 - auc: 0.6296 - val_loss: 0.6093 - val_accuracy: 0.6831 - val_auc: 0.6521\n","Epoch 4/5\n","25/25 [==============================] - 5s 201ms/step - loss: 0.5944 - accuracy: 0.6826 - auc: 0.6877 - val_loss: 0.6211 - val_accuracy: 0.6831 - val_auc: 0.6859\n","Epoch 5/5\n","25/25 [==============================] - 5s 197ms/step - loss: 0.5684 - accuracy: 0.7023 - auc: 0.7407 - val_loss: 0.7594 - val_accuracy: 0.6831 - val_auc: 0.6877\n","Epoch 1/5\n","25/25 [==============================] - 7s 231ms/step - loss: 4.1338 - accuracy: 0.5876 - auc: 0.5455 - val_loss: 0.7275 - val_accuracy: 0.6802 - val_auc: 0.5735\n","Epoch 2/5\n","25/25 [==============================] - 5s 221ms/step - loss: 0.6829 - accuracy: 0.6610 - auc: 0.5789 - val_loss: 0.6488 - val_accuracy: 0.6773 - val_auc: 0.6175\n","Epoch 3/5\n","25/25 [==============================] - 9s 352ms/step - loss: 0.6134 - accuracy: 0.6742 - auc: 0.6668 - val_loss: 0.5989 - val_accuracy: 0.6802 - val_auc: 0.6738\n","Epoch 4/5\n","25/25 [==============================] - 5s 198ms/step - loss: 0.5915 - accuracy: 0.6862 - auc: 0.7027 - val_loss: 0.5909 - val_accuracy: 0.6773 - val_auc: 0.6913\n","Epoch 5/5\n","25/25 [==============================] - 5s 204ms/step - loss: 0.5916 - accuracy: 0.6965 - auc: 0.7269 - val_loss: 0.6117 - val_accuracy: 0.6890 - val_auc: 0.6905\n","Epoch 1/5\n","25/25 [==============================] - 7s 233ms/step - loss: 2.5749 - accuracy: 0.5805 - auc: 0.5319 - val_loss: 0.6398 - val_accuracy: 0.6831 - val_auc: 0.6407\n","Epoch 2/5\n","25/25 [==============================] - 6s 237ms/step - loss: 0.6370 - accuracy: 0.6707 - auc: 0.6048 - val_loss: 0.6161 - val_accuracy: 0.6890 - val_auc: 0.6377\n","Epoch 3/5\n","25/25 [==============================] - 5s 215ms/step - loss: 0.6032 - accuracy: 0.6829 - auc: 0.6664 - val_loss: 0.5961 - val_accuracy: 0.6860 - val_auc: 0.6715\n","Epoch 4/5\n","25/25 [==============================] - 5s 212ms/step - loss: 0.5931 - accuracy: 0.6749 - auc: 0.6835 - val_loss: 0.6027 - val_accuracy: 0.6773 - val_auc: 0.6706\n","Epoch 5/5\n","25/25 [==============================] - 6s 226ms/step - loss: 0.5835 - accuracy: 0.6765 - auc: 0.7150 - val_loss: 0.5963 - val_accuracy: 0.6831 - val_auc: 0.6802\n","Epoch 1/5\n","25/25 [==============================] - 8s 219ms/step - loss: 3.4536 - accuracy: 0.5911 - auc: 0.5146 - val_loss: 0.6735 - val_accuracy: 0.6599 - val_auc: 0.5618\n","Epoch 2/5\n","25/25 [==============================] - 5s 201ms/step - loss: 0.6388 - accuracy: 0.6606 - auc: 0.5981 - val_loss: 0.6432 - val_accuracy: 0.6657 - val_auc: 0.5528\n","Epoch 3/5\n","25/25 [==============================] - 5s 197ms/step - loss: 0.6382 - accuracy: 0.6616 - auc: 0.6246 - val_loss: 0.6080 - val_accuracy: 0.6802 - val_auc: 0.6459\n","Epoch 4/5\n","25/25 [==============================] - 5s 200ms/step - loss: 0.5926 - accuracy: 0.6842 - auc: 0.6899 - val_loss: 0.5954 - val_accuracy: 0.6890 - val_auc: 0.6902\n","Epoch 5/5\n","25/25 [==============================] - 5s 203ms/step - loss: 0.5673 - accuracy: 0.6917 - auc: 0.7375 - val_loss: 0.5926 - val_accuracy: 0.6919 - val_auc: 0.6896\n","Epoch 1/5\n","25/25 [==============================] - 7s 214ms/step - loss: 2.6731 - accuracy: 0.5795 - auc: 0.5031 - val_loss: 0.6879 - val_accuracy: 0.6715 - val_auc: 0.5343\n","Epoch 2/5\n","25/25 [==============================] - 5s 205ms/step - loss: 0.6630 - accuracy: 0.6571 - auc: 0.5982 - val_loss: 0.6247 - val_accuracy: 0.6831 - val_auc: 0.5846\n","Epoch 3/5\n","25/25 [==============================] - 5s 202ms/step - loss: 0.6162 - accuracy: 0.6726 - auc: 0.6483 - val_loss: 0.6007 - val_accuracy: 0.6890 - val_auc: 0.6618\n","Epoch 4/5\n","25/25 [==============================] - 5s 198ms/step - loss: 0.5995 - accuracy: 0.6833 - auc: 0.6810 - val_loss: 0.6026 - val_accuracy: 0.6831 - val_auc: 0.6724\n","Epoch 5/5\n","25/25 [==============================] - 5s 202ms/step - loss: 0.5658 - accuracy: 0.6984 - auc: 0.7394 - val_loss: 0.5948 - val_accuracy: 0.6890 - val_auc: 0.6843\n","Epoch 1/5\n","25/25 [==============================] - 7s 221ms/step - loss: 2.6600 - accuracy: 0.5417 - auc: 0.5000 - val_loss: 0.6164 - val_accuracy: 0.6715 - val_auc: 0.6074\n","Epoch 2/5\n","25/25 [==============================] - 5s 208ms/step - loss: 0.6761 - accuracy: 0.6484 - auc: 0.5843 - val_loss: 0.6438 - val_accuracy: 0.6628 - val_auc: 0.6369\n","Epoch 3/5\n","25/25 [==============================] - 5s 195ms/step - loss: 0.6233 - accuracy: 0.6639 - auc: 0.6448 - val_loss: 0.6349 - val_accuracy: 0.6657 - val_auc: 0.6229\n","Epoch 4/5\n","25/25 [==============================] - 5s 197ms/step - loss: 0.5971 - accuracy: 0.6800 - auc: 0.6992 - val_loss: 0.6054 - val_accuracy: 0.6773 - val_auc: 0.6791\n","Epoch 5/5\n","25/25 [==============================] - 5s 198ms/step - loss: 0.5664 - accuracy: 0.6994 - auc: 0.7372 - val_loss: 0.6021 - val_accuracy: 0.6948 - val_auc: 0.6888\n","Epoch 1/5\n","25/25 [==============================] - 7s 219ms/step - loss: 2.5732 - accuracy: 0.5779 - auc: 0.5090 - val_loss: 0.6879 - val_accuracy: 0.6686 - val_auc: 0.5807\n","Epoch 2/5\n","25/25 [==============================] - 5s 198ms/step - loss: 0.6411 - accuracy: 0.6645 - auc: 0.5978 - val_loss: 0.6221 - val_accuracy: 0.6773 - val_auc: 0.6302\n","Epoch 3/5\n","25/25 [==============================] - 5s 198ms/step - loss: 0.6049 - accuracy: 0.6755 - auc: 0.6682 - val_loss: 0.6131 - val_accuracy: 0.6802 - val_auc: 0.6627\n","Epoch 4/5\n","25/25 [==============================] - 5s 199ms/step - loss: 0.6021 - accuracy: 0.6816 - auc: 0.6863 - val_loss: 0.5960 - val_accuracy: 0.6715 - val_auc: 0.6727\n","Epoch 5/5\n","25/25 [==============================] - 5s 203ms/step - loss: 0.5743 - accuracy: 0.6910 - auc: 0.7273 - val_loss: 0.6088 - val_accuracy: 0.6657 - val_auc: 0.6730\n","Epoch 1/5\n","25/25 [==============================] - 7s 219ms/step - loss: 3.8925 - accuracy: 0.5604 - auc: 0.4935 - val_loss: 0.6604 - val_accuracy: 0.6744 - val_auc: 0.5165\n","Epoch 2/5\n","25/25 [==============================] - 5s 202ms/step - loss: 0.6801 - accuracy: 0.6526 - auc: 0.5695 - val_loss: 0.6119 - val_accuracy: 0.6802 - val_auc: 0.6244\n","Epoch 3/5\n","25/25 [==============================] - 5s 208ms/step - loss: 0.6338 - accuracy: 0.6684 - auc: 0.6405 - val_loss: 0.6125 - val_accuracy: 0.6802 - val_auc: 0.6526\n","Epoch 4/5\n","25/25 [==============================] - 5s 207ms/step - loss: 0.6058 - accuracy: 0.6716 - auc: 0.6673 - val_loss: 0.6034 - val_accuracy: 0.6773 - val_auc: 0.6758\n","Epoch 5/5\n","25/25 [==============================] - 5s 201ms/step - loss: 0.5824 - accuracy: 0.6875 - auc: 0.7059 - val_loss: 0.6517 - val_accuracy: 0.6860 - val_auc: 0.6799\n","Epoch 1/5\n","25/25 [==============================] - 7s 226ms/step - loss: 4.9008 - accuracy: 0.5663 - auc: 0.5274 - val_loss: 0.7584 - val_accuracy: 0.5698 - val_auc: 0.5564\n","Epoch 2/5\n","25/25 [==============================] - 5s 202ms/step - loss: 0.7499 - accuracy: 0.6454 - auc: 0.5571 - val_loss: 0.6380 - val_accuracy: 0.6657 - val_auc: 0.5872\n","Epoch 3/5\n","25/25 [==============================] - 5s 198ms/step - loss: 0.7764 - accuracy: 0.6496 - auc: 0.5699 - val_loss: 0.7585 - val_accuracy: 0.6308 - val_auc: 0.6328\n","Epoch 4/5\n","25/25 [==============================] - 5s 200ms/step - loss: 0.6173 - accuracy: 0.6855 - auc: 0.6681 - val_loss: 0.6167 - val_accuracy: 0.6890 - val_auc: 0.6697\n","Epoch 5/5\n","25/25 [==============================] - 5s 196ms/step - loss: 0.6650 - accuracy: 0.6752 - auc: 0.6717 - val_loss: 0.7828 - val_accuracy: 0.6512 - val_auc: 0.6817\n","Epoch 1/5\n","25/25 [==============================] - 7s 214ms/step - loss: 1.7056 - accuracy: 0.5915 - auc: 0.5269 - val_loss: 0.7462 - val_accuracy: 0.5610 - val_auc: 0.6078\n","Epoch 2/5\n","25/25 [==============================] - 5s 202ms/step - loss: 0.6439 - accuracy: 0.6707 - auc: 0.5802 - val_loss: 0.6226 - val_accuracy: 0.6831 - val_auc: 0.6194\n","Epoch 3/5\n","25/25 [==============================] - 5s 205ms/step - loss: 0.6154 - accuracy: 0.6784 - auc: 0.6438 - val_loss: 0.6133 - val_accuracy: 0.6773 - val_auc: 0.6632\n","Epoch 4/5\n","25/25 [==============================] - 5s 202ms/step - loss: 0.5876 - accuracy: 0.6836 - auc: 0.6993 - val_loss: 0.6013 - val_accuracy: 0.6802 - val_auc: 0.6765\n","Epoch 5/5\n","25/25 [==============================] - 5s 206ms/step - loss: 0.5599 - accuracy: 0.7046 - auc: 0.7466 - val_loss: 0.6355 - val_accuracy: 0.6831 - val_auc: 0.6735\n","Epoch 1/5\n","25/25 [==============================] - 7s 217ms/step - loss: 3.0620 - accuracy: 0.5672 - auc: 0.5159 - val_loss: 0.6734 - val_accuracy: 0.6744 - val_auc: 0.5102\n","Epoch 2/5\n","25/25 [==============================] - 5s 207ms/step - loss: 0.6898 - accuracy: 0.6606 - auc: 0.5807 - val_loss: 0.7096 - val_accuracy: 0.6715 - val_auc: 0.5750\n","Epoch 3/5\n","25/25 [==============================] - 5s 200ms/step - loss: 0.6862 - accuracy: 0.6600 - auc: 0.5845 - val_loss: 0.6150 - val_accuracy: 0.6773 - val_auc: 0.6545\n","Epoch 4/5\n","25/25 [==============================] - 5s 199ms/step - loss: 0.6260 - accuracy: 0.6710 - auc: 0.6603 - val_loss: 0.6002 - val_accuracy: 0.6831 - val_auc: 0.6587\n","Epoch 5/5\n","25/25 [==============================] - 5s 202ms/step - loss: 0.5950 - accuracy: 0.6807 - auc: 0.6976 - val_loss: 0.5901 - val_accuracy: 0.6802 - val_auc: 0.6872\n","Epoch 1/5\n","25/25 [==============================] - 7s 216ms/step - loss: 3.9171 - accuracy: 0.5856 - auc: 0.5272 - val_loss: 0.7199 - val_accuracy: 0.6657 - val_auc: 0.4941\n","Epoch 2/5\n","25/25 [==============================] - 5s 201ms/step - loss: 0.7404 - accuracy: 0.6632 - auc: 0.5929 - val_loss: 0.6110 - val_accuracy: 0.6686 - val_auc: 0.6590\n","Epoch 3/5\n","25/25 [==============================] - 5s 198ms/step - loss: 0.6375 - accuracy: 0.6703 - auc: 0.6658 - val_loss: 0.6786 - val_accuracy: 0.6599 - val_auc: 0.6700\n","Epoch 4/5\n","25/25 [==============================] - 5s 204ms/step - loss: 0.6918 - accuracy: 0.6749 - auc: 0.6590 - val_loss: 0.6109 - val_accuracy: 0.6831 - val_auc: 0.6824\n","Epoch 5/5\n","25/25 [==============================] - 5s 197ms/step - loss: 0.6019 - accuracy: 0.6994 - auc: 0.7352 - val_loss: 0.6349 - val_accuracy: 0.6657 - val_auc: 0.6710\n","Epoch 1/5\n","25/25 [==============================] - 7s 217ms/step - loss: 3.0466 - accuracy: 0.5769 - auc: 0.5065 - val_loss: 0.8265 - val_accuracy: 0.6657 - val_auc: 0.4470\n","Epoch 2/5\n","25/25 [==============================] - 5s 204ms/step - loss: 0.6797 - accuracy: 0.6671 - auc: 0.5468 - val_loss: 0.6872 - val_accuracy: 0.6628 - val_auc: 0.4842\n","Epoch 3/5\n","25/25 [==============================] - 5s 198ms/step - loss: 0.6483 - accuracy: 0.6739 - auc: 0.5667 - val_loss: 0.6231 - val_accuracy: 0.6686 - val_auc: 0.6074\n","Epoch 4/5\n","25/25 [==============================] - 5s 194ms/step - loss: 0.6126 - accuracy: 0.6749 - auc: 0.6345 - val_loss: 0.6163 - val_accuracy: 0.6802 - val_auc: 0.6312\n","Epoch 5/5\n","25/25 [==============================] - 5s 196ms/step - loss: 0.5961 - accuracy: 0.6813 - auc: 0.6799 - val_loss: 0.5989 - val_accuracy: 0.6831 - val_auc: 0.6853\n","Epoch 1/5\n","25/25 [==============================] - 7s 219ms/step - loss: 4.5273 - accuracy: 0.5921 - auc: 0.5254 - val_loss: 0.7853 - val_accuracy: 0.6657 - val_auc: 0.4446\n","Epoch 2/5\n","25/25 [==============================] - 5s 209ms/step - loss: 0.7613 - accuracy: 0.6587 - auc: 0.5443 - val_loss: 0.6757 - val_accuracy: 0.6773 - val_auc: 0.6110\n","Epoch 3/5\n","25/25 [==============================] - 5s 199ms/step - loss: 0.6511 - accuracy: 0.6719 - auc: 0.6004 - val_loss: 0.6433 - val_accuracy: 0.6831 - val_auc: 0.6267\n","Epoch 4/5\n","25/25 [==============================] - 5s 201ms/step - loss: 0.6152 - accuracy: 0.6739 - auc: 0.6647 - val_loss: 0.6606 - val_accuracy: 0.6802 - val_auc: 0.6514\n","Epoch 5/5\n","25/25 [==============================] - 5s 204ms/step - loss: 0.6104 - accuracy: 0.6913 - auc: 0.7186 - val_loss: 0.6421 - val_accuracy: 0.6686 - val_auc: 0.6477\n","Epoch 1/5\n","25/25 [==============================] - 8s 219ms/step - loss: 3.7306 - accuracy: 0.5844 - auc: 0.5119 - val_loss: 0.6445 - val_accuracy: 0.6744 - val_auc: 0.5951\n","Epoch 2/5\n","25/25 [==============================] - 5s 202ms/step - loss: 0.6539 - accuracy: 0.6642 - auc: 0.5985 - val_loss: 0.6208 - val_accuracy: 0.6773 - val_auc: 0.6324\n","Epoch 3/5\n","25/25 [==============================] - 5s 199ms/step - loss: 0.6287 - accuracy: 0.6752 - auc: 0.6417 - val_loss: 0.6222 - val_accuracy: 0.6773 - val_auc: 0.6464\n","Epoch 4/5\n","25/25 [==============================] - 5s 199ms/step - loss: 0.6140 - accuracy: 0.6803 - auc: 0.6796 - val_loss: 0.6114 - val_accuracy: 0.6831 - val_auc: 0.6637\n","Epoch 5/5\n","25/25 [==============================] - 5s 216ms/step - loss: 0.5772 - accuracy: 0.6952 - auc: 0.7249 - val_loss: 0.5922 - val_accuracy: 0.6860 - val_auc: 0.6866\n","Epoch 1/5\n","25/25 [==============================] - 7s 223ms/step - loss: 4.5425 - accuracy: 0.5847 - auc: 0.5176 - val_loss: 0.6939 - val_accuracy: 0.6831 - val_auc: 0.5232\n","Epoch 2/5\n","25/25 [==============================] - 5s 205ms/step - loss: 0.6923 - accuracy: 0.6519 - auc: 0.5525 - val_loss: 0.6322 - val_accuracy: 0.6831 - val_auc: 0.5536\n","Epoch 3/5\n","25/25 [==============================] - 5s 201ms/step - loss: 0.6402 - accuracy: 0.6765 - auc: 0.5879 - val_loss: 0.6132 - val_accuracy: 0.6831 - val_auc: 0.6510\n","Epoch 4/5\n","25/25 [==============================] - 5s 196ms/step - loss: 0.6011 - accuracy: 0.6736 - auc: 0.6726 - val_loss: 0.5867 - val_accuracy: 0.6773 - val_auc: 0.6796\n","Epoch 5/5\n","25/25 [==============================] - 5s 198ms/step - loss: 0.5907 - accuracy: 0.6875 - auc: 0.6961 - val_loss: 0.5903 - val_accuracy: 0.6744 - val_auc: 0.6831\n","Epoch 1/5\n","25/25 [==============================] - 8s 260ms/step - loss: 3.9043 - accuracy: 0.5656 - auc: 0.5284 - val_loss: 0.7677 - val_accuracy: 0.6657 - val_auc: 0.5425\n","Epoch 2/5\n","25/25 [==============================] - 6s 248ms/step - loss: 0.6546 - accuracy: 0.6574 - auc: 0.6255 - val_loss: 0.6155 - val_accuracy: 0.6802 - val_auc: 0.6242\n","Epoch 3/5\n","25/25 [==============================] - 6s 235ms/step - loss: 0.6418 - accuracy: 0.6781 - auc: 0.6610 - val_loss: 0.7782 - val_accuracy: 0.6657 - val_auc: 0.5988\n","Epoch 4/5\n","25/25 [==============================] - 6s 235ms/step - loss: 0.6388 - accuracy: 0.6884 - auc: 0.6731 - val_loss: 0.6123 - val_accuracy: 0.6686 - val_auc: 0.6650\n","Epoch 5/5\n","25/25 [==============================] - 6s 238ms/step - loss: 0.5754 - accuracy: 0.7101 - auc: 0.7312 - val_loss: 0.6156 - val_accuracy: 0.6802 - val_auc: 0.6607\n","Epoch 1/5\n","25/25 [==============================] - 7s 220ms/step - loss: 3.8313 - accuracy: 0.5983 - auc: 0.5299 - val_loss: 0.6652 - val_accuracy: 0.6366 - val_auc: 0.6259\n","Epoch 2/5\n","25/25 [==============================] - 5s 202ms/step - loss: 0.6979 - accuracy: 0.6616 - auc: 0.5947 - val_loss: 0.6335 - val_accuracy: 0.6599 - val_auc: 0.6156\n","Epoch 3/5\n","25/25 [==============================] - 5s 210ms/step - loss: 0.6567 - accuracy: 0.6629 - auc: 0.6154 - val_loss: 0.6761 - val_accuracy: 0.6628 - val_auc: 0.6290\n","Epoch 4/5\n","25/25 [==============================] - 5s 207ms/step - loss: 0.6218 - accuracy: 0.6823 - auc: 0.6780 - val_loss: 0.6184 - val_accuracy: 0.6831 - val_auc: 0.6506\n","Epoch 5/5\n","25/25 [==============================] - 5s 213ms/step - loss: 0.5812 - accuracy: 0.6939 - auc: 0.7180 - val_loss: 0.5961 - val_accuracy: 0.6802 - val_auc: 0.6729\n","Epoch 1/5\n","25/25 [==============================] - 8s 241ms/step - loss: 2.5413 - accuracy: 0.6228 - auc: 0.5427 - val_loss: 0.6404 - val_accuracy: 0.6831 - val_auc: 0.5618\n","Epoch 2/5\n","25/25 [==============================] - 5s 203ms/step - loss: 0.6500 - accuracy: 0.6568 - auc: 0.5697 - val_loss: 0.6176 - val_accuracy: 0.6831 - val_auc: 0.6197\n","Epoch 3/5\n","25/25 [==============================] - 5s 203ms/step - loss: 0.6111 - accuracy: 0.6833 - auc: 0.6404 - val_loss: 0.6153 - val_accuracy: 0.6831 - val_auc: 0.6523\n","Epoch 4/5\n","25/25 [==============================] - 5s 205ms/step - loss: 0.5867 - accuracy: 0.6829 - auc: 0.6977 - val_loss: 0.6094 - val_accuracy: 0.6948 - val_auc: 0.6777\n","Epoch 5/5\n","25/25 [==============================] - 5s 203ms/step - loss: 0.5532 - accuracy: 0.7133 - auc: 0.7542 - val_loss: 0.6215 - val_accuracy: 0.6686 - val_auc: 0.6757\n","Epoch 1/5\n","25/25 [==============================] - 8s 254ms/step - loss: 3.5067 - accuracy: 0.5569 - auc: 0.5000 - val_loss: 0.7088 - val_accuracy: 0.6744 - val_auc: 0.5994\n","Epoch 2/5\n","25/25 [==============================] - 5s 205ms/step - loss: 0.6726 - accuracy: 0.6687 - auc: 0.6145 - val_loss: 0.6164 - val_accuracy: 0.6831 - val_auc: 0.6198\n","Epoch 3/5\n","25/25 [==============================] - 5s 198ms/step - loss: 0.6123 - accuracy: 0.6784 - auc: 0.6532 - val_loss: 0.5993 - val_accuracy: 0.6919 - val_auc: 0.6564\n","Epoch 4/5\n","25/25 [==============================] - 5s 202ms/step - loss: 0.6325 - accuracy: 0.6729 - auc: 0.6424 - val_loss: 0.6196 - val_accuracy: 0.6657 - val_auc: 0.6642\n","Epoch 5/5\n","25/25 [==============================] - 5s 205ms/step - loss: 0.5922 - accuracy: 0.6823 - auc: 0.6986 - val_loss: 0.6073 - val_accuracy: 0.6802 - val_auc: 0.6899\n","Epoch 1/5\n","25/25 [==============================] - 7s 220ms/step - loss: 2.6486 - accuracy: 0.5989 - auc: 0.5402 - val_loss: 0.6307 - val_accuracy: 0.6773 - val_auc: 0.6406\n","Epoch 2/5\n","25/25 [==============================] - 5s 197ms/step - loss: 0.6253 - accuracy: 0.6781 - auc: 0.6328 - val_loss: 0.6429 - val_accuracy: 0.6860 - val_auc: 0.6678\n","Epoch 3/5\n","25/25 [==============================] - 5s 197ms/step - loss: 0.6027 - accuracy: 0.6871 - auc: 0.6918 - val_loss: 0.5823 - val_accuracy: 0.6919 - val_auc: 0.6977\n","Epoch 4/5\n","25/25 [==============================] - 5s 203ms/step - loss: 0.5689 - accuracy: 0.7020 - auc: 0.7297 - val_loss: 0.5980 - val_accuracy: 0.6802 - val_auc: 0.6912\n","Epoch 5/5\n","25/25 [==============================] - 5s 200ms/step - loss: 0.5483 - accuracy: 0.7227 - auc: 0.7707 - val_loss: 0.6037 - val_accuracy: 0.6773 - val_auc: 0.6863\n","Epoch 1/5\n","25/25 [==============================] - 7s 218ms/step - loss: 3.1665 - accuracy: 0.5882 - auc: 0.5263 - val_loss: 0.7622 - val_accuracy: 0.6715 - val_auc: 0.5948\n","Epoch 2/5\n","25/25 [==============================] - 5s 202ms/step - loss: 0.7056 - accuracy: 0.6574 - auc: 0.5758 - val_loss: 0.6127 - val_accuracy: 0.6890 - val_auc: 0.6552\n","Epoch 3/5\n","25/25 [==============================] - 5s 202ms/step - loss: 0.6594 - accuracy: 0.6687 - auc: 0.6273 - val_loss: 0.6376 - val_accuracy: 0.6831 - val_auc: 0.6591\n","Epoch 4/5\n","25/25 [==============================] - 5s 203ms/step - loss: 0.6155 - accuracy: 0.6836 - auc: 0.6801 - val_loss: 0.6262 - val_accuracy: 0.6773 - val_auc: 0.6800\n","Epoch 5/5\n","25/25 [==============================] - 5s 208ms/step - loss: 0.5923 - accuracy: 0.6800 - auc: 0.7087 - val_loss: 0.5978 - val_accuracy: 0.6686 - val_auc: 0.6814\n","Epoch 1/5\n","25/25 [==============================] - 7s 223ms/step - loss: 3.4884 - accuracy: 0.5850 - auc: 0.5119 - val_loss: 0.6640 - val_accuracy: 0.6570 - val_auc: 0.5234\n","Epoch 2/5\n","25/25 [==============================] - 5s 204ms/step - loss: 0.7236 - accuracy: 0.6464 - auc: 0.5523 - val_loss: 0.6302 - val_accuracy: 0.6715 - val_auc: 0.6432\n","Epoch 3/5\n","25/25 [==============================] - 5s 202ms/step - loss: 0.6131 - accuracy: 0.6703 - auc: 0.6536 - val_loss: 0.6320 - val_accuracy: 0.6686 - val_auc: 0.6223\n","Epoch 4/5\n","25/25 [==============================] - 5s 209ms/step - loss: 0.6318 - accuracy: 0.6703 - auc: 0.6489 - val_loss: 0.6045 - val_accuracy: 0.6831 - val_auc: 0.6806\n","Epoch 5/5\n","25/25 [==============================] - 5s 211ms/step - loss: 0.5921 - accuracy: 0.6816 - auc: 0.6978 - val_loss: 0.5951 - val_accuracy: 0.6860 - val_auc: 0.6858\n"]}]},{"cell_type":"code","source":["val_F1_mean, val_F1_sd = mean_and_sd(val_F1)\n","val_acc_mean, val_acc_sd = mean_and_sd(val_acc)\n","val_pr_auc_mean, val_pr_auc_sd = mean_and_sd(val_pr_auc)\n","val_roc_auc_mean, val_roc_auc_sd = mean_and_sd(val_roc_auc)\n","\n","test_F1_mean, test_F1_sd = mean_and_sd(test_F1)\n","test_acc_mean, test_acc_sd = mean_and_sd(test_acc)\n","test_pr_auc_mean, test_pr_auc_sd = mean_and_sd(test_pr_auc)\n","test_roc_auc_mean, test_roc_auc_sd = mean_and_sd(test_roc_auc)"],"metadata":{"id":"z-ueEJk4rjoj"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["print(f\"validation F1: {round(val_F1_mean, 3)} +/- {round(val_F1_sd, 3)}\")\n","print(f\"test F1: {round(test_F1_mean, 3)} +/- {round(test_F1_sd, 3)}\")\n","\n","print(f\"validation accuracy: {round(val_acc_mean, 3)} +/- {round(val_acc_sd, 3)}\")\n","print(f\"test accuracy: {round(test_acc_mean, 3)} +/- {round(test_acc_sd, 3)}\")\n","\n","print(f\"validation PR-AUC: {round(val_pr_auc_mean, 3)} +/- {round(val_pr_auc_sd, 3)}\")\n","print(f\"test PR-AUC: {round(test_pr_auc_mean, 3)} +/- {round(test_pr_auc_sd, 3)}\")\n","\n","print(f\"validation ROC-AUC: {round(val_roc_auc_mean, 3)} +/- {round(val_roc_auc_sd, 3)}\")\n","print(f\"test ROC-AUC: {round(test_roc_auc_mean, 3)} +/- {round(test_roc_auc_sd, 3)}\")"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"rlBVo3xdrno9","executionInfo":{"status":"ok","timestamp":1651530561603,"user_tz":240,"elapsed":13,"user":{"displayName":"Benedikt Geiger","userId":"17925887631246406508"}},"outputId":"2572a1c9-cb77-4436-accb-0e79c11e724f"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["validation F1: 0.787 +/- 0.015\n","test F1: 0.824 +/- 0.035\n","validation accuracy: 0.68 +/- 0.011\n","test accuracy: 0.721 +/- 0.035\n","validation PR-AUC: 0.802 +/- 0.008\n","test PR-AUC: 0.84 +/- 0.008\n","validation ROC-AUC: 0.681 +/- 0.011\n","test ROC-AUC: 0.654 +/- 0.019\n"]}]}]}