--- a
+++ b/notebooks/Training.ipynb
@@ -0,0 +1,4203 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "VERSION = 33\n",
+    "\n",
+    "FOCAL_LOSS = 0\n",
+    "CLOUD_SINGLE = True\n",
+    "MIXUP = False\n",
+    "NO_BLACK_LOSS = True\n",
+    "DATA_SMALL = False\n",
+    "\n",
+    "# VERSION 31 old features, no stage2 training\n",
+    "# VERSION 32 old features, no stage2 training, fine-tuned weighted\n",
+    "# VERSION 33 old features, with stage2 training\n",
+    "# VERSION 34 old features, with stage2 training, fine-tuned weighted\n",
+    "# VERSION 35 new features, with stage2 training\n",
+    "# VERSION 35 new features, with stage2 training, fine-tuned weighted"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "if VERSION in [31,32]:\n",
+    "    TRAIN_ON_STAGE_1 = False\n",
+    "else:\n",
+    "    TRAIN_ON_STAGE_1 = True\n",
+    "\n",
+    "if VERSION in [32,34,36]:\n",
+    "    WEIGHTED = True\n",
+    "else:\n",
+    "    WEIGHTED = False"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%run ./Code.ipynb"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "if VERSION in [31,32]:\n",
+    "    # old features, no stage2 training\n",
+    "    train_md, test_md = loadMetadata()\n",
+    "elif VERSION in [33,34]:\n",
+    "    # old features, with stage2 training\n",
+    "    train_md, test_md = loadMetadata3()\n",
+    "elif VERSION in [35,36]:\n",
+    "    # new features\n",
+    "    train_md, test_md = loadMetadata2()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "1.111931    230422\n",
+       "0.864417    230399\n",
+       "1.200910    221421\n",
+       "0.446933     70301\n",
+       "0.737883       129\n",
+       "0.000000       124\n",
+       "0.796930         1\n",
+       "Name: weights, dtype: int64"
+      ]
+     },
+     "execution_count": 9,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "train_md.weights.value_counts()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "0"
+      ]
+     },
+     "execution_count": 10,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "train_md.weights.isnull().sum()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "(752797, 103)"
+      ]
+     },
+     "execution_count": 11,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "train_md.shape"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Pre-processing features"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 29,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "_,_ = loadMetadata(True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "_,_ = loadMetadata3(True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "_,_ = loadMetadata2(True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "[7, 9]"
+      ]
+     },
+     "execution_count": 12,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "my_datasets3"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 18,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "dataset 7 fold 0 feats size torch.Size([2697008, 552])\n",
+      "dataset 7 fold 1 feats size torch.Size([2697008, 552])\n",
+      "dataset 7 fold 2 feats size torch.Size([2697008, 552])\n"
+     ]
+    }
+   ],
+   "source": [
+    "preprocessedData(7,do_train=True,do_test=False)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 19,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "dataset 9 fold 0 feats size torch.Size([2697008, 256])\n",
+      "dataset 9 fold 1 feats size torch.Size([2697008, 256])\n",
+      "dataset 9 fold 2 feats size torch.Size([2697008, 256])\n"
+     ]
+    }
+   ],
+   "source": [
+    "preprocessedData(9,do_train=True,do_test=False)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 20,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "[11, 12, 13]"
+      ]
+     },
+     "execution_count": 20,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "my_datasets5"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 23,
+   "metadata": {
+    "scrolled": false
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "dataset 11 fold 0 feats size torch.Size([2697008, 256])\n",
+      "dataset 11 fold 1 feats size torch.Size([2697008, 256])\n",
+      "dataset 11 fold 2 feats size torch.Size([2697008, 256])\n",
+      "dataset 11 fold 3 feats size torch.Size([2697008, 256])\n",
+      "dataset 11 fold 4 feats size torch.Size([2697008, 256])\n"
+     ]
+    }
+   ],
+   "source": [
+    "preprocessedData(11,do_train=True,do_test=False, folds=range(5))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 24,
+   "metadata": {
+    "scrolled": false
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "dataset 12 fold 0 feats size torch.Size([2697008, 256])\n",
+      "dataset 12 fold 1 feats size torch.Size([2697008, 256])\n",
+      "dataset 12 fold 2 feats size torch.Size([2697008, 256])\n",
+      "dataset 12 fold 3 feats size torch.Size([2697008, 256])\n",
+      "dataset 12 fold 4 feats size torch.Size([2697008, 256])\n"
+     ]
+    }
+   ],
+   "source": [
+    "preprocessedData(12,do_train=True,do_test=False, folds=range(5))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 26,
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "dataset 13 fold 0 feats size torch.Size([2697008, 256])\n",
+      "dataset 13 fold 1 feats size torch.Size([2697008, 256])\n",
+      "dataset 13 fold 2 feats size torch.Size([2697008, 256])\n",
+      "dataset 13 fold 3 feats size torch.Size([2697008, 256])\n",
+      "dataset 13 fold 4 feats size torch.Size([2697008, 256])\n"
+     ]
+    }
+   ],
+   "source": [
+    "preprocessedData(13,do_train=True,do_test=False, folds=range(5))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "dataset 7 fold 0 feats size torch.Size([2697008, 552])\n",
+      "dataset 7 fold 1 feats size torch.Size([2697008, 552])\n",
+      "dataset 7 fold 2 feats size torch.Size([2697008, 552])\n",
+      "dataset 9 fold 0 feats size torch.Size([2697008, 256])\n",
+      "dataset 9 fold 1 feats size torch.Size([2697008, 256])\n",
+      "dataset 9 fold 2 feats size torch.Size([2697008, 256])\n",
+      "dataset 11 fold 0 feats size torch.Size([2697008, 256])\n",
+      "dataset 11 fold 1 feats size torch.Size([2697008, 256])\n",
+      "dataset 11 fold 2 feats size torch.Size([2697008, 256])\n",
+      "dataset 11 fold 3 feats size torch.Size([2697008, 256])\n",
+      "dataset 11 fold 4 feats size torch.Size([2697008, 256])\n",
+      "dataset 12 fold 0 feats size torch.Size([2697008, 256])\n",
+      "dataset 12 fold 1 feats size torch.Size([2697008, 256])\n",
+      "dataset 12 fold 2 feats size torch.Size([2697008, 256])\n",
+      "dataset 12 fold 3 feats size torch.Size([2697008, 256])\n",
+      "dataset 12 fold 4 feats size torch.Size([2697008, 256])\n",
+      "dataset 13 fold 0 feats size torch.Size([2697008, 256])\n",
+      "dataset 13 fold 1 feats size torch.Size([2697008, 256])\n",
+      "dataset 13 fold 2 feats size torch.Size([2697008, 256])\n",
+      "dataset 13 fold 3 feats size torch.Size([2697008, 256])\n",
+      "dataset 13 fold 4 feats size torch.Size([2697008, 256])\n"
+     ]
+    }
+   ],
+   "source": [
+    "for ds in my_datasets3:\n",
+    "    preprocessedData(ds)\n",
+    "\n",
+    "for ds in my_datasets5:\n",
+    "    preprocessedData(ds, folds=range(5))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "dataset 14 fold 0 feats size torch.Size([3011188, 256])\n"
+     ]
+    },
+    {
+     "ename": "FileNotFoundError",
+     "evalue": "[Errno 2] No such file or directory: '/mnt/edisk/running/yuval/model_se_resnet101_version_new_splits_stage2_type_features_test_split_0.pkl'",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[0;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
+      "\u001b[0;32m<ipython-input-10-5a07cc4fa1a8>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mpreprocessedData\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m14\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mfold_col\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'fold5'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mdo_test\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mdo_train\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
+      "\u001b[0;32m<ipython-input-6-51e1d35e10f9>\u001b[0m in \u001b[0;36mpreprocessedData\u001b[0;34m(dataset, folds, fold_col, do_test, do_test2, do_train)\u001b[0m\n\u001b[1;32m     62\u001b[0m                 .format(dataset_name.replace('_5n','').replace('_5f','').replace('_5',''),\n\u001b[1;32m     63\u001b[0m                         filename_add,dsft,focal,ds_num,test_fix,fold)\n\u001b[0;32m---> 64\u001b[0;31m             \u001b[0mfeats\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpickle\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilename\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m'rb'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     65\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     66\u001b[0m             \u001b[0;32massert\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfeats\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m8\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtest_md\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: '/mnt/edisk/running/yuval/model_se_resnet101_version_new_splits_stage2_type_features_test_split_0.pkl'"
+     ]
+    }
+   ],
+   "source": [
+    "preprocessedData(14,fold_col='fold5',do_test=True,do_train=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Running"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 28,
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "completed epochs: 0 starting now: 4\n",
+      "DataSet 14 train size 17355 fold 2\n",
+      "adding dummy serieses 27\n",
+      "DataSet 14 valid size 4416 fold 2\n",
+      "setFeats, augmentation 0\n",
+      "dataset train: 17355 valid: 4416 loader train: 542 valid: 138\n",
+      "starting from scratch\n",
+      "setFeats, augmentation -1\n",
+      "Batch 50 device: cuda time passed: 22.728 time per batch: 0.455\n",
+      "Batch 100 device: cuda time passed: 39.605 time per batch: 0.396\n",
+      "Batch 150 device: cuda time passed: 55.920 time per batch: 0.373\n",
+      "Batch 200 device: cuda time passed: 72.699 time per batch: 0.363\n",
+      "Batch 250 device: cuda time passed: 89.437 time per batch: 0.358\n",
+      "Batch 300 device: cuda time passed: 105.583 time per batch: 0.352\n",
+      "Batch 350 device: cuda time passed: 122.869 time per batch: 0.351\n",
+      "Batch 400 device: cuda time passed: 139.694 time per batch: 0.349\n",
+      "Batch 450 device: cuda time passed: 158.701 time per batch: 0.353\n",
+      "Batch 500 device: cuda time passed: 176.083 time per batch: 0.352\n",
+      "Batch 50 device: cuda time passed: 8.131 time per batch: 0.163\n",
+      "Batch 100 device: cuda time passed: 14.930 time per batch: 0.149\n",
+      "v35, d14, e1, f2, trn ll: 0.0580, val ll: 0.0650, ll_w: 0.0575, cor: 0.8338, auc: 0.9870, lr: 0.0002\n",
+      "setFeats, augmentation -1\n",
+      "Batch 50 device: cuda time passed: 22.873 time per batch: 0.457\n",
+      "Batch 100 device: cuda time passed: 39.907 time per batch: 0.399\n",
+      "Batch 150 device: cuda time passed: 57.763 time per batch: 0.385\n",
+      "Batch 200 device: cuda time passed: 75.045 time per batch: 0.375\n",
+      "Batch 250 device: cuda time passed: 92.730 time per batch: 0.371\n",
+      "Batch 300 device: cuda time passed: 109.667 time per batch: 0.366\n",
+      "Batch 350 device: cuda time passed: 126.574 time per batch: 0.362\n",
+      "Batch 400 device: cuda time passed: 143.725 time per batch: 0.359\n",
+      "Batch 450 device: cuda time passed: 161.952 time per batch: 0.360\n",
+      "Batch 500 device: cuda time passed: 178.315 time per batch: 0.357\n",
+      "Batch 50 device: cuda time passed: 8.428 time per batch: 0.169\n",
+      "Batch 100 device: cuda time passed: 15.068 time per batch: 0.151\n",
+      "v35, d14, e2, f2, trn ll: 0.0356, val ll: 0.0640, ll_w: 0.0562, cor: 0.8351, auc: 0.9872, lr: 0.0002\n",
+      "setFeats, augmentation -1\n",
+      "Batch 50 device: cuda time passed: 22.887 time per batch: 0.458\n",
+      "Batch 100 device: cuda time passed: 39.853 time per batch: 0.399\n",
+      "Batch 150 device: cuda time passed: 56.609 time per batch: 0.377\n",
+      "Batch 200 device: cuda time passed: 73.060 time per batch: 0.365\n",
+      "Batch 250 device: cuda time passed: 90.475 time per batch: 0.362\n",
+      "Batch 300 device: cuda time passed: 107.703 time per batch: 0.359\n",
+      "Batch 350 device: cuda time passed: 124.235 time per batch: 0.355\n",
+      "Batch 400 device: cuda time passed: 141.549 time per batch: 0.354\n",
+      "Batch 450 device: cuda time passed: 159.349 time per batch: 0.354\n",
+      "Batch 500 device: cuda time passed: 175.549 time per batch: 0.351\n",
+      "Batch 50 device: cuda time passed: 8.712 time per batch: 0.174\n",
+      "Batch 100 device: cuda time passed: 15.378 time per batch: 0.154\n",
+      "v35, d14, e3, f2, trn ll: 0.0341, val ll: 0.0622, ll_w: 0.0549, cor: 0.8409, auc: 0.9887, lr: 0.0002\n",
+      "setFeats, augmentation -1\n",
+      "Batch 50 device: cuda time passed: 23.339 time per batch: 0.467\n",
+      "Batch 100 device: cuda time passed: 40.154 time per batch: 0.402\n",
+      "Batch 150 device: cuda time passed: 56.698 time per batch: 0.378\n",
+      "Batch 200 device: cuda time passed: 74.060 time per batch: 0.370\n",
+      "Batch 250 device: cuda time passed: 91.302 time per batch: 0.365\n",
+      "Batch 300 device: cuda time passed: 108.675 time per batch: 0.362\n",
+      "Batch 350 device: cuda time passed: 126.067 time per batch: 0.360\n",
+      "Batch 400 device: cuda time passed: 143.346 time per batch: 0.358\n",
+      "Batch 450 device: cuda time passed: 162.692 time per batch: 0.362\n",
+      "Batch 500 device: cuda time passed: 179.814 time per batch: 0.360\n",
+      "Batch 50 device: cuda time passed: 7.879 time per batch: 0.158\n",
+      "Batch 100 device: cuda time passed: 14.703 time per batch: 0.147\n",
+      "v35, d14, e4, f2, trn ll: 0.0335, val ll: 0.0613, ll_w: 0.0539, cor: 0.8421, auc: 0.9887, lr: 0.0002\n",
+      "total running time 900.058856010437\n",
+      "completed epochs: 4 starting now: 4\n",
+      "DataSet 14 train size 17355 fold 2\n",
+      "adding dummy serieses 27\n",
+      "DataSet 14 valid size 4416 fold 2\n",
+      "setFeats, augmentation 0\n",
+      "dataset train: 17355 valid: 4416 loader train: 542 valid: 138\n",
+      "loading model model.b4.f2.d14.v35\n",
+      "setFeats, augmentation -1\n",
+      "Batch 50 device: cuda time passed: 22.315 time per batch: 0.446\n",
+      "Batch 100 device: cuda time passed: 38.974 time per batch: 0.390\n",
+      "Batch 150 device: cuda time passed: 56.082 time per batch: 0.374\n",
+      "Batch 200 device: cuda time passed: 73.546 time per batch: 0.368\n",
+      "Batch 250 device: cuda time passed: 90.717 time per batch: 0.363\n",
+      "Batch 300 device: cuda time passed: 107.371 time per batch: 0.358\n",
+      "Batch 350 device: cuda time passed: 124.145 time per batch: 0.355\n",
+      "Batch 400 device: cuda time passed: 140.102 time per batch: 0.350\n",
+      "Batch 450 device: cuda time passed: 158.547 time per batch: 0.352\n",
+      "Batch 500 device: cuda time passed: 176.664 time per batch: 0.353\n",
+      "Batch 50 device: cuda time passed: 8.467 time per batch: 0.169\n",
+      "Batch 100 device: cuda time passed: 15.208 time per batch: 0.152\n",
+      "v35, d14, e5, f2, trn ll: 0.0319, val ll: 0.0602, ll_w: 0.0530, cor: 0.8445, auc: 0.9892, lr: 2e-05\n",
+      "setFeats, augmentation -1\n",
+      "Batch 50 device: cuda time passed: 23.093 time per batch: 0.462\n",
+      "Batch 100 device: cuda time passed: 40.190 time per batch: 0.402\n",
+      "Batch 150 device: cuda time passed: 57.287 time per batch: 0.382\n",
+      "Batch 200 device: cuda time passed: 74.458 time per batch: 0.372\n",
+      "Batch 250 device: cuda time passed: 91.213 time per batch: 0.365\n",
+      "Batch 300 device: cuda time passed: 108.222 time per batch: 0.361\n",
+      "Batch 350 device: cuda time passed: 125.171 time per batch: 0.358\n",
+      "Batch 400 device: cuda time passed: 142.199 time per batch: 0.355\n",
+      "Batch 450 device: cuda time passed: 162.403 time per batch: 0.361\n",
+      "Batch 500 device: cuda time passed: 179.653 time per batch: 0.359\n",
+      "Batch 50 device: cuda time passed: 8.772 time per batch: 0.175\n",
+      "Batch 100 device: cuda time passed: 15.212 time per batch: 0.152\n",
+      "v35, d14, e6, f2, trn ll: 0.0316, val ll: 0.0603, ll_w: 0.0531, cor: 0.8444, auc: 0.9892, lr: 2e-05\n",
+      "setFeats, augmentation -1\n",
+      "Batch 50 device: cuda time passed: 21.633 time per batch: 0.433\n",
+      "Batch 100 device: cuda time passed: 38.635 time per batch: 0.386\n",
+      "Batch 150 device: cuda time passed: 55.617 time per batch: 0.371\n",
+      "Batch 200 device: cuda time passed: 72.396 time per batch: 0.362\n",
+      "Batch 250 device: cuda time passed: 89.274 time per batch: 0.357\n",
+      "Batch 300 device: cuda time passed: 106.187 time per batch: 0.354\n",
+      "Batch 350 device: cuda time passed: 122.741 time per batch: 0.351\n",
+      "Batch 400 device: cuda time passed: 139.218 time per batch: 0.348\n",
+      "Batch 450 device: cuda time passed: 158.183 time per batch: 0.352\n",
+      "Batch 500 device: cuda time passed: 175.811 time per batch: 0.352\n",
+      "Batch 50 device: cuda time passed: 8.583 time per batch: 0.172\n",
+      "Batch 100 device: cuda time passed: 15.243 time per batch: 0.152\n",
+      "v35, d14, e7, f2, trn ll: 0.0315, val ll: 0.0602, ll_w: 0.0530, cor: 0.8446, auc: 0.9892, lr: 2e-05\n",
+      "setFeats, augmentation -1\n",
+      "Batch 50 device: cuda time passed: 23.204 time per batch: 0.464\n",
+      "Batch 100 device: cuda time passed: 40.589 time per batch: 0.406\n",
+      "Batch 150 device: cuda time passed: 58.004 time per batch: 0.387\n",
+      "Batch 200 device: cuda time passed: 75.275 time per batch: 0.376\n",
+      "Batch 250 device: cuda time passed: 92.072 time per batch: 0.368\n",
+      "Batch 300 device: cuda time passed: 109.143 time per batch: 0.364\n",
+      "Batch 350 device: cuda time passed: 125.692 time per batch: 0.359\n",
+      "Batch 400 device: cuda time passed: 143.440 time per batch: 0.359\n",
+      "Batch 450 device: cuda time passed: 164.424 time per batch: 0.365\n",
+      "Batch 500 device: cuda time passed: 181.490 time per batch: 0.363\n",
+      "Batch 50 device: cuda time passed: 8.635 time per batch: 0.173\n",
+      "Batch 100 device: cuda time passed: 15.140 time per batch: 0.151\n",
+      "v35, d14, e8, f2, trn ll: 0.0314, val ll: 0.0601, ll_w: 0.0529, cor: 0.8448, auc: 0.9892, lr: 2e-05\n",
+      "total running time 871.9309167861938\n",
+      "completed epochs: 8 starting now: 3\n",
+      "DataSet 14 train size 17355 fold 2\n",
+      "adding dummy serieses 27\n",
+      "DataSet 14 valid size 4416 fold 2\n",
+      "setFeats, augmentation 0\n",
+      "dataset train: 17355 valid: 4416 loader train: 542 valid: 138\n",
+      "loading model model.b8.f2.d14.v35\n",
+      "setFeats, augmentation -1\n",
+      "Batch 50 device: cuda time passed: 22.052 time per batch: 0.441\n",
+      "Batch 100 device: cuda time passed: 38.728 time per batch: 0.387\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Batch 150 device: cuda time passed: 56.020 time per batch: 0.373\n",
+      "Batch 200 device: cuda time passed: 73.183 time per batch: 0.366\n",
+      "Batch 250 device: cuda time passed: 90.324 time per batch: 0.361\n",
+      "Batch 300 device: cuda time passed: 107.782 time per batch: 0.359\n",
+      "Batch 350 device: cuda time passed: 124.543 time per batch: 0.356\n",
+      "Batch 400 device: cuda time passed: 141.478 time per batch: 0.354\n",
+      "Batch 450 device: cuda time passed: 162.319 time per batch: 0.361\n",
+      "Batch 500 device: cuda time passed: 179.659 time per batch: 0.359\n",
+      "Batch 50 device: cuda time passed: 7.857 time per batch: 0.157\n",
+      "Batch 100 device: cuda time passed: 14.956 time per batch: 0.150\n",
+      "v35, d14, e9, f2, trn ll: 0.0313, val ll: 0.0599, ll_w: 0.0527, cor: 0.8454, auc: 0.9892, lr: 5e-06\n",
+      "setFeats, augmentation -1\n",
+      "Batch 50 device: cuda time passed: 23.852 time per batch: 0.477\n",
+      "Batch 100 device: cuda time passed: 41.405 time per batch: 0.414\n",
+      "Batch 150 device: cuda time passed: 59.015 time per batch: 0.393\n",
+      "Batch 200 device: cuda time passed: 76.762 time per batch: 0.384\n",
+      "Batch 250 device: cuda time passed: 94.142 time per batch: 0.377\n",
+      "Batch 300 device: cuda time passed: 111.812 time per batch: 0.373\n",
+      "Batch 350 device: cuda time passed: 129.992 time per batch: 0.371\n",
+      "Batch 400 device: cuda time passed: 147.178 time per batch: 0.368\n",
+      "Batch 450 device: cuda time passed: 168.899 time per batch: 0.375\n",
+      "Batch 500 device: cuda time passed: 184.992 time per batch: 0.370\n",
+      "Batch 50 device: cuda time passed: 8.119 time per batch: 0.162\n",
+      "Batch 100 device: cuda time passed: 14.013 time per batch: 0.140\n",
+      "v35, d14, e10, f2, trn ll: 0.0313, val ll: 0.0600, ll_w: 0.0528, cor: 0.8451, auc: 0.9893, lr: 5e-06\n",
+      "setFeats, augmentation -1\n",
+      "Batch 50 device: cuda time passed: 21.209 time per batch: 0.424\n",
+      "Batch 100 device: cuda time passed: 37.676 time per batch: 0.377\n",
+      "Batch 150 device: cuda time passed: 53.211 time per batch: 0.355\n",
+      "Batch 200 device: cuda time passed: 68.886 time per batch: 0.344\n",
+      "Batch 250 device: cuda time passed: 84.449 time per batch: 0.338\n",
+      "Batch 300 device: cuda time passed: 100.285 time per batch: 0.334\n",
+      "Batch 350 device: cuda time passed: 116.165 time per batch: 0.332\n",
+      "Batch 400 device: cuda time passed: 131.615 time per batch: 0.329\n",
+      "Batch 450 device: cuda time passed: 150.426 time per batch: 0.334\n",
+      "Batch 500 device: cuda time passed: 166.304 time per batch: 0.333\n",
+      "Batch 50 device: cuda time passed: 7.404 time per batch: 0.148\n",
+      "Batch 100 device: cuda time passed: 13.992 time per batch: 0.140\n",
+      "v35, d14, e11, f2, trn ll: 0.0312, val ll: 0.0600, ll_w: 0.0528, cor: 0.8450, auc: 0.9893, lr: 5e-06\n",
+      "total running time 644.8997066020966\n",
+      "completed epochs: 11 starting now: 2\n",
+      "DataSet 14 train size 17355 fold 2\n",
+      "adding dummy serieses 27\n",
+      "DataSet 14 valid size 4416 fold 2\n",
+      "setFeats, augmentation 0\n",
+      "dataset train: 17355 valid: 4416 loader train: 542 valid: 138\n",
+      "loading model model.b11.f2.d14.v35\n",
+      "setFeats, augmentation -1\n",
+      "Batch 50 device: cuda time passed: 21.014 time per batch: 0.420\n",
+      "Batch 100 device: cuda time passed: 36.560 time per batch: 0.366\n",
+      "Batch 150 device: cuda time passed: 53.272 time per batch: 0.355\n",
+      "Batch 200 device: cuda time passed: 68.574 time per batch: 0.343\n",
+      "Batch 250 device: cuda time passed: 83.938 time per batch: 0.336\n",
+      "Batch 300 device: cuda time passed: 100.283 time per batch: 0.334\n",
+      "Batch 350 device: cuda time passed: 116.620 time per batch: 0.333\n",
+      "Batch 400 device: cuda time passed: 132.783 time per batch: 0.332\n",
+      "Batch 450 device: cuda time passed: 152.425 time per batch: 0.339\n",
+      "Batch 500 device: cuda time passed: 168.627 time per batch: 0.337\n",
+      "Batch 50 device: cuda time passed: 8.257 time per batch: 0.165\n",
+      "Batch 100 device: cuda time passed: 14.836 time per batch: 0.148\n",
+      "v35, d14, e12, f2, trn ll: 0.0311, val ll: 0.0600, ll_w: 0.0528, cor: 0.8451, auc: 0.9893, lr: 2e-06\n",
+      "setFeats, augmentation -1\n",
+      "Batch 50 device: cuda time passed: 21.311 time per batch: 0.426\n",
+      "Batch 100 device: cuda time passed: 38.735 time per batch: 0.387\n",
+      "Batch 150 device: cuda time passed: 54.760 time per batch: 0.365\n",
+      "Batch 200 device: cuda time passed: 71.364 time per batch: 0.357\n",
+      "Batch 250 device: cuda time passed: 87.975 time per batch: 0.352\n",
+      "Batch 300 device: cuda time passed: 104.126 time per batch: 0.347\n",
+      "Batch 350 device: cuda time passed: 120.536 time per batch: 0.344\n",
+      "Batch 400 device: cuda time passed: 136.741 time per batch: 0.342\n",
+      "Batch 450 device: cuda time passed: 154.903 time per batch: 0.344\n",
+      "Batch 500 device: cuda time passed: 173.138 time per batch: 0.346\n",
+      "Batch 50 device: cuda time passed: 8.686 time per batch: 0.174\n",
+      "Batch 100 device: cuda time passed: 15.208 time per batch: 0.152\n",
+      "v35, d14, e13, f2, trn ll: 0.0311, val ll: 0.0599, ll_w: 0.0527, cor: 0.8454, auc: 0.9892, lr: 2e-06\n",
+      "total running time 421.16584634780884\n",
+      "total time 2838.7946536540985\n",
+      "completed epochs: 0 starting now: 4\n",
+      "DataSet 14 train size 17408 fold 3\n",
+      "adding dummy serieses 16\n",
+      "DataSet 14 valid size 4352 fold 3\n",
+      "setFeats, augmentation 0\n",
+      "dataset train: 17408 valid: 4352 loader train: 544 valid: 136\n",
+      "starting from scratch\n",
+      "setFeats, augmentation -1\n",
+      "Batch 50 device: cuda time passed: 21.909 time per batch: 0.438\n",
+      "Batch 100 device: cuda time passed: 38.295 time per batch: 0.383\n",
+      "Batch 150 device: cuda time passed: 54.459 time per batch: 0.363\n",
+      "Batch 200 device: cuda time passed: 71.059 time per batch: 0.355\n",
+      "Batch 250 device: cuda time passed: 87.285 time per batch: 0.349\n",
+      "Batch 300 device: cuda time passed: 103.278 time per batch: 0.344\n",
+      "Batch 350 device: cuda time passed: 119.758 time per batch: 0.342\n",
+      "Batch 400 device: cuda time passed: 136.023 time per batch: 0.340\n",
+      "Batch 450 device: cuda time passed: 156.381 time per batch: 0.348\n",
+      "Batch 500 device: cuda time passed: 172.724 time per batch: 0.345\n",
+      "Batch 50 device: cuda time passed: 8.381 time per batch: 0.168\n",
+      "Batch 100 device: cuda time passed: 15.352 time per batch: 0.154\n",
+      "v35, d14, e1, f3, trn ll: 0.0574, val ll: 0.0673, ll_w: 0.0595, cor: 0.8311, auc: 0.9867, lr: 0.0002\n",
+      "setFeats, augmentation -1\n",
+      "Batch 50 device: cuda time passed: 21.941 time per batch: 0.439\n",
+      "Batch 100 device: cuda time passed: 38.796 time per batch: 0.388\n",
+      "Batch 150 device: cuda time passed: 55.281 time per batch: 0.369\n",
+      "Batch 200 device: cuda time passed: 72.261 time per batch: 0.361\n",
+      "Batch 250 device: cuda time passed: 88.752 time per batch: 0.355\n",
+      "Batch 300 device: cuda time passed: 105.021 time per batch: 0.350\n",
+      "Batch 350 device: cuda time passed: 121.855 time per batch: 0.348\n",
+      "Batch 400 device: cuda time passed: 137.611 time per batch: 0.344\n",
+      "Batch 450 device: cuda time passed: 158.361 time per batch: 0.352\n",
+      "Batch 500 device: cuda time passed: 175.196 time per batch: 0.350\n",
+      "Batch 50 device: cuda time passed: 7.713 time per batch: 0.154\n",
+      "Batch 100 device: cuda time passed: 15.676 time per batch: 0.157\n",
+      "v35, d14, e2, f3, trn ll: 0.0356, val ll: 0.0659, ll_w: 0.0581, cor: 0.8330, auc: 0.9871, lr: 0.0002\n",
+      "setFeats, augmentation -1\n",
+      "Batch 50 device: cuda time passed: 22.308 time per batch: 0.446\n",
+      "Batch 100 device: cuda time passed: 38.613 time per batch: 0.386\n",
+      "Batch 150 device: cuda time passed: 55.122 time per batch: 0.367\n",
+      "Batch 200 device: cuda time passed: 71.951 time per batch: 0.360\n",
+      "Batch 250 device: cuda time passed: 88.792 time per batch: 0.355\n",
+      "Batch 300 device: cuda time passed: 105.939 time per batch: 0.353\n",
+      "Batch 350 device: cuda time passed: 121.911 time per batch: 0.348\n",
+      "Batch 400 device: cuda time passed: 138.362 time per batch: 0.346\n",
+      "Batch 450 device: cuda time passed: 158.974 time per batch: 0.353\n",
+      "Batch 500 device: cuda time passed: 175.745 time per batch: 0.351\n",
+      "Batch 50 device: cuda time passed: 8.061 time per batch: 0.161\n",
+      "Batch 100 device: cuda time passed: 14.448 time per batch: 0.144\n",
+      "v35, d14, e3, f3, trn ll: 0.0344, val ll: 0.0648, ll_w: 0.0572, cor: 0.8354, auc: 0.9880, lr: 0.0002\n",
+      "setFeats, augmentation -1\n",
+      "Batch 50 device: cuda time passed: 22.337 time per batch: 0.447\n",
+      "Batch 100 device: cuda time passed: 39.126 time per batch: 0.391\n",
+      "Batch 150 device: cuda time passed: 56.197 time per batch: 0.375\n",
+      "Batch 200 device: cuda time passed: 72.948 time per batch: 0.365\n",
+      "Batch 250 device: cuda time passed: 89.869 time per batch: 0.359\n",
+      "Batch 300 device: cuda time passed: 106.597 time per batch: 0.355\n",
+      "Batch 350 device: cuda time passed: 123.356 time per batch: 0.352\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Batch 400 device: cuda time passed: 140.828 time per batch: 0.352\n",
+      "Batch 450 device: cuda time passed: 161.360 time per batch: 0.359\n",
+      "Batch 500 device: cuda time passed: 178.210 time per batch: 0.356\n",
+      "Batch 50 device: cuda time passed: 8.594 time per batch: 0.172\n",
+      "Batch 100 device: cuda time passed: 15.062 time per batch: 0.151\n",
+      "v35, d14, e4, f3, trn ll: 0.0338, val ll: 0.0646, ll_w: 0.0566, cor: 0.8354, auc: 0.9876, lr: 0.0002\n",
+      "total running time 878.9186632633209\n",
+      "completed epochs: 4 starting now: 4\n",
+      "DataSet 14 train size 17408 fold 3\n",
+      "adding dummy serieses 16\n",
+      "DataSet 14 valid size 4352 fold 3\n",
+      "setFeats, augmentation 0\n",
+      "dataset train: 17408 valid: 4352 loader train: 544 valid: 136\n",
+      "loading model model.b4.f3.d14.v35\n",
+      "setFeats, augmentation -1\n",
+      "Batch 50 device: cuda time passed: 22.351 time per batch: 0.447\n",
+      "Batch 100 device: cuda time passed: 39.285 time per batch: 0.393\n",
+      "Batch 150 device: cuda time passed: 56.332 time per batch: 0.376\n",
+      "Batch 200 device: cuda time passed: 73.401 time per batch: 0.367\n",
+      "Batch 250 device: cuda time passed: 90.373 time per batch: 0.361\n",
+      "Batch 300 device: cuda time passed: 107.953 time per batch: 0.360\n",
+      "Batch 350 device: cuda time passed: 125.840 time per batch: 0.360\n",
+      "Batch 400 device: cuda time passed: 141.813 time per batch: 0.355\n",
+      "Batch 450 device: cuda time passed: 163.658 time per batch: 0.364\n",
+      "Batch 500 device: cuda time passed: 180.005 time per batch: 0.360\n",
+      "Batch 50 device: cuda time passed: 9.095 time per batch: 0.182\n",
+      "Batch 100 device: cuda time passed: 15.365 time per batch: 0.154\n",
+      "v35, d14, e5, f3, trn ll: 0.0323, val ll: 0.0630, ll_w: 0.0553, cor: 0.8398, auc: 0.9887, lr: 2e-05\n",
+      "setFeats, augmentation -1\n",
+      "Batch 50 device: cuda time passed: 22.265 time per batch: 0.445\n",
+      "Batch 100 device: cuda time passed: 38.806 time per batch: 0.388\n",
+      "Batch 150 device: cuda time passed: 55.889 time per batch: 0.373\n",
+      "Batch 200 device: cuda time passed: 72.973 time per batch: 0.365\n",
+      "Batch 250 device: cuda time passed: 89.595 time per batch: 0.358\n",
+      "Batch 300 device: cuda time passed: 106.073 time per batch: 0.354\n",
+      "Batch 350 device: cuda time passed: 123.445 time per batch: 0.353\n",
+      "Batch 400 device: cuda time passed: 139.717 time per batch: 0.349\n",
+      "Batch 450 device: cuda time passed: 159.922 time per batch: 0.355\n",
+      "Batch 500 device: cuda time passed: 177.361 time per batch: 0.355\n",
+      "Batch 50 device: cuda time passed: 8.455 time per batch: 0.169\n",
+      "Batch 100 device: cuda time passed: 15.167 time per batch: 0.152\n",
+      "v35, d14, e6, f3, trn ll: 0.0319, val ll: 0.0627, ll_w: 0.0550, cor: 0.8406, auc: 0.9888, lr: 2e-05\n",
+      "setFeats, augmentation -1\n",
+      "Batch 50 device: cuda time passed: 21.948 time per batch: 0.439\n",
+      "Batch 100 device: cuda time passed: 38.589 time per batch: 0.386\n",
+      "Batch 150 device: cuda time passed: 56.699 time per batch: 0.378\n",
+      "Batch 200 device: cuda time passed: 74.066 time per batch: 0.370\n",
+      "Batch 250 device: cuda time passed: 90.412 time per batch: 0.362\n",
+      "Batch 300 device: cuda time passed: 108.005 time per batch: 0.360\n",
+      "Batch 350 device: cuda time passed: 124.540 time per batch: 0.356\n",
+      "Batch 400 device: cuda time passed: 141.433 time per batch: 0.354\n",
+      "Batch 450 device: cuda time passed: 158.940 time per batch: 0.353\n",
+      "Batch 500 device: cuda time passed: 176.269 time per batch: 0.353\n",
+      "Batch 50 device: cuda time passed: 7.920 time per batch: 0.158\n",
+      "Batch 100 device: cuda time passed: 14.641 time per batch: 0.146\n",
+      "v35, d14, e7, f3, trn ll: 0.0317, val ll: 0.0628, ll_w: 0.0550, cor: 0.8408, auc: 0.9888, lr: 2e-05\n",
+      "setFeats, augmentation -1\n",
+      "Batch 50 device: cuda time passed: 21.956 time per batch: 0.439\n",
+      "Batch 100 device: cuda time passed: 38.717 time per batch: 0.387\n",
+      "Batch 150 device: cuda time passed: 54.847 time per batch: 0.366\n",
+      "Batch 200 device: cuda time passed: 73.123 time per batch: 0.366\n",
+      "Batch 250 device: cuda time passed: 89.668 time per batch: 0.359\n",
+      "Batch 300 device: cuda time passed: 105.873 time per batch: 0.353\n",
+      "Batch 350 device: cuda time passed: 122.045 time per batch: 0.349\n",
+      "Batch 400 device: cuda time passed: 138.273 time per batch: 0.346\n",
+      "Batch 450 device: cuda time passed: 157.145 time per batch: 0.349\n",
+      "Batch 500 device: cuda time passed: 173.020 time per batch: 0.346\n",
+      "Batch 50 device: cuda time passed: 8.091 time per batch: 0.162\n",
+      "Batch 100 device: cuda time passed: 14.363 time per batch: 0.144\n",
+      "v35, d14, e8, f3, trn ll: 0.0317, val ll: 0.0624, ll_w: 0.0548, cor: 0.8413, auc: 0.9889, lr: 2e-05\n",
+      "total running time 867.7460985183716\n",
+      "completed epochs: 8 starting now: 3\n",
+      "DataSet 14 train size 17408 fold 3\n",
+      "adding dummy serieses 16\n",
+      "DataSet 14 valid size 4352 fold 3\n",
+      "setFeats, augmentation 0\n",
+      "dataset train: 17408 valid: 4352 loader train: 544 valid: 136\n",
+      "loading model model.b8.f3.d14.v35\n",
+      "setFeats, augmentation -1\n",
+      "Batch 50 device: cuda time passed: 20.904 time per batch: 0.418\n",
+      "Batch 100 device: cuda time passed: 37.911 time per batch: 0.379\n",
+      "Batch 150 device: cuda time passed: 54.975 time per batch: 0.367\n",
+      "Batch 200 device: cuda time passed: 71.442 time per batch: 0.357\n",
+      "Batch 250 device: cuda time passed: 87.521 time per batch: 0.350\n",
+      "Batch 300 device: cuda time passed: 104.238 time per batch: 0.347\n",
+      "Batch 350 device: cuda time passed: 121.406 time per batch: 0.347\n",
+      "Batch 400 device: cuda time passed: 137.836 time per batch: 0.345\n",
+      "Batch 450 device: cuda time passed: 157.379 time per batch: 0.350\n",
+      "Batch 500 device: cuda time passed: 173.441 time per batch: 0.347\n",
+      "Batch 50 device: cuda time passed: 8.016 time per batch: 0.160\n",
+      "Batch 100 device: cuda time passed: 14.483 time per batch: 0.145\n",
+      "v35, d14, e9, f3, trn ll: 0.0313, val ll: 0.0623, ll_w: 0.0547, cor: 0.8414, auc: 0.9889, lr: 5e-06\n",
+      "setFeats, augmentation -1\n",
+      "Batch 50 device: cuda time passed: 22.463 time per batch: 0.449\n",
+      "Batch 100 device: cuda time passed: 39.561 time per batch: 0.396\n",
+      "Batch 150 device: cuda time passed: 56.056 time per batch: 0.374\n",
+      "Batch 200 device: cuda time passed: 72.486 time per batch: 0.362\n",
+      "Batch 250 device: cuda time passed: 88.576 time per batch: 0.354\n",
+      "Batch 300 device: cuda time passed: 105.400 time per batch: 0.351\n",
+      "Batch 350 device: cuda time passed: 122.470 time per batch: 0.350\n",
+      "Batch 400 device: cuda time passed: 138.730 time per batch: 0.347\n",
+      "Batch 450 device: cuda time passed: 158.273 time per batch: 0.352\n",
+      "Batch 500 device: cuda time passed: 174.807 time per batch: 0.350\n",
+      "Batch 50 device: cuda time passed: 8.288 time per batch: 0.166\n",
+      "Batch 100 device: cuda time passed: 14.541 time per batch: 0.145\n",
+      "v35, d14, e10, f3, trn ll: 0.0314, val ll: 0.0623, ll_w: 0.0547, cor: 0.8414, auc: 0.9889, lr: 5e-06\n",
+      "setFeats, augmentation -1\n",
+      "Batch 50 device: cuda time passed: 23.117 time per batch: 0.462\n",
+      "Batch 100 device: cuda time passed: 40.982 time per batch: 0.410\n",
+      "Batch 150 device: cuda time passed: 61.655 time per batch: 0.411\n",
+      "Batch 200 device: cuda time passed: 78.830 time per batch: 0.394\n",
+      "Batch 250 device: cuda time passed: 94.886 time per batch: 0.380\n",
+      "Batch 300 device: cuda time passed: 111.050 time per batch: 0.370\n",
+      "Batch 350 device: cuda time passed: 127.401 time per batch: 0.364\n",
+      "Batch 400 device: cuda time passed: 144.495 time per batch: 0.361\n",
+      "Batch 450 device: cuda time passed: 163.904 time per batch: 0.364\n",
+      "Batch 500 device: cuda time passed: 180.762 time per batch: 0.362\n",
+      "Batch 50 device: cuda time passed: 9.296 time per batch: 0.186\n",
+      "Batch 100 device: cuda time passed: 15.943 time per batch: 0.159\n",
+      "v35, d14, e11, f3, trn ll: 0.0314, val ll: 0.0623, ll_w: 0.0548, cor: 0.8414, auc: 0.9890, lr: 5e-06\n",
+      "total running time 650.1911387443542\n",
+      "completed epochs: 11 starting now: 2\n",
+      "DataSet 14 train size 17408 fold 3\n",
+      "adding dummy serieses 16\n",
+      "DataSet 14 valid size 4352 fold 3\n",
+      "setFeats, augmentation 0\n",
+      "dataset train: 17408 valid: 4352 loader train: 544 valid: 136\n",
+      "loading model model.b11.f3.d14.v35\n",
+      "setFeats, augmentation -1\n",
+      "Batch 50 device: cuda time passed: 22.074 time per batch: 0.441\n",
+      "Batch 100 device: cuda time passed: 38.499 time per batch: 0.385\n",
+      "Batch 150 device: cuda time passed: 55.562 time per batch: 0.370\n",
+      "Batch 200 device: cuda time passed: 71.852 time per batch: 0.359\n",
+      "Batch 250 device: cuda time passed: 88.012 time per batch: 0.352\n",
+      "Batch 300 device: cuda time passed: 105.386 time per batch: 0.351\n",
+      "Batch 350 device: cuda time passed: 123.305 time per batch: 0.352\n",
+      "Batch 400 device: cuda time passed: 140.065 time per batch: 0.350\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Batch 450 device: cuda time passed: 162.016 time per batch: 0.360\n",
+      "Batch 500 device: cuda time passed: 179.990 time per batch: 0.360\n",
+      "Batch 50 device: cuda time passed: 8.722 time per batch: 0.174\n",
+      "Batch 100 device: cuda time passed: 15.752 time per batch: 0.158\n",
+      "v35, d14, e12, f3, trn ll: 0.0313, val ll: 0.0622, ll_w: 0.0546, cor: 0.8417, auc: 0.9890, lr: 2e-06\n",
+      "setFeats, augmentation -1\n",
+      "Batch 50 device: cuda time passed: 24.253 time per batch: 0.485\n",
+      "Batch 100 device: cuda time passed: 41.866 time per batch: 0.419\n",
+      "Batch 150 device: cuda time passed: 60.387 time per batch: 0.403\n",
+      "Batch 200 device: cuda time passed: 77.524 time per batch: 0.388\n",
+      "Batch 250 device: cuda time passed: 95.133 time per batch: 0.381\n",
+      "Batch 300 device: cuda time passed: 113.060 time per batch: 0.377\n",
+      "Batch 350 device: cuda time passed: 130.037 time per batch: 0.372\n",
+      "Batch 400 device: cuda time passed: 146.372 time per batch: 0.366\n",
+      "Batch 450 device: cuda time passed: 166.916 time per batch: 0.371\n",
+      "Batch 500 device: cuda time passed: 182.928 time per batch: 0.366\n",
+      "Batch 50 device: cuda time passed: 7.564 time per batch: 0.151\n",
+      "Batch 100 device: cuda time passed: 14.464 time per batch: 0.145\n",
+      "v35, d14, e13, f3, trn ll: 0.0313, val ll: 0.0621, ll_w: 0.0546, cor: 0.8417, auc: 0.9890, lr: 2e-06\n",
+      "total running time 444.2737958431244\n",
+      "total time 5680.686395645142\n",
+      "completed epochs: 0 starting now: 4\n",
+      "DataSet 14 train size 17376 fold 4\n",
+      "adding dummy serieses 16\n",
+      "DataSet 14 valid size 4384 fold 4\n",
+      "setFeats, augmentation 0\n",
+      "dataset train: 17376 valid: 4384 loader train: 543 valid: 137\n",
+      "starting from scratch\n",
+      "setFeats, augmentation -1\n",
+      "Batch 50 device: cuda time passed: 21.233 time per batch: 0.425\n",
+      "Batch 100 device: cuda time passed: 37.840 time per batch: 0.378\n",
+      "Batch 150 device: cuda time passed: 54.944 time per batch: 0.366\n",
+      "Batch 200 device: cuda time passed: 72.824 time per batch: 0.364\n",
+      "Batch 250 device: cuda time passed: 89.779 time per batch: 0.359\n",
+      "Batch 300 device: cuda time passed: 106.930 time per batch: 0.356\n",
+      "Batch 350 device: cuda time passed: 123.451 time per batch: 0.353\n",
+      "Batch 400 device: cuda time passed: 139.893 time per batch: 0.350\n",
+      "Batch 450 device: cuda time passed: 157.031 time per batch: 0.349\n",
+      "Batch 500 device: cuda time passed: 173.987 time per batch: 0.348\n",
+      "Batch 50 device: cuda time passed: 8.944 time per batch: 0.179\n",
+      "Batch 100 device: cuda time passed: 14.936 time per batch: 0.149\n",
+      "v35, d14, e1, f4, trn ll: 0.0625, val ll: 0.0667, ll_w: 0.0586, cor: 0.8303, auc: 0.9854, lr: 0.0002\n",
+      "setFeats, augmentation -1\n",
+      "Batch 50 device: cuda time passed: 21.754 time per batch: 0.435\n",
+      "Batch 100 device: cuda time passed: 38.725 time per batch: 0.387\n",
+      "Batch 150 device: cuda time passed: 55.236 time per batch: 0.368\n",
+      "Batch 200 device: cuda time passed: 72.718 time per batch: 0.364\n",
+      "Batch 250 device: cuda time passed: 88.735 time per batch: 0.355\n",
+      "Batch 300 device: cuda time passed: 105.104 time per batch: 0.350\n",
+      "Batch 350 device: cuda time passed: 122.012 time per batch: 0.349\n",
+      "Batch 400 device: cuda time passed: 137.964 time per batch: 0.345\n",
+      "Batch 450 device: cuda time passed: 156.284 time per batch: 0.347\n",
+      "Batch 500 device: cuda time passed: 173.980 time per batch: 0.348\n",
+      "Batch 50 device: cuda time passed: 8.316 time per batch: 0.166\n",
+      "Batch 100 device: cuda time passed: 14.987 time per batch: 0.150\n",
+      "v35, d14, e2, f4, trn ll: 0.0359, val ll: 0.0657, ll_w: 0.0577, cor: 0.8311, auc: 0.9863, lr: 0.0002\n",
+      "setFeats, augmentation -1\n",
+      "Batch 50 device: cuda time passed: 22.364 time per batch: 0.447\n",
+      "Batch 100 device: cuda time passed: 40.987 time per batch: 0.410\n",
+      "Batch 150 device: cuda time passed: 58.507 time per batch: 0.390\n",
+      "Batch 200 device: cuda time passed: 77.763 time per batch: 0.389\n",
+      "Batch 250 device: cuda time passed: 95.192 time per batch: 0.381\n",
+      "Batch 300 device: cuda time passed: 111.754 time per batch: 0.373\n",
+      "Batch 350 device: cuda time passed: 129.000 time per batch: 0.369\n",
+      "Batch 400 device: cuda time passed: 146.293 time per batch: 0.366\n",
+      "Batch 450 device: cuda time passed: 165.444 time per batch: 0.368\n",
+      "Batch 500 device: cuda time passed: 183.020 time per batch: 0.366\n",
+      "Batch 50 device: cuda time passed: 7.861 time per batch: 0.157\n",
+      "Batch 100 device: cuda time passed: 15.124 time per batch: 0.151\n",
+      "v35, d14, e3, f4, trn ll: 0.0343, val ll: 0.0632, ll_w: 0.0554, cor: 0.8399, auc: 0.9873, lr: 0.0002\n",
+      "setFeats, augmentation -1\n",
+      "Batch 50 device: cuda time passed: 21.633 time per batch: 0.433\n",
+      "Batch 100 device: cuda time passed: 37.934 time per batch: 0.379\n",
+      "Batch 150 device: cuda time passed: 55.032 time per batch: 0.367\n",
+      "Batch 200 device: cuda time passed: 72.134 time per batch: 0.361\n",
+      "Batch 250 device: cuda time passed: 89.244 time per batch: 0.357\n",
+      "Batch 300 device: cuda time passed: 107.215 time per batch: 0.357\n",
+      "Batch 350 device: cuda time passed: 125.071 time per batch: 0.357\n",
+      "Batch 400 device: cuda time passed: 142.857 time per batch: 0.357\n",
+      "Batch 450 device: cuda time passed: 164.440 time per batch: 0.365\n",
+      "Batch 500 device: cuda time passed: 181.712 time per batch: 0.363\n",
+      "Batch 50 device: cuda time passed: 8.480 time per batch: 0.170\n",
+      "Batch 100 device: cuda time passed: 15.129 time per batch: 0.151\n",
+      "v35, d14, e4, f4, trn ll: 0.0335, val ll: 0.0664, ll_w: 0.0588, cor: 0.8346, auc: 0.9881, lr: 0.0002\n",
+      "total running time 953.7901792526245\n",
+      "completed epochs: 4 starting now: 4\n",
+      "DataSet 14 train size 17376 fold 4\n",
+      "adding dummy serieses 16\n",
+      "DataSet 14 valid size 4384 fold 4\n",
+      "setFeats, augmentation 0\n",
+      "dataset train: 17376 valid: 4384 loader train: 543 valid: 137\n",
+      "loading model model.b4.f4.d14.v35\n",
+      "setFeats, augmentation -1\n",
+      "Batch 50 device: cuda time passed: 23.867 time per batch: 0.477\n",
+      "Batch 100 device: cuda time passed: 41.869 time per batch: 0.419\n",
+      "Batch 150 device: cuda time passed: 59.945 time per batch: 0.400\n",
+      "Batch 200 device: cuda time passed: 78.035 time per batch: 0.390\n",
+      "Batch 250 device: cuda time passed: 95.686 time per batch: 0.383\n",
+      "Batch 300 device: cuda time passed: 113.281 time per batch: 0.378\n",
+      "Batch 350 device: cuda time passed: 130.764 time per batch: 0.374\n",
+      "Batch 400 device: cuda time passed: 148.630 time per batch: 0.372\n",
+      "Batch 450 device: cuda time passed: 170.233 time per batch: 0.378\n",
+      "Batch 500 device: cuda time passed: 188.222 time per batch: 0.376\n",
+      "Batch 50 device: cuda time passed: 8.176 time per batch: 0.164\n",
+      "Batch 100 device: cuda time passed: 14.735 time per batch: 0.147\n",
+      "v35, d14, e5, f4, trn ll: 0.0317, val ll: 0.0608, ll_w: 0.0533, cor: 0.8448, auc: 0.9887, lr: 2e-05\n",
+      "setFeats, augmentation -1\n",
+      "Batch 50 device: cuda time passed: 21.737 time per batch: 0.435\n",
+      "Batch 100 device: cuda time passed: 37.853 time per batch: 0.379\n",
+      "Batch 150 device: cuda time passed: 55.337 time per batch: 0.369\n",
+      "Batch 200 device: cuda time passed: 71.157 time per batch: 0.356\n",
+      "Batch 250 device: cuda time passed: 87.010 time per batch: 0.348\n",
+      "Batch 300 device: cuda time passed: 103.209 time per batch: 0.344\n",
+      "Batch 350 device: cuda time passed: 119.077 time per batch: 0.340\n",
+      "Batch 400 device: cuda time passed: 135.723 time per batch: 0.339\n",
+      "Batch 450 device: cuda time passed: 154.558 time per batch: 0.343\n",
+      "Batch 500 device: cuda time passed: 170.455 time per batch: 0.341\n",
+      "Batch 50 device: cuda time passed: 7.589 time per batch: 0.152\n",
+      "Batch 100 device: cuda time passed: 14.520 time per batch: 0.145\n",
+      "v35, d14, e6, f4, trn ll: 0.0314, val ll: 0.0611, ll_w: 0.0535, cor: 0.8444, auc: 0.9886, lr: 2e-05\n",
+      "setFeats, augmentation -1\n",
+      "Batch 50 device: cuda time passed: 20.682 time per batch: 0.414\n",
+      "Batch 100 device: cuda time passed: 36.792 time per batch: 0.368\n",
+      "Batch 150 device: cuda time passed: 52.970 time per batch: 0.353\n",
+      "Batch 200 device: cuda time passed: 69.095 time per batch: 0.345\n",
+      "Batch 250 device: cuda time passed: 85.616 time per batch: 0.342\n",
+      "Batch 300 device: cuda time passed: 101.163 time per batch: 0.337\n",
+      "Batch 350 device: cuda time passed: 117.018 time per batch: 0.334\n",
+      "Batch 400 device: cuda time passed: 133.178 time per batch: 0.333\n",
+      "Batch 450 device: cuda time passed: 151.207 time per batch: 0.336\n",
+      "Batch 500 device: cuda time passed: 168.530 time per batch: 0.337\n",
+      "Batch 50 device: cuda time passed: 7.677 time per batch: 0.154\n",
+      "Batch 100 device: cuda time passed: 14.814 time per batch: 0.148\n",
+      "v35, d14, e7, f4, trn ll: 0.0312, val ll: 0.0609, ll_w: 0.0534, cor: 0.8449, auc: 0.9886, lr: 2e-05\n",
+      "setFeats, augmentation -1\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Batch 50 device: cuda time passed: 21.214 time per batch: 0.424\n",
+      "Batch 100 device: cuda time passed: 37.518 time per batch: 0.375\n",
+      "Batch 150 device: cuda time passed: 54.071 time per batch: 0.360\n",
+      "Batch 200 device: cuda time passed: 69.747 time per batch: 0.349\n",
+      "Batch 250 device: cuda time passed: 85.969 time per batch: 0.344\n",
+      "Batch 300 device: cuda time passed: 102.031 time per batch: 0.340\n",
+      "Batch 350 device: cuda time passed: 119.208 time per batch: 0.341\n",
+      "Batch 400 device: cuda time passed: 134.735 time per batch: 0.337\n",
+      "Batch 450 device: cuda time passed: 154.078 time per batch: 0.342\n",
+      "Batch 500 device: cuda time passed: 170.411 time per batch: 0.341\n",
+      "Batch 50 device: cuda time passed: 8.323 time per batch: 0.166\n",
+      "Batch 100 device: cuda time passed: 14.637 time per batch: 0.146\n",
+      "v35, d14, e8, f4, trn ll: 0.0311, val ll: 0.0610, ll_w: 0.0534, cor: 0.8453, auc: 0.9885, lr: 2e-05\n",
+      "total running time 855.8814930915833\n",
+      "completed epochs: 8 starting now: 3\n",
+      "DataSet 14 train size 17376 fold 4\n",
+      "adding dummy serieses 16\n",
+      "DataSet 14 valid size 4384 fold 4\n",
+      "setFeats, augmentation 0\n",
+      "dataset train: 17376 valid: 4384 loader train: 543 valid: 137\n",
+      "loading model model.b8.f4.d14.v35\n",
+      "setFeats, augmentation -1\n",
+      "Batch 50 device: cuda time passed: 21.184 time per batch: 0.424\n",
+      "Batch 100 device: cuda time passed: 37.215 time per batch: 0.372\n",
+      "Batch 150 device: cuda time passed: 53.050 time per batch: 0.354\n",
+      "Batch 200 device: cuda time passed: 69.171 time per batch: 0.346\n",
+      "Batch 250 device: cuda time passed: 85.650 time per batch: 0.343\n",
+      "Batch 300 device: cuda time passed: 101.736 time per batch: 0.339\n",
+      "Batch 350 device: cuda time passed: 117.355 time per batch: 0.335\n",
+      "Batch 400 device: cuda time passed: 133.001 time per batch: 0.333\n",
+      "Batch 450 device: cuda time passed: 152.997 time per batch: 0.340\n",
+      "Batch 500 device: cuda time passed: 169.252 time per batch: 0.339\n",
+      "Batch 50 device: cuda time passed: 7.640 time per batch: 0.153\n",
+      "Batch 100 device: cuda time passed: 14.457 time per batch: 0.145\n",
+      "v35, d14, e9, f4, trn ll: 0.0310, val ll: 0.0609, ll_w: 0.0532, cor: 0.8452, auc: 0.9886, lr: 5e-06\n",
+      "setFeats, augmentation -1\n",
+      "Batch 50 device: cuda time passed: 20.694 time per batch: 0.414\n",
+      "Batch 100 device: cuda time passed: 37.491 time per batch: 0.375\n",
+      "Batch 150 device: cuda time passed: 53.734 time per batch: 0.358\n",
+      "Batch 200 device: cuda time passed: 69.955 time per batch: 0.350\n",
+      "Batch 250 device: cuda time passed: 85.941 time per batch: 0.344\n",
+      "Batch 300 device: cuda time passed: 101.942 time per batch: 0.340\n",
+      "Batch 350 device: cuda time passed: 117.779 time per batch: 0.337\n",
+      "Batch 400 device: cuda time passed: 133.697 time per batch: 0.334\n",
+      "Batch 450 device: cuda time passed: 151.893 time per batch: 0.338\n",
+      "Batch 500 device: cuda time passed: 169.235 time per batch: 0.338\n",
+      "Batch 50 device: cuda time passed: 8.345 time per batch: 0.167\n",
+      "Batch 100 device: cuda time passed: 14.826 time per batch: 0.148\n",
+      "v35, d14, e10, f4, trn ll: 0.0308, val ll: 0.0608, ll_w: 0.0532, cor: 0.8453, auc: 0.9887, lr: 5e-06\n",
+      "setFeats, augmentation -1\n",
+      "Batch 50 device: cuda time passed: 21.099 time per batch: 0.422\n",
+      "Batch 100 device: cuda time passed: 36.777 time per batch: 0.368\n",
+      "Batch 150 device: cuda time passed: 53.978 time per batch: 0.360\n",
+      "Batch 200 device: cuda time passed: 69.323 time per batch: 0.347\n",
+      "Batch 250 device: cuda time passed: 85.573 time per batch: 0.342\n",
+      "Batch 300 device: cuda time passed: 102.080 time per batch: 0.340\n",
+      "Batch 350 device: cuda time passed: 118.682 time per batch: 0.339\n",
+      "Batch 400 device: cuda time passed: 135.171 time per batch: 0.338\n",
+      "Batch 450 device: cuda time passed: 152.631 time per batch: 0.339\n",
+      "Batch 500 device: cuda time passed: 167.938 time per batch: 0.336\n",
+      "Batch 50 device: cuda time passed: 8.310 time per batch: 0.166\n",
+      "Batch 100 device: cuda time passed: 14.317 time per batch: 0.143\n",
+      "v35, d14, e11, f4, trn ll: 0.0308, val ll: 0.0607, ll_w: 0.0532, cor: 0.8454, auc: 0.9887, lr: 5e-06\n",
+      "total running time 623.4176635742188\n",
+      "completed epochs: 11 starting now: 2\n",
+      "DataSet 14 train size 17376 fold 4\n",
+      "adding dummy serieses 16\n",
+      "DataSet 14 valid size 4384 fold 4\n",
+      "setFeats, augmentation 0\n",
+      "dataset train: 17376 valid: 4384 loader train: 543 valid: 137\n",
+      "loading model model.b11.f4.d14.v35\n",
+      "setFeats, augmentation -1\n",
+      "Batch 50 device: cuda time passed: 21.779 time per batch: 0.436\n",
+      "Batch 100 device: cuda time passed: 38.087 time per batch: 0.381\n",
+      "Batch 150 device: cuda time passed: 53.619 time per batch: 0.357\n",
+      "Batch 200 device: cuda time passed: 69.523 time per batch: 0.348\n",
+      "Batch 250 device: cuda time passed: 85.874 time per batch: 0.343\n",
+      "Batch 300 device: cuda time passed: 101.439 time per batch: 0.338\n",
+      "Batch 350 device: cuda time passed: 117.336 time per batch: 0.335\n",
+      "Batch 400 device: cuda time passed: 133.035 time per batch: 0.333\n",
+      "Batch 450 device: cuda time passed: 152.595 time per batch: 0.339\n",
+      "Batch 500 device: cuda time passed: 169.052 time per batch: 0.338\n",
+      "Batch 50 device: cuda time passed: 8.452 time per batch: 0.169\n",
+      "Batch 100 device: cuda time passed: 14.411 time per batch: 0.144\n",
+      "v35, d14, e12, f4, trn ll: 0.0307, val ll: 0.0609, ll_w: 0.0532, cor: 0.8456, auc: 0.9886, lr: 2e-06\n",
+      "setFeats, augmentation -1\n",
+      "Batch 50 device: cuda time passed: 20.947 time per batch: 0.419\n",
+      "Batch 100 device: cuda time passed: 37.425 time per batch: 0.374\n",
+      "Batch 150 device: cuda time passed: 52.922 time per batch: 0.353\n",
+      "Batch 200 device: cuda time passed: 69.971 time per batch: 0.350\n",
+      "Batch 250 device: cuda time passed: 85.655 time per batch: 0.343\n",
+      "Batch 300 device: cuda time passed: 101.705 time per batch: 0.339\n",
+      "Batch 350 device: cuda time passed: 117.494 time per batch: 0.336\n",
+      "Batch 400 device: cuda time passed: 134.239 time per batch: 0.336\n",
+      "Batch 450 device: cuda time passed: 154.606 time per batch: 0.344\n",
+      "Batch 500 device: cuda time passed: 171.257 time per batch: 0.343\n",
+      "Batch 50 device: cuda time passed: 8.181 time per batch: 0.164\n",
+      "Batch 100 device: cuda time passed: 14.339 time per batch: 0.143\n",
+      "v35, d14, e13, f4, trn ll: 0.0308, val ll: 0.0607, ll_w: 0.0532, cor: 0.8454, auc: 0.9887, lr: 2e-06\n",
+      "total running time 417.99802231788635\n",
+      "total time 8532.541821241379\n",
+      "total time 8532.541992664337\n",
+      "total time 8532.54287481308\n"
+     ]
+    }
+   ],
+   "source": [
+    "weight_decay = 1e-4\n",
+    "lrs = np.array([2e-4, 2e-5, 5e-6, 2e-6])\n",
+    "epochs = np.array([4, 4, 3, 2])\n",
+    "stg = time.time()\n",
+    "for ds in [14]:\n",
+    "    #folds = getNFolds(ds)\n",
+    "    for f in [2,3,4]:#range(folds):\n",
+    "        for i,lr in enumerate(lrs):\n",
+    "            learning_rate = lr\n",
+    "            model, predictions, val_results = train_one(dataset=ds, epochs=epochs[i], bs=32, fold=f)\n",
+    "        print('total time', time.time() - stg)\n",
+    "    print('total time', time.time() - stg)\n",
+    "print('total time', time.time() - stg)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "completed epochs: 0 starting now: 2\n",
+      "DataSet 14 train size 17369 fold 0\n",
+      "adding dummy serieses 9\n",
+      "DataSet 14 valid size 4384 fold 0\n",
+      "setFeats, augmentation 0\n",
+      "WeightedRandomSampler\n",
+      "dataset train: 17369 valid: 4384 loader train: 542 valid: 137\n",
+      "loading model model.b13.f0.d14.v35\n",
+      "setFeats, augmentation -1\n",
+      "Batch 50 device: cuda time passed: 21.626 time per batch: 0.433\n",
+      "Batch 100 device: cuda time passed: 37.191 time per batch: 0.372\n",
+      "Batch 150 device: cuda time passed: 53.924 time per batch: 0.359\n",
+      "Batch 200 device: cuda time passed: 69.954 time per batch: 0.350\n",
+      "Batch 250 device: cuda time passed: 85.591 time per batch: 0.342\n",
+      "Batch 300 device: cuda time passed: 101.332 time per batch: 0.338\n",
+      "Batch 350 device: cuda time passed: 118.288 time per batch: 0.338\n",
+      "Batch 400 device: cuda time passed: 134.411 time per batch: 0.336\n",
+      "Batch 450 device: cuda time passed: 155.215 time per batch: 0.345\n",
+      "Batch 500 device: cuda time passed: 171.357 time per batch: 0.343\n",
+      "Batch 50 device: cuda time passed: 8.519 time per batch: 0.170\n",
+      "Batch 100 device: cuda time passed: 14.697 time per batch: 0.147\n",
+      "v36, d14, e1, f0, trn ll: 0.0293, val ll: 0.0595, ll_w: 0.0575, cor: 0.8474, auc: 0.9893, lr: 1e-05\n",
+      "setFeats, augmentation -1\n",
+      "Batch 50 device: cuda time passed: 20.234 time per batch: 0.405\n",
+      "Batch 100 device: cuda time passed: 37.254 time per batch: 0.373\n",
+      "Batch 150 device: cuda time passed: 53.470 time per batch: 0.356\n",
+      "Batch 200 device: cuda time passed: 69.176 time per batch: 0.346\n",
+      "Batch 250 device: cuda time passed: 85.543 time per batch: 0.342\n",
+      "Batch 300 device: cuda time passed: 102.066 time per batch: 0.340\n",
+      "Batch 350 device: cuda time passed: 118.857 time per batch: 0.340\n",
+      "Batch 400 device: cuda time passed: 134.521 time per batch: 0.336\n",
+      "Batch 450 device: cuda time passed: 153.072 time per batch: 0.340\n",
+      "Batch 500 device: cuda time passed: 170.818 time per batch: 0.342\n",
+      "Batch 50 device: cuda time passed: 8.401 time per batch: 0.168\n",
+      "Batch 100 device: cuda time passed: 14.616 time per batch: 0.146\n",
+      "v36, d14, e2, f0, trn ll: 0.0302, val ll: 0.0595, ll_w: 0.0574, cor: 0.8477, auc: 0.9893, lr: 1e-05\n",
+      "total running time 458.30859661102295\n",
+      "completed epochs: 2 starting now: 1\n",
+      "DataSet 14 train size 17369 fold 0\n",
+      "adding dummy serieses 9\n",
+      "DataSet 14 valid size 4384 fold 0\n",
+      "setFeats, augmentation 0\n",
+      "WeightedRandomSampler\n",
+      "dataset train: 17369 valid: 4384 loader train: 542 valid: 137\n",
+      "loading model model.b2.f0.d14.v36\n",
+      "setFeats, augmentation -1\n",
+      "Batch 50 device: cuda time passed: 21.426 time per batch: 0.429\n",
+      "Batch 100 device: cuda time passed: 38.129 time per batch: 0.381\n",
+      "Batch 150 device: cuda time passed: 53.906 time per batch: 0.359\n",
+      "Batch 200 device: cuda time passed: 70.144 time per batch: 0.351\n",
+      "Batch 250 device: cuda time passed: 86.070 time per batch: 0.344\n",
+      "Batch 300 device: cuda time passed: 102.134 time per batch: 0.340\n",
+      "Batch 350 device: cuda time passed: 118.405 time per batch: 0.338\n",
+      "Batch 400 device: cuda time passed: 134.645 time per batch: 0.337\n",
+      "Batch 450 device: cuda time passed: 155.131 time per batch: 0.345\n",
+      "Batch 500 device: cuda time passed: 171.499 time per batch: 0.343\n",
+      "Batch 50 device: cuda time passed: 8.219 time per batch: 0.164\n",
+      "Batch 100 device: cuda time passed: 14.580 time per batch: 0.146\n",
+      "v36, d14, e3, f0, trn ll: 0.0298, val ll: 0.0595, ll_w: 0.0575, cor: 0.8472, auc: 0.9893, lr: 5e-06\n",
+      "total running time 213.19121074676514\n",
+      "total time 671.833601474762\n",
+      "completed epochs: 0 starting now: 2\n",
+      "DataSet 14 train size 17468 fold 1\n",
+      "adding dummy serieses 12\n",
+      "DataSet 14 valid size 4288 fold 1\n",
+      "setFeats, augmentation 0\n",
+      "WeightedRandomSampler\n",
+      "dataset train: 17468 valid: 4288 loader train: 545 valid: 134\n",
+      "loading model model.b13.f1.d14.v35\n",
+      "setFeats, augmentation -1\n",
+      "Batch 50 device: cuda time passed: 21.207 time per batch: 0.424\n",
+      "Batch 100 device: cuda time passed: 37.348 time per batch: 0.373\n",
+      "Batch 150 device: cuda time passed: 53.336 time per batch: 0.356\n",
+      "Batch 200 device: cuda time passed: 69.851 time per batch: 0.349\n",
+      "Batch 250 device: cuda time passed: 86.062 time per batch: 0.344\n",
+      "Batch 300 device: cuda time passed: 102.322 time per batch: 0.341\n",
+      "Batch 350 device: cuda time passed: 119.087 time per batch: 0.340\n",
+      "Batch 400 device: cuda time passed: 135.502 time per batch: 0.339\n",
+      "Batch 450 device: cuda time passed: 154.983 time per batch: 0.344\n",
+      "Batch 500 device: cuda time passed: 172.005 time per batch: 0.344\n",
+      "Batch 50 device: cuda time passed: 8.149 time per batch: 0.163\n",
+      "Batch 100 device: cuda time passed: 14.844 time per batch: 0.148\n",
+      "v36, d14, e1, f1, trn ll: 0.0290, val ll: 0.0593, ll_w: 0.0580, cor: 0.8483, auc: 0.9901, lr: 1e-05\n",
+      "setFeats, augmentation -1\n",
+      "Batch 50 device: cuda time passed: 20.749 time per batch: 0.415\n",
+      "Batch 100 device: cuda time passed: 37.137 time per batch: 0.371\n",
+      "Batch 150 device: cuda time passed: 53.310 time per batch: 0.355\n",
+      "Batch 200 device: cuda time passed: 69.517 time per batch: 0.348\n",
+      "Batch 250 device: cuda time passed: 85.902 time per batch: 0.344\n",
+      "Batch 300 device: cuda time passed: 101.682 time per batch: 0.339\n",
+      "Batch 350 device: cuda time passed: 118.095 time per batch: 0.337\n",
+      "Batch 400 device: cuda time passed: 134.853 time per batch: 0.337\n",
+      "Batch 450 device: cuda time passed: 155.186 time per batch: 0.345\n",
+      "Batch 500 device: cuda time passed: 171.021 time per batch: 0.342\n",
+      "Batch 50 device: cuda time passed: 7.627 time per batch: 0.153\n",
+      "Batch 100 device: cuda time passed: 13.832 time per batch: 0.138\n",
+      "v36, d14, e2, f1, trn ll: 0.0294, val ll: 0.0592, ll_w: 0.0579, cor: 0.8485, auc: 0.9901, lr: 1e-05\n",
+      "total running time 450.5849003791809\n",
+      "completed epochs: 2 starting now: 1\n",
+      "DataSet 14 train size 17468 fold 1\n",
+      "adding dummy serieses 12\n",
+      "DataSet 14 valid size 4288 fold 1\n",
+      "setFeats, augmentation 0\n",
+      "WeightedRandomSampler\n",
+      "dataset train: 17468 valid: 4288 loader train: 545 valid: 134\n",
+      "loading model model.b2.f1.d14.v36\n",
+      "setFeats, augmentation -1\n",
+      "Batch 50 device: cuda time passed: 21.513 time per batch: 0.430\n",
+      "Batch 100 device: cuda time passed: 37.783 time per batch: 0.378\n",
+      "Batch 150 device: cuda time passed: 53.955 time per batch: 0.360\n",
+      "Batch 200 device: cuda time passed: 70.656 time per batch: 0.353\n",
+      "Batch 250 device: cuda time passed: 86.881 time per batch: 0.348\n",
+      "Batch 300 device: cuda time passed: 102.464 time per batch: 0.342\n",
+      "Batch 350 device: cuda time passed: 119.210 time per batch: 0.341\n",
+      "Batch 400 device: cuda time passed: 135.295 time per batch: 0.338\n",
+      "Batch 450 device: cuda time passed: 155.318 time per batch: 0.345\n",
+      "Batch 500 device: cuda time passed: 171.368 time per batch: 0.343\n",
+      "Batch 50 device: cuda time passed: 8.426 time per batch: 0.169\n",
+      "Batch 100 device: cuda time passed: 14.857 time per batch: 0.149\n",
+      "v36, d14, e3, f1, trn ll: 0.0287, val ll: 0.0593, ll_w: 0.0580, cor: 0.8484, auc: 0.9900, lr: 5e-06\n",
+      "total running time 213.8144974708557\n",
+      "total time 1336.568071603775\n",
+      "completed epochs: 0 starting now: 2\n",
+      "DataSet 14 train size 17355 fold 2\n",
+      "adding dummy serieses 27\n",
+      "DataSet 14 valid size 4416 fold 2\n",
+      "setFeats, augmentation 0\n",
+      "WeightedRandomSampler\n",
+      "dataset train: 17355 valid: 4416 loader train: 542 valid: 138\n",
+      "loading model model.b13.f2.d14.v35\n",
+      "setFeats, augmentation -1\n",
+      "Batch 50 device: cuda time passed: 21.352 time per batch: 0.427\n",
+      "Batch 100 device: cuda time passed: 37.326 time per batch: 0.373\n",
+      "Batch 150 device: cuda time passed: 53.162 time per batch: 0.354\n",
+      "Batch 200 device: cuda time passed: 68.927 time per batch: 0.345\n",
+      "Batch 250 device: cuda time passed: 84.995 time per batch: 0.340\n",
+      "Batch 300 device: cuda time passed: 101.175 time per batch: 0.337\n",
+      "Batch 350 device: cuda time passed: 117.550 time per batch: 0.336\n",
+      "Batch 400 device: cuda time passed: 134.080 time per batch: 0.335\n",
+      "Batch 450 device: cuda time passed: 153.286 time per batch: 0.341\n",
+      "Batch 500 device: cuda time passed: 169.635 time per batch: 0.339\n",
+      "Batch 50 device: cuda time passed: 8.047 time per batch: 0.161\n",
+      "Batch 100 device: cuda time passed: 14.520 time per batch: 0.145\n",
+      "v36, d14, e1, f2, trn ll: 0.0284, val ll: 0.0606, ll_w: 0.0577, cor: 0.8449, auc: 0.9889, lr: 1e-05\n",
+      "setFeats, augmentation -1\n",
+      "Batch 50 device: cuda time passed: 20.672 time per batch: 0.413\n",
+      "Batch 100 device: cuda time passed: 37.469 time per batch: 0.375\n",
+      "Batch 150 device: cuda time passed: 54.116 time per batch: 0.361\n",
+      "Batch 200 device: cuda time passed: 69.517 time per batch: 0.348\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Batch 250 device: cuda time passed: 86.149 time per batch: 0.345\n",
+      "Batch 300 device: cuda time passed: 102.999 time per batch: 0.343\n",
+      "Batch 350 device: cuda time passed: 119.363 time per batch: 0.341\n",
+      "Batch 400 device: cuda time passed: 135.685 time per batch: 0.339\n",
+      "Batch 450 device: cuda time passed: 152.529 time per batch: 0.339\n",
+      "Batch 500 device: cuda time passed: 172.402 time per batch: 0.345\n",
+      "Batch 50 device: cuda time passed: 8.181 time per batch: 0.164\n",
+      "Batch 100 device: cuda time passed: 14.485 time per batch: 0.145\n",
+      "v36, d14, e2, f2, trn ll: 0.0292, val ll: 0.0600, ll_w: 0.0571, cor: 0.8455, auc: 0.9891, lr: 1e-05\n",
+      "total running time 443.0156455039978\n",
+      "completed epochs: 2 starting now: 1\n",
+      "DataSet 14 train size 17355 fold 2\n",
+      "adding dummy serieses 27\n",
+      "DataSet 14 valid size 4416 fold 2\n",
+      "setFeats, augmentation 0\n",
+      "WeightedRandomSampler\n",
+      "dataset train: 17355 valid: 4416 loader train: 542 valid: 138\n",
+      "loading model model.b2.f2.d14.v36\n",
+      "setFeats, augmentation -1\n",
+      "Batch 50 device: cuda time passed: 20.755 time per batch: 0.415\n",
+      "Batch 100 device: cuda time passed: 36.901 time per batch: 0.369\n",
+      "Batch 150 device: cuda time passed: 52.968 time per batch: 0.353\n",
+      "Batch 200 device: cuda time passed: 69.214 time per batch: 0.346\n",
+      "Batch 250 device: cuda time passed: 85.613 time per batch: 0.342\n",
+      "Batch 300 device: cuda time passed: 102.234 time per batch: 0.341\n",
+      "Batch 350 device: cuda time passed: 118.636 time per batch: 0.339\n",
+      "Batch 400 device: cuda time passed: 135.389 time per batch: 0.338\n",
+      "Batch 450 device: cuda time passed: 155.574 time per batch: 0.346\n",
+      "Batch 500 device: cuda time passed: 171.428 time per batch: 0.343\n",
+      "Batch 50 device: cuda time passed: 8.167 time per batch: 0.163\n",
+      "Batch 100 device: cuda time passed: 14.550 time per batch: 0.145\n",
+      "v36, d14, e3, f2, trn ll: 0.0292, val ll: 0.0600, ll_w: 0.0571, cor: 0.8454, auc: 0.9892, lr: 5e-06\n",
+      "total running time 213.9665548801422\n",
+      "total time 1993.8867774009705\n",
+      "completed epochs: 0 starting now: 2\n",
+      "DataSet 14 train size 17408 fold 3\n",
+      "adding dummy serieses 16\n",
+      "DataSet 14 valid size 4352 fold 3\n",
+      "setFeats, augmentation 0\n",
+      "WeightedRandomSampler\n",
+      "dataset train: 17408 valid: 4352 loader train: 544 valid: 136\n",
+      "loading model model.b13.f3.d14.v35\n",
+      "setFeats, augmentation -1\n",
+      "Batch 50 device: cuda time passed: 21.250 time per batch: 0.425\n",
+      "Batch 100 device: cuda time passed: 37.196 time per batch: 0.372\n",
+      "Batch 150 device: cuda time passed: 53.011 time per batch: 0.353\n",
+      "Batch 200 device: cuda time passed: 69.612 time per batch: 0.348\n",
+      "Batch 250 device: cuda time passed: 85.879 time per batch: 0.344\n",
+      "Batch 300 device: cuda time passed: 102.447 time per batch: 0.341\n",
+      "Batch 350 device: cuda time passed: 118.245 time per batch: 0.338\n",
+      "Batch 400 device: cuda time passed: 135.061 time per batch: 0.338\n",
+      "Batch 450 device: cuda time passed: 153.103 time per batch: 0.340\n",
+      "Batch 500 device: cuda time passed: 170.886 time per batch: 0.342\n",
+      "Batch 50 device: cuda time passed: 8.188 time per batch: 0.164\n",
+      "Batch 100 device: cuda time passed: 14.597 time per batch: 0.146\n",
+      "v36, d14, e1, f3, trn ll: 0.0290, val ll: 0.0626, ll_w: 0.0604, cor: 0.8411, auc: 0.9889, lr: 1e-05\n",
+      "setFeats, augmentation -1\n",
+      "Batch 50 device: cuda time passed: 20.856 time per batch: 0.417\n",
+      "Batch 100 device: cuda time passed: 36.492 time per batch: 0.365\n",
+      "Batch 150 device: cuda time passed: 52.552 time per batch: 0.350\n",
+      "Batch 200 device: cuda time passed: 68.865 time per batch: 0.344\n",
+      "Batch 250 device: cuda time passed: 85.231 time per batch: 0.341\n",
+      "Batch 300 device: cuda time passed: 102.115 time per batch: 0.340\n",
+      "Batch 350 device: cuda time passed: 118.472 time per batch: 0.338\n",
+      "Batch 400 device: cuda time passed: 134.599 time per batch: 0.336\n",
+      "Batch 450 device: cuda time passed: 152.687 time per batch: 0.339\n",
+      "Batch 500 device: cuda time passed: 170.115 time per batch: 0.340\n",
+      "Batch 50 device: cuda time passed: 8.548 time per batch: 0.171\n",
+      "Batch 100 device: cuda time passed: 14.807 time per batch: 0.148\n",
+      "v36, d14, e2, f3, trn ll: 0.0295, val ll: 0.0629, ll_w: 0.0605, cor: 0.8408, auc: 0.9889, lr: 1e-05\n",
+      "total running time 447.1415765285492\n",
+      "completed epochs: 2 starting now: 1\n",
+      "DataSet 14 train size 17408 fold 3\n",
+      "adding dummy serieses 16\n",
+      "DataSet 14 valid size 4352 fold 3\n",
+      "setFeats, augmentation 0\n",
+      "WeightedRandomSampler\n",
+      "dataset train: 17408 valid: 4352 loader train: 544 valid: 136\n",
+      "loading model model.b2.f3.d14.v36\n",
+      "setFeats, augmentation -1\n",
+      "Batch 50 device: cuda time passed: 20.516 time per batch: 0.410\n",
+      "Batch 100 device: cuda time passed: 36.334 time per batch: 0.363\n",
+      "Batch 150 device: cuda time passed: 53.084 time per batch: 0.354\n",
+      "Batch 200 device: cuda time passed: 69.037 time per batch: 0.345\n",
+      "Batch 250 device: cuda time passed: 85.908 time per batch: 0.344\n",
+      "Batch 300 device: cuda time passed: 102.228 time per batch: 0.341\n",
+      "Batch 350 device: cuda time passed: 118.540 time per batch: 0.339\n",
+      "Batch 400 device: cuda time passed: 134.860 time per batch: 0.337\n",
+      "Batch 450 device: cuda time passed: 153.588 time per batch: 0.341\n",
+      "Batch 500 device: cuda time passed: 170.057 time per batch: 0.340\n",
+      "Batch 50 device: cuda time passed: 8.858 time per batch: 0.177\n",
+      "Batch 100 device: cuda time passed: 15.062 time per batch: 0.151\n",
+      "v36, d14, e3, f3, trn ll: 0.0290, val ll: 0.0624, ll_w: 0.0601, cor: 0.8414, auc: 0.9889, lr: 5e-06\n",
+      "total running time 212.96735048294067\n",
+      "total time 2654.3308753967285\n",
+      "completed epochs: 0 starting now: 2\n",
+      "DataSet 14 train size 17376 fold 4\n",
+      "adding dummy serieses 16\n",
+      "DataSet 14 valid size 4384 fold 4\n",
+      "setFeats, augmentation 0\n",
+      "WeightedRandomSampler\n",
+      "dataset train: 17376 valid: 4384 loader train: 543 valid: 137\n",
+      "loading model model.b13.f4.d14.v35\n",
+      "setFeats, augmentation -1\n",
+      "Batch 50 device: cuda time passed: 20.938 time per batch: 0.419\n",
+      "Batch 100 device: cuda time passed: 36.742 time per batch: 0.367\n",
+      "Batch 150 device: cuda time passed: 53.118 time per batch: 0.354\n",
+      "Batch 200 device: cuda time passed: 69.207 time per batch: 0.346\n",
+      "Batch 250 device: cuda time passed: 85.657 time per batch: 0.343\n",
+      "Batch 300 device: cuda time passed: 102.577 time per batch: 0.342\n",
+      "Batch 350 device: cuda time passed: 118.931 time per batch: 0.340\n",
+      "Batch 400 device: cuda time passed: 135.803 time per batch: 0.340\n",
+      "Batch 450 device: cuda time passed: 152.349 time per batch: 0.339\n",
+      "Batch 500 device: cuda time passed: 171.693 time per batch: 0.343\n",
+      "Batch 50 device: cuda time passed: 8.196 time per batch: 0.164\n",
+      "Batch 100 device: cuda time passed: 14.573 time per batch: 0.146\n",
+      "v36, d14, e1, f4, trn ll: 0.0293, val ll: 0.0609, ll_w: 0.0584, cor: 0.8450, auc: 0.9886, lr: 1e-05\n",
+      "setFeats, augmentation -1\n",
+      "Batch 50 device: cuda time passed: 20.888 time per batch: 0.418\n",
+      "Batch 100 device: cuda time passed: 37.072 time per batch: 0.371\n",
+      "Batch 150 device: cuda time passed: 52.913 time per batch: 0.353\n",
+      "Batch 200 device: cuda time passed: 69.316 time per batch: 0.347\n",
+      "Batch 250 device: cuda time passed: 85.225 time per batch: 0.341\n",
+      "Batch 300 device: cuda time passed: 101.227 time per batch: 0.337\n",
+      "Batch 350 device: cuda time passed: 117.882 time per batch: 0.337\n",
+      "Batch 400 device: cuda time passed: 134.412 time per batch: 0.336\n",
+      "Batch 450 device: cuda time passed: 153.441 time per batch: 0.341\n",
+      "Batch 500 device: cuda time passed: 169.712 time per batch: 0.339\n",
+      "Batch 50 device: cuda time passed: 8.684 time per batch: 0.174\n",
+      "Batch 100 device: cuda time passed: 15.135 time per batch: 0.151\n",
+      "v36, d14, e2, f4, trn ll: 0.0288, val ll: 0.0610, ll_w: 0.0585, cor: 0.8453, auc: 0.9885, lr: 1e-05\n",
+      "total running time 445.4769241809845\n",
+      "completed epochs: 2 starting now: 1\n",
+      "DataSet 14 train size 17376 fold 4\n",
+      "adding dummy serieses 16\n",
+      "DataSet 14 valid size 4384 fold 4\n",
+      "setFeats, augmentation 0\n",
+      "WeightedRandomSampler\n",
+      "dataset train: 17376 valid: 4384 loader train: 543 valid: 137\n",
+      "loading model model.b2.f4.d14.v36\n",
+      "setFeats, augmentation -1\n",
+      "Batch 50 device: cuda time passed: 20.951 time per batch: 0.419\n",
+      "Batch 100 device: cuda time passed: 37.131 time per batch: 0.371\n",
+      "Batch 150 device: cuda time passed: 53.374 time per batch: 0.356\n",
+      "Batch 200 device: cuda time passed: 69.208 time per batch: 0.346\n",
+      "Batch 250 device: cuda time passed: 85.747 time per batch: 0.343\n",
+      "Batch 300 device: cuda time passed: 102.371 time per batch: 0.341\n",
+      "Batch 350 device: cuda time passed: 118.719 time per batch: 0.339\n",
+      "Batch 400 device: cuda time passed: 134.930 time per batch: 0.337\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Batch 450 device: cuda time passed: 154.019 time per batch: 0.342\n",
+      "Batch 500 device: cuda time passed: 170.163 time per batch: 0.340\n",
+      "Batch 50 device: cuda time passed: 7.538 time per batch: 0.151\n",
+      "Batch 100 device: cuda time passed: 15.825 time per batch: 0.158\n",
+      "v36, d14, e3, f4, trn ll: 0.0284, val ll: 0.0610, ll_w: 0.0585, cor: 0.8453, auc: 0.9885, lr: 5e-06\n",
+      "total running time 213.95540952682495\n",
+      "total time 3314.099096775055\n",
+      "total time 3314.0992426872253\n",
+      "total time 3314.099636554718\n"
+     ]
+    }
+   ],
+   "source": [
+    "weight_decay = 1e-4\n",
+    "lrs = np.array([1e-5, 5e-6])\n",
+    "epochs = np.array([2, 1])\n",
+    "stg = time.time()\n",
+    "for ds in [14]:#my_datasets3+my_datasets5:\n",
+    "    folds = getNFolds(ds)\n",
+    "    #folds = 2\n",
+    "    for f in range(folds):\n",
+    "        for i,lr in enumerate(lrs):\n",
+    "            learning_rate = lr\n",
+    "            model, predictions, val_results = train_one(dataset=ds, epochs=epochs[i], bs=32, fold=f, init_ver=35)\n",
+    "        print('total time', time.time() - stg)\n",
+    "    print('total time', time.time() - stg)\n",
+    "print('total time', time.time() - stg)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "31 0 65 5 0\n",
+      "31 1 65 5 0\n",
+      "31 2 65 5 0\n",
+      "31 3 39 3 0\n",
+      "31 4 39 3 0\n",
+      "33 0 65 5 0\n",
+      "33 1 65 5 0\n",
+      "33 2 65 5 0\n",
+      "33 3 39 3 0\n",
+      "33 4 39 3 0\n",
+      "34 0 15 0 0\n",
+      "34 1 15 0 0\n",
+      "34 2 15 0 0\n",
+      "34 3 9 0 0\n",
+      "34 4 9 0 0\n",
+      "35 0 13 1 0\n",
+      "35 1 13 1 0\n",
+      "35 2 13 1 0\n",
+      "35 3 13 1 0\n",
+      "35 4 13 1 0\n",
+      "36 0 3 0 0\n",
+      "36 1 3 0 0\n",
+      "36 2 3 0 0\n",
+      "36 3 3 0 0\n",
+      "36 4 3 0 0\n"
+     ]
+    }
+   ],
+   "source": [
+    "for ver in [31,33,34,35,36]:\n",
+    "    for i in range(5):\n",
+    "        stats_filename = PATH_WORK/'stats.f{}.v{}'.format(i,ver)\n",
+    "        stats = pd.read_csv(stats_filename)\n",
+    "        #stats = stats.loc[stats.epoch != 13]\n",
+    "        #stats.to_csv(stats_filename, index=False)\n",
+    "        print(ver,i,len(stats),len(stats.loc[stats.epoch==13]),len(stats.loc[stats.epoch>13]))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 59,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "stats1 = pd.concat([pd.read_csv(PATH_WORK/'stats.f{}.v{}'.format(i,31)) for i in range(5)] +\n",
+    "                   [pd.read_csv(PATH_WORK/'stats.f{}.v{}'.format(i,33)) for i in range(5)] + \n",
+    "                   [pd.read_csv(PATH_WORK/'stats.f{}.v{}'.format(i,35)) for i in range(5)], axis=0, sort=False)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 70,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "stats1 = pd.concat([pd.read_csv(PATH_WORK/'stats.f{}.v{}'.format(i,33)) for i in range(5)] + \n",
+    "                   [pd.read_csv(PATH_WORK/'stats.f{}.v{}'.format(i,35)) for i in range(5)], axis=0, sort=False)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 71,
+   "metadata": {
+    "scrolled": false
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th>val_loss</th>\n",
+       "      <th>val_w_loss</th>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>dataset</th>\n",
+       "      <th>ver</th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <td>7</td>\n",
+       "      <td>33.0</td>\n",
+       "      <td>0.062216</td>\n",
+       "      <td>0.059966</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>9</td>\n",
+       "      <td>33.0</td>\n",
+       "      <td>0.061798</td>\n",
+       "      <td>0.059631</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>11</td>\n",
+       "      <td>33.0</td>\n",
+       "      <td>0.060567</td>\n",
+       "      <td>0.058444</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>12</td>\n",
+       "      <td>33.0</td>\n",
+       "      <td>0.060435</td>\n",
+       "      <td>0.058339</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>13</td>\n",
+       "      <td>33.0</td>\n",
+       "      <td>0.060730</td>\n",
+       "      <td>0.058572</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>14</td>\n",
+       "      <td>35.0</td>\n",
+       "      <td>0.060279</td>\n",
+       "      <td>0.058197</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "              val_loss  val_w_loss\n",
+       "dataset ver                       \n",
+       "7       33.0  0.062216    0.059966\n",
+       "9       33.0  0.061798    0.059631\n",
+       "11      33.0  0.060567    0.058444\n",
+       "12      33.0  0.060435    0.058339\n",
+       "13      33.0  0.060730    0.058572\n",
+       "14      35.0  0.060279    0.058197"
+      ]
+     },
+     "execution_count": 71,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "stats1.loc[stats1.epoch==13].groupby(['dataset','ver'])['val_loss','val_w_loss'].mean()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 76,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "epoch\n",
+       "1     0.069373\n",
+       "2     0.066851\n",
+       "3     0.066766\n",
+       "4     0.065325\n",
+       "5     0.063819\n",
+       "6     0.063702\n",
+       "7     0.063659\n",
+       "8     0.063762\n",
+       "9     0.063641\n",
+       "10    0.063510\n",
+       "11    0.063465\n",
+       "12    0.063588\n",
+       "13    0.058714\n",
+       "Name: val_w_loss, dtype: float64"
+      ]
+     },
+     "execution_count": 76,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "stats1.groupby('epoch')['val_w_loss'].mean()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 77,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "epoch\n",
+       "1    0.058880\n",
+       "2    0.058748\n",
+       "3    0.058762\n",
+       "Name: val_w_loss, dtype: float64"
+      ]
+     },
+     "execution_count": 77,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "stats2.groupby('epoch')['val_w_loss'].mean()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 78,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>ver</th>\n",
+       "      <th>dataset</th>\n",
+       "      <th>epoch</th>\n",
+       "      <th>fold</th>\n",
+       "      <th>train_loss</th>\n",
+       "      <th>val_loss</th>\n",
+       "      <th>val_w_loss</th>\n",
+       "      <th>val_loss2</th>\n",
+       "      <th>val_w_loss2</th>\n",
+       "      <th>cor</th>\n",
+       "      <th>any</th>\n",
+       "      <th>epidural</th>\n",
+       "      <th>intraparenchymal</th>\n",
+       "      <th>intraventricular</th>\n",
+       "      <th>subarachnoid</th>\n",
+       "      <th>subdural</th>\n",
+       "      <th>train_sz</th>\n",
+       "      <th>val_sz</th>\n",
+       "      <th>bs</th>\n",
+       "      <th>train_time</th>\n",
+       "      <th>valid_time</th>\n",
+       "      <th>lr</th>\n",
+       "      <th>wd</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <td>0</td>\n",
+       "      <td>34</td>\n",
+       "      <td>7</td>\n",
+       "      <td>1</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0.026721</td>\n",
+       "      <td>0.064065</td>\n",
+       "      <td>0.061943</td>\n",
+       "      <td>0.058069</td>\n",
+       "      <td>0.041959</td>\n",
+       "      <td>0.841664</td>\n",
+       "      <td>0.094596</td>\n",
+       "      <td>0.017690</td>\n",
+       "      <td>0.048768</td>\n",
+       "      <td>0.022488</td>\n",
+       "      <td>0.050100</td>\n",
+       "      <td>0.078243</td>\n",
+       "      <td>14526</td>\n",
+       "      <td>7232</td>\n",
+       "      <td>32</td>\n",
+       "      <td>188.513041</td>\n",
+       "      <td>42.595724</td>\n",
+       "      <td>0.000010</td>\n",
+       "      <td>0.0001</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1</td>\n",
+       "      <td>34</td>\n",
+       "      <td>7</td>\n",
+       "      <td>2</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0.026779</td>\n",
+       "      <td>0.063986</td>\n",
+       "      <td>0.061799</td>\n",
+       "      <td>0.058277</td>\n",
+       "      <td>0.042019</td>\n",
+       "      <td>0.841992</td>\n",
+       "      <td>0.095012</td>\n",
+       "      <td>0.017891</td>\n",
+       "      <td>0.048536</td>\n",
+       "      <td>0.022399</td>\n",
+       "      <td>0.050454</td>\n",
+       "      <td>0.078633</td>\n",
+       "      <td>14526</td>\n",
+       "      <td>7232</td>\n",
+       "      <td>32</td>\n",
+       "      <td>148.325198</td>\n",
+       "      <td>42.840445</td>\n",
+       "      <td>0.000010</td>\n",
+       "      <td>0.0001</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>2</td>\n",
+       "      <td>34</td>\n",
+       "      <td>7</td>\n",
+       "      <td>3</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0.026806</td>\n",
+       "      <td>0.063772</td>\n",
+       "      <td>0.061700</td>\n",
+       "      <td>0.057858</td>\n",
+       "      <td>0.041902</td>\n",
+       "      <td>0.841932</td>\n",
+       "      <td>0.094341</td>\n",
+       "      <td>0.017474</td>\n",
+       "      <td>0.048568</td>\n",
+       "      <td>0.022377</td>\n",
+       "      <td>0.050502</td>\n",
+       "      <td>0.077406</td>\n",
+       "      <td>14526</td>\n",
+       "      <td>7232</td>\n",
+       "      <td>32</td>\n",
+       "      <td>148.716054</td>\n",
+       "      <td>42.719312</td>\n",
+       "      <td>0.000005</td>\n",
+       "      <td>0.0001</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>3</td>\n",
+       "      <td>34</td>\n",
+       "      <td>9</td>\n",
+       "      <td>1</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0.027112</td>\n",
+       "      <td>0.063366</td>\n",
+       "      <td>0.061400</td>\n",
+       "      <td>0.059194</td>\n",
+       "      <td>0.042473</td>\n",
+       "      <td>0.842161</td>\n",
+       "      <td>0.095822</td>\n",
+       "      <td>0.016753</td>\n",
+       "      <td>0.048794</td>\n",
+       "      <td>0.022736</td>\n",
+       "      <td>0.053430</td>\n",
+       "      <td>0.081001</td>\n",
+       "      <td>14526</td>\n",
+       "      <td>7232</td>\n",
+       "      <td>32</td>\n",
+       "      <td>154.520063</td>\n",
+       "      <td>41.107329</td>\n",
+       "      <td>0.000010</td>\n",
+       "      <td>0.0001</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>4</td>\n",
+       "      <td>34</td>\n",
+       "      <td>9</td>\n",
+       "      <td>2</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0.028070</td>\n",
+       "      <td>0.063492</td>\n",
+       "      <td>0.061455</td>\n",
+       "      <td>0.059902</td>\n",
+       "      <td>0.042804</td>\n",
+       "      <td>0.842046</td>\n",
+       "      <td>0.097219</td>\n",
+       "      <td>0.016762</td>\n",
+       "      <td>0.048958</td>\n",
+       "      <td>0.023702</td>\n",
+       "      <td>0.053533</td>\n",
+       "      <td>0.081922</td>\n",
+       "      <td>14526</td>\n",
+       "      <td>7232</td>\n",
+       "      <td>32</td>\n",
+       "      <td>136.198384</td>\n",
+       "      <td>41.837500</td>\n",
+       "      <td>0.000010</td>\n",
+       "      <td>0.0001</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1</td>\n",
+       "      <td>36</td>\n",
+       "      <td>14</td>\n",
+       "      <td>2</td>\n",
+       "      <td>3</td>\n",
+       "      <td>0.029522</td>\n",
+       "      <td>0.062853</td>\n",
+       "      <td>0.060485</td>\n",
+       "      <td>0.063770</td>\n",
+       "      <td>0.045538</td>\n",
+       "      <td>0.840799</td>\n",
+       "      <td>0.101509</td>\n",
+       "      <td>0.012685</td>\n",
+       "      <td>0.049486</td>\n",
+       "      <td>0.034019</td>\n",
+       "      <td>0.059630</td>\n",
+       "      <td>0.087555</td>\n",
+       "      <td>17408</td>\n",
+       "      <td>4352</td>\n",
+       "      <td>32</td>\n",
+       "      <td>195.092067</td>\n",
+       "      <td>19.647925</td>\n",
+       "      <td>0.000010</td>\n",
+       "      <td>0.0001</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>2</td>\n",
+       "      <td>36</td>\n",
+       "      <td>14</td>\n",
+       "      <td>3</td>\n",
+       "      <td>3</td>\n",
+       "      <td>0.029033</td>\n",
+       "      <td>0.062372</td>\n",
+       "      <td>0.060134</td>\n",
+       "      <td>0.062729</td>\n",
+       "      <td>0.044978</td>\n",
+       "      <td>0.841355</td>\n",
+       "      <td>0.099212</td>\n",
+       "      <td>0.012550</td>\n",
+       "      <td>0.048394</td>\n",
+       "      <td>0.033793</td>\n",
+       "      <td>0.059500</td>\n",
+       "      <td>0.086445</td>\n",
+       "      <td>17408</td>\n",
+       "      <td>4352</td>\n",
+       "      <td>32</td>\n",
+       "      <td>188.747055</td>\n",
+       "      <td>19.592905</td>\n",
+       "      <td>0.000005</td>\n",
+       "      <td>0.0001</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>0</td>\n",
+       "      <td>36</td>\n",
+       "      <td>14</td>\n",
+       "      <td>1</td>\n",
+       "      <td>4</td>\n",
+       "      <td>0.029257</td>\n",
+       "      <td>0.060873</td>\n",
+       "      <td>0.058421</td>\n",
+       "      <td>0.061626</td>\n",
+       "      <td>0.045347</td>\n",
+       "      <td>0.845014</td>\n",
+       "      <td>0.102233</td>\n",
+       "      <td>0.018479</td>\n",
+       "      <td>0.052477</td>\n",
+       "      <td>0.020628</td>\n",
+       "      <td>0.056853</td>\n",
+       "      <td>0.078477</td>\n",
+       "      <td>17376</td>\n",
+       "      <td>4384</td>\n",
+       "      <td>32</td>\n",
+       "      <td>208.909939</td>\n",
+       "      <td>19.676935</td>\n",
+       "      <td>0.000010</td>\n",
+       "      <td>0.0001</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1</td>\n",
+       "      <td>36</td>\n",
+       "      <td>14</td>\n",
+       "      <td>2</td>\n",
+       "      <td>4</td>\n",
+       "      <td>0.028809</td>\n",
+       "      <td>0.060978</td>\n",
+       "      <td>0.058489</td>\n",
+       "      <td>0.061709</td>\n",
+       "      <td>0.045331</td>\n",
+       "      <td>0.845300</td>\n",
+       "      <td>0.102328</td>\n",
+       "      <td>0.018075</td>\n",
+       "      <td>0.052644</td>\n",
+       "      <td>0.021467</td>\n",
+       "      <td>0.056481</td>\n",
+       "      <td>0.078641</td>\n",
+       "      <td>17376</td>\n",
+       "      <td>4384</td>\n",
+       "      <td>32</td>\n",
+       "      <td>192.605031</td>\n",
+       "      <td>19.740591</td>\n",
+       "      <td>0.000010</td>\n",
+       "      <td>0.0001</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>2</td>\n",
+       "      <td>36</td>\n",
+       "      <td>14</td>\n",
+       "      <td>3</td>\n",
+       "      <td>4</td>\n",
+       "      <td>0.028370</td>\n",
+       "      <td>0.060988</td>\n",
+       "      <td>0.058506</td>\n",
+       "      <td>0.061573</td>\n",
+       "      <td>0.045219</td>\n",
+       "      <td>0.845341</td>\n",
+       "      <td>0.102488</td>\n",
+       "      <td>0.018332</td>\n",
+       "      <td>0.052359</td>\n",
+       "      <td>0.020422</td>\n",
+       "      <td>0.056385</td>\n",
+       "      <td>0.078535</td>\n",
+       "      <td>17376</td>\n",
+       "      <td>4384</td>\n",
+       "      <td>32</td>\n",
+       "      <td>189.093271</td>\n",
+       "      <td>20.038952</td>\n",
+       "      <td>0.000005</td>\n",
+       "      <td>0.0001</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>78 rows × 23 columns</p>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "    ver  dataset  epoch  fold  train_loss  val_loss  val_w_loss  val_loss2  \\\n",
+       "0    34        7      1     0    0.026721  0.064065    0.061943   0.058069   \n",
+       "1    34        7      2     0    0.026779  0.063986    0.061799   0.058277   \n",
+       "2    34        7      3     0    0.026806  0.063772    0.061700   0.057858   \n",
+       "3    34        9      1     0    0.027112  0.063366    0.061400   0.059194   \n",
+       "4    34        9      2     0    0.028070  0.063492    0.061455   0.059902   \n",
+       "..  ...      ...    ...   ...         ...       ...         ...        ...   \n",
+       "1    36       14      2     3    0.029522  0.062853    0.060485   0.063770   \n",
+       "2    36       14      3     3    0.029033  0.062372    0.060134   0.062729   \n",
+       "0    36       14      1     4    0.029257  0.060873    0.058421   0.061626   \n",
+       "1    36       14      2     4    0.028809  0.060978    0.058489   0.061709   \n",
+       "2    36       14      3     4    0.028370  0.060988    0.058506   0.061573   \n",
+       "\n",
+       "    val_w_loss2       cor       any  epidural  intraparenchymal  \\\n",
+       "0      0.041959  0.841664  0.094596  0.017690          0.048768   \n",
+       "1      0.042019  0.841992  0.095012  0.017891          0.048536   \n",
+       "2      0.041902  0.841932  0.094341  0.017474          0.048568   \n",
+       "3      0.042473  0.842161  0.095822  0.016753          0.048794   \n",
+       "4      0.042804  0.842046  0.097219  0.016762          0.048958   \n",
+       "..          ...       ...       ...       ...               ...   \n",
+       "1      0.045538  0.840799  0.101509  0.012685          0.049486   \n",
+       "2      0.044978  0.841355  0.099212  0.012550          0.048394   \n",
+       "0      0.045347  0.845014  0.102233  0.018479          0.052477   \n",
+       "1      0.045331  0.845300  0.102328  0.018075          0.052644   \n",
+       "2      0.045219  0.845341  0.102488  0.018332          0.052359   \n",
+       "\n",
+       "    intraventricular  subarachnoid  subdural  train_sz  val_sz  bs  \\\n",
+       "0           0.022488      0.050100  0.078243     14526    7232  32   \n",
+       "1           0.022399      0.050454  0.078633     14526    7232  32   \n",
+       "2           0.022377      0.050502  0.077406     14526    7232  32   \n",
+       "3           0.022736      0.053430  0.081001     14526    7232  32   \n",
+       "4           0.023702      0.053533  0.081922     14526    7232  32   \n",
+       "..               ...           ...       ...       ...     ...  ..   \n",
+       "1           0.034019      0.059630  0.087555     17408    4352  32   \n",
+       "2           0.033793      0.059500  0.086445     17408    4352  32   \n",
+       "0           0.020628      0.056853  0.078477     17376    4384  32   \n",
+       "1           0.021467      0.056481  0.078641     17376    4384  32   \n",
+       "2           0.020422      0.056385  0.078535     17376    4384  32   \n",
+       "\n",
+       "    train_time  valid_time        lr      wd  \n",
+       "0   188.513041   42.595724  0.000010  0.0001  \n",
+       "1   148.325198   42.840445  0.000010  0.0001  \n",
+       "2   148.716054   42.719312  0.000005  0.0001  \n",
+       "3   154.520063   41.107329  0.000010  0.0001  \n",
+       "4   136.198384   41.837500  0.000010  0.0001  \n",
+       "..         ...         ...       ...     ...  \n",
+       "1   195.092067   19.647925  0.000010  0.0001  \n",
+       "2   188.747055   19.592905  0.000005  0.0001  \n",
+       "0   208.909939   19.676935  0.000010  0.0001  \n",
+       "1   192.605031   19.740591  0.000010  0.0001  \n",
+       "2   189.093271   20.038952  0.000005  0.0001  \n",
+       "\n",
+       "[78 rows x 23 columns]"
+      ]
+     },
+     "execution_count": 78,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "stats2"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 18,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "stats1 = stats1.loc[stats1.epoch==13]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 64,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "stats2 = pd.concat([pd.read_csv(PATH_WORK/'stats.f{}.v{}'.format(i,34)) for i in range(5)] +\n",
+    "                   [pd.read_csv(PATH_WORK/'stats.f{}.v{}'.format(i,36)) for i in range(5)], axis=0, sort=False)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 65,
+   "metadata": {
+    "scrolled": false
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th>val_loss</th>\n",
+       "      <th>val_w_loss</th>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>dataset</th>\n",
+       "      <th>ver</th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <td>7</td>\n",
+       "      <td>34</td>\n",
+       "      <td>0.062340</td>\n",
+       "      <td>0.060022</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>9</td>\n",
+       "      <td>34</td>\n",
+       "      <td>0.061991</td>\n",
+       "      <td>0.059712</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>11</td>\n",
+       "      <td>34</td>\n",
+       "      <td>0.060771</td>\n",
+       "      <td>0.058518</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>12</td>\n",
+       "      <td>34</td>\n",
+       "      <td>0.060592</td>\n",
+       "      <td>0.058365</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>13</td>\n",
+       "      <td>34</td>\n",
+       "      <td>0.060883</td>\n",
+       "      <td>0.058596</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>14</td>\n",
+       "      <td>36</td>\n",
+       "      <td>0.060440</td>\n",
+       "      <td>0.058245</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "             val_loss  val_w_loss\n",
+       "dataset ver                      \n",
+       "7       34   0.062340    0.060022\n",
+       "9       34   0.061991    0.059712\n",
+       "11      34   0.060771    0.058518\n",
+       "12      34   0.060592    0.058365\n",
+       "13      34   0.060883    0.058596\n",
+       "14      36   0.060440    0.058245"
+      ]
+     },
+     "execution_count": 65,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "stats2.loc[stats2.epoch==3].groupby(['dataset','ver'])['val_loss','val_w_loss'].mean()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 19,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "stats2 = stats2.loc[stats2.epoch==3]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 24,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "stats1['weighted'] = False\n",
+    "stats2['weighted'] = True"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 25,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "stats = pd.concat([stats1,stats2],axis=0,sort=False)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 28,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "stats['name'] = [getDSName(ds) for ds in stats.dataset.values]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 31,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "stats['type'] = np.where(stats.ver.isin([31,32]), 'old feats, no stage1', \n",
+    "                         np.where(stats.ver.isin([33,34]), 'old feats, with stage1', 'new feats, with stage 1'))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 41,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "stats['name'] = pd.Categorical(stats['name'], \\\n",
+    "    ['Densenet161_F3','se_resnext101_32x4d_F3','se_resnet101_F5','se_resnet101_focal_F5','se_resnext101_32x4d_F5'])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 40,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "stats['type'] = pd.Categorical(stats['type'], \\\n",
+    "    ['old feats, no stage1', 'old feats, with stage1', 'new feats, with stage 1'])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 44,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "stats.val_loss2 = np.where(stats.val_loss2 == 0, np.nan, stats.val_loss2)\n",
+    "stats.val_w_loss2 = np.where(stats.val_w_loss2 == 0, np.nan, stats.val_w_loss2)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 46,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th>val_loss</th>\n",
+       "      <th>val_w_loss</th>\n",
+       "      <th>val_loss2</th>\n",
+       "      <th>val_w_loss2</th>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>name</th>\n",
+       "      <th>weighted</th>\n",
+       "      <th>type</th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <td rowspan=\"6\" valign=\"top\">Densenet161_F3</td>\n",
+       "      <td rowspan=\"3\" valign=\"top\">False</td>\n",
+       "      <td>old feats, no stage1</td>\n",
+       "      <td>0.062832</td>\n",
+       "      <td>0.060421</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>old feats, with stage1</td>\n",
+       "      <td>0.062216</td>\n",
+       "      <td>0.059966</td>\n",
+       "      <td>0.063667</td>\n",
+       "      <td>0.045864</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>new feats, with stage 1</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td rowspan=\"3\" valign=\"top\">True</td>\n",
+       "      <td>old feats, no stage1</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>old feats, with stage1</td>\n",
+       "      <td>0.062340</td>\n",
+       "      <td>0.060022</td>\n",
+       "      <td>0.064111</td>\n",
+       "      <td>0.045960</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>new feats, with stage 1</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td rowspan=\"6\" valign=\"top\">se_resnext101_32x4d_F3</td>\n",
+       "      <td rowspan=\"3\" valign=\"top\">False</td>\n",
+       "      <td>old feats, no stage1</td>\n",
+       "      <td>0.062391</td>\n",
+       "      <td>0.060030</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>old feats, with stage1</td>\n",
+       "      <td>0.061798</td>\n",
+       "      <td>0.059631</td>\n",
+       "      <td>0.063498</td>\n",
+       "      <td>0.045743</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>new feats, with stage 1</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td rowspan=\"3\" valign=\"top\">True</td>\n",
+       "      <td>old feats, no stage1</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>old feats, with stage1</td>\n",
+       "      <td>0.061991</td>\n",
+       "      <td>0.059712</td>\n",
+       "      <td>0.064051</td>\n",
+       "      <td>0.045793</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>new feats, with stage 1</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td rowspan=\"6\" valign=\"top\">se_resnet101_F5</td>\n",
+       "      <td rowspan=\"3\" valign=\"top\">False</td>\n",
+       "      <td>old feats, no stage1</td>\n",
+       "      <td>0.060777</td>\n",
+       "      <td>0.058524</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>old feats, with stage1</td>\n",
+       "      <td>0.060435</td>\n",
+       "      <td>0.058339</td>\n",
+       "      <td>0.062473</td>\n",
+       "      <td>0.045330</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>new feats, with stage 1</td>\n",
+       "      <td>0.060279</td>\n",
+       "      <td>0.058197</td>\n",
+       "      <td>0.059201</td>\n",
+       "      <td>0.042987</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td rowspan=\"3\" valign=\"top\">True</td>\n",
+       "      <td>old feats, no stage1</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>old feats, with stage1</td>\n",
+       "      <td>0.060592</td>\n",
+       "      <td>0.058365</td>\n",
+       "      <td>0.063225</td>\n",
+       "      <td>0.045505</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>new feats, with stage 1</td>\n",
+       "      <td>0.060440</td>\n",
+       "      <td>0.058245</td>\n",
+       "      <td>0.059749</td>\n",
+       "      <td>0.043057</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td rowspan=\"6\" valign=\"top\">se_resnet101_focal_F5</td>\n",
+       "      <td rowspan=\"3\" valign=\"top\">False</td>\n",
+       "      <td>old feats, no stage1</td>\n",
+       "      <td>0.061076</td>\n",
+       "      <td>0.058784</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>old feats, with stage1</td>\n",
+       "      <td>0.060730</td>\n",
+       "      <td>0.058572</td>\n",
+       "      <td>0.062898</td>\n",
+       "      <td>0.045629</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>new feats, with stage 1</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td rowspan=\"3\" valign=\"top\">True</td>\n",
+       "      <td>old feats, no stage1</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>old feats, with stage1</td>\n",
+       "      <td>0.060883</td>\n",
+       "      <td>0.058596</td>\n",
+       "      <td>0.063621</td>\n",
+       "      <td>0.045774</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>new feats, with stage 1</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td rowspan=\"6\" valign=\"top\">se_resnext101_32x4d_F5</td>\n",
+       "      <td rowspan=\"3\" valign=\"top\">False</td>\n",
+       "      <td>old feats, no stage1</td>\n",
+       "      <td>0.061160</td>\n",
+       "      <td>0.058838</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>old feats, with stage1</td>\n",
+       "      <td>0.060567</td>\n",
+       "      <td>0.058444</td>\n",
+       "      <td>0.063178</td>\n",
+       "      <td>0.045846</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>new feats, with stage 1</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td rowspan=\"3\" valign=\"top\">True</td>\n",
+       "      <td>old feats, no stage1</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>old feats, with stage1</td>\n",
+       "      <td>0.060771</td>\n",
+       "      <td>0.058518</td>\n",
+       "      <td>0.063957</td>\n",
+       "      <td>0.046079</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>new feats, with stage 1</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                                                         val_loss  val_w_loss  \\\n",
+       "name                   weighted type                                            \n",
+       "Densenet161_F3         False    old feats, no stage1     0.062832    0.060421   \n",
+       "                                old feats, with stage1   0.062216    0.059966   \n",
+       "                                new feats, with stage 1       NaN         NaN   \n",
+       "                       True     old feats, no stage1          NaN         NaN   \n",
+       "                                old feats, with stage1   0.062340    0.060022   \n",
+       "                                new feats, with stage 1       NaN         NaN   \n",
+       "se_resnext101_32x4d_F3 False    old feats, no stage1     0.062391    0.060030   \n",
+       "                                old feats, with stage1   0.061798    0.059631   \n",
+       "                                new feats, with stage 1       NaN         NaN   \n",
+       "                       True     old feats, no stage1          NaN         NaN   \n",
+       "                                old feats, with stage1   0.061991    0.059712   \n",
+       "                                new feats, with stage 1       NaN         NaN   \n",
+       "se_resnet101_F5        False    old feats, no stage1     0.060777    0.058524   \n",
+       "                                old feats, with stage1   0.060435    0.058339   \n",
+       "                                new feats, with stage 1  0.060279    0.058197   \n",
+       "                       True     old feats, no stage1          NaN         NaN   \n",
+       "                                old feats, with stage1   0.060592    0.058365   \n",
+       "                                new feats, with stage 1  0.060440    0.058245   \n",
+       "se_resnet101_focal_F5  False    old feats, no stage1     0.061076    0.058784   \n",
+       "                                old feats, with stage1   0.060730    0.058572   \n",
+       "                                new feats, with stage 1       NaN         NaN   \n",
+       "                       True     old feats, no stage1          NaN         NaN   \n",
+       "                                old feats, with stage1   0.060883    0.058596   \n",
+       "                                new feats, with stage 1       NaN         NaN   \n",
+       "se_resnext101_32x4d_F5 False    old feats, no stage1     0.061160    0.058838   \n",
+       "                                old feats, with stage1   0.060567    0.058444   \n",
+       "                                new feats, with stage 1       NaN         NaN   \n",
+       "                       True     old feats, no stage1          NaN         NaN   \n",
+       "                                old feats, with stage1   0.060771    0.058518   \n",
+       "                                new feats, with stage 1       NaN         NaN   \n",
+       "\n",
+       "                                                         val_loss2  \\\n",
+       "name                   weighted type                                 \n",
+       "Densenet161_F3         False    old feats, no stage1           NaN   \n",
+       "                                old feats, with stage1    0.063667   \n",
+       "                                new feats, with stage 1        NaN   \n",
+       "                       True     old feats, no stage1           NaN   \n",
+       "                                old feats, with stage1    0.064111   \n",
+       "                                new feats, with stage 1        NaN   \n",
+       "se_resnext101_32x4d_F3 False    old feats, no stage1           NaN   \n",
+       "                                old feats, with stage1    0.063498   \n",
+       "                                new feats, with stage 1        NaN   \n",
+       "                       True     old feats, no stage1           NaN   \n",
+       "                                old feats, with stage1    0.064051   \n",
+       "                                new feats, with stage 1        NaN   \n",
+       "se_resnet101_F5        False    old feats, no stage1           NaN   \n",
+       "                                old feats, with stage1    0.062473   \n",
+       "                                new feats, with stage 1   0.059201   \n",
+       "                       True     old feats, no stage1           NaN   \n",
+       "                                old feats, with stage1    0.063225   \n",
+       "                                new feats, with stage 1   0.059749   \n",
+       "se_resnet101_focal_F5  False    old feats, no stage1           NaN   \n",
+       "                                old feats, with stage1    0.062898   \n",
+       "                                new feats, with stage 1        NaN   \n",
+       "                       True     old feats, no stage1           NaN   \n",
+       "                                old feats, with stage1    0.063621   \n",
+       "                                new feats, with stage 1        NaN   \n",
+       "se_resnext101_32x4d_F5 False    old feats, no stage1           NaN   \n",
+       "                                old feats, with stage1    0.063178   \n",
+       "                                new feats, with stage 1        NaN   \n",
+       "                       True     old feats, no stage1           NaN   \n",
+       "                                old feats, with stage1    0.063957   \n",
+       "                                new feats, with stage 1        NaN   \n",
+       "\n",
+       "                                                         val_w_loss2  \n",
+       "name                   weighted type                                  \n",
+       "Densenet161_F3         False    old feats, no stage1             NaN  \n",
+       "                                old feats, with stage1      0.045864  \n",
+       "                                new feats, with stage 1          NaN  \n",
+       "                       True     old feats, no stage1             NaN  \n",
+       "                                old feats, with stage1      0.045960  \n",
+       "                                new feats, with stage 1          NaN  \n",
+       "se_resnext101_32x4d_F3 False    old feats, no stage1             NaN  \n",
+       "                                old feats, with stage1      0.045743  \n",
+       "                                new feats, with stage 1          NaN  \n",
+       "                       True     old feats, no stage1             NaN  \n",
+       "                                old feats, with stage1      0.045793  \n",
+       "                                new feats, with stage 1          NaN  \n",
+       "se_resnet101_F5        False    old feats, no stage1             NaN  \n",
+       "                                old feats, with stage1      0.045330  \n",
+       "                                new feats, with stage 1     0.042987  \n",
+       "                       True     old feats, no stage1             NaN  \n",
+       "                                old feats, with stage1      0.045505  \n",
+       "                                new feats, with stage 1     0.043057  \n",
+       "se_resnet101_focal_F5  False    old feats, no stage1             NaN  \n",
+       "                                old feats, with stage1      0.045629  \n",
+       "                                new feats, with stage 1          NaN  \n",
+       "                       True     old feats, no stage1             NaN  \n",
+       "                                old feats, with stage1      0.045774  \n",
+       "                                new feats, with stage 1          NaN  \n",
+       "se_resnext101_32x4d_F5 False    old feats, no stage1             NaN  \n",
+       "                                old feats, with stage1      0.045846  \n",
+       "                                new feats, with stage 1          NaN  \n",
+       "                       True     old feats, no stage1             NaN  \n",
+       "                                old feats, with stage1      0.046079  \n",
+       "                                new feats, with stage 1          NaN  "
+      ]
+     },
+     "execution_count": 46,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "stats.groupby(['name','weighted','type'])['val_loss','val_w_loss','val_loss2','val_w_loss2'].mean()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 53,
+   "metadata": {
+    "scrolled": false
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th>val_loss</th>\n",
+       "      <th>val_w_loss</th>\n",
+       "      <th>val_loss2</th>\n",
+       "      <th>val_w_loss2</th>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>fold</th>\n",
+       "      <th>weighted</th>\n",
+       "      <th>type</th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <td rowspan=\"6\" valign=\"top\">0</td>\n",
+       "      <td rowspan=\"3\" valign=\"top\">False</td>\n",
+       "      <td>old feats, no stage1</td>\n",
+       "      <td>0.059207</td>\n",
+       "      <td>0.057173</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>old feats, with stage1</td>\n",
+       "      <td>0.059376</td>\n",
+       "      <td>0.057322</td>\n",
+       "      <td>0.071586</td>\n",
+       "      <td>0.052126</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>new feats, with stage 1</td>\n",
+       "      <td>0.059336</td>\n",
+       "      <td>0.057380</td>\n",
+       "      <td>0.067331</td>\n",
+       "      <td>0.048938</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td rowspan=\"3\" valign=\"top\">True</td>\n",
+       "      <td>old feats, no stage1</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>old feats, with stage1</td>\n",
+       "      <td>0.059580</td>\n",
+       "      <td>0.057412</td>\n",
+       "      <td>0.072157</td>\n",
+       "      <td>0.052435</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>new feats, with stage 1</td>\n",
+       "      <td>0.059511</td>\n",
+       "      <td>0.057492</td>\n",
+       "      <td>0.067691</td>\n",
+       "      <td>0.049024</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td rowspan=\"6\" valign=\"top\">1</td>\n",
+       "      <td rowspan=\"3\" valign=\"top\">False</td>\n",
+       "      <td>old feats, no stage1</td>\n",
+       "      <td>0.059903</td>\n",
+       "      <td>0.058524</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>old feats, with stage1</td>\n",
+       "      <td>0.059548</td>\n",
+       "      <td>0.058294</td>\n",
+       "      <td>0.055482</td>\n",
+       "      <td>0.040188</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>new feats, with stage 1</td>\n",
+       "      <td>0.059209</td>\n",
+       "      <td>0.057936</td>\n",
+       "      <td>0.052200</td>\n",
+       "      <td>0.038043</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td rowspan=\"3\" valign=\"top\">True</td>\n",
+       "      <td>old feats, no stage1</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>old feats, with stage1</td>\n",
+       "      <td>0.059670</td>\n",
+       "      <td>0.058308</td>\n",
+       "      <td>0.056408</td>\n",
+       "      <td>0.040399</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>new feats, with stage 1</td>\n",
+       "      <td>0.059323</td>\n",
+       "      <td>0.057955</td>\n",
+       "      <td>0.052909</td>\n",
+       "      <td>0.038214</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td rowspan=\"6\" valign=\"top\">2</td>\n",
+       "      <td rowspan=\"3\" valign=\"top\">False</td>\n",
+       "      <td>old feats, no stage1</td>\n",
+       "      <td>0.060600</td>\n",
+       "      <td>0.057828</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>old feats, with stage1</td>\n",
+       "      <td>0.060391</td>\n",
+       "      <td>0.057761</td>\n",
+       "      <td>0.056373</td>\n",
+       "      <td>0.040194</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>new feats, with stage 1</td>\n",
+       "      <td>0.059884</td>\n",
+       "      <td>0.057178</td>\n",
+       "      <td>0.053371</td>\n",
+       "      <td>0.037849</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td rowspan=\"3\" valign=\"top\">True</td>\n",
+       "      <td>old feats, no stage1</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>old feats, with stage1</td>\n",
+       "      <td>0.060492</td>\n",
+       "      <td>0.057737</td>\n",
+       "      <td>0.056875</td>\n",
+       "      <td>0.040117</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>new feats, with stage 1</td>\n",
+       "      <td>0.060004</td>\n",
+       "      <td>0.057138</td>\n",
+       "      <td>0.053842</td>\n",
+       "      <td>0.037853</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td rowspan=\"6\" valign=\"top\">3</td>\n",
+       "      <td rowspan=\"3\" valign=\"top\">False</td>\n",
+       "      <td>old feats, no stage1</td>\n",
+       "      <td>0.062978</td>\n",
+       "      <td>0.060445</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>old feats, with stage1</td>\n",
+       "      <td>0.062238</td>\n",
+       "      <td>0.060029</td>\n",
+       "      <td>0.063756</td>\n",
+       "      <td>0.045654</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>new feats, with stage 1</td>\n",
+       "      <td>0.062283</td>\n",
+       "      <td>0.060144</td>\n",
+       "      <td>0.062267</td>\n",
+       "      <td>0.044982</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td rowspan=\"3\" valign=\"top\">True</td>\n",
+       "      <td>old feats, no stage1</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>old feats, with stage1</td>\n",
+       "      <td>0.062409</td>\n",
+       "      <td>0.060069</td>\n",
+       "      <td>0.064277</td>\n",
+       "      <td>0.045677</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>new feats, with stage 1</td>\n",
+       "      <td>0.062372</td>\n",
+       "      <td>0.060134</td>\n",
+       "      <td>0.062729</td>\n",
+       "      <td>0.044978</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td rowspan=\"6\" valign=\"top\">4</td>\n",
+       "      <td rowspan=\"3\" valign=\"top\">False</td>\n",
+       "      <td>old feats, no stage1</td>\n",
+       "      <td>0.061198</td>\n",
+       "      <td>0.058650</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>old feats, with stage1</td>\n",
+       "      <td>0.060621</td>\n",
+       "      <td>0.058288</td>\n",
+       "      <td>0.065170</td>\n",
+       "      <td>0.048486</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>new feats, with stage 1</td>\n",
+       "      <td>0.060685</td>\n",
+       "      <td>0.058348</td>\n",
+       "      <td>0.060837</td>\n",
+       "      <td>0.045122</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td rowspan=\"3\" valign=\"top\">True</td>\n",
+       "      <td>old feats, no stage1</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>old feats, with stage1</td>\n",
+       "      <td>0.060810</td>\n",
+       "      <td>0.058297</td>\n",
+       "      <td>0.066407</td>\n",
+       "      <td>0.048896</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>new feats, with stage 1</td>\n",
+       "      <td>0.060988</td>\n",
+       "      <td>0.058506</td>\n",
+       "      <td>0.061573</td>\n",
+       "      <td>0.045219</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                                       val_loss  val_w_loss  val_loss2  \\\n",
+       "fold weighted type                                                       \n",
+       "0    False    old feats, no stage1     0.059207    0.057173        NaN   \n",
+       "              old feats, with stage1   0.059376    0.057322   0.071586   \n",
+       "              new feats, with stage 1  0.059336    0.057380   0.067331   \n",
+       "     True     old feats, no stage1          NaN         NaN        NaN   \n",
+       "              old feats, with stage1   0.059580    0.057412   0.072157   \n",
+       "              new feats, with stage 1  0.059511    0.057492   0.067691   \n",
+       "1    False    old feats, no stage1     0.059903    0.058524        NaN   \n",
+       "              old feats, with stage1   0.059548    0.058294   0.055482   \n",
+       "              new feats, with stage 1  0.059209    0.057936   0.052200   \n",
+       "     True     old feats, no stage1          NaN         NaN        NaN   \n",
+       "              old feats, with stage1   0.059670    0.058308   0.056408   \n",
+       "              new feats, with stage 1  0.059323    0.057955   0.052909   \n",
+       "2    False    old feats, no stage1     0.060600    0.057828        NaN   \n",
+       "              old feats, with stage1   0.060391    0.057761   0.056373   \n",
+       "              new feats, with stage 1  0.059884    0.057178   0.053371   \n",
+       "     True     old feats, no stage1          NaN         NaN        NaN   \n",
+       "              old feats, with stage1   0.060492    0.057737   0.056875   \n",
+       "              new feats, with stage 1  0.060004    0.057138   0.053842   \n",
+       "3    False    old feats, no stage1     0.062978    0.060445        NaN   \n",
+       "              old feats, with stage1   0.062238    0.060029   0.063756   \n",
+       "              new feats, with stage 1  0.062283    0.060144   0.062267   \n",
+       "     True     old feats, no stage1          NaN         NaN        NaN   \n",
+       "              old feats, with stage1   0.062409    0.060069   0.064277   \n",
+       "              new feats, with stage 1  0.062372    0.060134   0.062729   \n",
+       "4    False    old feats, no stage1     0.061198    0.058650        NaN   \n",
+       "              old feats, with stage1   0.060621    0.058288   0.065170   \n",
+       "              new feats, with stage 1  0.060685    0.058348   0.060837   \n",
+       "     True     old feats, no stage1          NaN         NaN        NaN   \n",
+       "              old feats, with stage1   0.060810    0.058297   0.066407   \n",
+       "              new feats, with stage 1  0.060988    0.058506   0.061573   \n",
+       "\n",
+       "                                       val_w_loss2  \n",
+       "fold weighted type                                  \n",
+       "0    False    old feats, no stage1             NaN  \n",
+       "              old feats, with stage1      0.052126  \n",
+       "              new feats, with stage 1     0.048938  \n",
+       "     True     old feats, no stage1             NaN  \n",
+       "              old feats, with stage1      0.052435  \n",
+       "              new feats, with stage 1     0.049024  \n",
+       "1    False    old feats, no stage1             NaN  \n",
+       "              old feats, with stage1      0.040188  \n",
+       "              new feats, with stage 1     0.038043  \n",
+       "     True     old feats, no stage1             NaN  \n",
+       "              old feats, with stage1      0.040399  \n",
+       "              new feats, with stage 1     0.038214  \n",
+       "2    False    old feats, no stage1             NaN  \n",
+       "              old feats, with stage1      0.040194  \n",
+       "              new feats, with stage 1     0.037849  \n",
+       "     True     old feats, no stage1             NaN  \n",
+       "              old feats, with stage1      0.040117  \n",
+       "              new feats, with stage 1     0.037853  \n",
+       "3    False    old feats, no stage1             NaN  \n",
+       "              old feats, with stage1      0.045654  \n",
+       "              new feats, with stage 1     0.044982  \n",
+       "     True     old feats, no stage1             NaN  \n",
+       "              old feats, with stage1      0.045677  \n",
+       "              new feats, with stage 1     0.044978  \n",
+       "4    False    old feats, no stage1             NaN  \n",
+       "              old feats, with stage1      0.048486  \n",
+       "              new feats, with stage 1     0.045122  \n",
+       "     True     old feats, no stage1             NaN  \n",
+       "              old feats, with stage1      0.048896  \n",
+       "              new feats, with stage 1     0.045219  "
+      ]
+     },
+     "execution_count": 53,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "stats.loc[stats.name=='se_resnet101_F5']\\\n",
+    "    .groupby(['fold','weighted','type'])['val_loss','val_w_loss','val_loss2','val_w_loss2'].mean()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 21,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "14    5\n",
+       "13    5\n",
+       "12    5\n",
+       "11    5\n",
+       "9     3\n",
+       "7     3\n",
+       "Name: dataset, dtype: int64"
+      ]
+     },
+     "execution_count": 21,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "stats2.dataset.value_counts()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 39,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "stats2 = pd.concat([pd.read_csv(PATH_WORK/'stats.f{}.v{}'.format(0,32)),\n",
+    "                    pd.read_csv(PATH_WORK/'stats.f{}.v{}'.format(1,32)),\n",
+    "                    pd.read_csv(PATH_WORK/'stats.f{}.v{}'.format(0,34)),\n",
+    "                    pd.read_csv(PATH_WORK/'stats.f{}.v{}'.format(1,34))], axis=0, sort=False)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 40,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>dataset</th>\n",
+       "      <th>epoch</th>\n",
+       "      <th>fold</th>\n",
+       "      <th>train_loss</th>\n",
+       "      <th>val_loss</th>\n",
+       "      <th>val_w_loss</th>\n",
+       "      <th>val_loss2</th>\n",
+       "      <th>val_w_loss2</th>\n",
+       "      <th>cor</th>\n",
+       "      <th>any</th>\n",
+       "      <th>epidural</th>\n",
+       "      <th>intraparenchymal</th>\n",
+       "      <th>intraventricular</th>\n",
+       "      <th>subarachnoid</th>\n",
+       "      <th>subdural</th>\n",
+       "      <th>train_sz</th>\n",
+       "      <th>val_sz</th>\n",
+       "      <th>bs</th>\n",
+       "      <th>train_time</th>\n",
+       "      <th>valid_time</th>\n",
+       "      <th>lr</th>\n",
+       "      <th>wd</th>\n",
+       "      <th>ver</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <td>12</td>\n",
+       "      <td>12</td>\n",
+       "      <td>13</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0.031208</td>\n",
+       "      <td>0.059201</td>\n",
+       "      <td>0.064282</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.847493</td>\n",
+       "      <td>0.094982</td>\n",
+       "      <td>0.014763</td>\n",
+       "      <td>0.044750</td>\n",
+       "      <td>0.025390</td>\n",
+       "      <td>0.064951</td>\n",
+       "      <td>0.074592</td>\n",
+       "      <td>15619</td>\n",
+       "      <td>3936</td>\n",
+       "      <td>32</td>\n",
+       "      <td>85.367667</td>\n",
+       "      <td>10.784082</td>\n",
+       "      <td>0.000002</td>\n",
+       "      <td>0.0001</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>25</td>\n",
+       "      <td>14</td>\n",
+       "      <td>13</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0.031730</td>\n",
+       "      <td>0.059394</td>\n",
+       "      <td>0.063065</td>\n",
+       "      <td>0.067177</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>0.847427</td>\n",
+       "      <td>0.114279</td>\n",
+       "      <td>0.018076</td>\n",
+       "      <td>0.043661</td>\n",
+       "      <td>0.037124</td>\n",
+       "      <td>0.069421</td>\n",
+       "      <td>0.073398</td>\n",
+       "      <td>17369</td>\n",
+       "      <td>4384</td>\n",
+       "      <td>32</td>\n",
+       "      <td>101.753389</td>\n",
+       "      <td>11.775365</td>\n",
+       "      <td>0.000002</td>\n",
+       "      <td>0.0001</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>12</td>\n",
+       "      <td>12</td>\n",
+       "      <td>13</td>\n",
+       "      <td>1</td>\n",
+       "      <td>0.031786</td>\n",
+       "      <td>0.059864</td>\n",
+       "      <td>0.058066</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.846629</td>\n",
+       "      <td>0.097579</td>\n",
+       "      <td>0.014948</td>\n",
+       "      <td>0.040305</td>\n",
+       "      <td>0.025653</td>\n",
+       "      <td>0.063112</td>\n",
+       "      <td>0.079871</td>\n",
+       "      <td>15699</td>\n",
+       "      <td>3840</td>\n",
+       "      <td>32</td>\n",
+       "      <td>85.526455</td>\n",
+       "      <td>10.400574</td>\n",
+       "      <td>0.000002</td>\n",
+       "      <td>0.0001</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>25</td>\n",
+       "      <td>14</td>\n",
+       "      <td>13</td>\n",
+       "      <td>1</td>\n",
+       "      <td>0.031240</td>\n",
+       "      <td>0.059391</td>\n",
+       "      <td>0.057033</td>\n",
+       "      <td>0.052189</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>0.848211</td>\n",
+       "      <td>0.093570</td>\n",
+       "      <td>0.009090</td>\n",
+       "      <td>0.037775</td>\n",
+       "      <td>0.020982</td>\n",
+       "      <td>0.047940</td>\n",
+       "      <td>0.062393</td>\n",
+       "      <td>17468</td>\n",
+       "      <td>4288</td>\n",
+       "      <td>32</td>\n",
+       "      <td>103.794368</td>\n",
+       "      <td>11.503633</td>\n",
+       "      <td>0.000002</td>\n",
+       "      <td>0.0001</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>12</td>\n",
+       "      <td>12</td>\n",
+       "      <td>13</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0.031486</td>\n",
+       "      <td>0.059400</td>\n",
+       "      <td>0.063696</td>\n",
+       "      <td>0.071379</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>0.847333</td>\n",
+       "      <td>0.121656</td>\n",
+       "      <td>0.020638</td>\n",
+       "      <td>0.043654</td>\n",
+       "      <td>0.038130</td>\n",
+       "      <td>0.077089</td>\n",
+       "      <td>0.076830</td>\n",
+       "      <td>17369</td>\n",
+       "      <td>4384</td>\n",
+       "      <td>32</td>\n",
+       "      <td>99.668121</td>\n",
+       "      <td>10.958552</td>\n",
+       "      <td>0.000002</td>\n",
+       "      <td>0.0001</td>\n",
+       "      <td>33.0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>12</td>\n",
+       "      <td>12</td>\n",
+       "      <td>13</td>\n",
+       "      <td>1</td>\n",
+       "      <td>0.032215</td>\n",
+       "      <td>0.059726</td>\n",
+       "      <td>0.057000</td>\n",
+       "      <td>0.055488</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>0.846940</td>\n",
+       "      <td>0.098863</td>\n",
+       "      <td>0.011571</td>\n",
+       "      <td>0.040962</td>\n",
+       "      <td>0.021129</td>\n",
+       "      <td>0.051022</td>\n",
+       "      <td>0.066003</td>\n",
+       "      <td>17468</td>\n",
+       "      <td>4288</td>\n",
+       "      <td>32</td>\n",
+       "      <td>103.740352</td>\n",
+       "      <td>10.660052</td>\n",
+       "      <td>0.000002</td>\n",
+       "      <td>0.0001</td>\n",
+       "      <td>33.0</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "    dataset  epoch  fold  train_loss  val_loss  val_w_loss  val_loss2  \\\n",
+       "12       12     13     0    0.031208  0.059201    0.064282   0.000000   \n",
+       "25       14     13     0    0.031730  0.059394    0.063065   0.067177   \n",
+       "12       12     13     1    0.031786  0.059864    0.058066   0.000000   \n",
+       "25       14     13     1    0.031240  0.059391    0.057033   0.052189   \n",
+       "12       12     13     0    0.031486  0.059400    0.063696   0.071379   \n",
+       "12       12     13     1    0.032215  0.059726    0.057000   0.055488   \n",
+       "\n",
+       "    val_w_loss2       cor       any  epidural  intraparenchymal  \\\n",
+       "12          0.0  0.847493  0.094982  0.014763          0.044750   \n",
+       "25          NaN  0.847427  0.114279  0.018076          0.043661   \n",
+       "12          0.0  0.846629  0.097579  0.014948          0.040305   \n",
+       "25          NaN  0.848211  0.093570  0.009090          0.037775   \n",
+       "12          NaN  0.847333  0.121656  0.020638          0.043654   \n",
+       "12          NaN  0.846940  0.098863  0.011571          0.040962   \n",
+       "\n",
+       "    intraventricular  subarachnoid  subdural  train_sz  val_sz  bs  \\\n",
+       "12          0.025390      0.064951  0.074592     15619    3936  32   \n",
+       "25          0.037124      0.069421  0.073398     17369    4384  32   \n",
+       "12          0.025653      0.063112  0.079871     15699    3840  32   \n",
+       "25          0.020982      0.047940  0.062393     17468    4288  32   \n",
+       "12          0.038130      0.077089  0.076830     17369    4384  32   \n",
+       "12          0.021129      0.051022  0.066003     17468    4288  32   \n",
+       "\n",
+       "    train_time  valid_time        lr      wd   ver  \n",
+       "12   85.367667   10.784082  0.000002  0.0001   NaN  \n",
+       "25  101.753389   11.775365  0.000002  0.0001   NaN  \n",
+       "12   85.526455   10.400574  0.000002  0.0001   NaN  \n",
+       "25  103.794368   11.503633  0.000002  0.0001   NaN  \n",
+       "12   99.668121   10.958552  0.000002  0.0001  33.0  \n",
+       "12  103.740352   10.660052  0.000002  0.0001  33.0  "
+      ]
+     },
+     "execution_count": 40,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "stats1.loc[stats1.epoch == 13]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 41,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>dataset</th>\n",
+       "      <th>epoch</th>\n",
+       "      <th>fold</th>\n",
+       "      <th>train_loss</th>\n",
+       "      <th>val_loss</th>\n",
+       "      <th>val_w_loss</th>\n",
+       "      <th>val_loss2</th>\n",
+       "      <th>val_w_loss2</th>\n",
+       "      <th>cor</th>\n",
+       "      <th>any</th>\n",
+       "      <th>epidural</th>\n",
+       "      <th>intraparenchymal</th>\n",
+       "      <th>intraventricular</th>\n",
+       "      <th>subarachnoid</th>\n",
+       "      <th>subdural</th>\n",
+       "      <th>train_sz</th>\n",
+       "      <th>val_sz</th>\n",
+       "      <th>bs</th>\n",
+       "      <th>train_time</th>\n",
+       "      <th>valid_time</th>\n",
+       "      <th>lr</th>\n",
+       "      <th>wd</th>\n",
+       "      <th>ver</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <td>5</td>\n",
+       "      <td>14</td>\n",
+       "      <td>3</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0.032872</td>\n",
+       "      <td>0.061043</td>\n",
+       "      <td>0.062914</td>\n",
+       "      <td>0.066554</td>\n",
+       "      <td>0.066554</td>\n",
+       "      <td>0.844028</td>\n",
+       "      <td>0.112525</td>\n",
+       "      <td>0.017403</td>\n",
+       "      <td>0.044180</td>\n",
+       "      <td>0.036614</td>\n",
+       "      <td>0.069472</td>\n",
+       "      <td>0.073159</td>\n",
+       "      <td>17369</td>\n",
+       "      <td>4384</td>\n",
+       "      <td>32</td>\n",
+       "      <td>101.308828</td>\n",
+       "      <td>11.260699</td>\n",
+       "      <td>0.000005</td>\n",
+       "      <td>0.0001</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>8</td>\n",
+       "      <td>12</td>\n",
+       "      <td>3</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0.033388</td>\n",
+       "      <td>0.060360</td>\n",
+       "      <td>0.063039</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.845265</td>\n",
+       "      <td>0.097268</td>\n",
+       "      <td>0.014937</td>\n",
+       "      <td>0.045735</td>\n",
+       "      <td>0.025769</td>\n",
+       "      <td>0.065610</td>\n",
+       "      <td>0.075936</td>\n",
+       "      <td>15619</td>\n",
+       "      <td>3936</td>\n",
+       "      <td>32</td>\n",
+       "      <td>82.747959</td>\n",
+       "      <td>10.047513</td>\n",
+       "      <td>0.000005</td>\n",
+       "      <td>0.0001</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>2</td>\n",
+       "      <td>14</td>\n",
+       "      <td>3</td>\n",
+       "      <td>1</td>\n",
+       "      <td>0.033915</td>\n",
+       "      <td>0.060324</td>\n",
+       "      <td>0.056965</td>\n",
+       "      <td>0.051529</td>\n",
+       "      <td>0.051529</td>\n",
+       "      <td>0.846076</td>\n",
+       "      <td>0.091700</td>\n",
+       "      <td>0.009166</td>\n",
+       "      <td>0.037559</td>\n",
+       "      <td>0.020248</td>\n",
+       "      <td>0.049256</td>\n",
+       "      <td>0.061077</td>\n",
+       "      <td>17468</td>\n",
+       "      <td>4288</td>\n",
+       "      <td>32</td>\n",
+       "      <td>102.540290</td>\n",
+       "      <td>10.781580</td>\n",
+       "      <td>0.000005</td>\n",
+       "      <td>0.0001</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>5</td>\n",
+       "      <td>12</td>\n",
+       "      <td>3</td>\n",
+       "      <td>1</td>\n",
+       "      <td>0.034051</td>\n",
+       "      <td>0.060730</td>\n",
+       "      <td>0.057532</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.844560</td>\n",
+       "      <td>0.099342</td>\n",
+       "      <td>0.015278</td>\n",
+       "      <td>0.041240</td>\n",
+       "      <td>0.025786</td>\n",
+       "      <td>0.063912</td>\n",
+       "      <td>0.080207</td>\n",
+       "      <td>15699</td>\n",
+       "      <td>3840</td>\n",
+       "      <td>32</td>\n",
+       "      <td>83.967543</td>\n",
+       "      <td>9.888514</td>\n",
+       "      <td>0.000005</td>\n",
+       "      <td>0.0001</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>2</td>\n",
+       "      <td>12</td>\n",
+       "      <td>3</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0.032771</td>\n",
+       "      <td>0.060638</td>\n",
+       "      <td>0.063045</td>\n",
+       "      <td>0.070920</td>\n",
+       "      <td>0.070920</td>\n",
+       "      <td>0.844702</td>\n",
+       "      <td>0.120086</td>\n",
+       "      <td>0.020185</td>\n",
+       "      <td>0.044548</td>\n",
+       "      <td>0.037809</td>\n",
+       "      <td>0.075970</td>\n",
+       "      <td>0.077759</td>\n",
+       "      <td>17369</td>\n",
+       "      <td>4384</td>\n",
+       "      <td>32</td>\n",
+       "      <td>95.730286</td>\n",
+       "      <td>10.998797</td>\n",
+       "      <td>0.000005</td>\n",
+       "      <td>0.0001</td>\n",
+       "      <td>34.0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>2</td>\n",
+       "      <td>12</td>\n",
+       "      <td>3</td>\n",
+       "      <td>1</td>\n",
+       "      <td>0.035328</td>\n",
+       "      <td>0.060772</td>\n",
+       "      <td>0.057422</td>\n",
+       "      <td>0.055588</td>\n",
+       "      <td>0.055588</td>\n",
+       "      <td>0.844641</td>\n",
+       "      <td>0.098851</td>\n",
+       "      <td>0.011631</td>\n",
+       "      <td>0.040441</td>\n",
+       "      <td>0.021717</td>\n",
+       "      <td>0.052439</td>\n",
+       "      <td>0.065184</td>\n",
+       "      <td>17468</td>\n",
+       "      <td>4288</td>\n",
+       "      <td>32</td>\n",
+       "      <td>98.306380</td>\n",
+       "      <td>10.804755</td>\n",
+       "      <td>0.000005</td>\n",
+       "      <td>0.0001</td>\n",
+       "      <td>34.0</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "   dataset  epoch  fold  train_loss  val_loss  val_w_loss  val_loss2  \\\n",
+       "5       14      3     0    0.032872  0.061043    0.062914   0.066554   \n",
+       "8       12      3     0    0.033388  0.060360    0.063039   0.000000   \n",
+       "2       14      3     1    0.033915  0.060324    0.056965   0.051529   \n",
+       "5       12      3     1    0.034051  0.060730    0.057532   0.000000   \n",
+       "2       12      3     0    0.032771  0.060638    0.063045   0.070920   \n",
+       "2       12      3     1    0.035328  0.060772    0.057422   0.055588   \n",
+       "\n",
+       "   val_w_loss2       cor       any  epidural  intraparenchymal  \\\n",
+       "5     0.066554  0.844028  0.112525  0.017403          0.044180   \n",
+       "8     0.000000  0.845265  0.097268  0.014937          0.045735   \n",
+       "2     0.051529  0.846076  0.091700  0.009166          0.037559   \n",
+       "5     0.000000  0.844560  0.099342  0.015278          0.041240   \n",
+       "2     0.070920  0.844702  0.120086  0.020185          0.044548   \n",
+       "2     0.055588  0.844641  0.098851  0.011631          0.040441   \n",
+       "\n",
+       "   intraventricular  subarachnoid  subdural  train_sz  val_sz  bs  train_time  \\\n",
+       "5          0.036614      0.069472  0.073159     17369    4384  32  101.308828   \n",
+       "8          0.025769      0.065610  0.075936     15619    3936  32   82.747959   \n",
+       "2          0.020248      0.049256  0.061077     17468    4288  32  102.540290   \n",
+       "5          0.025786      0.063912  0.080207     15699    3840  32   83.967543   \n",
+       "2          0.037809      0.075970  0.077759     17369    4384  32   95.730286   \n",
+       "2          0.021717      0.052439  0.065184     17468    4288  32   98.306380   \n",
+       "\n",
+       "   valid_time        lr      wd   ver  \n",
+       "5   11.260699  0.000005  0.0001   NaN  \n",
+       "8   10.047513  0.000005  0.0001   NaN  \n",
+       "2   10.781580  0.000005  0.0001   NaN  \n",
+       "5    9.888514  0.000005  0.0001   NaN  \n",
+       "2   10.998797  0.000005  0.0001  34.0  \n",
+       "2   10.804755  0.000005  0.0001  34.0  "
+      ]
+     },
+     "execution_count": 41,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "stats2.loc[stats2.epoch == 3]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 64,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "stats = pd.concat([stats1.loc[stats1.epoch == 13], stats2.loc[stats2.epoch == 3]], sort=False)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 68,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "stats['name'] = np.where(stats['dataset'] == 14, 'new feats', \n",
+    "                         np.where(stats['ver'].isnull(), 'old feats, train', 'old feats, test+train'))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 69,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "stats['weighted_training'] = stats.epoch == 3"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 70,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th>val_loss</th>\n",
+       "      <th>val_w_loss</th>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>fold</th>\n",
+       "      <th>weighted_training</th>\n",
+       "      <th>name</th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <td rowspan=\"6\" valign=\"top\">0</td>\n",
+       "      <td rowspan=\"3\" valign=\"top\">False</td>\n",
+       "      <td>new feats</td>\n",
+       "      <td>0.059394</td>\n",
+       "      <td>0.063065</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>old feats, test+train</td>\n",
+       "      <td>0.059400</td>\n",
+       "      <td>0.063696</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>old feats, train</td>\n",
+       "      <td>0.059201</td>\n",
+       "      <td>0.064282</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td rowspan=\"3\" valign=\"top\">True</td>\n",
+       "      <td>new feats</td>\n",
+       "      <td>0.061043</td>\n",
+       "      <td>0.062914</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>old feats, test+train</td>\n",
+       "      <td>0.060638</td>\n",
+       "      <td>0.063045</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>old feats, train</td>\n",
+       "      <td>0.060360</td>\n",
+       "      <td>0.063039</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td rowspan=\"6\" valign=\"top\">1</td>\n",
+       "      <td rowspan=\"3\" valign=\"top\">False</td>\n",
+       "      <td>new feats</td>\n",
+       "      <td>0.059391</td>\n",
+       "      <td>0.057033</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>old feats, test+train</td>\n",
+       "      <td>0.059726</td>\n",
+       "      <td>0.057000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>old feats, train</td>\n",
+       "      <td>0.059864</td>\n",
+       "      <td>0.058066</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td rowspan=\"3\" valign=\"top\">True</td>\n",
+       "      <td>new feats</td>\n",
+       "      <td>0.060324</td>\n",
+       "      <td>0.056965</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>old feats, test+train</td>\n",
+       "      <td>0.060772</td>\n",
+       "      <td>0.057422</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>old feats, train</td>\n",
+       "      <td>0.060730</td>\n",
+       "      <td>0.057532</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                                              val_loss  val_w_loss\n",
+       "fold weighted_training name                                       \n",
+       "0    False             new feats              0.059394    0.063065\n",
+       "                       old feats, test+train  0.059400    0.063696\n",
+       "                       old feats, train       0.059201    0.064282\n",
+       "     True              new feats              0.061043    0.062914\n",
+       "                       old feats, test+train  0.060638    0.063045\n",
+       "                       old feats, train       0.060360    0.063039\n",
+       "1    False             new feats              0.059391    0.057033\n",
+       "                       old feats, test+train  0.059726    0.057000\n",
+       "                       old feats, train       0.059864    0.058066\n",
+       "     True              new feats              0.060324    0.056965\n",
+       "                       old feats, test+train  0.060772    0.057422\n",
+       "                       old feats, train       0.060730    0.057532"
+      ]
+     },
+     "execution_count": 70,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "stats.groupby(['fold','weighted_training','name'])[['val_loss','val_w_loss']].mean()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Analysis"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>val_loss</th>\n",
+       "      <th>val_w_loss</th>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>dataset</th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <td>6</td>\n",
+       "      <td>0.063542</td>\n",
+       "      <td>0.069056</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>7</td>\n",
+       "      <td>0.062726</td>\n",
+       "      <td>0.068622</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>8</td>\n",
+       "      <td>0.064020</td>\n",
+       "      <td>0.069777</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>9</td>\n",
+       "      <td>0.062367</td>\n",
+       "      <td>0.067991</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>10</td>\n",
+       "      <td>0.062012</td>\n",
+       "      <td>0.067773</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>11</td>\n",
+       "      <td>0.061149</td>\n",
+       "      <td>0.066677</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>12</td>\n",
+       "      <td>0.060799</td>\n",
+       "      <td>0.066053</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>13</td>\n",
+       "      <td>0.061087</td>\n",
+       "      <td>0.066270</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "         val_loss  val_w_loss\n",
+       "dataset                      \n",
+       "6        0.063542    0.069056\n",
+       "7        0.062726    0.068622\n",
+       "8        0.064020    0.069777\n",
+       "9        0.062367    0.067991\n",
+       "10       0.062012    0.067773\n",
+       "11       0.061149    0.066677\n",
+       "12       0.060799    0.066053\n",
+       "13       0.061087    0.066270"
+      ]
+     },
+     "execution_count": 13,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "stats = pd.concat([pd.read_csv(PATH_WORK/'stats.f{}.v{}'.format(f,31)) for f in range(5)],axis=0)\n",
+    "stats.loc[stats.epoch==13].groupby('dataset')[['val_loss','val_w_loss']].mean()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>val_loss</th>\n",
+       "      <th>val_w_loss</th>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>dataset</th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <td>6</td>\n",
+       "      <td>0.064378</td>\n",
+       "      <td>0.068053</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>7</td>\n",
+       "      <td>0.063824</td>\n",
+       "      <td>0.067963</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>8</td>\n",
+       "      <td>0.065129</td>\n",
+       "      <td>0.068768</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>9</td>\n",
+       "      <td>0.062979</td>\n",
+       "      <td>0.067057</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>10</td>\n",
+       "      <td>0.062737</td>\n",
+       "      <td>0.067054</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>11</td>\n",
+       "      <td>0.061973</td>\n",
+       "      <td>0.065464</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>12</td>\n",
+       "      <td>0.061613</td>\n",
+       "      <td>0.064652</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>13</td>\n",
+       "      <td>0.061888</td>\n",
+       "      <td>0.064962</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "         val_loss  val_w_loss\n",
+       "dataset                      \n",
+       "6        0.064378    0.068053\n",
+       "7        0.063824    0.067963\n",
+       "8        0.065129    0.068768\n",
+       "9        0.062979    0.067057\n",
+       "10       0.062737    0.067054\n",
+       "11       0.061973    0.065464\n",
+       "12       0.061613    0.064652\n",
+       "13       0.061888    0.064962"
+      ]
+     },
+     "execution_count": 11,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "stats = pd.concat([pd.read_csv(PATH_WORK/'stats.f{}.v{}'.format(f,32)) for f in range(5)],axis=0)\n",
+    "stats.loc[stats.epoch==3].groupby('dataset')[['val_loss','val_w_loss']].mean()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "[<matplotlib.lines.Line2D at 0x7efd58fd4b90>]"
+      ]
+     },
+     "execution_count": 14,
+     "metadata": {},
+     "output_type": "execute_result"
+    },
+    {
+     "data": {
+      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAD4CAYAAADlwTGnAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3de5xdZX3v8c939t5zyWUmZBgIuUCYEMQAEcOAGaXlkgOCWlIUMZRSbKnIUaytbRGPtq8eL1VsS7CntJaKiqJyq7axRKJys9UQGSIJhBAIEcgkQCa3SUJmMrff+WOvxJ3JTGZnZpI9e/b3/XrNa+/1rGdlfguS9d3PevZaSxGBmZmVnrJCF2BmZoXhADAzK1EOADOzEuUAMDMrUQ4AM7MSlS50AYfi6KOPjunTpxe6DDOzovLkk09ujoi63u1FFQDTp0+nqamp0GWYmRUVSS/31e5TQGZmJcoBYGZWovIKAEkXS1ojaa2km/pYXyHpnmT9MknTc9bNlrRU0ipJT0uqTNqvTJZXSnpQ0tHDtVNmZjawAQNAUgq4DbgEmAVcKWlWr27XAtsi4iRgIXBzsm0auAu4PiJOBc4DOpP2rwDnR8RsYCVww7DskZmZ5SWfEcDZwNqIWBcRHcDdwPxefeYDdybv7wfmSRJwEbAyIlYARMSWiOgGlPyMTfpVAxuHvDdmZpa3fAJgCrA+Z7k5aeuzT0R0Aa1ALXAyEJKWSFou6cakTyfwv4GnyR74ZwF39PXLJV0nqUlSU0tLS947ZmZmB5dPAKiPtt63EO2vTxo4B7gqeb1M0jxJGbIB8FZgMtlTQJ/q65dHxO0R0RARDXV1B3yN1czMBimfAGgGpuUsT+XA0zX7+iTn92uArUn7YxGxOSJ2A4uBOcAZABHxYmTvR30v8PYh7MdBfXvpS/xwhc8wmZnlyicAngBmSjpRUjmwAFjUq88i4Jrk/eXAw8mBfQkwW9KYJBjOBZ4FNgCzJO39SH8hsHpou9K/+55s5ttL+7wOwsysZA14JXBEdEm6gezBPAV8PSJWSfos0BQRi8iev/+2pLVkP/kvSLbdJukWsiESwOKIeABA0v8FfiapE3gZ+OCw712isb6Wb/z8Jdo6uqkqTx2uX2NmVlRUTE8Ea2hoiMHcCuKRNZv4w288wXf++G284yRfbmBmpUXSkxHR0Lu9JK4EPmv6RFJlYumLWwpdipnZiFESATCuIs3pU2pYus4BYGa2V0kEAEDjjFpWrN/OG3u6Cl2KmdmIUDoBUF9LV0/Q9PK2QpdiZjYilEwANEw/inSZeNyngczMgBIKgDHlad4ybYIngs3MEiUTAJA9DfT0hlZ2eR7AzKzEAmBGLd09wRO/3lroUszMCq6kAuDME46iPFXmr4OamVFiAVCZSXHG8RM8EWxmRokFAGTnAZ7Z0MqO9s5Cl2JmVlAlFwBz62vpCfjlOs8DmFlpK7kAeOvxEyhPex7AzKzkAqAyk+LM44/y9QBmVvJKLgAg+3XQ1a/tYPvujkKXYmZWMCUbABGwzNcDmFkJK8kAmD21hspMmU8DmVlJK8kAqEinaDhhoq8HMLOSllcASLpY0hpJayXd1Mf6Ckn3JOuXSZqes262pKWSVkl6WlKlpPGSnsr52Szp1uHbrYE1zqjludd2smXXniP5a83MRowBA0BSCrgNuASYBVwpaVavbtcC2yLiJGAhcHOybRq4C7g+Ik4FzgM6I2JnRJyx94fsQ+G/P0z7lJe59bWA5wHMrHTlMwI4G1gbEesiogO4G5jfq8984M7k/f3APEkCLgJWRsQKgIjYEhHduRtKmgkcA/z34Hfj0M2eWsOY8pTnAcysZOUTAFOA9TnLzUlbn30iogtoBWqBk4GQtETSckk39vHnXwncExHR1y+XdJ2kJklNLS0teZSbn0yqjIbpngcws9KVTwCoj7beB+v++qSBc4CrktfLJM3r1W8B8L3+fnlE3B4RDRHRUFdXl0e5+Wusr+WFTbto2el5ADMrPfkEQDMwLWd5KrCxvz7Jef8aYGvS/lhEbI6I3cBiYM7ejSS9BUhHxJOD3oMhaJyRnQfwKMDMSlE+AfAEMFPSiZLKyX5iX9SrzyLgmuT95cDDySmdJcBsSWOSYDgXeDZnuys5yKf/w+20ydWMq0j7vkBmVpLSA3WIiC5JN5A9mKeAr0fEKkmfBZoiYhFwB/BtSWvJfvJfkGy7TdItZEMkgMUR8UDOH38F8K5h3aNDkE6VcfaJE3ncE8FmVoIGDACAiFhM9vRNbttf57xvB97fz7Z3kf0qaF/r6vOu9DBprK/l4ec28fqOdo6trix0OWZmR0xJXgmca+/1AJ4HMLNSU/IBMGtyNdWVaV8PYGYlp+QDIFUmzj6x1hPBZlZySj4AIPt10Je37Gbj9rZCl2JmdsQ4AMhOBAM+DWRmJcUBAJwyaTwTxmQ8EWxmJcUBAJSVibedONHzAGZWUhwAicb6Wpq3tbF+6+5Cl2JmdkQ4ABKNM44G8CjAzEqGAyBx8rHjmDi23LeFMLOS4QBISGJufXYeoJ9HE5iZjSoOgByN9bW82trOK54HMLMS4ADIsff5AL4ewMxKgQMgx4y6cdSNr/BEsJmVBAdAjuw8QC1LX/Q8gJmNfg6AXhrra9m0cw/rNr9R6FLMzA4rB0Avc+snAn4+gJmNfnkFgKSLJa2RtFbSTX2sr5B0T7J+maTpOetmS1oqaZWkpyVVJu3lkm6X9Lyk5yS9b7h2aihOPHosx1ZXeCLYzEa9AR8JKSkF3AZcCDQDT0haFBG5D3e/FtgWESdJWgDcDHwgeRD8XcDVEbFCUi3QmWzzaWBTRJwsqQyYOHy7NXiSaKyv5X/WZucBJBW6JDOzwyKfEcDZwNqIWBcRHcDdwPxefeYDdybv7wfmKXvkvAhYGRErACJiS0R0J/3+CPhi0t4TEZuHtivDp3FGLZt37WHtpl2FLsXM7LDJJwCmAOtzlpuTtj77REQX0ArUAicDIWmJpOWSbgSQNCHZ7nNJ+32Sjh3CfgyrxnrfF8jMRr98AqCvcyC9vyPZX580cA5wVfJ6maR5SftU4OcRMQdYCvx9n79cuk5Sk6SmlpaWPModumkTq5hcU+l5ADMb1fIJgGZgWs7yVGBjf32S8/41wNak/bGI2BwRu4HFwBxgC7Ab+EGy/X1J+wEi4vaIaIiIhrq6urx2aqgkMXdGLct+vZWeHl8PYGajUz4B8AQwU9KJksqBBcCiXn0WAdck7y8HHo7slVRLgNmSxiTBcC7wbLLuh8B5yTbzgGcZQRrra9n6RgfPb9pZ6FLMzA6LAb8FFBFdkm4gezBPAV+PiFWSPgs0RcQi4A7g25LWkv3kvyDZdpukW8iGSACLI+KB5I/+ZLLNrUAL8IfDvG9DkntfoFMmVRe4GjOz4adiuuVBQ0NDNDU1HbHf91tffpg3T6rm9j9oOGK/08xsuEl6MiIOOJD5SuCDmHui5wHMbPRyABxE44xaWts6Wf3ajkKXYmY27BwAB+HnA5jZaOYAOIjjaqqYXjvGN4Yzs1HJATCAxuR6gG7PA5jZKOMAGMDc+lp2tnexamNroUsxMxtWDoABNNZ7HsDMRicHwACOqa6kvm6s5wHMbNRxAOShsb6WJ17aRld3T6FLMTMbNg6APDTOqGXXni6e3uB5ADMbPRwAeZi7dx7Ap4HMbBRxAOTh6HEVzDxmnCeCzWxUcQDkqXFGLU0vbaOjy/MAZjY6OADy1FhfS1tnN09v2F7oUszMhoUDIE9v8/UAZjbKOADyNHFsOadMGu+JYDMbNRwAh2BufXYeYE9Xd6FLMTMbMgfAIWicUcuerh6eesXzAGZW/PIKAEkXS1ojaa2km/pYXyHpnmT9MknTc9bNlrRU0ipJT0uqTNofTf7Mp5KfY4Zrpw6XuSfWIsHj67YWuhQzsyEbMAAkpYDbgEuAWcCVkmb16nYtsC0iTgIWAjcn26aBu4DrI+JU4DygM2e7qyLijORn01B35nCrGZNh1nHVLF23udClmJkNWT4jgLOBtRGxLiI6gLuB+b36zAfuTN7fD8yTJOAiYGVErACIiC0RUdQn0Bvra1n+ynbaO4t6N8zM8gqAKcD6nOXmpK3PPhHRBbQCtcDJQEhaImm5pBt7bfeN5PTPXyWBcQBJ10lqktTU0tKSR7mH19z6Wjq6elj+yrZCl2JmNiT5BEBfB+bej8fqr08aOAe4Knm9TNK8ZP1VEXE68FvJz9V9/fKIuD0iGiKioa6uLo9yD6+z6ydSJnjc1wOYWZHLJwCagWk5y1OBjf31Sc771wBbk/bHImJzROwGFgNzACJiQ/K6E/gu2VNNI151ZYbTptT4egAzK3r5BMATwExJJ0oqBxYAi3r1WQRck7y/HHg4IgJYAsyWNCYJhnOBZyWlJR0NICkDvAd4Zui7c2Q01tfy1PrttHV4HsDMiteAAZCc07+B7MF8NXBvRKyS9FlJlybd7gBqJa0FPgHclGy7DbiFbIg8BSyPiAeACmCJpJVJ+wbg34Z1zw6juTNq6ewOnnzZ8wBmVrzS+XSKiMVkT9/ktv11zvt24P39bHsX2a+C5ra9AZx5qMWOFGdNn0iqTCxdt5lzZh5d6HLMzAbFVwIPwriKNKdPqfGN4cysqDkABqlxRi0rm1t5Y09XoUsxMxsUB8AgNdbX0tUT/PcLvirYzIqTA2CQzpo+kalHVfGX961gmb8SamZFyAEwSFXlKe67vpFjqiv4g6//kkeeG/G3MjIz248DYAiOq6ni3g83MvPYcXzoW03851MbCl2SmVneHABDVDuugu9+aC5zTjiKP73nKe56/OVCl2RmlhcHwDCorszwrT86m/PfdAyf+Y9n+OdH1xa6JDOzATkAhkllJsW/Xn0m88+YzJcfXMMXf7Sa7N0wzMxGpryuBLb8ZFJlLLziDMZXpvnXx9axo62Lz//uaaTK+rzTtZlZQTkAhllZmfjc/NOorszwz4++yI72ThZecQblaQ+2zGxkcQAcBpK48eJTqKnK8MUfPceu9i6++vtnUlWeKnRpZmb7+GPpYfThc2fwpfeezs9eaOHqO5bR2tY58EZmZkeIA+AwW3D28fzTlXNY0bydK29/nM279hS6JDMzwAFwRLx79nH82x80sG7zLq746lI2bG8rdElmZg6AI+W8Nx3DXde+jZZde3j/v/yCF1t2FbokMytxDoAjqGH6RO6+bi4d3T28/6tLeWZDa6FLMrMS5gA4wk6dXMO9H26kKpPiytsf55e/3lroksysROUVAJIulrRG0lpJN/WxvkLSPcn6ZZKm56ybLWmppFWSnpZU2WvbRZKK5oHww6G+bhz3Xd9IXXUFV9+xzHcSNbOCGDAAJKWA24BLgFnAlZJm9ep2LbAtIk4CFgI3J9umyT4P+PqIOBU4D9j3XUhJ7wVK8mT45AlV3JdzJ9FFKzYWuiQzKzH5jADOBtZGxLqI6ADuBub36jMfuDN5fz8wT5KAi4CVEbECICK2REQ3gKRxwCeAzw99N4pT7p1EP373r/jOMt9J1MyOnHwCYAqwPme5OWnrs09EdAGtQC1wMhCSlkhaLunGnG0+B/wDsPtgv1zSdZKaJDW1tLTkUW5xyb2T6Kd/4DuJmtmRk8+tIPq6k1nv21z21ycNnAOcRfZA/5CkJ4EtwEkR8We58wV9iYjbgdsBGhoaRuXtNffeSfTP713Blx9cQ8vOPbz79OOYPKGKY8ZXkE55rt7Mhl8+AdAMTMtZngr0PmG9t09zct6/BtiatD8WEZsBJC0G5pA973+mpJeSGo6R9GhEnDf4XSlumVQZCz9wBtVVab7x85f4xs9fAiBVJiZVVzJ5QiWTJ1Tt+5mSs1xdmSls8WZWlDTQPeuTA/rzwDxgA/AE8HsRsSqnz0eB0yPiekkLgPdGxBWSjgIeIjsK6AAeBBZGxAM5204H/isiThuo2IaGhmhqajq0PSxCL7bsYv3W3Wzc3s7G7W1s3N7Ghu1tbGxt49Xt7XT17P//bHxFmuNyAmHKhKpsYNRklyfVVJLxKMKsZEl6MiIaercPOAKIiC5JNwBLgBTw9YhYJemzQFNELALuAL4taS3ZT/4Lkm23SbqFbGgEsDj34G99m1E3jhl14/pc190TbN61JxsI+37a2bC9jVdb21jZ3MrWNzr220aCY8dX8s5Tj+VvLj2V7Py8mZW6AUcAI0mpjACGqq2jm42tbTmjh3ZWNm/n0TUtfPdDb+PtM44udIlmdgQNegRgxaeqPHXAKKK9s5tz/+4Rbv3JCzTW13oUYGa+FUSpqMyk+Mh5J/HLl7ay9MUthS7HzEYAB0AJ+cBZ05hUXcmtP33BD6w3MwdAKanMpPjI+TP45Utb+YVHAWYlzwFQYq5o2DsKeN6jALMS5wAoMXtHAU+8tM2jALMS5wAoQXvnAhb+xKMAs1LmAChBFekUHz1/Bk0vb+Pnaz0KMCtVDoASdcVZ0ziuppKFngswK1kOgBJVkU7xkfNP4smXt/E/azcXuhwzKwAHQAm7omEqx9X4ugCzUuUAKGG5o4D/fsGjALNS4wAocVc0TGVyja8LMCtFDoASt3cUsPyV7R4FmJUYB4Dx/mQU4G8EmZUWB4DtGwX86pXt/MyjALOS4QAwIHuPIM8FmJUWB4ABUJ4u46MXZEcBjz3fUuhyzOwIyCsAJF0saY2ktZJu6mN9haR7kvXLkge97103W9JSSaskPS2pMml/UNKKpP2rklLDtVM2OO8/cxpTJlT5ugCzEjFgACQH5tuAS4BZwJWSZvXqdi2wLSJOAhYCNyfbpoG7gOsj4lTgPKAz2eaKiHgLcBpQB7x/yHtjQ1KeLuOj55/EU+s9CjArBfmMAM4G1kbEuojoAO4G5vfqMx+4M3l/PzBP2YfOXgSsjIgVABGxJSK6k/c7kv5poBzwR84R4PIzpzJlQhULPQowG/XyCYApwPqc5eakrc8+EdEFtAK1wMlASFoiabmkG3M3krQE2ATsJBscB5B0naQmSU0tLf5UerjtHQWsWL+dRz0KMBvV8gkA9dHW+6Nhf33SwDnAVcnrZZLm7esQ8U7gOKACuKCvXx4Rt0dEQ0Q01NXV5VGuDdXeUYDnAsxGt3wCoBmYlrM8FdjYX5/kvH8NsDVpfywiNkfEbmAxMCd3w4hoBxZx4GklK5DydBk3XJCMAtZ4FGA2WuUTAE8AMyWdKKkcWED2gJ1rEXBN8v5y4OHIfnRcAsyWNCYJhnOBZyWNk3Qc7AuMdwHPDX13bLi8b87eUYCvCzAbrQYMgOSc/g1kD+argXsjYpWkz0q6NOl2B1AraS3wCeCmZNttwC1kQ+QpYHlEPACMBRZJWgmsIDsP8NVh3TMbkvJ0GR+74CRWNLd6FGA2SqmYPt01NDREU1NTocsoGR1dPVzwD48ycWw5//nRd5D9YpeZFRtJT0ZEQ+92Xwls/SpPl3HD+SexsrmVR9ZsKnQ5ZjbMHAB2UO87cypTj/I3gsxGIweAHVQmlZ0LWNncysPPeRRgNpo4AGxA750zlWkTPQowG20cADagTKqMj50/k6c3eBRgNpo4ACwvl82Z4lGA2SjjALC85I4CHlrtUYDZaOAAsLxdNmcKx08cw60P+epgs9HAAWB5y6Sy9wh6ZsMOfupRgFnRcwDYIbnsrckowPcIMit6DgA7JHuvC1i10aMAs2LnALBDdtlbp3BCrUcBZsXOAWCHLJ3K3iNo1cYd/OTZ1wtdjpkNkgPABuU3owBfF2BWrBwANijpVBkfu2Amz766g4U/fYHHnm9hzWs7ad3d6UAwKxLpQhdgxet3z5jMnb94iX986IX92iszZUyqruSY6komVVcyqaaSY5P3x1ZXcGx1drk87c8fZoXkALBBS6fK+P5H3s6r29t5fWc7r7W28/qO5HXnHl5vbeep9dt5bVU7HV09B2xfO7Y8CYmKXiGR/Zk4tpwJYzJUZlIF2Duz0c8BYEOSSZVxfO0Yjq8d02+fiKC1rZPXdvwmJF7fsYfXdrTzems7r+1o5+kNO9jyxh76OntUmSnjqDHlTBhTzoSqDEeNzVBTVc5RYzJMGJPJac++ThiTDY5MyiMMs4PJKwAkXQx8BUgBX4uIL/VaXwF8CzgT2AJ8ICJeStbNBv4VqAZ6gLPIzj3cB8wAuoEfRsRNw7A/NgJJSg7K5Zwyqbrffp3dPWzauYfXWtvZtKOdbbs72ba7g+27O9i+u5Ntuztpbevg+dd3sX13J9t3d9DV0/98w7iKdBIQGY4aU05NVSYJksx+77PL2UCpqcqQdnBYiRgwACSlgNuAC4Fm4AlJiyLi2Zxu1wLbIuIkSQuAm4EPSEoDdwFXR8QKSbVAJ1AB/H1EPCKpHHhI0iUR8aPh3T0rJplUGVMmVDFlQlVe/SOCXXu6kjDoZHtbRzYkdnfsC4/WvSHS1knztja27e5gR1snB8kNxlekqekVHBPGZJhQVb7fiGNCzgikujLjOQ0rOvmMAM4G1kbEOgBJdwPzgdwAmA/8TfL+fuCflH2C+EXAyohYARARW5I+u4FHkrYOScuBqUPbFSs1khhfmWF8ZYZpE/Pfrqcn2Nnexfa2jiQ4OveNMvYGSWvSvm13Bxu2te3rc7DgqEiXMb4yzbiKNOP2vlZkGF+Z3q99/L71GcZVpA9YX5EeWXMeEUF3T9DZHXR099DR1UNn929+OrqCzu4eeiIYX5mhujJNdVWGinQZ2cPAyNDdE+xq72JHeye79nRRXzd2xP23PtLyCYApwPqc5Wbgbf31iYguSa1ALXAyEJKWAHXA3RHx5dwNJU0AfofsKaYDSLoOuA7g+OOPz6Ncs4MrKxM1YzLUjMlwQm3+2/X0BDv3dCXhsH94tO7uZFdHF7vau9i1p4ud7dn3G7a3sWtPJ7vas20HO2W1V3mqbF+AZFKiTNkfKRt6AsrKyLaRtImkH4jfLOe+at966OoJOrp66Nh7IE8O4h37Duo9+w74nd09fc7N5LMf1VXp/UKhujIbiNn32dfxlWmqKzMHrB9bntoXIBFBe2cPO9o72dneSWtbFzvbO9nRnrwmyzuTA/zO9i52tO2/vGtP1371vXfOFG654oxD37FRJJ8A6CvCe/916K9PGjiH7Hn/3WRP9TwZEQ8BJKeIvgf8494RxgF/SMTtwO0ADQ0N/oK5FUxZmaipys4THE//k979iQj2dPXsOxjtau9iZxIO+0Jj32v2oNXVHQRBTw/0RBDJnxORXe4J+mjLvu/uiX3b7G2LZJtMSmRSZYyrSFOeKiOTKiOTLiOT0r7l8nTymvTNpH/Tvq9tX18hxM492QPvjpyD8o7237S92tq+731754HfDNvvv7eguiqDIK/wTJWJ6sokcKrSjK/IMP3oMckoMb1fuCxbt5V/X97MB98+ndlTJxzy/8vRIp8AaAam5SxPBTb206c5OajXAFuT9sciYjOApMXAHOChZLvbgRci4tZB74FZkZBEZSZFZSZF3fiKQpdTcB1dPQcExM72/QNkR3snPRHJwTuzb0Sxb9SQM4qoyqTyPuV0yWmTeHTNJj7/X6u558NzR9SpqiMpnwB4Apgp6URgA7AA+L1efRYB1wBLgcuBhyNi76mfGyWNATqAc4GFAJI+TzYo/ng4dsTMikt5uozacRXUjjvyYTi+MsOfXXgyn/mPZ1iy6nUuPm3SEa9hJBjwawsR0QXcACwBVgP3RsQqSZ+VdGnS7Q6gVtJa4BPATcm224BbyIbIU8DyiHhA0lTg08AsYLmkpyQ5CMzsiFlw1jRmHjOOL/5odZ8XKpYCFdN9WxoaGqKpqanQZZjZKPHomk188BtP8Jl3v5k//q36Qpdz2CRzrw292/3FZTMrWee96Rh+a+bR/L+H17J9d0ehyzniHABmVtI+/e43s7O9k6/0uqlhKXAAmFlJO2VSNR8463i+vfRl1rXsKnQ5R5QDwMxK3icuPJmKdBlf+tFzhS7liHIAmFnJqxtfwUfOP4kfP/s6S1/cMvAGo4QDwMwMuPacE5lcU8kXFj9LTx637BgNHABmZkBlJsUnLzmFZzbs4Ae/2lDoco4IB4CZWeJ3Zk/mLVNr+Lsla9jd0TXwBkXOAWBmligrE595zyxe29HOv/3s14Uu57BzAJiZ5Thr+kTedfokvvrYi7y+o73Q5RxWDgAzs14+efEpdPX08A8/XlPoUg4rB4CZWS8n1I7lg2+fzn1PNrNqY2uhyzlsHABmZn244YKZTKjK8IUHVlNMN808FA4AM7M+1FRl+Pi8mfzixS08/NymQpdzWDgAzMz6cdXcE6g/eixfWLyazu7R98wAB4CZWT8yqTL+z7vezLqWN/juslcKXc6wcwCYmR3EvDcfQ2N9Lbf+9Hla2zoLXc6wyisAJF0saY2ktZJu6mN9haR7kvXLJE3PWTdb0lJJqyQ9Lakyaf+CpPWSSuv+q2ZWVCTx6Xe/me1tndz2yNpClzOsBgwASSngNuASss/wvVLSrF7drgW2RcRJZB/6fnOybRq4C7g+Ik4FzgP2RugPgbOHYR/MzA6r06bUcPmcqXzz5y/xypbdhS5n2OQzAjgbWBsR6yKiA7gbmN+rz3zgzuT9/cA8SQIuAlZGxAqAiNgSEd3J+8cj4tXh2Akzs8PtL975JlJl4uYHR88zA/IJgCnA+pzl5qStzz4R0QW0ArXAyUBIWiJpuaQbh16ymdmRd2x1JR8+t54Hnn6Vppe2FrqcYZFPAKiPtt5XRfTXJw2cA1yVvF4mad6hFCjpOklNkppaWloOZVMzs2F13W/Xc2x1BZ97YPWoeGZAPgHQDEzLWZ4KbOyvT3LevwbYmrQ/FhGbI2I3sBiYcygFRsTtEdEQEQ11dXWHsqmZ2bAaU57mLy56EyvWb+eHK3sfBotPPgHwBDBT0omSyoEFwKJefRYB1yTvLwcejuy100uA2ZLGJMFwLvDs8JRuZnbkvW/OVE6dXM2XH1xDe2d3ocsZkgEDIDmnfwPZg/lq4N6IWCXps5IuTbrdAdRKWgt8Argp2XYbcAvZEHkKWB4RDwBI+rKkZmCMpGZJfzO8u2ZmNvzKyrJfC92wvY07/qe4nxmgYrrJUUNDQzQ1NRW6DDMzPvStJk+dSY4AAAVCSURBVH6xdjOP/uX51I2vKHQ5ByXpyYho6N3uK4HNzAbhU5ecwp6uHhb+9PlClzJoDgAzs0GorxvH7889gbt/+QprXttZ6HIGxQFgZjZIH583k3EVab6weHWhSxkUB4CZ2SAdNbacP5k3k58938Kja4rvmQEOADOzIbi68QROqB3D3y5eTVeRPTPAAWBmNgQV6RSfuuQUnn99F/c0rR94gxEkXegCzMyK3TtPncTZ0yey8CfP09bRTUW6jPJ0GRXpFOXpMspTZVRk9r6mei2XUZFK7VsuK+vrzjqHhwPAzGyIJPFX75nF733tcT7/wNAmhDMp9RkUP/zYOVRmUsNUcZYDwMxsGJw+tYblf3UhbZ3d7OnsoaO7hz2d3XR099DR1cOerr2v3fuW9/TT3tdy+jCMDBwAZmbDJJMqI5Mqg8pCV5IfTwKbmZUoB4CZWYlyAJiZlSgHgJlZiXIAmJmVKAeAmVmJcgCYmZUoB4CZWYkqqkdCSmoBXi50Hb0cDWwudBF5KqZaobjqLaZaobjqLaZaYWTWe0JE1PVuLKoAGIkkNfX1rM2RqJhqheKqt5hqheKqt5hqheKq16eAzMxKlAPAzKxEOQCG7vZCF3AIiqlWKK56i6lWKK56i6lWKKJ6PQdgZlaiPAIwMytRDgAzsxLlABgESdMkPSJptaRVkj5e6JoGIikl6VeS/qvQtQxE0gRJ90t6Lvlv3Fjomg5G0p8lfw+ekfQ9SSPmcSCSvi5pk6RnctomSvqJpBeS16MKWWOufur9u+TvwkpJP5A0oZA17tVXrTnr/kJSSDq6ELXlywEwOF3An0fEm4G5wEclzSpwTQP5ODC0h5UeOV8BHoyIU4C3MILrljQF+BOgISJOA1LAgsJWtZ9vAhf3arsJeCgiZgIPJcsjxTc5sN6fAKdFxGzgeeBTR7qofnyTA2tF0jTgQuCVI13QoXIADEJEvBoRy5P3O8keoKYUtqr+SZoKvBv4WqFrGYikauC3gTsAIqIjIrYXtqoBpYEqSWlgDLCxwPXsExE/A7b2ap4P3Jm8vxP43SNa1EH0VW9E/DgiupLFx4GpR7ywPvTz3xZgIXAjMOK/YeMAGCJJ04G3AssKW8lB3Ur2L2RPoQvJQz3QAnwjOWX1NUljC11UfyJiA/D3ZD/tvQq0RsSPC1vVgI6NiFch+2EGOKbA9RyKPwJ+VOgi+iPpUmBDRKwodC35cAAMgaRxwL8DfxoROwpdT18kvQfYFBFPFrqWPKWBOcC/RMRbgTcYWaco9pOcP58PnAhMBsZK+v3CVjU6Sfo02dOv3yl0LX2RNAb4NPDXha4lXw6AQZKUIXvw/05EfL/Q9RzEO4BLJb0E3A1cIOmuwpZ0UM1Ac0TsHVHdTzYQRqr/Bfw6IloiohP4PvD2Atc0kNclHQeQvG4qcD0DknQN8B7gqhi5Fy/NIPtBYEXy720qsFzSpIJWdRAOgEGQJLLnqFdHxC2FrudgIuJTETE1IqaTnZx8OCJG7CfUiHgNWC/pTUnTPODZApY0kFeAuZLGJH8v5jGCJ60Ti4BrkvfXAP9ZwFoGJOli4JPApRGxu9D19Ccino6IYyJievLvrRmYk/ydHpEcAIPzDuBqsp+mn0p+3lXookaRjwHfkbQSOAP42wLX069kpHI/sBx4muy/qRFzKwBJ3wOWAm+S1CzpWuBLwIWSXiD7bZUvFbLGXP3U+0/AeOAnyb+1rxa0yEQ/tRYV3wrCzKxEeQRgZlaiHABmZiXKAWBmVqIcAGZmJcoBYGZWohwAZmYlygFgZlai/j/ydXzjs9/peQAAAABJRU5ErkJggg==\n",
+      "text/plain": [
+       "<Figure size 432x288 with 1 Axes>"
+      ]
+     },
+     "metadata": {
+      "needs_background": "light"
+     },
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "plt.plot(stats.groupby('epoch').mean().val_loss)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 34,
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "(4368, 60, 6)"
+      ]
+     },
+     "execution_count": 34,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "predictions.shape"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 35,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "array([0.26533565, 0.14922237, 0.21536556, 0.21641548, 0.21140262,\n",
+       "       0.19743267], dtype=float32)"
+      ]
+     },
+     "execution_count": 35,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "predictions.mean((0,1))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 36,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "[<matplotlib.lines.Line2D at 0x7f7449493790>]"
+      ]
+     },
+     "execution_count": 36,
+     "metadata": {},
+     "output_type": "execute_result"
+    },
+    {
+     "data": {
+      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD4CAYAAADiry33AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3dd3hUZd7/8fc3nYSSQmiBFCCU0BISehMUBUWQ1VURhUWUxQV1F7fo6vrs6j5bdNey+2NVRMSyiKCCiAgqSlNaKAEChBJaCJAECCW93L8/MvpEDGQg5cyc+b6uKxeZM2dmPjeED4dT7iPGGJRSStmXl9UBlFJK1S0teqWUsjkteqWUsjkteqWUsjkteqWUsjkfqwNcqmnTpiY6OtrqGEop5Va2bNmSY4wJr+o5lyv66OhokpOTrY6hlFJuRUSOXO453XWjlFI2p0WvlFI2p0WvlFI2p0WvlFI2p0WvlFI2p0WvlFI2p0WvlFI251TRi8gIEUkTkQMi8vgV1rtDRIyIJDkeR4tIgYhsd3y9WlvBL5WbX8yLX+xj78nzdfURSinllqq9YEpEvIGZwHAgA9gsIkuMMbsvWa8R8Aiw8ZK3OGiMia+lvJfPifDKqoNcKCzl6Vvj6vrjlFLKbTizRd8bOGCMSTfGFAPzgTFVrPcs8BxQWIv5nNYk0JdhnZqxJCWT0rJyKyIopZRLcqboI4BjlR5nOJZ9T0QSgDbGmKVVvD5GRLaJyGoRGVTVB4jIFBFJFpHk7OxsZ7P/yNieEeRcLGLtgZxrfg+llLIbZ4peqlj2/f0HRcQLeBF4rIr1TgCRxpgEYAYwT0Qa/+jNjJlljEkyxiSFh1c5J49ThnZsRnCgL4u2Hr/m91BKKbtxpugzgDaVHrcGMis9bgR0BVaJyGGgL7BERJKMMUXGmNMAxpgtwEGgQ20Er4qfjxejurfk890nuVhUWlcfo5RSbsWZot8MxIpIjIj4AXcDS7570hhzzhjT1BgTbYyJBjYAo40xySIS7jiYi4i0BWKB9FofRSVjE1pTWFLOZztP1OXHKKWU26i26I0xpcB0YAWwB1hgjEkVkWdEZHQ1Lx8M7BCRFOADYKox5kxNQ19Jz8hgosMCWbRNd98opRQ4OR+9MWYZsOySZU9fZt3rKn3/IfBhDfJdNRHhtoQIXl65nxPnCmjZpEF9frxSSrkcW14ZOzYhAmNg8bbM6ldWSimbs2XRR4UF0TMymEXbMjDGVP8CpZSyMVsWPcDYnq3Zd+oiqZk6JYJSyrPZtuhHdWuJr7foQVmllMezbdGHBPkxtGMzPt6uUyIopTybbYse4CeOKRHW6ZQISikPZuuiH9qpGU0a+LIwOcPqKEopZRlbF72/jzd3JrVmRepJTp6zZFJNpZSynK2LHuDevlGUGcO8TUetjqKUUpawfdFHhQVxXYdw3tt0lOJSPSirlPI8ti96gAn9osm+UMSK1JNWR1FKqXrnEUU/pEM4kaGBvL3+sNVRlFKq3nlE0Xt5Cff1jWLz4bPs1itllVIexiOKHuCnSa0J8PXinQ2HrY6ilFL1ymOKPjjQjzE9Ili8LZNz+SVWx1FKqXrjMUUPcF+/KApKyli45Vj1KyullE14VNF3jWhCYlQI7244Qnm5Tl+slPIMHlX0ABP6RXH4dD5r9mdbHUUppeqFxxX9yK4tadrQnzfWHbI6ilJK1QuPK3o/Hy+mDmnL2v05rN6nW/VKKfvzuKKHioOykaGB/OXTPZTpvnqllM15ZNH7+3jzxMhOpJ26wIJkPQNHKWVvHln0ACO6tqBXdAj//DyNi0WlVsdRSqk647FFLyI8eUscOReLeWXVAavjKKVUnfHYogeIbxPMmPhWzF57iOO5BVbHUUqpOuHRRQ/w2xGdAHh++V6LkyilVN3w+KKPCG7A5IExLN6eScqxXKvjKKVUrfP4ogd46Lp2NG3ox18/22N1FKWUqnVOFb2IjBCRNBE5ICKPX2G9O0TEiEhSpWVPOF6XJiI31Ubo2tYowJdJA2LYkH5GbyKulLKdaoteRLyBmcBIIA4YJyJxVazXCHgE2FhpWRxwN9AFGAH8x/F+Lmd4XHMAVu49ZXESpZSqXc5s0fcGDhhj0o0xxcB8YEwV6z0LPAdU3iQeA8w3xhQZYw4BBxzv53JimzWkTWgDvtytRa+Ushdnij4CqHz5aIZj2fdEJAFoY4xZerWvdbx+iogki0hydrY188+ICDd0bs43B0+TX6wXUCml7MOZopcqln0/QYyIeAEvAo9d7Wu/X2DMLGNMkjEmKTw83IlIdeOGzs0pLi1n7f4cyzIopVRtc6boM4A2lR63BjIrPW4EdAVWichhoC+wxHFAtrrXupTeMaE0CvBh5R7dfaOUsg9nin4zECsiMSLiR8XB1SXfPWmMOWeMaWqMiTbGRAMbgNHGmGTHeneLiL+IxACxwKZaH0Ut8fX2YkiHcL7am6V3oFJK2Ua1RW+MKQWmAyuAPcACY0yqiDwjIqOreW0qsADYDSwHphljymoeu+4Mj2tOzsVitmfoxVNKKXvwcWYlY8wyYNkly56+zLrXXfL4f4H/vcZ89e66Ds3w9hK+3H2KnpEhVsdRSqka0ytjL9Ek0Jde0SGs3JNldRSllKoVWvRVuKFzc9JOXeDYmXyroyilVI1p0Vfh+s4VV8l+qWffKKVsQIu+CjFNg2gXHqRFr5SyBS36y7ghrjkb089wvrDE6ihKKVUjWvSXcUPn5pSWG9bss2ZKBqWUqi1a9JfRMzKEkEBfneRMKeX2tOgvw9tLGNqpGV+nZVNcWm51HKWUumZa9FcwukcrzhWU8EmKy07Po5RS1dKiv4IhHcLp2LwRs9akY4zOfaOUck9a9FcgIkwZ3Ja0UxdYpQdllVJuSou+Grf2aEWLxgG8tvqg1VGUUuqaaNFXw8/Hi8kDK24cnnJMZ7RUSrkfLXon3N27DY38fZi1Jt3qKEopddW06J3QKMCX8X2j+GzXCY6e1onOlFLuRYveSZMGROPtJcxep1v1Sin3okXvpOaNAxibEMGC5GOcySu2Oo5SSjlNi/4qTBnclsKSct5ef9jqKEopm8k6X1hnV+Fr0V+F9s0acUPnZrz17WHyi0utjqOUspHHP9rJ2P98UycXZ2rRX6WHrmvP2fwSnvlkt9VRlFI2kZ59ka/2ZjE8rjkiUuvvr0V/lRKjQvjFde2Yv/kYH28/bnUcpZQNvPnNYfy8vRjfJ6pO3l+L/hrMGN6BpKgQfv/RTtKzL1odRynlxs7ll/DBlgzGxLcivJF/nXyGFv018PH24l/jEvD18WL6vG0UlpRZHUkp5abe23yUgpIyJg2IqbPP0KK/Rq2CG/DCnT3YfeI8f1m2x+o4Sik3VFJWzlvfHqZ/uzDiWjWus8/Roq+BYZ2a8+CgGN5ef4RlO09YHUcp5WaW7zrJiXOF3F+HW/OgRV9jv7mpE/FtgvndBzvIOKvTIyilnDfnm0NEhwUyrFOzOv0cLfoa8vPx4t/jEigoKeOd9UesjqOUchNbj55l29FcJg2Iwcur9k+prEyLvha0CQ3kuo7hfLw9k7JyvROVUqp6c9YdolGAD3cktq7zz3Kq6EVkhIikicgBEXm8iuenishOEdkuIutEJM6xPFpEChzLt4vIq7U9AFcxJj6Ck+cL2Zh+2uooSikXdzy3gM92nWRc70iC/H3q/POqLXoR8QZmAiOBOGDcd0VeyTxjTDdjTDzwHPBCpecOGmPiHV9Tayu4q7mhc3Ma+vuwWC+iUkpV47v5sib2j66Xz3Nmi743cMAYk26MKQbmA2Mqr2CMOV/pYRDgcfsvGvh5M6JrCz7beVLPq1dKXVZeUSnvbTzKiC4tiAhuUC+f6UzRRwDHKj3OcCz7ARGZJiIHqdiif6TSUzEisk1EVovIoKo+QESmiEiyiCRnZ7vvTbhvi4/gQlEpK/dkWR1FKeWiPtyawfnCUu4fWLenVFbmTNFXdTj4R1vsxpiZxph2wO+ApxyLTwCRxpgEYAYwT0R+dFWAMWaWMSbJGJMUHh7ufHoX069dGM0a+evuG6VUlcrLDXPWHSK+TTCJUSH19rnOFH0G0KbS49ZA5hXWnw/cBmCMKTLGnHZ8vwU4CHS4tqiuz9tLGBPfilVpWZzVm5MopS6xcm8Wh0/n88Cg+tuaB+eKfjMQKyIxIuIH3A0sqbyCiMRWengLsN+xPNxxMBcRaQvEAra+F9+Y+AhKygyf6pWySqlLvLEunYjgBozo0qJeP7faojfGlALTgRXAHmCBMSZVRJ4RkdGO1aaLSKqIbKdiF81Ex/LBwA4RSQE+AKYaY87U+ihcSJdWjYlt1lCnMFZK/cCu4+fYkH6Gif2j8PGu30uYnDqB0xizDFh2ybKnK33/6GVe9yHwYU0CuhsR4baECJ5fkcaxM/m0CQ20OpJSygXMWXeIID9v7uoVWe+frVfG1oHRPVoBsCTlSocylFKe4tT5Qj7ZkclPk9rQpIFvvX++Fn0daBMaSO/oUD7amlEn939USrmXt9cfprTcMGlAtCWfr0VfR8YktOJgdh6pmeerX1kpZVsFxWX8d+NRhnduTlRYkCUZtOjryC3dWuLn7cXcbw9bHUUpZaGPtmWQm1/CA4PaWpZBi76OBAf6MWlANB9syWDr0bNWx1FKWaC83PDGukN0i2hCr+j6u0DqUlr0dejh62Np3tifpz/epdMXK+WBlqeeJD07jwcGxSBSt3POX4kWfR1q6O/Dk7fEsev4eeZvPmp1HKVUPSovN/xr5X7ahgcxqnsrS7No0dexW7u3pE9MKM+vSNNpEZTyIJ/vPsXekxd4eFh7vOv4DlLV0aKvYyLCM2O6cqGwlOdWpFkdRylVD4yp2JqPDgvkVou35kGLvl50bNGIif2imb/5KDsycq2Oo5SqY1/uyWL3ifNMHxZb79MdVMX6BB7il8NjCQvy5+mPUynXA7NK2ZYxhpdX7iMyNJDb4q3fmgct+nrTOMCX39/cie3Hcnk/+Vj1L1BKuaWv07LYdfw804e2d4mtedCir1djEyLo2zaU/1mSyga9ibhStmOM4eUv99M6pAFje/7oRnyW0aKvRyLCK+MTiQwN5MG3k9lzQqdHUMpOVu3LJiXjHNOGtsfXRbbmQYu+3oUE+fH2/b0J8vNh4pxNHDuTb3UkpVQt+G5rPiK4Abf3bG11nB/QordAq+AGvD25N4UlZUycs4nTF4usjqSUqqFVadlsP5bLL4a2w8/HtarVtdJ4kA7NGzHnZ704nlvA/XM3k1dUanUkpdQ1Kis3/O2zvUSFBfLTxDbVv6CeadFbKCk6lJn39GRX5nkenb9N565Xyk19uDWDtFMX+O1NnVxuax606C13Q1xznhjZiS/3ZLF0h95QXCl3U1hSxotf7KNHm2Bu7la/N/12lha9C5g0IIZuEU14duluzheWWB1HKXUV3vzmMCfOFfLEyE6WzlB5JVr0LsDbS/jL2G7kXCzihc/3WR1HKeWks3nF/GfVAYZ1akbftmFWx7ksLXoX0a11E+7rG8Xb6w+zM+Oc1XGUUk74f18fIK+olN+N6GR1lCvSonchj93UkbCG/jy5eKfeqEQpF3fsTD7vrD/CHYmt6diikdVxrkiL3oU0DvDlD6Pi2JFxjv9uPGJ1HKXUFfzz8zRE4FfDO1gdpVpa9C7m1u4tGRTblOeXp5F1vtDqOEqpKuzIyGXx9kzuHxhDyyYNrI5TLS16F/PdjUqKysp5Zuluq+MopS5RVFrGbxbuILyRP1OHtLM6jlO06F1QTNMgpg9tz9IdJ5i9Nt3qOEqpSl7+cj9ppy7w99u70aSBr9VxnOJjdQBVtelD25N28gJ//nQPzRoHMLqHa9zAQClPtvXoWV5dfZA7k1ozrFNzq+M4zaktehEZISJpInJARB6v4vmpIrJTRLaLyDoRiav03BOO16WJyE21Gd7OvLyEf97Zg94xofx6QQrrD+r89UpZqaC4jF8vSKFF4wCeGhVX/QtcSLVFLyLewExgJBAHjKtc5A7zjDHdjDHxwHPAC47XxgF3A12AEcB/HO+nnBDg683r9yURFRbIlHeS2XtS569XyirPr0gjPSeP5+7oQeMA99hl8x1ntuh7AweMMenGmGJgPjCm8grGmMoNFAR8dxL4GGC+MabIGHMIOOB4P+WkJoG+zL2/N4F+3vxszmYycwusjqSUx9mQfpo53xxiQr8oBsY2tTrOVXOm6COAyjc5zXAs+wERmSYiB6nYon/kKl87RUSSRSQ5Ozvb2eweIyK4AXMn9SavqJSfvbmJs3nFVkdSymNcLCrlNx+kEBUWyOMjXfsK2MtxpuirmqXnR5dtGmNmGmPaAb8DnrrK184yxiQZY5LCw8OdiOR5OrdszGv3JXL4dD73zN7IGS17perFX5btIeNsAf/4aQ8C/dzz/BVnij4DqDyTfmsg8wrrzwduu8bXqivo374psyckkZ59kXte36B3plKqjq3el828jUd5YGAMvaJDrY5zzZwp+s1ArIjEiIgfFQdXl1ReQURiKz28Bdjv+H4JcLeI+ItIDBALbKp5bM81uEM4b0zsxeHTedzz+kZytOyVqhPn8kv43Qc7aN+sIY/d2NHqODVSbdEbY0qB6cAKYA+wwBiTKiLPiMhox2rTRSRVRLYDM4CJjtemAguA3cByYJoxpqwOxuFRBsY2Zc7EXhw5k8e4WRvIvqBlr1Rt+9MnqWRfLOKFO3sQ4OveJwuKq92+LikpySQnJ1sdwy1sSD/NpDc30yo4gAU/70dYQ3+rIyllC8t3nWTqu1t45PpYZrjBpGUAIrLFGJNU1XM6BYIb69s2jLmTepFxtoAZC1Io16mNlaqx0xeLeHLRTrq0aszDw9pbHadWaNG7uT5tw/jDqDhW78tm9jqdF0epmjDG8OSiXVwoLOWFO+Px9bZHRdpjFB5ufJ9IRnZtwXPL09h+LNfqOEq5rY+3Z7I89SQzbuzg8jcTuRpa9DYgIvztJ91p3jiAh9/bqjcYV+oapJ28wO8X7SQpKoQHB7W1Ok6t0qK3iSaBvvxrXAKZuYU88eFOXO0gu1Ku7Fx+CVPeSaahvw8zx/fE26uqaz3dlxa9jSRGhfDYjR34dOcJ5m8+Vv0LlFKUlRsenr+NzNwCXrk3keaNA6yOVOu06G1m6uB2DGzflD8uSWXb0bNWx1HK5f3j8zTW7MvmmTFdSYwKsTpOndCitxkvL+GFu3oQGuTHHa+u5y/L9lBQrNeoKVWVpTsyeWXVQe7pE8m43pFWx6kzWvQ21KxRAMsfHcxPE1sza006N760mrX7dVZQpSrbc+I8v1m4g8SoEP54axer49QpLXqbahLoy99u7878KX3x9fLivjc2MeP97TrrpVJAZm4BD7yVTOMGPrwyvid+PvauQnuPTtG3bRjLHh3Ew8PasyQlk5Evr2FDut6WUHmurAuFjJ+9kfMFJbwxsRfNbHjw9VJa9B4gwNebx27syMfTBxDo58M9r2/gXyv3U6ZTJigPk5tfzIQ3NnHyXCFz7+9F14gmVkeqF1r0HqRLqyZ88vBAbu3Rihe+2MeEORvJulBodSyl6sWFwhImztlEek4esycmkRjlvvPLXy0teg/T0N+Hl+6K5++3d2PLkbPc/PI61h/UXTnK3gqKy5g8N5nUzPP8556eDGjvfvd9rQkteg8kItzVK5KPpw2kSQMfpryTzKnzumWv7KmkrJyp725h85EzvHhXPDfENbc6Ur3TovdgHVs0YvbEXhSXlvM/H6daHUepWmeM4alFu1i9L5u/ju3GrT1aWR3JElr0Hi6maRC/vKEDy1NPsnzXCavjKFWrXll9kPeTj/HwsPbcbeMLoqqjRa94YFAMnVs25umPUzlXoDNfKntYuiOT55anMbpHK7e5S1Rd0aJX+Hp78ffbu5FzsYi/fbbX6jhK1diWI2eZsSCFpKgQnrujOyL2mo3yamnRKwC6tw5m8sAY3tt0VC+oUm7t6Ol8Hnw7mVZNApg1Icntb+xdG7To1fd+NbwDbUIb8PuPdlJYohOhKfdzvrCESXM3UW4Mb07qTWiQn9WRXIIWvfpeoJ8PfxnbjfScPF78cp/evES5lfJyw2MLUjhyOp/X7k0kpmmQ1ZFchha9+oFBseHckdia11anc9drG0g+fMbqSEo55ZXVB/li9ymevKUzfdqGWR3HpWjRqx/560+68extXTl0Oo87Xl3PA29tZu/J81bHUuqy1u7P5p+fV5xh87P+0VbHcTniav89T0pKMsnJyVbHUEB+cSlvfnOYV1cf5GJRKWPjI/jtiE60aGL/2f6U+zieW8Cof60lvJE/i6dVTNzniURkizEmqarndIteXVagnw/ThrZn7W+HMmVwW5buPMH1/1zFa6sPUlxabnU8pSgqLeMX726hpMzw6r2JHlvy1dGiV9UKDvTjiZGd+fJXQ+jXLoy/fraXES+v0btWKcv96ZPdpGSc4x8/7UHb8IZWx3FZWvTKaZFhgcye2Is5P0uirNxw3xubeOjdLWTphGjKAou2ZTBv41Eeuq4dI7q2sDqOS3Oq6EVkhIikicgBEXm8iudniMhuEdkhIitFJKrSc2Uist3xtaQ2wytrDOvUnBW/HMyvb+zAV3uzuOmlNTpPjqpXh3PyeGrRLnpHh/KYh09v4Ixqi15EvIGZwEggDhgnInGXrLYNSDLGdAc+AJ6r9FyBMSbe8TW6lnIriwX4ejN9WCyfPjKINqGBTH13K48tSOFCoc6Vo+pWcWk5j87fhreX8OLd8fh4646J6jjzO9QbOGCMSTfGFAPzgTGVVzDGfG2MyXc83AC0rt2YylW1b9aQDx/qzyPD2rNoWwYjXlrLpkN67r2qO//8Io2UjHP8/fbuRAQ3sDqOW3Cm6COAY5UeZziWXc5k4LNKjwNEJFlENojIbVW9QESmONZJzs7WA3zuxtfbixk3dmTh1P74eAt3zVrPOxuOWB1L2dDa/dm8tjqdcb0jGdmtpdVx3IYzRV/VtG9VnnwvIvcCScDzlRZHOs7tvAd4SUTa/ejNjJlljEkyxiSFh4c7EUm5osSoEJY9MohhHZvx9Me7WJKSaXUkZSOnLxYxY0EK7Zs15OlRl+49VlfiTNFnAG0qPW4N/OhvsIjcADwJjDbGFH233BiT6fg1HVgFJNQgr3JxQf4+zBzfk17Rocx4fzur0rKsjqRswBjDrxemcK6ghH/dnUADP52R8mo4U/SbgVgRiRERP+Bu4Adnz4hIAvAaFSWfVWl5iIj4O75vCgwAdtdWeOWaAny9mT0xiQ7NG/HQu1vZcuSs1ZGUm5v59QG+Tsvm9yM7EdeqsdVx3E61RW+MKQWmAyuAPcACY0yqiDwjIt+dRfM80BBYeMlplJ2BZBFJAb4G/maM0aL3AI0DfHnr/t40b+zP/XM3k3bygtWRlJuat/Eo//h8H6N7tGKizmNzTXSuG1Wnjp3J5/ZXvgXgg6n9iQwLtDiRcifLdp5g2rytDOkQzusTkvDVUykvS+e6UZZpExrI25N7U1hSxm3/+UanTVBOW7s/m0fnbyMxMoRXxidqydeA/s6pOtepRWMWTRtA04Z+TJiziZe/3E95uWv9T1K5lm1Hz/Lzd7bQLrwhb0zspQdfa0iLXtWLduENWTxtALfFR/Dil/v42dzNnMkrtjqWckH7T11g0tzNNG3oz9v396ZJoK/VkdyeFr2qN4F+PrxwZw/+d2xXNhw8zah/rWXbUT0jR/2fs3nF3P/WZny9vXh3ch+aNdZ7H9QGLXpVr0SE8X2i+PCh/nh5CXe+tp431h3S+9MqSsvKeWT+Nk6dK2LWfYl64L4WadErS3Rr3YRPHx7EkA7NeHbpbqa+u4VzBTohmid7fkUaa/fn8OxtXUiIDLE6jq1o0SvLNAn05fUJiTx5c2dW7sli1L/XsiMj1+pYygIfbz/Oa2vSua9vFHf1irQ6ju1o0StLiQgPDm7L+z/vR1mZ4Y5XdEI0T5OaeY7ffbiDXtEh/EHnsKkTWvTKJSRGhfDpI4MY0D6MPyzexR+XpFKmp2Da3tm8Yn7+zhaCG/jxn/GJ+PloJdUF/V1VLiMkyI/ZE3vxwMAY5n57mClvJ5NXVGp1LFVHDuXkMfHNTWRdKOK1+xIJb+RvdSTb0qJXLsXbS3hqVBzPjunC12lZ3Pnaek7pPWltpbzcMPebQ4x8eQ2Hc/L4f+MS6NEm2OpYtqZFr1zSff2imT0xiUM5edw28xv2nDhvdSRVCzLO5nPvGxv54ye76ds2jM9/NYQbu+iNveuaFr1yWcM6NWfh1H6UG8Odr67nYPZFqyOpGliYfIwRL60l5Vguf/tJN978WS9aNNELouqDFr1yaV1aNeHDh/rj6+PFQ+9uIb9Y99m7m7Jyw58+SeU3H+ygS6vGLP/lYO7uHYlIVTevU3VBi165vNYhgbx0Vzz7sy7y1OJdehWtG7lYVMoDb23mzW8Oc/+AGOY92Jc2oXrFa33TolduYXCHcB4ZFstHW4/z/uZj1b9AWS7jbD53vPIta/bn8OfbuvL0rXF4e+lWvBV8rA6glLMeuT6WrUfP8vSSVLpGNKFrRBOrI6nL2Hb0LA++vYWi0jLmTurFoNhwqyN5NN2iV27D20t46a54QgP9+MV/t+rcOC7IGMN7m45y16wNNPDz4qOH+mvJuwAteuVWwhr6M3N8Apm5Bfx6YYrur3cheUWlzFiQwhMf7aRPTCiLfzGA2OaNrI6l0KJXbigxKpTHR3bii92neG5FmtVxFBU3Cxkz8xsWbz/OjOEdmDupN2EN9UpXV6H76JVbmjwwhvScPF5ZdZCwID8eGNTW6kge66OtGTy5aBdB/t68O7kPA9o3tTqSuoQWvXJLIsKzY7pyNq+YP3+6h7CGfoxNaG11LI9yMPsizy7dzaq0bPrEhPLvcQl6RygXpUWv3Ja3l/DS3fHkztnMbxbuIDjQj6Edm1kdy/YuFJbw768O8OY3h/D38ebJmzszaUA0Pt66J9hV6Z+Mcmv+Pt7MmpBIxxaNeOjdLWw5ovegrStl5YaFyccY+o/VzFqTztiECL7+9XU8OLitlryLE1c7ayEpKckkJydbHUO5mewLRdzx6u9Vb/8AAApSSURBVLfk5pcwd1IvvRVdLco6X8iC5GO8t+kYx3MLSIgM5o+3dtEZJ12MiGwxxiRV+ZwWvbKLo6fzGff6Bk6dL+RXwzswdUg7vRLzGpWXG9YdyGHexqN8uecUpeWG/u3CuK9vFDd1aYGX/r66nCsVve6jV7YRGRbIskcH8dTiXTy/Io01+7J58a54WgU3sDqayzPGsO/URdYfzGF9+mk2HjpDbn4JIYG+3D8whnG9I4lpGmR1THWNdIte2Y4xho+2Hufpj3fh7SX89SfduaV7S6tjuZTCkjJ2HT/H1qNn2Xokl82Hz3A6rxiA1iEN6Nc2jCEdwxke1xx/H2+L0ypn1HiLXkRGAC8D3sBsY8zfLnl+BvAAUApkA/cbY444npsIPOVY9c/GmLeuaRRKOUlEuD2xNYlRITz6/namzdvKom3NmD4slngP3a9sjGHX8fMs3ZHJhvTT7D5xnpKyio28NqENGNwhnH7twujXNkxnl7SharfoRcQb2AcMBzKAzcA4Y8zuSusMBTYaY/JF5CHgOmPMXSISCiQDSYABtgCJxpjLnhqhW/SqNpWUlTNrTTqz1qRzrqCEAe3DmDa0Pf3ahnnEfOgHsi6yJCWTT1IyOZSTh6+3kBAZQs/IEBIig0mIDKZZIz333Q5qukXfGzhgjEl3vNl8YAzwfdEbY76utP4G4F7H9zcBXxhjzjhe+wUwAnjvageh1LXw9fZi2tD2TOwfzbyNR3h97SHueX0jCZHBPHlzZ5KiQ62OWKvKyg3bj+WyKi2LL/dksefEeUSgb0wYUwa3ZWTXFgQH+lkdU9UzZ4o+Aqg8AXgG0OcK608GPrvCayMufYGITAGmAERGRjoRSamr09DfhymD2zGhXzQLt2Tw6qqDjJ+9kdcnJDG4g3vPrnihsISv9mbx9d4sVu/L5mx+Cd5eQs/IYP4wKo5R3VvSXK9Y9WjOFH1V/7+tcn+PiNxLxW6aIVfzWmPMLGAWVOy6cSKTUtckwNeb+/pGcUu3loyfvZEH305m9sQkt5tK90JhCSv3ZLF0xwnW7M+muLSc0KCKK4Ov69SMIbHhNAn0tTqmchHOFH0G0KbS49ZA5qUricgNwJPAEGNMUaXXXnfJa1ddS1ClalNokB//faAP97y+gQfeco+yP5dfwsq9p1i28+T35d6icQDj+0RyS7eWJESG6HUDqkrOHIz1oeJg7PXAcSoOxt5jjEmttE4C8AEwwhizv9LyUCoOwPZ0LNpKxcHYM5f7PD0Yq+rTmbxi7nl9A4dy8nhjYi8GxrrWzIunLxbx+e5TfLbrJN8eyKG03NCicQAju7VgVPeWJLQJ0YuXFFDDg7HGmFIRmQ6soOL0yjnGmFQReQZINsYsAZ4HGgILHWcyHDXGjDbGnBGRZ6n4xwHgmSuVvFL1LTTIj3kP9uWe1zcw+a3NPDUqjob+3hQUl1NYUkZBSRmBft6M6NqClk3q58KrrAuFrNh1kmU7T7Lx0GnKDUSGBjJ5YAwjuragR+tgLXd1VfSCKaX4vy37vScvVPm8CPRrG8bYhAhGdmtJQ//avaj89MUilu08wac7T7Dx0BmMgXbhQdzcrSUjurYgrmVjjzgdVF07netGKScUlZaRnp1HgK83DXy9CfD1IsDXm5PnClm8/TiLth3nyOl8Any9GB7Xgv7twugVHUK78IbXXMJ7TpznzW8OsXh7JsWl5bRv1pCbu7Xklm4t6dD82t9XeR4teqVqgTGGrUfP8tHW4yzfdfL7KQNCAn1JjAqhV3QoY3tGVHsBUnm54eu0LN5Yd4hvD56mga83tydGcG/fKDq1aFwfQ1E2pEWvVC0zxnAoJ4/kI2dJPnyG5CNnHf8b8OLePlH8fEg7whv98J6pufnFLEzO4N2NRzhyOp+WTQKY0C+acb3b6EVMqsa06JWqB4dz8vj3VwdYtC0DPx8v7usbxZTB7cjMLeCdDUf4JCWTotJykqJCmNA/mpFdW+CrN+xQtUSLXql6dCgnj39/tZ/F244jIpSVG4L8vLktoWL3TOeWuntG1T4teqUscCgnj/9uOEJkWCBjEyJoFKBXqqq6ozceUcoCMU2DeGpUnNUxlNKbgyullN1p0SullM1p0SullM1p0SullM1p0SullM1p0SullM1p0SullM1p0SullM253JWxIpINHKnBWzQFcmopjtXsNBaw13jsNBbQ8bgyZ8cSZYyp8n6YLlf0NSUiyZe7DNjd2GksYK/x2GksoONxZbUxFt11o5RSNqdFr5RSNmfHop9ldYBaZKexgL3GY6exgI7HldV4LLbbR6+UUuqH7LhFr5RSqhIteqWUsjnbFL2IjBCRNBE5ICKPW53naonIHBHJEpFdlZaFisgXIrLf8WuIlRmdJSJtRORrEdkjIqki8qhjubuOJ0BENolIimM8f3IsjxGRjY7xvC8ibnOHbxHxFpFtIrLU8didx3JYRHaKyHYRSXYsc8ufNQARCRaRD0Rkr+PvUL+ajscWRS8i3sBMYCQQB4wTEXe7tc9cYMQlyx4HVhpjYoGVjsfuoBR4zBjTGegLTHP8ebjreIqAYcaYHkA8MEJE+gJ/B150jOcsMNnCjFfrUWBPpcfuPBaAocaY+Ernm7vrzxrAy8ByY0wnoAcVf041G48xxu2/gH7AikqPnwCesDrXNYwjGthV6XEa0NLxfUsgzeqM1ziuj4HhdhgPEAhsBfpQcbWij2P5D34GXfkLaO0oi2HAUkDcdSyOvIeBppcsc8ufNaAxcAjHiTK1NR5bbNEDEcCxSo8zHMvcXXNjzAkAx6/NLM5z1UQkGkgANuLG43Hs6tgOZAFfAAeBXGNMqWMVd/qZewn4LVDueByG+44FwACfi8gWEZniWOauP2ttgWzgTceutdkiEkQNx2OXopcqlul5oxYTkYbAh8AvjTHnrc5TE8aYMmNMPBVbw72BzlWtVr+prp6IjAKyjDFbKi+uYlWXH0slA4wxPanYdTtNRAZbHagGfICewCvGmAQgj1rY7WSXos8A2lR63BrItChLbTolIi0BHL9mWZzHaSLiS0XJ/9cY85FjsduO5zvGmFxgFRXHHoJFxMfxlLv8zA0ARovIYWA+FbtvXsI9xwKAMSbT8WsWsIiKf4jd9WctA8gwxmx0PP6AiuKv0XjsUvSbgVjHmQN+wN3AEosz1YYlwETH9xOp2Nft8kREgDeAPcaYFyo95a7jCReRYMf3DYAbqDhA9jVwh2M1txiPMeYJY0xrY0w0FX9PvjLGjMcNxwIgIkEi0ui774EbgV246c+aMeYkcExEOjoWXQ/spqbjsfrgQy0exLgZ2EfFvtMnrc5zDfnfA04AJVT8qz6Zin2nK4H9jl9Drc7p5FgGUvFf/x3AdsfXzW48nu7ANsd4dgFPO5a3BTYBB4CFgL/VWa9yXNcBS915LI7cKY6v1O/+7rvrz5ojezyQ7Ph5WwyE1HQ8OgWCUkrZnF123SillLoMLXqllLI5LXqllLI5LXqllLI5LXqllLI5LXqllLI5LXqllLK5/w8YUfT+2+pufgAAAABJRU5ErkJggg==\n",
+      "text/plain": [
+       "<Figure size 432x288 with 1 Axes>"
+      ]
+     },
+     "metadata": {
+      "needs_background": "light"
+     },
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "plt.plot(predictions.mean(0)[:,0])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 37,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "[<matplotlib.lines.Line2D at 0x7f744941c650>]"
+      ]
+     },
+     "execution_count": 37,
+     "metadata": {},
+     "output_type": "execute_result"
+    },
+    {
+     "data": {
+      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD4CAYAAADiry33AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3deXxU9b3/8dcnk5WQkD0kJEASwhKQzbDJKiLgUrdaldZq60K1tdraXpfeVvuzt7dXvbd6tbYu1autWjdapYIiCoLshH1LIAkBkkBWCHuSyXx/f2SwMQaYJJOcmTOf5+ORR2bOnDP5fCF55+R7vuf7FWMMSiml7CvI6gKUUkp1LQ16pZSyOQ16pZSyOQ16pZSyOQ16pZSyuWCrC2gtISHB9O/f3+oylFLKr2zYsKHaGJPY1ms+F/T9+/cnLy/P6jKUUsqviMi+s72mXTdKKWVzGvRKKWVzGvRKKWVzGvRKKWVzGvRKKWVzGvRKKWVzGvRKKWVzPjeOXlnreL2TL3ZXUX28nllDe5MUHW51SUqpTtKgVxyqO82nuypYvLOC1UU1NDS5AHh0/g6mDEzkutFpzMxJJjzEYXGlSqmO0KAPYE0uwwPvbWXexlIA+sX34JYJ/bg0J5m4yFDe31zGPzaWce/fNhEVFswVw1O4bnQauf1iCQoSi6tXSnlKfG2FqdzcXKNTIHQ9Ywy/fH87b6zdz20TM5gzNp0BST0R+WqAu1yGNcU1zNtYxkfbD3KyoYm02AiuHdWHa0f1ITOxp0UtUEq1JCIbjDG5bb6mQR+Yfr94N898tocfTM3k4cuGeHTMyQYni3Yc4u8by1hZWI3LwKi+MTx5/QgGJGngK2UlDXr1Fa+tKuHR+Tu4ITeNx785/Gtn8Z6oOHqa+ZvLeWF5ESLCW3PHk6Vn90pZ5lxBr8MrA8z8LeX8+p87mDEkmf+89oIOhTxAcnQ4d07J5G93jscYw5wX11BcddzL1SqlvEGDPoAs313Fz97ZzJj+cfzh26MIdnT+vz87OYo37xxPk8sw56U17K0+4YVKlVLepEEfIE42OPnJ25vJSuzJS7fkenWo5MDkKN64cxyNTc1n9iUa9kr5FA36APH2+gPUnmjgP64ZRq+IEK+//+De0bxxxzjqnU3MeWkNB2pPev1rKKU6xqOgF5HZIlIgIoUi8lAbr98vIjtFZKuIfCYi/Vq81iQim90f871ZvPJMg9PFS8uLGds/jtz+cV32dYakRPPGHeM5Ue/k7jc2UO9s6rKvpZTy3HmDXkQcwHPAZUAOMEdEclrttgnINcYMB94Dnmjx2iljzEj3x1Veqlu1w/wt5ZTXnebuaVld/rVyUqP5nxtGsr3sKP+5YFeXfz2l1Pl5ckY/Fig0xhQbYxqAt4CrW+5gjFlqjDnzt/oaIM27ZaqOcrkMzy8rYnDvKKYNanPdYK+7NCeZOyZl8NrqfSzcdrBbvqZS6uw8Cfo+wIEWz0vd287mduCjFs/DRSRPRNaIyDVtHSAic9375FVVVXlQkvLU4l0VFFYe5+5pWR0eStkRD8wezMj0GB58byv7avTirFJW8iTo20qHNu+yEpGbgVzgyRab+7oH8X8beFpEvtZ/YIx50RiTa4zJTUzsnrPOQGCM4Y+fF9E3rgdXXJDSrV87NDiIP3x7FCLwozc3an+9UhbyJOhLgfQWz9OA8tY7icgM4N+Bq4wx9We2G2PK3Z+Lgc+BUZ2oV7XD6uIathw4wg+mZnplzHx7pcX20P56pXyAJz/964FsEckQkVDgJuAro2dEZBTwAs0hX9lie6yIhLkfJwATgZ3eKl6d258+LyIxKoxvjrbukknL/vqPtx+yrA6lAtl5g94Y4wTuARYBu4B3jDE7ROQxETkziuZJoCfwbqthlEOAPBHZAiwF/ssYo0HfDbaV1vHFnmpun5Rh+TzyD142mMG9o3ji43yaXL41t5JSgcCj+eiNMQuBha22PdLi8YyzHLcKuKAzBaqO+dOyQqLCg/nOuL5Wl0KII4h7pg/gnjc3sXjnIWYP697rBUoFOr0z1oYO1Z3mo+2HuHl8P6LCvX8XbEdcNiyFfvE9+NOyYnxtxlSl7E6D3obmbynDGLghN/38O3cTR5Bw5+RMthw4wuriGqvLUSqgaNDb0AebyxmR1ouMhEirS/mK6y9MI6FnKM8vK7a6FKUCiga9zRRWHmNH+VGuHnmue9qsER7i4PsTM1i+u4od5XVWl6NUwNCgt5n3N5UTJHDlCN+84Hnz+H70DAvmBT2rV6rbaNDbiDGGD7aUMXFAAklR4VaX06ZeESF8e1xfPtxarlMZK9VNNOhtZOP+IxyoPeWT3TYt3TYxA0eQ8NIXelavVHfQoLeRDzaXERYcxKyhyVaXck69e4Vz7ag+vL3+ANXH689/gFKqUzTobaKxycWCrQeZMSTZZ8bOn8vcKVk0NLl4bVWJ1aUoZXsa9DaxorCamhMNXD0y1epSPDIgqSezh/bm1ZUlHD7RYHU5StmaBr1NfLCpjF4RIUwblGR1KR776aUDOd7g5PllRVaXopStadDbwMkGJ5/srODyC1IIDfaf/9KByVFcO7IPr64qoeLoaavLUcq2/CcV1Fkt3lnByYYmv+m2aemnlw7EZQzPLtljdSlK2ZYGvQ18sLmc1F7hjO0fZ3Up7ZYe14ObxvTlrXUHdMlBpbqIBr2fO3yigeW7q/jGyFSCgrpvTVhv+vH0AQQ7hKc/1bN6pbqCBr2fW1pQidNlun1NWG9Kig7n1ov68/7mMgoOHbO6HKVsR4Pezy3JryQxKoxhqb2sLqVT7p6aRc/QYP77kwKrS1HKdjTo/ZizycXy3VVMG5jot902Z8T0CGXulEwW76xg0/7DVpejlK1o0PuxDfsOc/S0k+mD/Wfs/Ll8f1IG8ZGhelavlJdp0PuxpQVVhDiESdkJVpfiFT3Dgrl7WhYrC2vYsK/W6nKUsg0Nej+2NL+SMf3j/GJuG0/NGduX2B4h/HGp3i2rlLdo0PupsiOnKKg4ZptumzMiw4L5/sQMPsuvZGf5UavLUcoWNOj91JL8SgC/mtvGU7dO6E9kqIM/6Rw4SnmFBr2fWppfSd+4HmQl+tYC4N7Qq0cIN0/ox4Kt5eyt1rtlleosDXo/dLqxiVVF1UwfnISIfw+rPJs7JmUS4gjiBT2rV6rTNOj90OriGk43urjYZv3zLSVGhXHjmHTmbSyl/Mgpq8tRyq9p0PuhpfmVRIQ4GJfhf5OYtcfcKZkYg64tq1QnadD7GWMMS/IrmTggnvAQh9XldKm02B5cPbIPf1u3nxpdW1apDvMo6EVktogUiEihiDzUxuv3i8hOEdkqIp+JSL8Wr90qInvcH7d6s/hAVFh5nNLDp2zdbdPS3dOyqHe6+L+VJVaXopTfOm/Qi4gDeA64DMgB5ohITqvdNgG5xpjhwHvAE+5j44BHgXHAWOBREYn1XvmB58ywyottOKyyLQOSenLZsN68tqqE4/VOq8tRyi95ckY/Fig0xhQbYxqAt4CrW+5gjFlqjDnpfroGSHM/ngUsNsbUGmMOA4uB2d4pPTAtya9kcO8oUmMirC6l29wxOZNj9U7e31RmdSlK+SVPgr4PcKDF81L3trO5HfioPceKyFwRyRORvKqqKg9KCkxHTzeSt++w7e6GPZ9R6THkpETz+pp9GGOsLkcpv+NJ0Lc1ULvNnzYRuRnIBZ5sz7HGmBeNMbnGmNzExEQPSgpMX+yupsllAqZ//gwR4ebx/cg/dIyNOoWxUu3mSdCXAuktnqcB5a13EpEZwL8DVxlj6ttzrPLMst2VRIcHMyo9xupSut3VI1PpGRbM62v2W12KUn7Hk6BfD2SLSIaIhAI3AfNb7iAio4AXaA75yhYvLQJmikis+yLsTPc21U7GGJbtrmJydiLBjsAbFRsZFsx1o/uwYOtBak80WF2OUn7lvIlhjHEC99Ac0LuAd4wxO0TkMRG5yr3bk0BP4F0R2Swi893H1gK/ofmXxXrgMfc21U75h45RcbSeqYMCt2vr5vH9aGhy8W7egfPvrJT6UrAnOxljFgILW217pMXjGec49hXglY4WqJp9XtB8kXrqwMAN+oHJUYzNiOONtfu5c3Km3y+fqFR3Cbw+AD+1bHclQ1KiSY4Ot7oUS908vh/7a0+yfI+OzlLKUxr0fuDY6UbySg4H9Nn8GbOH9iahZ6helFWqHTTo/cCqohqcLsO0AO6fPyM0OIgbctNZkl9Bmc5qqZRHNOj9wOcFVfQMC+bCfjp7BDSvK2uAt9bpWb1SntCg93HGGJbvrmLigHhCAnBYZVvS43pw8aAk3lp/gMYml9XlKOXzNDl8XGHlccqOnGLqwMC6G/Z8vjOuL1XH6lmxp9rqUpTyeRr0Pm7ZbvewSu2f/4pJ2Qn0CHV8OZunUursNOh93LLdVWQn9aRPAM1W6YmwYAcTBySwJL9SJzpT6jw06H3YyQYna4trdVjlWUwfnETZkVPsrjhudSlK+TQNeh+2priGhiYX0wJkkZH2OrP4inbfKHVuGvQ+7POCKiJCHIzJ0GGVbendK5yhqdEsya+wuhSlfJoGvQ9btruKi7LiCQu29yLgnTF9cBIb9h3myEmd0VKps9Gg91F7q0+wr+akjrY5j+mDk3CZf41OUkp9nQa9j1pW0NzvPE3Hz5/TiLQY4iNDtZ9eqXPQoPdRSwqqyEyIpG98D6tL8WlBQcLUQYks211Fk0uHWSrVFg16H3Si3smaopqAWwS8o6YPTuLIyUY26XqySrVJg94HrSyspqHJpUHvocnZiQQHCZ9p941SbdKg90FLCyrpGRZMbv84q0vxC70iQsjtH8tSDXql2qRB72OMMSzJr2TKwARCg/W/x1PTByeRf+iYzlGvVBs0SXzMjvKjVByt//KuT+WZ6YOTAb1LVqm2aND7mCX5lYig0x60U1ZiJH3jemj3jVJt0KD3MUvyKxmeFkNiVJjVpfgVEWH64CRWFlZzqqHJ6nKU8ika9D6k+ng9W0qPMF3P5jtk+uAk6p0uVhfrYiRKtaRB70M+L6jCGLhkiAZ9R4zLjCMixMGyAp0OQamWNOh9yNL8SpKiwhiaGm11KX4pLNjBhKx4nfdGqVY06H1EY5OL5buruHhQEiJidTl+a+rAREpqTlJSfcLqUpTyGRr0PmJ9SS3H6p1M126bTjmzGtfyPXpWr9QZGvQ+Yml+JaGOICYNSLC6FL/WPyGSfvE9tJ9eqRY8CnoRmS0iBSJSKCIPtfH6FBHZKCJOEbm+1WtNIrLZ/THfW4XbzWf5lYzLjCMyLNjqUvze1IGJrCqqod6pwyyVAg+CXkQcwHPAZUAOMEdEclrtth/4HvBmG29xyhgz0v1xVSfrtaV9NScorjqhk5h5ydSBiZxqbCKvRGezVAo8O6MfCxQaY4qNMQ3AW8DVLXcwxpQYY7YCri6o0fbO3LavQe8d4zPjCXGIjr5Rys2ToO8DHGjxvNS9zVPhIpInImtE5Jp2VRcgluRXkpkYSb/4SKtLsYXIsGDG9I/Tfnql3DwJ+rbG+rVnKZ++xphc4NvA0yKS9bUvIDLX/csgr6oqsH44T9Q7WVtcq3fDetnUgYkUVBzjYJ3OZqmUJ0FfCqS3eJ4GlHv6BYwx5e7PxcDnwKg29nnRGJNrjMlNTAysxbBXFdXQ0OTiYu228aozi6ov1+4bpTwK+vVAtohkiEgocBPg0egZEYkVkTD34wRgIrCzo8Xa0ZL8SiJDHYzRRUa8alByFMnRYdpPrxQeBL0xxgncAywCdgHvGGN2iMhjInIVgIiMEZFS4FvACyKyw334ECBPRLYAS4H/MsZo0LsZY/i8oJJJ2brIiLeJCFMHJvLFnmqcTTpGQAU2jwZtG2MWAgtbbXukxeP1NHfptD5uFXBBJ2u0reY+5NP8ZEa21aXY0tSBSbyTV8rmA0d0WUYV0PQ00kJnhlXqIiNdY9KABIIE7b5RAU+D3kKf51cxNDWa5Ohwq0uxpV49QhjVN1aDXgU8DXqL1J1sZMP+w7o2bBebOjCRraV1VB+vt7oUpSyjQW+R5XuqaHIZLh4cWMNJu9uZ2SxX7NFVp1Tg0qC3yNKCSmJ6hDAyPdbqUmztgj69iO0RouPpVUDToLeAy2VYVlDF1IGJOIJ0kZGuFBQkTByQwIrCaoxpzw3dStmHBr0FtpbVUXOiQScx6yaTsxOoPFbP7orjVpeilCU06C2wNL+SIIEp2do/3x0muf+dv9BVp1SA0qC3wNKCSkb1jSU2MtTqUgJCn5gIMhMiWVGoF2RVYNKg72ZVx+rZWlrHxYP0bL47TcpOYG1xra46pQKSBn03O3Pzjs5W2b0mDUjgVGMTG/cdsboUpbqdBn03W5pfSVJUGDkp0VaXElDGZ8XjCBJWFGo/vQo8GvTdyOUyrCisZurARER0WGV3ig4PYVR6jN44pQKSBn032nnwKHWnGpk4IMHqUgLSpOwEtpbVceRkg9WlKNWtNOi70eqiGgAmZMVbXElgmpydgDGwsrDG6lKU6lYa9N1oVVE1WYmROlulRUakxRAVFqz99CrgaNB3k8YmF+v21urZvIWCHUGMz4rniz06HYIKLBr03WRbWR0nGpq4KEv75600OTuB0sOn2Fdz0upSlOo2GvTd5Ez//PhMPaO30iT3hfAv9C5ZFUA06LvJqqJqhqREE6fTHlgqIyGSPjERrNB5b1QA0aDvBqcbm8grOcxF2j9vORFhcnYCq4pqcDa5rC5HqW6hQd8NNu0/Qr3TpUHvIyZlJ3DstJOtZXVWl6JUt9Cg7wari6oJEhiTEWd1KQqYmJWAiC4vqAKHBn03WF1cwwVpMUSHh1hdigJiI0MZmR7D3zeW0uDU7htlfxr0Xexkg5NN+49ot42PuXd6NiU1J3l9zT6rS1Gqy2nQd7H1JYdxuowGvY+ZNiiRydkJPLNkD3UnG60uR6kupUHfxVYVVRPiEHL7af+8LxERfnH5EOpONfLskj1Wl6NUl9Kg72Kri2oY1TeWiFCH1aWoVoakRHPDhem8trqEkuoTVpejVJfRoO9Cdaca2V5Wp902PuxnMwcS4gji8Y/zrS5FqS7jUdCLyGwRKRCRQhF5qI3Xp4jIRhFxisj1rV67VUT2uD9u9Vbh/mDd3lpcBibotAc+Kyk6nLumZvHR9kOsL6m1uhylusR5g15EHMBzwGVADjBHRHJa7bYf+B7wZqtj44BHgXHAWOBREYntfNn+YVVRNeEhQYzsG2N1Keoc7pycSe/ocP7jw524XDqrpbIfT87oxwKFxphiY0wD8BZwdcsdjDElxpitQOtBybOAxcaYWmPMYWAxMNsLdfuF1UU1jOkfR1iw9s/7sohQBz+fNYgtpXX8c2u51eUo5XWeBH0f4ECL56XubZ7w6FgRmSsieSKSV1Vlj8mmqo7Vk3/omM4/7yeuG9WHoanRPLV4t85Vr2zHk6BvaxVrT38SPDrWGPOiMSbXGJObmJjo4Vv7tlVFzbfXT9L1Yf1CUJBwy4R+lNScZHvZUavLUcqrPAn6UiC9xfM0wNO/bztzrF9bsaeamB4hDE3tZXUpykMzc3rjCBIWbDtodSlKeZUnQb8eyBaRDBEJBW4C5nv4/ouAmSIS674IO9O9zdaMMawsrOairHgcQW39UaN8UWxkKBdlxbNgW7l23yhbOW/QG2OcwD00B/Qu4B1jzA4ReUxErgIQkTEiUgp8C3hBRHa4j60FfkPzL4v1wGPubbZWXH2C8rrTTNRuG79z5fAUDtSe0u4bZSvBnuxkjFkILGy17ZEWj9fT3C3T1rGvAK90oka/s7JQ++f91cyc3vziH9tZsO0gF6Rpt5uyB70ztgus2FNNelwE/eIjrS5FtdOZ7puF2w5q942yDQ16L3M2uVhdXKNn837sigtS2F97kh3l2n2j7EGD3su2ltVx7LRT++f92MyhOvpG2YsGvZet3FONCFyUpUHvr+K0+0bZjAa9l60orGZoajRxkaFWl6I64fILUthXo903yh406L3oRL2TjfsPa7eNDcxyd98s1O4bZQMa9F60rqSWxiajF2JtIC4ylAmZ2n2j7EGD3otW7qkmNDiIMf112UA7uPyCFEq0+0bZgAa9F60orGZM/1jCQ3RaYjuYNTRZu2+ULWjQe0nlsdPkHzqm/fM2Et8zTLtvlC1o0HvJ6qIaQKc9sJtrR/WhpOYkn+yssLoUpTpMg95LvthTTa8InZbYbq4emUpmQiT/vaiAJl1mUPkpDXovODMt8cQBOi2x3QQ7grh/5kD2VB7ng81lVpejVIdo0HtBUdVxDuq0xLZ1+bCU5mUGP91Ng7P1sshK+T4Nei/4dFclABcPSrK4EtUVgoKEn88axIHaU7y9fr/V5SjVbhr0XvDpzgpyUqJJjYmwuhTVRaYNTGRs/zieWVLIqYYmq8tRql006Dup9kQDG/cfZkZOstWlqC4kIvzb7EFUHavn1VUlVpejVLto0HfS0vxKXAZmDNFuG7sb0z+Oiwcl8vyyIupONVpdTkDZXlbHT97axJ+/KGZr6RGcTXqtpD08WkpQnd2nuypIigpjmA6rDAg/nzWIK55ZwUvLi/n5rEFWlxMwXlhezIdby3l/czkAkaEORveLZXxmPDeNSSe+Z5jFFfo2PaPvhHpnE8t3V3HJkGSCdFhlQBia2osrh6fwysq9VB2rt7qcgFDvbGJpfiU35qaz5uFLeGbOKK4bnUbVsXqeXFTA5CeW8sTH+Rw+0WB1qT5Lg74T1hbXcqKhSbttAsz9lw6k3uniuaWFVpcSEFYV1nC83smsob3p3Sucq0ak8ptrhvHxT6bw6f1TuGRIMn9aVsTkJ5byP58UUHdSu9Va06DvhE93VRAeEqTj5wNMZmJPvnVhGm+u3U/p4ZNWl2N7H28/RM+wYC4aEP+11wYkRfHsnFF8fN8UpgxM4NklhUx6Ygk7dcbRr9Cg7yBjDJ/tqmTSgESdrTIA3XtJNgg889mec+6nFw07p8llWLyrgumDkwgLPvvP2aDeUfzxOxey8N7JOJsMb67b141V+j4N+g7adfAYZUdOabdNgEqNieC74/vx3oZSCiuPt7nPnopjjP3Pz/jtgp3dXJ19rC+ppfZEA7OG9vZo/5zUaKYNSmTRjgpcOjfRlzToO+izXc2zGU4frEEfqH44LYuIEAdPLd79tddqjtdz22vrqTvVyEtf7GX+lnILKvR/H28/RGhwENMGJXp8zOxhvak6Vs+G/Ye7sDL/okHfQZ/mVzIiPYak6HCrS1EWie8Zxu2TM1mw7SDby+q+3F7vbOIHf91A5dF63p47ntx+sTw0byt7Ko5ZWK3/McbwyY5DTMlOJDLM85Hg0wcnEeoI4qNth7qwOv+iQd8BlUdPs+XAEWbo2XzAu2NyBjE9QnhyUQHQHE4Pz9tG3r7D/M8NI8jtH8dz3xlNj1AHP3h9A8frnRZX3PWMMbyz/gB5JbWdWrBlW1kd5XWnmTW0fXedR4WHMDk7gUU7DumCMW4a9B2wJL95EjOd9kBFh4fww2lZLNtdxdriGv74eRF/31TG/ZcO5MrhqQAkR4fz7JzRlFSf4MH3tto+fLaXHeWBeVu5/vnVzHxqOa+s2MuRk+0f4/7x9kM4goQZQ9r/czZrWG/KjpxiW4u/tAKZR0EvIrNFpEBECkXkoTZeDxORt92vrxWR/u7t/UXklIhsdn88793yrfHprkr6xEQwuHeU1aUoH3DLhP4kR4dx/ztbeHJRAdeMTOXH0wd8ZZ8JWfE8MHswC7Yd5JWVJdYU2k2W7W4+EXr0Gzn0CAvmsQ93MvY/P+Onb29my4EjHr/Poh2HGJcRR2xkaLtruHRI83q/H21vu/vGGMMP39jAHa/lUVJ9ot3v72/OG/Qi4gCeAy4DcoA5IpLTarfbgcPGmAHAU8DjLV4rMsaMdH/c5aW6LXO6sYkVhVVcMiQJEb0bVkF4iIN7L8mm7MgpLuwXy399c3ib3xs/mJLJzJxkfrdwFx9vP2TbUSHLdldxQZ9efH9iBh/8aCIL7p3EjbnpfLqzgqufW8ktr6xj3d7ac75HYeUxiqpOMHuYZ6NtWouNDGVCZjwfb2+7+2bBtoMs3HaIzwsqmfn0cn6/eDenG+07K6knZ/RjgUJjTLExpgF4C7i61T5XA6+5H78HXCI2TcFlu6s43ejikg78Oans68bcdJ64fjh/viX3rPdViAj/fcMI+sb34K7XNzDp8SU8/nG+rS7S1p1qZOP+I0wd+K9RMkNTe/Gba4ax6uHpPDh7MDvK6rjhhdXc8MJqvthT1WYQf+w+E5+Z07Ggh+bRN3urT1DQ6t/3dGMTv1uYz5CUaL548GJmD+3NM5/t4dKnln05ms5uPAn6PsCBFs9L3dva3McY4wTqgDO3sWWIyCYRWSYik9v6AiIyV0TyRCSvqqqqXQ3obu/mlZIYFcbErK/fpacCV7AjiBty08/bzRAdHsKCH0/mmTmjGNg7iheXF3PpU8v5xrMr+Mem0m6qtuusKqymyWWY2sZwyKjwEO6elsWKB6fzyJU57Ks5wXdfXsc1z63k4+0Hv/IXzqIdFYxMj6F3r46Paps5NBmRf/3SOOPPXxRTduQUv7pyCCm9InhmzijevGMcoY4gbn8tjx/8NY/q4/aax8iToG/rzLz1r+Cz7XMQ6GuMGQXcD7wpItFf29GYF40xucaY3MREz8fLdrfKo6dZWlDJdaP7EOzQ69iqYyJCHVw1IpVXvz+WNQ9fwq+uzKGxycVP397CL9/fRqMf3027fE8VUWHBjEyPOes+EaEObpuUwfIHLua31w7j8MlG7np9IzOeWsY76w+wt/oE28rqOtxtc0ZSVDi5/WK/EvQVR0/zx8+LmDU0mYuy/jV1yUUDEvjovik8MHsQS/OrmP30chbvtM/ZvSdpVQqkt3ieBrS+++PLfUQkGOgF1Bpj6o0xNQDGmA1AETCws0Vb5e+bymhyGW7ITT//zkp5IDEqjNsnZbDg3sn8YGomr6/Zz3dfXkutH87EaIxhWUEVEwckEOLBiVBYsIPvjOvHkp9N5dk5owgPdvDAvK3Meno5gMd3w57L7GEp5B86xl73BdcnFxXgbDL84kai5uoAAA+SSURBVPIhX9s3NDiIH04bwPwfTyQxKpw7/5LHQ/O22mJIrCdBvx7IFpEMEQkFbgLmt9pnPnCr+/H1wBJjjBGRRPfFXEQkE8gGir1Tevc6MzY4t18sWYk9rS5H2YwjSHj4siE8deMINu4/wlV/WEH+oX9NzFV3qpEPNpdx79828Y1nV1BY6b1+/S0Hjpz34qgnCiuPU153us1um3MJdgTxjRGpLLh3En+5bSxj+scyY0gyGQmRna7pzF8FH20/yLbSOt7bUMr3J/WnX/zZ33tw72je/9FF3DU1i7fzDnD5/37Bhn2d//ex0nlvNzPGOEXkHmAR4ABeMcbsEJHHgDxjzHzgZeCvIlII1NL8ywBgCvCYiDiBJuAuY4xf/ott2HeY4uoT3DUty+pSlI1dOyqNjISezP1LHtf9cRXfn9ifTfubg9jpMsRHhuIyhjkvreWtueM7fdLR2ORi7l/zqDnewEu35HJxJ24CXLa7+fralIEd634VEaYMTOzw8W3pExPBiLRefLTtEEvzK0noGco9Fw8473FhwQ4eumww0wcncf87m/nuy+tY84tLiA4P8Vpt3cmjjmZjzEJjzEBjTJYx5rfubY+4Qx5jzGljzLeMMQOMMWONMcXu7fOMMUONMSOMMaONMf/suqZ0rbfXHyAy1MEVF6RYXYqyuZHpMfzzx5PITo7iuaVFVB+vZ+6UTObdfRHr/n0G7941AWMMc15cQ3FV2xOqeWrRjkNUHK0nLjKUu9/YQF5Jx8/Dlu2uYkBST/rERHSqJm+bNaw328rqWF9ymJ/NHERUO8J6bEYc/3vTSE42NPn1iBy9ouiB4/VOFmw7yJXDU9s154ZSHZUcHc7f776IvF/O4JOfTuWB2YO5sF8sjiBhQFIUb945niaXYc5Lazp1w89rq0roG9eDD++dREqvCG57df1Xuow8daqhibV7a78yrNJXXDas+eRsSEp0h66vjUqPJbVXOB9uOejt0rqNBr0HFmwt52RDEzeMSbO6FBVAHEFCwlnWQh2YHMUbd46jwelizktr2F/T/gVQdpQ3n+XeMqEfSVHh/OW2sUSEOrjl5XUcqG3f+63ZW0OD0+WTQZ+REMmvv5HD0zeOxNGBJT+DgoTLL0hh+Z4qv10UXoPeA+/klZKVGMnovrFWl6LUlwb3juaNO8ZzqrGJm15czetr9rVrxavXVpUQEeLgW+6z3PS4HvzltnHUO1189+W17VoTd1lBFeEhQYzNiGt3O7rD9yZmMKgTU5ZcMTyFxibjt0MuNejPo7DyGBv2HeaG3HSd8kD5nJzUaF6/fRyhwUH88v3tTHp8KZf+fhm/XbCTVYXVZ51mofZEAx9sLufa0X3oFfGvPutBvaN45Xu5HDp6mm89v4q31u3nVMP5pwZYvqeKcRnxtl1tbWR6DH1iIvhwq3+uK6BBfx7v5pXiCBKuHd36ZmClfMOwPr1Y+vNpfHr/FH55xRCSo8N5bdU+vv3ntfz8vS1tTjHw1vr91Dtd3Dqh/9deu7BfHK98bwzhIQ4e+vs2JvzXZ/xu4a6zduccqD1JcdUJn+y28RYR4crhKazYU92hmTitpkF/Do1NLuZtLGX64CSSonSBEeW7RJov0t4xOZPX7xjHpkcu5UcXZ/H3jWU84Z4r/wxnk4vXV+/joqz4s3ZnXJSVwEf3TebtueO5KCueP6/Yy9QnlzL3L3lsK/3q1L9nhlW2d/y8v7lyeCpOl2HRDv9b0ESHkJzDpzsrqD7eoHfCKr8TGRbMz2cO4vDJRv70eRG9o8O59aL+AHy6q4LyutM8etXQc76HiDAuM55xmfGUHznFm2v389c1+/jkDyuYMSSZn8zIZlifXizbXUWfmAgyvXCDky8b1ie6eYTS1oPcOKav1eW0iwb9WZxscPLbhbvITIhs13qVSvkKEeGxq4ZSebSeX/9zB0lRYVx2QQqvriqhT0xEuxb0SI2J4OezBvGDqZm8urKEl74o5spnK7g0J5nVRTVcNTLV9tewznTfvLC8mNoTDcR1YJ58q2jXzVn8zye7KT18it9dd4FH83Yo5YuCHUE8O2cUo9JjuO/tzfx1dQlrimv57oR+HRpqGBUewo8vyWbFQ9P56YyBrCmu4Xi9k2k27p9v6YrhKTS5zNdmxPR1mmBt2HzgCP+3ci/fGdeXcZk6HbHybxGhDl6+dQxpsRH86oMdhAUHcWMnuyOjw0O4b0Y2Kx6czku35HZouT9/lJMSTUZCJAu2+dfoGw36VhqcLh58bytJUeE8dNlgq8tRyitiI0N57ftj6RMTwXfG9evQ8nxt6RURwqU5yQR14K8Df3Sm+2Z1UY1fzVmvQd/K88uKKKg4xn9cM6xdc2Io5evS43qw7N+m8asrvz5Fr/LcFcNTcBnOuh6tL9Kgb6Gw8hh/WFLIlcNTmJETGH+KqsAS7Aiy/UXTrjYoOYoBST1Z4Ec3T2nQu7lchgfnbaNHmINfn2fYmVIqcIkIV1yQwtq9tVQcPW11OR7RoHd7dVUJG/Yd5ldX5Jx1IimllAK4emQqxsD7m8qsLsUjGvTAhn21/O6jXVwyOInrdKoDpdR5ZCb2ZHTfGOZtLG1ziglfE/BBX3n0NHe/vpHUmAh+f8NI7b9USnnkmxemsbviONvK6s6/s8UCOugbnC5++MZGjp128vzNF9Krh46yUUp55srhqYQGBzFvQ6nVpZxXQAf9bxfsJG/fYR6/fjhDUqKtLkcp5Ud6RYQwMyeZD7aUU+88/1TOVgrYoJ+3oZTXVu/jjkkZXDUi1epylFJ+6JsXpnHkZCNL8yutLuWcAjLot5fV8Yt/bGN8Zpze/aqU6rDJAxJIigrjvQ2+PfomoGavPFR3mueXFfHmuv3ER4byh2+PJlgnLFNKdVCwI4hrR/Xh5RV7qT5e77NDswMi5Q7WneKRD7Yz5YmlvL5mH9eMTOXduyb47H+KUsp/fPPCNJwuwwebffdOWVue0btchsKq42zaf5i1xbV8uPUgLmO4/sI0fnTxANLjelhdolLKJgYmRzE8rRfvbSjl9kkZVpfTJtsE/eETDbyyci+b9h9hy4EjHKt3As1Xxr95YRo/nJalAa+U6hLfHJ3Go/N3sLP8KDmpvjeCzzZBH+wQXlhWTHZyT64elcqo9FhG9Y0hIyFSb4JSSnWpq0ak8h8LdjJvYyk5qTlWl/M1tgn6qPAQtv56JuEhDqtLUUoFmNjIUC4ZnMz7m8p46LLBPrcqnW9V00ka8kopq1x/YRo1Jxr4ydubqTrmW4uSeBT0IjJbRApEpFBEHmrj9TARedv9+loR6d/itYfd2wtEZJb3SldKKd9xyZAkfjpjIIt3VDDj98t4e/1+n5nw7LxBLyIO4DngMiAHmCMirTuhbgcOG2MGAE8Bj7uPzQFuAoYCs4E/ut9PKaVsRUS4b0Y2C++bzKDeUTw4bxs3vriGwsrjVpfmUR/9WKDQGFMMICJvAVcDO1vsczXwa/fj94A/SPMV0KuBt4wx9cBeESl0v99q75SvlFK+ZUBST966czzvbjjAbxfs4vL//YI+sREIgIDQ/EvBZQxNLoOzyf3ZZRiaGs1rt431ek2eBH0f4ECL56XAuLPtY4xxikgdEO/evqbVsV+b8F1E5gJzAfr27etp7Uop5ZOCgoQbx/Rl+uBk/vR5EVXH6zHGYAAMGAxBIgQHCY6gIEIcgiNI6NtFQ8A9Cfq2xia27ng62z6eHIsx5kXgRYDc3Fzf6NRSSqlOSowK45FvWD/c0pOLsaVAeovnaUDre32/3EdEgoFeQK2HxyqllOpCngT9eiBbRDJEJJTmi6vzW+0zH7jV/fh6YIlpvtw8H7jJPSonA8gG1nmndKWUUp44b9eNu8/9HmAR4ABeMcbsEJHHgDxjzHzgZeCv7outtTT/MsC93zs0X7h1Aj8yxvj2DP1KKWUz4ivjPM/Izc01eXl5VpehlFJ+RUQ2GGNy23rNVnfGKqWU+joNeqWUsjkNeqWUsjkNeqWUsjmfuxgrIlXAvk68RQJQ7aVyrGantoC92mOntoC2x5d52pZ+xpjEtl7wuaDvLBHJO9uVZ39jp7aAvdpjp7aAtseXeaMt2nWjlFI2p0GvlFI2Z8egf9HqArzITm0Be7XHTm0BbY8v63RbbNdHr5RS6qvseEavlFKqBQ16pZSyOdsE/fkWMPd1IvKKiFSKyPYW2+JEZLGI7HF/jrWyRk+JSLqILBWRXSKyQ0Tuc2/31/aEi8g6Ednibs//c2/PEJG17va87Z7G2y+IiENENonIh+7n/tyWEhHZJiKbRSTPvc0vv9cARCRGRN4TkXz3z9CEzrbHFkHv4QLmvu5VmhdQb+kh4DNjTDbwmfu5P3ACPzPGDAHGAz9y/3/4a3vqgenGmBHASGC2iIwHHgeecrfnMHC7hTW2133ArhbP/bktABcbY0a2GG/ur99rAP8LfGyMGQyMoPn/qXPtMcb4/QcwAVjU4vnDwMNW19WBdvQHtrd4XgCkuB+nAAVW19jBdn0AXGqH9gA9gI00r5tcDQS7t3/le9CXP2he6e0zYDrwIc1LfvplW9z1lgAJrbb55fcaEA3sxT1QxlvtscUZPW0vYP61Rcj9ULIx5iCA+3OSxfW0m4j0B0YBa/Hj9ri7OjYDlcBioAg4Yoxxunfxp++5p4EHAJf7eTz+2xZoXof6ExHZICJz3dv89XstE6gC/s/dtfZnEYmkk+2xS9B7tAi56l4i0hOYB/zEGHPU6no6wxjTZIwZSfPZ8FhgSFu7dW9V7SciVwKVxpgNLTe3savPt6WFicaY0TR33f5IRKZYXVAnBAOjgT8ZY0YBJ/BCt5Ndgt6ui5BXiEgKgPtzpcX1eExEQmgO+TeMMX93b/bb9pxhjDkCfE7ztYcYETmzHKe/fM9NBK4SkRLgLZq7b57GP9sCgDGm3P25EvgHzb+I/fV7rRQoNcasdT9/j+bg71R77BL0nixg7o9aLrp+K8193T5PRITmdYR3GWN+3+Ilf21PoojEuB9HADNovkC2FLjevZtftMcY87AxJs0Y05/mn5Mlxpjv4IdtARCRSBGJOvMYmAlsx0+/14wxh4ADIjLIvekSmtfc7lx7rL744MWLGJcDu2nuO/13q+vpQP1/Aw4CjTT/Vr+d5r7Tz4A97s9xVtfpYVsm0fyn/1Zgs/vjcj9uz3Bgk7s924FH3NszgXVAIfAuEGZ1re1s1zTgQ39ui7vuLe6PHWd+9v31e81d+0ggz/399j4Q29n26BQISillc3bpulFKKXUWGvRKKWVzGvRKKWVzGvRKKWVzGvRKKWVzGvRKKWVzGvRKKWVz/x+R4dzubPrqGAAAAABJRU5ErkJggg==\n",
+      "text/plain": [
+       "<Figure size 432x288 with 1 Axes>"
+      ]
+     },
+     "metadata": {
+      "needs_background": "light"
+     },
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "dd = train_md.loc[train_md.fold5==4].copy()\n",
+    "dd['res'] = val_results[:,0]\n",
+    "\n",
+    "plt.plot(dd[['res','pos_idx']].groupby('pos_idx').mean().values)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.7.4"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}