Downloads: 1
Diff of
/ndv/training.ipynb
[000000]
..
[64faee]
Switch to side-by-side view
--- a +++ b/ndv/training.ipynb @@ -0,0 +1,886 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "47 items written into valid.txt.\n" + ] + } + ], + "source": [ + "import torch, fastai, sys, os\n", + "from fastai.vision import *\n", + "import ants\n", + "from ants.core.ants_image import ANTsImage\n", + "from jupyterthemes import jtplot\n", + "sys.path.insert(0, './exp')\n", + "jtplot.style(theme='gruvboxd')\n", + "\n", + "import model\n", + "from model import SoftDiceLoss, KLDivergence, L2Loss\n", + "import dataloader \n", + "from dataloader import data" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "2" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "torch.cuda.set_device(2)\n", + "torch.cuda.current_device()" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "autounet = model.autounet.cuda()\n", + "sdl = SoftDiceLoss()\n", + "kld = KLDivergence()\n", + "l2l = L2Loss()" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "class AutoUNetCallback(LearnerCallback):\n", + " \"Custom callback for implementing `AutoUNet` training loop\"\n", + " _order=0\n", + " \n", + " def __init__(self, learn:Learner):\n", + " super().__init__(learn)\n", + " \n", + " def on_batch_begin(self, last_input:Tensor, last_target:Tensor, **kwargs):\n", + " \"Store the states to be later used to calculate the loss\"\n", + " self.top_y, self.bottom_y = last_target.data, last_input.data\n", + " \n", + " def on_loss_begin(self, last_output:Tuple[Tensor,Tensor], **kwargs):\n", + " \"Stroe the states to be later used to calculate the loss\"\n", + " self.top_res, self.bottom_res = last_output\n", + " self.z_mean, self.z_log_var = model.hooks.stored[3], model.hooks.stored[4]\n", + " return {'last_output': (self.top_res, self.bottom_res,\n", + " self.z_mean, self.z_log_var,\n", + " self.top_y, self.bottom_y)}" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "class AutoUNetLoss(nn.Module):\n", + " \"Combining all the loss functions defined for `AutoUNet`\"\n", + " def __init__(self):\n", + " super().__init__()\n", + " \n", + " def forward(self, top_res, bottom_res, z_mean, z_log_var, top_y, bottom_y):\n", + " return sdl(top_res, top_y) + (0.1 * kld(z_mean, z_log_var)) + (0.1 * l2l(bottom_res, bottom_y))" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "code_folding": [ + 1 + ] + }, + "outputs": [], + "source": [ + "#monkey-patch\n", + "def mp_loss_batch(model:nn.Module, xb:Tensor, yb:Tensor, loss_func:OptLossFunc=None, opt:OptOptimizer=None,\n", + " cb_handler:Optional[CallbackHandler]=None)->Tuple[Union[Tensor,int,float,str]]:\n", + " \"Calculate loss and metrics for a batch, call out to callbacks as necessary.\"\n", + " cb_handler = ifnone(cb_handler, CallbackHandler())\n", + " if not is_listy(xb): xb = [xb]\n", + " if not is_listy(yb): yb = [yb]\n", + " out = model(*xb)\n", + " out = cb_handler.on_loss_begin(out)\n", + "\n", + " if not loss_func: return to_detach(out), to_detach(yb[0])\n", + " loss = loss_func(*out) #modified\n", + "\n", + " if opt is not None:\n", + " loss,skip_bwd = cb_handler.on_backward_begin(loss)\n", + " if not skip_bwd: loss.backward()\n", + " if not cb_handler.on_backward_end(): opt.step()\n", + " if not cb_handler.on_step_end(): opt.zero_grad()\n", + "\n", + " return loss.detach().cpu()" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": { + "code_folding": [ + 1 + ] + }, + "outputs": [], + "source": [ + "#monkey-patch\n", + "def mp_fit(epochs:int, learn:Learner, callbacks:Optional[CallbackList]=None, metrics:OptMetrics=None)->None:\n", + " \"Fit the `model` on `data` and learn using `loss_func` and `opt`.\"\n", + " assert len(learn.data.train_dl) != 0, f\"\"\"Your training dataloader is empty, can't train a model.\n", + " Use a smaller batch size (batch size={learn.data.train_dl.batch_size} for {len(learn.data.train_dl.dataset)} elements).\"\"\"\n", + " cb_handler = CallbackHandler(callbacks, metrics)\n", + " pbar = master_bar(range(epochs))\n", + " cb_handler.on_train_begin(epochs, pbar=pbar, metrics=metrics)\n", + "\n", + " exception=False\n", + " try:\n", + " for epoch in pbar:\n", + " learn.model.train()\n", + " cb_handler.set_dl(learn.data.train_dl)\n", + " cb_handler.on_epoch_begin()\n", + " for xb,yb in progress_bar(learn.data.train_dl, parent=pbar):\n", + " xb, yb = cb_handler.on_batch_begin(xb, yb)\n", + " loss = loss_batch(learn.model, xb, yb, learn.loss_func, learn.opt, cb_handler) #modified\n", + " if cb_handler.on_batch_end(loss): break\n", + "\n", + " if not cb_handler.skip_validate and not learn.data.empty_val:\n", + " val_loss = validate(learn.model, learn.data.valid_dl, loss_func=learn.loss_func,\n", + " cb_handler=cb_handler, pbar=pbar)\n", + " else: val_loss=None\n", + " if cb_handler.on_epoch_end(val_loss): break\n", + " except Exception as e:\n", + " exception = e\n", + " raise\n", + " finally: cb_handler.on_train_end(exception)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": { + "code_folding": [ + 1 + ] + }, + "outputs": [], + "source": [ + " #monkey-patch\n", + "def mp_learner_fit(self, epochs:int, lr:Union[Floats,slice]=defaults.lr,\n", + " wd:Floats=None, callbacks:Collection[Callback]=None)->None:\n", + " \"Fit the model on this learner with `lr` learning rate, `wd` weight decay for `epochs` with `callbacks`.\"\n", + " lr = self.lr_range(lr)\n", + " if wd is None: wd = self.wd\n", + " if not getattr(self, 'opt', False): self.create_opt(lr, wd)\n", + " else: self.opt.lr,self.opt.wd = lr,wd\n", + " callbacks = [cb(self) for cb in self.callback_fns + listify(defaults.extra_callback_fns)] + listify(callbacks)\n", + " fit(epochs, self, metrics=self.metrics, callbacks=self.callbacks+callbacks)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": { + "code_folding": [ + 1 + ] + }, + "outputs": [], + "source": [ + "#monkey-patch\n", + "def mp_validate(model:nn.Module, dl:DataLoader, loss_func:OptLossFunc=None, cb_handler:Optional[CallbackHandler]=None,\n", + " pbar:Optional[PBar]=None, average=True, n_batch:Optional[int]=None)->Iterator[Tuple[Union[Tensor,int],...]]:\n", + " \"Calculate `loss_func` of `model` on `dl` in evaluation mode.\"\n", + " model.eval()\n", + " with torch.no_grad():\n", + " val_losses,nums = [],[]\n", + " if cb_handler: cb_handler.set_dl(dl)\n", + " for xb,yb in progress_bar(dl, parent=pbar, leave=(pbar is not None)):\n", + " if cb_handler: xb, yb = cb_handler.on_batch_begin(xb, yb, train=False)\n", + " val_loss = loss_batch(model, xb, yb, loss_func, cb_handler=cb_handler) #modified\n", + " val_losses.append(val_loss)\n", + " if not is_listy(yb): yb = [yb]\n", + " nums.append(first_el(yb).shape[0])\n", + " if cb_handler and cb_handler.on_batch_end(val_losses[-1]): break\n", + " if n_batch and (len(nums)>=n_batch): break\n", + " nums = np.array(nums, dtype=np.float32)\n", + " if average: return (to_np(torch.stack(val_losses)) * nums).sum() / nums.sum()\n", + " else: return val_losses" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": { + "code_folding": [ + 1 + ] + }, + "outputs": [], + "source": [ + "#monkey-patch\n", + "def mp_learner_validate(self, dl=None, callbacks=None, metrics=None):\n", + " \"Validate on `dl` with potential `callbacks` and `metrics`.\"\n", + " dl = ifnone(dl, self.data.valid_dl)\n", + " metrics = ifnone(metrics, self.metrics)\n", + " cb_handler = CallbackHandler(self.callbacks + ifnone(callbacks, []), metrics)\n", + " cb_handler.on_train_begin(1, None, metrics); cb_handler.on_epoch_begin()\n", + " val_metrics = validate(self.model, dl, self.loss_func, cb_handler)\n", + " cb_handler.on_epoch_end(val_metrics)\n", + " return cb_handler.state_dict['last_metrics']" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "from fastai.basic_train import loss_batch, fit, validate" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [], + "source": [ + "loss_batch = mp_loss_batch\n", + "fit = mp_fit\n", + "validate = mp_validate\n", + "Learner.fit = mp_learner_fit\n", + "Learner.validate = mp_learner_validate" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [], + "source": [ + "def dice_coefficient(last_output:Tensor, last_target:Tensor):\n", + " \"Metric based on dice coefficient\"\n", + " pred, targ = last_output[0], last_target\n", + " return 2 * (pred * targ).sum() / ((pred**2).sum() + (targ**2).sum())" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [], + "source": [ + "auto_unet_loss = AutoUNetLoss()" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [], + "source": [ + "learner = Learner(data, autounet, loss_func=auto_unet_loss)" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [], + "source": [ + "autounet_cb = AutoUNetCallback(learner)" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [], + "source": [ + "learner.callbacks.append(autounet_cb)" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + " <div>\n", + " <style>\n", + " /* Turns off some styling */\n", + " progress {\n", + " /* gets rid of default border in Firefox and Opera. */\n", + " border: none;\n", + " /* Needs to be in here for Safari polyfill so background images work as expected. */\n", + " background-size: auto;\n", + " }\n", + " .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n", + " background: #F44336;\n", + " }\n", + " </style>\n", + " <progress value='0' class='' max='1', style='width:300px; height:20px; vertical-align: middle;'></progress>\n", + " 0.00% [0/1 00:00<00:00]\n", + " </div>\n", + " \n", + "<table border=\"1\" class=\"dataframe\">\n", + " <thead>\n", + " <tr style=\"text-align: left;\">\n", + " <th>epoch</th>\n", + " <th>train_loss</th>\n", + " <th>valid_loss</th>\n", + " <th>time</th>\n", + " </tr>\n", + " </thead>\n", + " <tbody>\n", + " </tbody>\n", + "</table><p>\n", + "\n", + " <div>\n", + " <style>\n", + " /* Turns off some styling */\n", + " progress {\n", + " /* gets rid of default border in Firefox and Opera. */\n", + " border: none;\n", + " /* Needs to be in here for Safari polyfill so background images work as expected. */\n", + " background-size: auto;\n", + " }\n", + " .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n", + " background: #F44336;\n", + " }\n", + " </style>\n", + " <progress value='58' class='' max='288', style='width:300px; height:20px; vertical-align: middle;'></progress>\n", + " 20.14% [58/288 05:56<23:34 176342.1875]\n", + " </div>\n", + " " + ], + "text/plain": [ + "<IPython.core.display.HTML object>" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "LR Finder is complete, type {learner_name}.recorder.plot() to see the graph.\n" + ] + } + ], + "source": [ + "learner.lr_find()" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaQAAAEMCAYAAACLA8K2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3XmYFNW5+PFvb9PLMMywqCiCKIJJqSyiiJQLZru52ZOb3OQmsRLjFqNEgyGJSuISRNSIid7kl7jEpIxZ1GuMWTQuiVsBiiKIFKtsA8g6zML03l2/P6ownZEZGKarq7rn/TxPPdNTp7rr7UMzb59Tp84JWJaFEEII4bWg1wEIIYQQIAlJCCGET0hCEkII4QuSkIQQQviCJCQhhBC+EPY6gGplGnoAOBpo9zoWIYSoIgOBzYqqvWuItySkQ3c0sMnrIIQQogqNBJq77pSEdOjaAS6e+b+k0llPAggAgxob2NPWgdxNVn5Sv+6TOnaX3+o3Hqvj7tsuh256liQh9VEqnSWZynhy7gAQj0ZJpjK++LDVGqlf90kdu6va6lcGNQghhPAFSUhCCCF8QRKSEEIIX6joNSTT0C8DvgKMAxYqqjatpOxG4JOAAjyoqNpXuzz3SOAe4FygFfiRomp3VKpcCCGEuyrdQnobmAvs7w/9OuAa4LfdPPdBYCdwBPAx4FrT0D9RwXIhhBAuqmgLSVG1RwFMQx+5n7JfOWUfBJpKy0xDPw44BzhKUbW9wOumod8DXAA87nZ5ueuBUB0UvBkqLoQQflUtw77HAVsVVdtesm8x8IUKlXcr4Gy9EZtwCVY+RWbZr6GY6+Wz//3cpT9FeUn9uk/q2F1+q98DxVEtCakBaOuyr9XZX4nybg1qbCAejR7osH9T3PoE+fdcSN2Zs4isvI9AtrVXz+9qcNPAPj1f9Ezq131Sx+7yS/3GYnU9lldLQuoAGrvsa3L2V6K8W3vaOnp/Y2xrO+y8gdiEiymcPIP04p9S3LOmd6+B/W1jcNNAWlrbq+Kmt2oj9es+qWN3+a1+E/Gev7xXy7DvN4CjTEM/vGTfRGBZhcq7ZR3qlk+RevVOcs0vED99JuGR0w7pdfoUg2xSvz7YpI77V/32pNLDvsPOOcNA0DT0GFBUVC1rGnoECO3bnLKComo5RdXWmYb+PHCzaejfBMYAF2EPOsDtcvdYZFc/SrF9E7FxFxAceAyZ5b8Bq+DuaYUQwocq3UKaBaSA24CznMdPOWX3OL9fDnzZeXxPyXO/BAwDdgB/A+YoqvZ4Bctdk9/2KskFNxEeqpA4/TsE6vzR3yuEEJUUsKwDNaLE/piG3gi0njd9XvkmV43UE594KcHE4SQXzcPq3Nbj4QFgSNNAdvukf7jWSP26T+rYXX6r30Q8ygN3zQBoUlSt60CyqrmG1D/kOkktuoPCntUkzriaYOOxXkckhBAVIwnJb6wC6aX3kd9skDj9O4QOO9nriIQQoiIkIfmSRWblQ2TWPEZ80nTCw6d6HZAQQriuWu5D6pdy6/+OlWkjNu5rZOoGklv/pNchCSGEayQh+Vx+60JS2b3ET/kGwVgjmRUPceDR/EIIUX2ky64KFHa9SfLlWwkfNZXY+AshEPI6JCGEKDtJSFWi2LaB5MI5hAaPJT7pcghGvA5JCCHKShJSFbE6t5NcMIdA4nDip82AcNzrkIQQomwkIVUZK72H1MK5BMJR4qfPxArXex2SEEKUhSSkKmRlO0i+fBvkM+RO+iaB2CCvQxJCiD6ThFSt8ilSi+YRSO8ifsbVBOqP8DoiIYToE0lI1ayYI7zqPgota0hMuZrgwHetDC+EEFVDElKVC1hFMkvvJb/tVRKnzyTYOMrrkIQQ4pBIQqoJFpnlvyG3eT6Jyd8m2Hic1wEJIUSvSUKqIZkVvyPX/CKJyTMINo32OhwhhOgVSUg1JrPyD+Q2PU/iNElKQojqIgmpBmVWPUx20z9ITL6K0KAxXocjhBAHRRJSjcqu+j+yG54mftq3CA0a63U4QghxQJKQalh29R/Jrv+7nZQGv8frcIQQokeSkGpcds2fyK77G/HTvkV42GlehyOEEN2S9ZD6gezaP2Nl2olNuIjs6iFk18lCf0II/5GE1E/kmp+nmG4hPvFSAvHDyJgPglX0OiwhhHiHdNn1I4Wdy0guuJnwEROIT5oOoajXIQkhxDsq2kIyDf0y4CvAOGChomrTSsoGAD8HPgFkgPuAqxVVs/xQXiuKHc0k588mfuqVJKZ8j9SrP8HKtHodlhBCVLyF9DYwF7hjP2V3AocBxwCTgc8C031UXjOs9B6SC+diZTtITL2W4IDhXockhBCVTUiKqj2qqNqjwPbS/aahJ4AvAtcqqrZHUbX1wG3ABX4o70nA4+2QY8inSL/6Ewq7TBJnXE1o0FjP34sfNz/8G9f6JnXcv+q3J34Z1DAWqAOWlOxbDJxoGnrI63JF1QrdBT6osYF41NtrMYObBh7yc63m/6NAksTpVxFerRNqWVbGyGpDX+pXHBypY3f5pX5jsboey/2SkBqApKJq+ZJ9rUAIiPugfG93ge9p6yCZyvTirZZPAPuD1tLaTp8udLX+gUj7Tqz3foW9y39DvvmFMkVY3cpWv6JbUsfu8lv9JuI9f3n3yyi7DiBhGnppgmwCCkDKB+XdsjzeyhVDduM/SC+5h6jyJSKjP+b5+/LL5od/41rfpI77V/32xC8JaTWQBcaX7JsImE53mdfl/UJ+2yJSr/6YutEfIap8kQP3+AohRPlUNCGZhh42DT2G3VUYNA09Zhp6naJqSeC3wGzT0JtMQx8FfBu4F8Dr8v6ksHsFyYVzCR95GrEJl0DQL726QohaV+kW0izsLrDbgLOcx085ZVcAu4FNwKvAo8BdJc/1urzfKLZvIrngZkKNo4ifeqXcQCuEqIiAZR2oV0/sj2nojUDredPneTqoYUjTQHa7dMEyUDeQ+OSroJAhuegOyPd4Oa3muF2/QurYbX6r30Q8ygN3zQBoUlStrWu5X64hCR+ysu0kX74VAkESp3+HQF2D1yEJIWqYJCTRs1wnyZd/hJVPEz/9uwSiTV5HJISoUZKQxIEV0qQW3YGV2k3ijKsJxId6HZEQogZJQhIHp5gltfguCu0bSUz5HsH6YV5HJISoMZKQxMEr5km//nMKu1cSn/I9gg0jvI5ICFFDJCGJ3rGKpN+4j/y2V0lM+S7BptFeRySEqBGSkMQhsMgs/w3ZTc+RmPxtQkNP9DogIUQNkIQkDll21SNk1z5OfNI3CQ+b5HU4QogqJwlJ9El23RNkzN8Sm3AJkaPP8jocIUQVk4nKRJ/lmp/HyieJjb8IIgly6//udUhCiCokCUmURf7tRaTyaeKnfINAJEF29R+9DkkIUWWky06UTWHnMlKvzKPumPcTPfHLyPIVQojekIQkyqqwZw3Jl28lPGwSsYmXQjDidUhCiCohCUmUXbF9E8n5cwg2DCdx+kwCkQFehySEqAKSkIQrrNROkgvmYFlFElOvJZA43OuQhBA+JwlJuCfXSeqVH1Fo20jijGsJNh7ndURCCB+ThCTcVcyTXvIL8lteIjFlJuEjJnodkRDCpyQhiQqwyKx8mMzKh4lNvJTIMe/3OiAhhA/JfUiiYnIb/4GVaiE28RJCg0aTXv4g5Dq9DksI0RuBkP3TKpT9paWFJCoqv2MJyZduJJg4gvqzbpSJWYWoMuFhk6g/+yZXXlsSkqi4YufbJBfMIbfpeeKnXmHfRBuq8zosIcRBCEQSWPmkK68tCUl4wyqQXfs4yQVzCA15D/Vn3iBrKwlRBQKRBJZLXe2+uoZkGvpo4E7gDCAH/BK4VlG1omnoYeB24DzsRPoIcJmiahnnua6WC3cU2zaQfOkGoid8lsSU75Jd9yTZNX9ypX9aCNF3gXA9Vq7GW0imoYeAx4E3gSOBU4GPADOdQ64BzgFOAsYAJwI3l7yE2+XCLcUcmRW/I7VoHpHhZ5A442oC8cO8jkoIsT8utpB8k5CAE5ztB4qqZRRVawbuAL7ulF8IzFZUbauiajuB64HzTUMPVqhcuKyweyWdL11PMd1K/ZnXER52qtchCSG6CETcayH5qctuf1NDB4BRpqGPBEYAr5eULQaagBGmobe5WQ5s7Clor+a0DnT5WRNynWQW/y/FY95HbPxF5IcqZMzfQzFb8VBqsn59RurYXW7UbyCSgFznIb3mgZ7jp4S0CngL+KFp6NcBRwBXOGWW87Ot5PhW52cDUHS5vFuDGhuIR6M9HeK6wU0DPT2/K9pepbjsbYInfIW6s68jvOpXBFPbPQmlJuvXZ6SO3VXO+s3GGohFLEKH8JqxWM+jaX2TkBRVy5uG/nHgx9gtkhbgPmAc/0oYjcAu53GT87PD2dws79aetg6SKW/GPQSwP2gtre3vZOya0toO268jeuKXKY6bQWb5b8lvfrFip6/5+vUBqWN3uVG/iWCMVNtu8q3tvX9uvOcv7766PqKo2mpF1T6iqNrhiqq9B0gCixRV2wI0AxNKDp+I3YppVlSt1c3ynmK2PN78EIOrWyFD+o37SL/5AFHlf4ic8Fmp3xrbpI6rq34DkQTFfGef4umOb1pIAKahjwPWAWngXGAWoDnF9wLXmoa+AHtI+PXA/YqqFStULjyU3zKfZOd2EpO/Dbm9ZNc96XVIQvRDAQjH+8WgBoDPAd8AYsBK4EJF1Z52yuYAQ4Hl2C27h7GHalOhcuGxYutbpBb/L/FJ38TKJck1v+B1SEL0L+E4gUDQtWHfAcs6UCNK7I9p6I1A63nT53l6DWlI00B297P+9/Cw04hNuMhe1mLba66dp7/WbyVJHbur3PUbiB/GgHNvoePp6Yc0MXIiHuWBu2YANCmq1ta13FfXkIQ4GPlti8gsf5DY+IsJDVW8DkeIfiMQSdgPan2mBiF6I9f8PNk1fyJ+ynSCTbISrRCV8K+bYt1pz0pCElUru+5v5Db+g8SpVxIcMNzrcISoefbEqu60jkASkqhymVUPk9u+mPjkqwgkDvc6HCFqmt1Ccm9RTUlIoupllv2aQssq6qfOIjT4BK/DEaJmBSLuDfkGSUiiJlikl9xNduOzxCdfRWTEOV4HJERtitRj5d1rIfntPiQhDpFFds2fKO7dSmzcBQQbhpNZ8Xuw5L5mIcrF7WtIkpBETcm/vYhkcifxSdMJ1g8j9frPwaXlloXobwLheqz0btdeX7rsRM0ptm0gafyQQCRB/dRZBOqP8DokIWqCjLIT4hBYmVaSC2+h0Lae+qmzCB95mtchCVH13FycDyQhiVpWzJFeeg+ZtX8hdvL5JGQUnhB9IteQhOij3Pq/k9+ygLrjP0Z88lUUdplkVj1CsWOz16EJUVXkPiQhysDKtpMxf0vnC7Ow8ikSZ15PbNzXCMQGex2aENVDriEJUT5WcgfpJb8gOX82gdhg6s+5mbrjPuJ1WEL4XzhmLz0h9yEJUV7Ftg2kXvkRocPHEx9/MYH4YDLLH8StSSOFqHaBcL39wM8tJNPQB5QjECG8UNixlOTLtxAeNonYxEshKN/RhNiffUtP+KbLzjT0q0xD/3zJ7zrQZhr6etPQ31v26ISogGL7JpLz5xAaOIL4aTMgHPc6JCF8JxCpx8qnwSq4do7etpAuBbYBmIZ+FvAZ4IvAIuDW8oYmROVYqZ0kF8whEIqSmPI9AtEmr0MSwlfcHvINvU9IRwHrnccfBR5RVO0PwA3AlHIGJkSlWdkOki/fhpVpI3HGNTLDgxAl3B7yDb1PSJ1Ao/N4GvBP53EKSJQpJiG8U0iTevUnFPasITHlGooNshqtEIDrQ76h96PsngduNw39JWAi8KSz/wSguZyBCeEZq0B66b1ET/gvAiddTuyIZWTW/Ili2wavIxPCM3aXnb9aSFcAaexrR5coqrbd2f8R4JlyBiaEtyyyqx4h8vpcrFwniamziJ96BcHGY70OTAhPBCL1rs+c36sWkqJqW4BP7Gf/9HIEYxr6UcBdwDlAAHgJuExRtc2moYeB24HzsBPpI05Zxnmuq+Wifwqmd5BZei+ZtX8mOvrjJKZeS2HncjJr/0SxdZ3X4QlRMYFIAivT5uo5ejvsO2oaerTk9+GmoV9mGvq0MsXzMyACHAuMwL5mdY9Tdg12ojoJGAOcCNxc8ly3y0U/ZnVuJ/3GvXS+cC1Wtp3ElKuJTfwGhGNehyZERQTC7s70Db3vsnsMuATeuSH2FWA28LRp6F8tQzzHAX9QVK1DUbUk8CBwslN2ITBbUbWtiqrtBK4HzjcNPVihciGcxHQfnS/OIlh/uD0aLz7U67CEcJ0fh31PAp5zHn8K6ACOwE5SM8oQzzzgv01DbzINvQG7++yvpqE3YbeYXi85djHQBIxwu7yngAMeb36IoZa37uqXzu2kFtyM1bmD+qnfJzRojOexVuvWXR3L5rP6jdRDrrMs8XSnt6PsBgItzuP3A48pqpY1Df0Z4M5evtb+GMAFzjksYKlznganvLQDs9X52QAUXS7v1qDGBuLRaE+HuG5w00BPz1/reqpfa90DFEb+J4nTZxJe9zChHS9XMLLaIZ9hd5WjfjPRegZEIdSH14rF6nos721C2gKMMw39beBDwPnO/iagTxf/na6xp4GHgP9wdl8PPAF82Pm9EdhVck6wW2kdLpd3a09bB8mUN+MeAtgftJbWdpkS1AUHXb+tvye8awPRk88nGRxMduVDyCStB0c+w+4qZ/3Wh+K079lJsbX9kF8jEe/5y3tvu+zuA34HLMdOQPtujJ0MrOxtcF0MBo4B7lRULelcQ7oLOB07cTYDE0qOn4jdimlWVK3VzfKegrY83vwQQy1vB1u/ua0LSS68hfDwKcROvQIrHPc89mrZDraOZfOwfkN1BIJhrFyyLPF0p7fDvueYhr4SGAk8pKhazikqAj/qzWvt57V3mYa+FrjMNPQbnN3Tgc1O2b3AtaahLwBy2K2n+xVV29fd5na5ED0qtq0jadxIfNI3SUydRfr1n1PskPvFRfULROylJ9y+MbbXc+0rqvbofvb9sjzh8EngDuyuwQCwhH/d9zQHGIrdOgsCD2MP1aZC5UIckJXeQ3LhzUTf+z8kps4is+oRchue4cDfDYXwr3eWnnD5xtiAZfXuP4pp6GOB72Dfp2Nh/wG/VVG1NeUPz79MQ28EWs+bPs/Ta0hDmgayW/rfXdHX+g0Pm0TspK9QaFtPeul9WNlD73uvVfIZdle56jc0aCzxyTPY+/ev9ymeRDzKA3fNAGhSVO1dd9n29sbYDwLLsK+vLMS+D+kUYJlp6O/vU6RC1Jj8ttfofOk6CEZInHUDocNOPvCThPChStyDBL3vspsD/D9F1a4s3Wka+k+wZzWYXK7AhKgFVnoPqZdvo270R4lPmk5u4z/JrHoYinmvQxPi4FVg6QnofUI6Cfjyfvb/DLi47+EIUYsssm/9hfxuk/j4iwkNPZHs2sfJv70IubYkqkGlWki9Hfbdwf5nLjgGkA5yIXpQbF1Hp3E9+W2vETvpPOrPmUPk6LMg2OuxRUJUVCCSAB+2kP4I3G0a+qXAi86+s7FbSO8afSeE6CKfJrvmj2TXP0HdyGnUnfAZ6sZ8kuz6v5Nrfh4KWa8jFOJd7NVi/XcN6SrgfuzZE0r7Gh4GZpYrKCFqXj5Ndt2TZDc8S+ToM6k77sPUHf8xcuufJrv+SbnGJHwlEElgZX3WQlJUbS/wOdPQRwOKs3s59pIRr5TsE0IcjGKO3KZ/kmt+gfCRk4mO/RSRo6eSXvYrCi2rvY5OCMBuIRWTO10/zyF1Xiuq9hbw1r7fTUMfj72MuRDiUFgF8lsXkN/+GtExnyI+eSa5zS+RWfmw66t0CnEggbA/BzUIIdxUyJJZ+RDJ+bMJNY6i/uzZhIdN8joq0d9VaNi3JCQhfKjYvpHk/NnkNjxFbPxFxCZNJxAb5HVYop/y67BvIUSlWAWy656k88UfEAhFqT9rNuGjpngdleiH9i3O57aDuoZkGvpTBzhkQBliEULsh5XcQeqVHxEZOY3YyV8hf9hJpJf/BvJpr0MT/UEwTCAU8dWw7y0HccyqvgQihOhZbtNzFFpWE5twCfVn3kBqyS8otq7zOixR4yq19AQcZEJSVO38Ax8lhHBbce9WkvN/SPSEz5KY8j2yax4n+9ZfkSmIhFv+lZD800ISQvhFMU9mxe/J71pObNzXCA09kfTSu7HSe7yOTNSgQCSBVcxD0f1ZRGRQgxBVqrBzGckXr4NClvqzbiR02DivQxK1qEJDvkESkhBVzcq2k3r1x2Tf+ivxSZdTd/wnsJdlE6I8KnVTLEiXnRA1wCK77kkKbRuJTfg6ocZRpJbeA/mU14GJGlCpId8gLSQhakZh9wqSxg0Eoo3Uqz8gOGC41yGJGlCpm2JBEpIQNcVKt5BceDOFltUkps4ifORpXockqpydkCrTQpIuOyFqTTFPetn9RFrXERt3IbnGY8msegSsoteRiSpUyRaSJCQhalSu+XkKHc3ET7mcQGwQ6aX3SFISvRepr9gtBdJlJ0QNK7auI7lwLqFBY4hN+DoEQl6HJKpMv2whmYa+t8uuKLBCUbVxTnkYuB04DzuRPgJcpqhaphLlQlQrK7mD5MJbSEyZSWzipaSX/FxWpBUHLdAf70NSVG1A6QasAH5fcsg1wDnAScAY4ETg5gqWC1G1rNROkgtuITRwBPFTLoOgb76LCp8LhBPQn0fZmYY+GXs59F+V7L4QmK2o2lZF1XYC1wPnm4YerFD5fgU83vwQQy1vtVS/pHeTWjiXYP0w4pO+SSAY8TymWqtjP259rl/nPqRyxtMdv35NugB4QlG1rQCmoTcBI4DXS45ZDDQBI0xDb3OzHNjYXaCDGhuIR6OH8h7LZnDTQE/PX+tqq34LWOZPyZ14GZEpM4isvJdAMed1UDVWx/5zqPVrBUJkw1EGxgIEy/BvFIvV9Vjuu4RkGnoC+AKglexucH62lexrLSkrulzerT1tHSRT3lxmCmB/0Fpa22WuZxfUbv22E5h/M7HTZ5IfcwGpV38CBfkM16K+1m+groF6oLVlB1a6vc/xJOI9f3n3Y5fdfwNJ4K8l+zqcn40l+5pKytwu75bl8eaHGGp5q9X6LWbbSb18K0TqiZ8+EytSL3Vco1uf6tdZeqKYS5Y1nu74MSFdCPxaUbV3hgEpqtYKNAMTSo6biN2KaXa7vEzvSwhfsbIdJBfeAsUCiSnfIxBtOvCTRL9iLz1RgEJlVif2VZedaegnAFOBr+2n+F7gWtPQFwA57EEH9yuqVqxQuRC1J58kueh24hMvI3HGNSQX3Y7Vud3rqIRPBCL1WPnKjLAD/7WQLgBeVFRt9X7K5gAvAcuBtYCJPVS7UuVC1KZCltRrd1JofYvElKsJDhzpdUTCJwKRyg35BghY1oF69cT+mIbeCLSeN32ep4MahjQNZLdcEHZF/6vfANETv0TkqCmkXr2Twp79fS8s9xn7Wx1XVl/rN3LM+4gMn0py/uyyxJOIR3ngrhkATYqqtXUt91sLSQjhGYvM8t+Q3fAM8ckzCB0+3uuAhMcquTgfSEISQnSRXfMYmVWPED/lMsJHn+l1OMJDlZw2CHw2qEEI4Q+5Dc9gZTuInfw1svVHkF31KAcetCtqTgUnVgVJSEKIbuS3vkwqtZvYKdMJJo4gvfReKGa9DktUUCCSoNi5rWLnky47IUS3CnvWkpw/m2DDUSSmfJdAtPHATxI1wx5lV7kuO0lIQogeWamdJOffhJVPkpg6i2DDCK9DEhViX0OSQQ1CCD/Jp0gt+jH5nctInHE1ocPGeR2RqAB7cT4Z1CCE8BurQOZNnWLnNuKTLie34Vmy657AyvZ90k3hT4FwZVtIkpCEEL2SW/8UxfbNRN/zWeqPmUZu03Nk1z2JlXnXfY6imgWCBCJxaSEJIfytsNskadxI6LBxRMd8gvpp55Lb9LzdYsq0HvgFhP+F4wDSQhJCVIfCzjdI7nyD0NCTnMR0C7nmF8i+9VdJTFUu4Cw9IQlJCFFVCrveJLnrTUJDFaJjPkX92bPJrHyYXPMLyA211SkQSWBZRcinKnZOSUhCiLIp7DJJ7jKJHH0W0fd+nvBRk0kv+zVWcofXoYleCkTqnWRUuS8UMuxbCFF2uc0v0vnCLKxckvqzbqTuuA9DQP7cVJNAhacNAmkhCSFcYmVaSS/+KeFhk4ie+GXCR04m/cb9FDtkEeZqUOmJVUFaSEIIl+W3vUbnC7MotjeTUL9P3eiPeR2SOAjSQhJC1KZcJ+ll9xN6+xXiEy4h2HAU6Tfuh2LO68hEN+ybYqWFJISoUYVdy+lccBOhxlEkTp9JoG6g1yGJ7njQQpKEJISoKKtzO53zb8Iq5IirsygmjvQ6JLEf9kzfkpCEELUu10lq0TwKu0xyJ18hk7X6UCBSj5WXLjshRH9gFcgs+xWh5ieJTbqcyKgPeB2RKCGDGoQQ/U5463Ps3bWJ2ISLCQ0cSdr8LeTTXofV73kx7Nt3Cck09I8DNwBjgQ5gnqJqt5mGHgZuB87Dbtk9AlymqFrGeZ6r5UII9xR2LCG5YA6x8RdRf9YPSS+7n8Iu0+uw+jUvWki+6rIzDf3DwM+BmUATcALwhFN8DXAOcBIwBjgRuLnk6W6XCyFcVOzYTHL+D8ltnk/81CuJnqRBOOZ1WP1UAMKVXXoCfJaQgB8CsxVVe1ZRtbyiau2Kqr3plF3olG1VVG0ncD1wvmnowQqVCyHcVsyTXfNHkvNvIjToeOrPupHQUMXrqPqfcJxAINh/ryGZhl4PTAJ+bxq6CQwB5gNXAO3ACOD1kqcsxm5FjTANvc3NcmBjd3EHnM0LgS4/RXlJ/bqvuzq22jeSMm6k7viPEz/1SvKbXyKz8iG5ttRLh/oZDkQS9oNcsqyf/wO9lm8SEjAIO96vAB8BdgB3AP8HfMY5pnRJyn2LrTQARZfLuw+6sYF4NNrTIa4b3CQ3F7pJ6td93dbxjmcpdq4mePz/EJl2C6G3nyO0zSBQkMTUG739DBfrDycHDK4PE6B8n/9YrK7Hcj8lpA7n552Kqm0AMA39WmAnUHDKGoFdzuOmkud1uFzerT1tHSQK4K5jAAASLklEQVRT3ox7CGB/0Fpa22XFGRdI/brvoOq4dTlsvY7w8KnUHfef5I96P7lNz5Fb/xRWtr2C0VafQ/0Mh0IWsVyKltbyLrKYiPf85d0310cUVWvD7horrbfSx83AhJLfJ2K3YpoVVWt1s7ynuC2PNz/EUMub1K9P6tgqOEtaXGvPiTdUIXHubdSddB4kDvf8Pfh5O6j67fqcSAIr3+laPN3xUwsJ7BF2V5iG/hR2y+iHwKuKqm01Df1e4FrT0BcAOexBB/crqravu83tciGE5yzy214jv+01QkPeS93oj1B/zhzy2xaT2/wShV1vgiX/ZfvKvgepsgMawH8J6Vbsa0mLsVtvLwH/5ZTNAYYCy52yh7GHalOhciGEjxR2ryC1ewXBxlHUHfM+4hO/jlXIkN+6kNzm+bLuUh8EBwyn6MEqvwHLOlAjSuyPaeiNQOt50+d5eg1pSNNAdss1DldI/bqvrHUcqiN8xClEhk8lNFSh2LGZ3Jb55Lcs7LfXmg61fhPqdeS2GOQ2PFPWeBLxKA/cNQOgyblM82/81kISQohDU8iS37qQ/NaFBGKDiBx1BpERZxMd+1/kmp8n+9ZfsTLv+hsougrHCQ4cQeGNVZU/dcXPKIQQLrPSe8iu+xvZdX8jNEQhOvZT1E+7hdymf5J964l+22I6GKFBYyCfptixueLnloQkhKhphd0myQUmocNOJjrmU9RPm0Z247Pk1j2JldvrdXi+Exo8lsKeNRx4TFz5SUISQvQLhZ3LSO5cRujwCUTHfoq6Y95HdsPTZNc9ITNAlAgPHkt++2Jvzu3JWYUQwiOFHUtI7lhKeNgp1I39NJER55Bd8xi55hdkyHiojmDjKPLm7z05vSQkIUQ/5NzPtP11IiPOpm7Mp4iM+gCZlQ9T2LHU6+A8E2oaDcUCxfZup+90lSQkIUT/ZRXtaYi2LqTuuI8Qn3gphT1ryaz4Q7+8jyk0+AQKrWvBKhz4YBdIQhJCiHya7OpHyW16jugJnyFx5nXkt75MfvcKih1bKO7dAoWs11G6LjR4LIXdKzw7vyQkIYRwWOkW0kvvJbj+aeqO+w/qjv0wwfojCARDFDt3UNi7xU5Q7ZvIb3/ds5aEK4JhQk2jya55zLMQJCEJIUQXxfaNpJfcbf8SDBOsH0aw4WiCDcMJNRxNZOQ51I3+KOk37vPkfh03hBqPBaDQus6zGCQhCSFET4p5ih2b/z3xhBPElC+QUH9Adu1fyL7116pvLYUGj6XQtg6Kec9i8M3yE0IIUTXySdJv/JLUa3cRGXE2CfX7BBtGeB1Vn4QGn0ChZbWnMUhCEkKIQ1TYuYzOF79PoW0DCfX71I35JARCXofVe4EgoUHHe56QpMtOCCH6Ip8is+xX5N9eROzkrxI+4hQy5u8otKz0OrKDFhw4EoIRCnve8jYOT88uhBA1orBrud1aallFfPIM4qd9q2q68UKDT7Bvhi14O4WSJCQhhCiXfJqM+Vs6n78WK7uXxJnXERt/EYH4YV5H1qPQ4LEUWiq/3ERXkpCEEKLMrNRO0kvvIWncSKBuAPXn3ERU+SKBuoFeh7YfAcKDxpD3+PoRSEISQgjXFNs3kVp0B6lX5hFqOo76aXOJvufzBGJDvA7tHcGG4RBJUGhZ43UokpCEEMJthZaVJOfPJr3kboJNx1I/bS6xCV8n2Hic16ERGjzWvscqn/Q6FBllJ4QQlZLfsYT8jiUEG4+l7tgPkTjjaoqt68hueIr8tsV4sSieH+4/2kdaSEIIUWHFtvWkl/yCzue+S2HPWmInn0/9tLlERn0QwvGKxuKXAQ0gLSQhhPCMlW4hs+phMmv/TGTEmdQd836iYz9NbvNLZDc8g5Xc4er5A/XDCEYbfXH9CCQhCSGE9wppchueIbfhWUKHj6du1AeoP2cOhR1vkN3wtGtLQoQHj6Ww922sbLsrr99bvklIpqH/CvgiULroyLmKqi1yysPA7cB52F2NjwCXKaqWqUS5EEK4z6KwYwmpHUsINhxNZNQHiZ96JcXkdrLrnyK/ZUFZJ3H1U3cd+CghOX6mqNqV3ZRdA5wDnATkgMeBm4EZFSoXQoiKKXZsJrPsfrKrHiEy4hyiJ3yW6NhPk13/FLnm5yHf91kVQoPHkln1aBmiLY9qGtRwITBbUbWtiqrtBK4HzjcNPVih8v0KeLz5IYZa3qR+pY693sh2kHvrLyT/OZPs2j9TN/JcBpz7I6JjP0OwbuAh128wNoRgfCjFllUV//fujt9aSJpp6BrwNvBL4A5F1YqmoTcBI4DXS45dDDQBI0xDb3OzHNjYXcCDGhuIR6OH8l7LZnCTH+/+rh1Sv+6TOj5I7Yuxlr5Occh4gsPfT91x/0FwxyuEts8n0Lml2z/4XevXCtZRGP4+CundDI4VIFaZ+o/F6nos91NCuhOYCbQAk4E/AEXgDqDBOaat5PhW52eDc5yb5d3a09ZBMuXNZaYA9getpbXdg7sXap/Ur/ukjg9R6wvw1guEhihEjvswoXFXYWXaKexcRn7nGxR2mZBP/Vv9UjeQ0OHjCR8xgdDQE6GQJbvmMfa2Vm5AQyLe85d33yQkRdUWl/y6wDT0uYCGnZA6nP2NwC7ncZPzs6MC5d2y8OJWNv/FUMukft0ndXxo8rtN8rtNiNQTHnoS4cNPJnqiRiASp7BnjZ2gYjFi73kvwUGjsVK7yW9fQnbRjynsWVPxVW4P9G/s52tI+1otKKrWCjQDE0rKJ2K3YprdLi/XGxJCCFfkOsm//TLppffS+eyVJBfMpdCymvCwUykOnUB+15skX7qezue+S2aFs1aTD5dc900LyTT0/waexG6RTAK+B/y05JB7gWtNQ1+APQrueuB+RdWKFSoXQogqYFFsW0e2bR25NX9iSNNAOqqkS9Q3CQm4HLgbO6YtwM+w7wvaZw4wFFiO3bJ7GHuodqXKhRBCuChgWdWQN/3HNPRGoPW86fM8HdQwpGkgu6vk20+1kfp1n9Sxu/xWv4l4lAfumgHQpKhaW9dyP19DEkII0Y9IQhJCCOELkpCEEEL4giQkIYQQviAJSQghhC/4adh3VYofYG4mNwWw54ZKxKO+GEFTa6R+3Sd17C6/1e+B/l5KQjp0AwHuvu1yr+MQQohqM5B/nzsUkITUF5uBkYA/lloUQojqMBD77+e7yI2xQgghfEEGNQghhPAFSUhCCCF8QRKSEEIIX5CEJIQQwhdklJ2HTEO/DPgKMA5YqKjatD6+3seBHwEjgDeBixVVW1JSfjT2kh4fwr5FYYWiamf05Zx+Vsn6NQ19GvBPoLPkKfcqqnZlX87pd5X+DJccdzP2mmmfVlTtsb6c088q/Bn+GDAXGI69uOtrwAxF1Zb15Zy9IS0kb72N/QG4o68vZBr68cDvgJnAIOBR4C+moced8nrsP5grgFHYaz/V9B9LKli/jjZF1QaUbLVev1D5OsY09PHAJ5xz17pK1u9i4IOKqg0CDgf+AlQ02UsLyUOKqj0KYBr6yK5lpqEfht2a+QB2a+ZPwFWKqnV2PdahAc8pqva48/xbgenAh4E/Al8FWhRVu77kOS+X5Y34VIXrt1+qdB2bhh4C7sFe0PP+sr4ZH6pk/SqqtrXk2AB2K+kY09AjiqrlyvSWeiQtJB8yDT2A/c2kBRgDvBf7Jty5PTxtHPD6vl+cpdeXOPsBzgFWm4b+qGnou01DX2Ia+qfdiN/vXKpfgAGmoW81DX2zaegPmoY+vOzBVwkX6/hKwFRU7Z/ljrmauFW/pqGPNA29FUgDPwbmVCoZgSQkvzoV+wN2laJqnYqqtQLXA1/q4TkNvHsqjlZnP8Bg4IvAb4FhwHeBB53uj/7GjfpdCUzA7ps/Ffsb5p9NQ++v/8fKXsemoR8LfBP4dtmjrT5ufIZRVG2TompNQBNwBSUJrBKky86fRgGNwG7T0PftCwBR09AHYF+U/LKz/0VF1f4T6HCeU6oJWO087gAWKKr2iPP7301D/wfwUWCpG2/Cx0ZR5vpVVG0bsM3Zv8009Iux//OPxU5W/c0oyv8Z/gXwA0XVdrkYd7UYRfnr9x2KqnWYhv5TYJdp6EsUVVtf/rfwbpKQ/GkjsFNRtWHdlH/d2Uq9AUzc94vTpB8P3OfsWgq8v8xxVis36rer/j4nlxt1/EFgnGnotzi/Hwbcbxr6uYqqXVG2yKtDJT7DASCKnfwkIdU609DD2P8GYSBoGnoMKAKvAuudoa1zsSdwHQ5MVFTtz928nA7McIZuPsW/RtA94fz8NTDTNPRPAn8GzgXeB1xb9jfmE5WsX9PQzwU2ONsQYB6wHFhT9jfmIxX+DB/Z5fhFwPep4UElFf4Mfx57qPc67AlQZwNJ7NF3FdFf+7f9YhaQAm4DznIeP+VcbPwE9jfAN7G7fp4GTuzuhRRVW4t9jWiec/xngY8rqpZyytcDnwZuwv7w/hj4kqJqtdxdV7H6xf7m+QKwF1gGRICPKapWKP/b8pVKfoa3lW5AAWhVVO1dyxjUkEp+ho8FnsHu2luN3TL6YCXrV2b7FkII4QvSQhJCCOELkpCEEEL4giQkIYQQviAJSQghhC9IQhJCCOELkpCEEEL4gtwYK0SVMQ19FPad82cpqvaSx+EIUTaSkITowjT0XwFHK6r2Aa9j6UYz9qwFu90+UUny26cDWAvcoajaA718rVnAhYqqjSpbgKKmSJedED5hGnrdwRynqFrBma2gYssCAJ/EToKnYC/sppuG/qEKnl/0A9JCEqKXnPnFZmEvLX0k8BZwp6Jqvyg55grgfOB47OmEngO+paja2075NOwVfD8GXI29nMC3TUPfC9yLvX7VXcB7sOfE+7qiaq85zx1FSZddye+fB87DnkR3G3BdaSvGWb7hF8DZwA7gFuBzwFpF1S48wNtucabrAZhtGvq3gP/AnhNt30Sdd2PPj3gU9kqnvwduUFQtYxr6V4EfOsfumx7mBkXVrj+Y+hT9g7SQhOi9e4HPAJdgr0lzI3CLaegXdDnu28DJ2HMIjsT+A93V7cCtzuvsWy46CNyMvR7NKcAe4CHnD3dP5gIPYC+49hD2TNhj4J2E8Ufs5QfOxp4H7aOUzP58MExDD5mG/gXs9bWyJUUBYDv2XGnvxZ6483zgGqf8D9gJcDN20jkSe4kEOPj6FDVOWkhC9ILTytAARVG1fescrTcN/QTs5aDvA1BU7SclT1tvGvplwGLT0IcrqralpOymfUtKO68P9h/3KxVVW+zs+wGwABgNrOohvP9VVO0h5zmzsJf5fh/2jOMfwF5qYIwzySamoX8ZO0EcjKdMQy8CMSAE7MReShzn/RaxWzn7bDANfTTwDeyWWspp/RVKWloHXZ+if5CEJETv7FsN9tWShdHA/r/0zszeTpfc1YCCvQjavt6IY4DShPTKfs5h8e+LJu47/gh6TkhL9j1QVC1vGvp25zk4cezal4ycY1pMQ+/p9Uqdj700wbHYs0XfoKjautIDTEO/CLgQe5boepwlEw7wugdVn6J/kIQkRO/s+wM7FXutmFIWgGnoI4G/YXef3QjsAo7Gntq/68CFzv2co9hl2Yp911wO9Mc92+V3q8tz+jK1/xYnma11uuwWmob+5r5WjWnonwN+CnwPeB57iZPPYS930pMD1qfoPyQhCdE7rzk/Ryqq9pdujjkNiGN3u6UATEOfVIngemACh5mGfnxJl90g7CXWX+vxmV0oqrbcNPQ/Y6/R83Fn99nA64qqzXvnhPZgi1JZ7O6+UgdTn6KfkIQkxP4NMA19Qpd9aUXVVpqG/kvgHtPQv4N9bacemAQcpqjaLdjXbCzgKtPQH8S+dvODCsa+P89gdwPqzgjALHbrJc+htURuw74mpiqqZmB3JV7grEj8Jvbowc90ec56YJhp6Gdg11FSUbW1B1Gfop+QUXZC7N/pwOtdtn2j4C4G7sBe/t0EnsUesrwOQFG1N7AvyF/ilH+bfy0X7QlF1Szs0X6dwIvAX7CXrl4FpA/h9V7HTnJznV2/wO6ivB+7rk4Hru/ytMeAh4G/Yg+K+I6zv8f6FP2HrBgrRD9lGnoD9ii7WYqq3eV1PEJIl50Q/YRp6J/A7qJbARwOXIfdXfeQl3EJsY8kJCH6jwT2taxR2F13rwFnKqq23cughNhHuuyEEEL4ggxqEEII4QuSkIQQQviCJCQhhBC+IAlJCCGEL0hCEkII4QuSkIQQQvjC/wdV5HIuJYQVnQAAAABJRU5ErkJggg==\n", + "text/plain": [ + "<Figure size 432x288 with 1 Axes>" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "learner.recorder.plot()" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "PosixPath('/home/ubuntu/MultiCampus/MICCAI_BraTS_2019_Data_Training/models/trained_model_fit.pth')" + ] + }, + "execution_count": 20, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "learner.save(\"trained_model_fit\", return_path=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "learner.load(\"trained_model_fit\", device=2)" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [], + "text/plain": [ + "<IPython.core.display.HTML object>" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/plain": [ + "[10493.069, tensor(0.6972)]" + ] + }, + "execution_count": 19, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "learner.validate(metrics=[dice_coefficient])" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [], + "source": [ + "learner.metrics = [dice_coefficient]" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "<table border=\"1\" class=\"dataframe\">\n", + " <thead>\n", + " <tr style=\"text-align: left;\">\n", + " <th>epoch</th>\n", + " <th>train_loss</th>\n", + " <th>valid_loss</th>\n", + " <th>dice_coefficient</th>\n", + " <th>time</th>\n", + " </tr>\n", + " </thead>\n", + " <tbody>\n", + " <tr>\n", + " <td>0</td>\n", + " <td>10249.201172</td>\n", + " <td>10427.675781</td>\n", + " <td>0.699438</td>\n", + " <td>31:40</td>\n", + " </tr>\n", + " <tr>\n", + " <td>1</td>\n", + " <td>9863.964844</td>\n", + " <td>10486.278320</td>\n", + " <td>0.741603</td>\n", + " <td>31:53</td>\n", + " </tr>\n", + " <tr>\n", + " <td>2</td>\n", + " <td>9876.774414</td>\n", + " <td>10650.123047</td>\n", + " <td>0.712295</td>\n", + " <td>31:48</td>\n", + " </tr>\n", + " <tr>\n", + " <td>3</td>\n", + " <td>10385.082031</td>\n", + " <td>10679.890625</td>\n", + " <td>0.735404</td>\n", + " <td>31:32</td>\n", + " </tr>\n", + " <tr>\n", + " <td>4</td>\n", + " <td>9652.380859</td>\n", + " <td>10485.993164</td>\n", + " <td>0.668722</td>\n", + " <td>31:21</td>\n", + " </tr>\n", + " <tr>\n", + " <td>5</td>\n", + " <td>9648.621094</td>\n", + " <td>10375.062500</td>\n", + " <td>0.731086</td>\n", + " <td>31:28</td>\n", + " </tr>\n", + " <tr>\n", + " <td>6</td>\n", + " <td>9282.785156</td>\n", + " <td>10530.236328</td>\n", + " <td>0.752982</td>\n", + " <td>31:29</td>\n", + " </tr>\n", + " <tr>\n", + " <td>7</td>\n", + " <td>9316.895508</td>\n", + " <td>10471.415039</td>\n", + " <td>0.755438</td>\n", + " <td>31:32</td>\n", + " </tr>\n", + " <tr>\n", + " <td>8</td>\n", + " <td>9555.703125</td>\n", + " <td>10649.283203</td>\n", + " <td>0.713144</td>\n", + " <td>31:32</td>\n", + " </tr>\n", + " <tr>\n", + " <td>9</td>\n", + " <td>9621.862305</td>\n", + " <td>10841.560547</td>\n", + " <td>0.737293</td>\n", + " <td>31:39</td>\n", + " </tr>\n", + " </tbody>\n", + "</table>" + ], + "text/plain": [ + "<IPython.core.display.HTML object>" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "learner.fit(epochs=10, lr=1e-04)" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "PosixPath('/home/ubuntu/MultiCampus/MICCAI_BraTS_2019_Data_Training/models/trained_model_fit_2.pth')" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "learner.save(\"trained_model_fit_2\", return_path=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": {}, + "outputs": [], + "source": [ + "learner = learner.load(\"trained_model_fit_2\", device=2)" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "<table border=\"1\" class=\"dataframe\">\n", + " <thead>\n", + " <tr style=\"text-align: left;\">\n", + " <th>epoch</th>\n", + " <th>train_loss</th>\n", + " <th>valid_loss</th>\n", + " <th>dice_coefficient</th>\n", + " <th>time</th>\n", + " </tr>\n", + " </thead>\n", + " <tbody>\n", + " <tr>\n", + " <td>0</td>\n", + " <td>8968.473633</td>\n", + " <td>10620.648438</td>\n", + " <td>0.668884</td>\n", + " <td>31:43</td>\n", + " </tr>\n", + " <tr>\n", + " <td>1</td>\n", + " <td>8851.039062</td>\n", + " <td>10587.374023</td>\n", + " <td>0.722928</td>\n", + " <td>31:56</td>\n", + " </tr>\n", + " <tr>\n", + " <td>2</td>\n", + " <td>9223.566406</td>\n", + " <td>10720.088867</td>\n", + " <td>0.749549</td>\n", + " <td>31:20</td>\n", + " </tr>\n", + " <tr>\n", + " <td>3</td>\n", + " <td>9645.301758</td>\n", + " <td>10704.744141</td>\n", + " <td>0.754023</td>\n", + " <td>31:26</td>\n", + " </tr>\n", + " <tr>\n", + " <td>4</td>\n", + " <td>8731.027344</td>\n", + " <td>10556.869141</td>\n", + " <td>0.747595</td>\n", + " <td>31:17</td>\n", + " </tr>\n", + " </tbody>\n", + "</table>" + ], + "text/plain": [ + "<IPython.core.display.HTML object>" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "learner.fit(epochs=5, lr=1e-04)" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "<table border=\"1\" class=\"dataframe\">\n", + " <thead>\n", + " <tr style=\"text-align: left;\">\n", + " <th>epoch</th>\n", + " <th>train_loss</th>\n", + " <th>valid_loss</th>\n", + " <th>dice_coefficient</th>\n", + " <th>time</th>\n", + " </tr>\n", + " </thead>\n", + " <tbody>\n", + " <tr>\n", + " <td>0</td>\n", + " <td>8663.437500</td>\n", + " <td>10722.753906</td>\n", + " <td>0.748986</td>\n", + " <td>31:28</td>\n", + " </tr>\n", + " <tr>\n", + " <td>1</td>\n", + " <td>8925.670898</td>\n", + " <td>10696.874023</td>\n", + " <td>0.731256</td>\n", + " <td>31:27</td>\n", + " </tr>\n", + " <tr>\n", + " <td>2</td>\n", + " <td>9030.234375</td>\n", + " <td>10776.826172</td>\n", + " <td>0.758666</td>\n", + " <td>31:04</td>\n", + " </tr>\n", + " <tr>\n", + " <td>3</td>\n", + " <td>8400.864258</td>\n", + " <td>10698.207031</td>\n", + " <td>0.729448</td>\n", + " <td>31:09</td>\n", + " </tr>\n", + " <tr>\n", + " <td>4</td>\n", + " <td>8817.802734</td>\n", + " <td>10819.573242</td>\n", + " <td>0.762197</td>\n", + " <td>31:09</td>\n", + " </tr>\n", + " </tbody>\n", + "</table>" + ], + "text/plain": [ + "<IPython.core.display.HTML object>" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "learner.fit(epochs=5, lr=1e-04)" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "PosixPath('/home/ubuntu/MultiCampus/MICCAI_BraTS_2019_Data_Training/models/trained_model_fit_3.pth')" + ] + }, + "execution_count": 26, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "learner.save(\"trained_model_fit_3\", return_path=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": {}, + "outputs": [], + "source": [ + "learner = learner.load(\"trained_model_fit_3\", device=2)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + " <div>\n", + " <style>\n", + " /* Turns off some styling */\n", + " progress {\n", + " /* gets rid of default border in Firefox and Opera. */\n", + " border: none;\n", + " /* Needs to be in here for Safari polyfill so background images work as expected. */\n", + " background-size: auto;\n", + " }\n", + " .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n", + " background: #F44336;\n", + " }\n", + " </style>\n", + " <progress value='1' class='' max='5', style='width:300px; height:20px; vertical-align: middle;'></progress>\n", + " 20.00% [1/5 31:07<2:04:30]\n", + " </div>\n", + " \n", + "<table border=\"1\" class=\"dataframe\">\n", + " <thead>\n", + " <tr style=\"text-align: left;\">\n", + " <th>epoch</th>\n", + " <th>train_loss</th>\n", + " <th>valid_loss</th>\n", + " <th>dice_coefficient</th>\n", + " <th>time</th>\n", + " </tr>\n", + " </thead>\n", + " <tbody>\n", + " <tr>\n", + " <td>0</td>\n", + " <td>8293.036133</td>\n", + " <td>10747.576172</td>\n", + " <td>0.765893</td>\n", + " <td>31:07</td>\n", + " </tr>\n", + " </tbody>\n", + "</table><p>\n", + "\n", + " <div>\n", + " <style>\n", + " /* Turns off some styling */\n", + " progress {\n", + " /* gets rid of default border in Firefox and Opera. */\n", + " border: none;\n", + " /* Needs to be in here for Safari polyfill so background images work as expected. */\n", + " background-size: auto;\n", + " }\n", + " .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n", + " background: #F44336;\n", + " }\n", + " </style>\n", + " <progress value='77' class='' max='288', style='width:300px; height:20px; vertical-align: middle;'></progress>\n", + " 26.74% [77/288 07:52<21:33 8336.4150]\n", + " </div>\n", + " " + ], + "text/plain": [ + "<IPython.core.display.HTML object>" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "learner.fit(epochs=5, lr=1e-04)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.5" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +}