genomics / Git / [de45a9] /deprecated/DL_Genomics

Models:
MarcoTheBlack/
genomics
Downloads: 1
[de45a9]: / deprecated / DL_Genomics_v4.ipynb
History
Download this file
1944 lines (1943 with data), 132.6 kB

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Deep learning in genomics"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This notebook is based on the [jupyter notebook](https://github.com/abidlabs/deep-learning-genomics-primer/blob/master/A_Primer_on_Deep_Learning_in_Genomics_Public.ipynb) from the publication [\"A primer on deep learning in genomics\"](https://www.nature.com/articles/s41588-018-0295-5) but uses the [fastai](https://www.fast.ai) library based on [PyTorch](https://pytorch.org)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Notebook setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%reload_ext autoreload\n",
    "%autoreload 2\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from fastai import *\n",
    "from fastai.vision import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import LabelEncoder, OneHotEncoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'1.0.32.dev0'"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# fastai version\n",
    "__version__"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data setup"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "## Data preprocessing (not needed, go to \"Data loading\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "import requests"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "URL_seq = 'https://raw.githubusercontent.com/abidlabs/deep-learning-genomics-primer/master/sequences.txt'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# get data from URL\n",
    "seq_raw = requests.get(URL_seq).text.split('\\n')\n",
    "seq_raw = list(filter(None, seq_raw)) # Removes empty lists"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2000"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# check length\n",
    "len(seq_raw)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# setup df from list\n",
    "seq_df = pd.DataFrame(seq_raw, columns=['Sequences'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# show head of dataframe\n",
    "#seq_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "URL_labels = 'https://raw.githubusercontent.com/abidlabs/deep-learning-genomics-primer/master/labels.txt'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "seq_labels = requests.get(URL_labels).text.split('\\n')\n",
    "seq_labels = list(filter(None, seq_labels)) # Removes empty entries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2000"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(seq_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "seq_label_series = pd.Series(seq_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "seq_df['Target'] = seq_label_series.astype('int')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Sequences</th>\n",
       "      <th>Target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>CCGAGGGCTATGGTTTGGAAGTTAGAACCCTGGGGCTTCTCGCGGA...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>GAGTTTATATGGCGCGAGCCTAGTGGTTTTTGTACTTGTTTGTCGC...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>GATCAGTAGGGAAACAAACAGAGGGCCCAGCCACATCTAGCAGGTA...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>GTCCACGACCGAACTCCCACCTTGACCGCAGAGGTACCACCAGAGC...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>GGCGACCGAACTCCAACTAGAACCTGCATAACTGGCCTGGGAGATA...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                           Sequences  Target\n",
       "0  CCGAGGGCTATGGTTTGGAAGTTAGAACCCTGGGGCTTCTCGCGGA...       0\n",
       "1  GAGTTTATATGGCGCGAGCCTAGTGGTTTTTGTACTTGTTTGTCGC...       0\n",
       "2  GATCAGTAGGGAAACAAACAGAGGGCCCAGCCACATCTAGCAGGTA...       0\n",
       "3  GTCCACGACCGAACTCCCACCTTGACCGCAGAGGTACCACCAGAGC...       1\n",
       "4  GGCGACCGAACTCCAACTAGAACCTGCATAACTGGCCTGGGAGATA...       1"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "seq_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "seq_df.to_csv('seq_df.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data loading"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "seq_df = pd.read_csv('seq_df.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>Sequences</th>\n",
       "      <th>Target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>CCGAGGGCTATGGTTTGGAAGTTAGAACCCTGGGGCTTCTCGCGGA...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>GAGTTTATATGGCGCGAGCCTAGTGGTTTTTGTACTTGTTTGTCGC...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>GATCAGTAGGGAAACAAACAGAGGGCCCAGCCACATCTAGCAGGTA...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>GTCCACGACCGAACTCCCACCTTGACCGCAGAGGTACCACCAGAGC...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>GGCGACCGAACTCCAACTAGAACCTGCATAACTGGCCTGGGAGATA...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0                                          Sequences  Target\n",
       "0           0  CCGAGGGCTATGGTTTGGAAGTTAGAACCCTGGGGCTTCTCGCGGA...       0\n",
       "1           1  GAGTTTATATGGCGCGAGCCTAGTGGTTTTTGTACTTGTTTGTCGC...       0\n",
       "2           2  GATCAGTAGGGAAACAAACAGAGGGCCCAGCCACATCTAGCAGGTA...       0\n",
       "3           3  GTCCACGACCGAACTCCCACCTTGACCGCAGAGGTACCACCAGAGC...       1\n",
       "4           4  GGCGACCGAACTCCAACTAGAACCTGCATAACTGGCCTGGGAGATA...       1"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "seq_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 0, 0, 1, ..., 1, 0, 1, 1])"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "seq_df['Target'].values"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "## Data encoding test (incorporated into \"open_seq_image\" function)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# setup class instance to encode the four different bases to integer values (1D)\n",
    "int_enc = LabelEncoder()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# setup one hot encoder to encode integer encoded classes (1D) to one hot encoded array (4D)\n",
    "one_hot_enc = OneHotEncoder(categories=[range(4)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "seq_enc = []\n",
    "\n",
    "for s in seq:\n",
    "    enc = int_enc.fit_transform(list(s)) # bases (ACGT) to int (0,1,2,3)\n",
    "    enc = np.array(enc).reshape(-1,1) # reshape to get rank 2 array (from rank 1 array)\n",
    "    enc = one_hot_enc.fit_transform(enc) # encoded integer encoded bases to sparse matrix (sparse matrix dtype)\n",
    "    seq_enc.append(enc.toarray()) # export sparse matrix to np array"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2000"
      ]
     },
     "execution_count": 89,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(seq_enc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 462,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([[0., 0., 0., 1., ..., 0., 1., 0., 0.],\n",
       "        [1., 1., 0., 0., ..., 1., 0., 1., 1.],\n",
       "        [0., 0., 1., 0., ..., 0., 0., 0., 0.],\n",
       "        [0., 0., 0., 0., ..., 0., 0., 0., 0.]]), (4, 50))"
      ]
     },
     "execution_count": 462,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "seq_enc[0].T, seq_enc[0].T.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 311,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAADIAAAAECAMAAAD7wpwzAAADAFBMVEUAAAABAQECAgIDAwMEBAQFBQUGBgYHBwcICAgJCQkKCgoLCwsMDAwNDQ0ODg4PDw8QEBARERESEhITExMUFBQVFRUWFhYXFxcYGBgZGRkaGhobGxscHBwdHR0eHh4fHx8gICAhISEiIiIjIyMkJCQlJSUmJiYnJycoKCgpKSkqKiorKyssLCwtLS0uLi4vLy8wMDAxMTEyMjIzMzM0NDQ1NTU2NjY3Nzc4ODg5OTk6Ojo7Ozs8PDw9PT0+Pj4/Pz9AQEBBQUFCQkJDQ0NERERFRUVGRkZHR0dISEhJSUlKSkpLS0tMTExNTU1OTk5PT09QUFBRUVFSUlJTU1NUVFRVVVVWVlZXV1dYWFhZWVlaWlpbW1tcXFxdXV1eXl5fX19gYGBhYWFiYmJjY2NkZGRlZWVmZmZnZ2doaGhpaWlqampra2tsbGxtbW1ubm5vb29wcHBxcXFycnJzc3N0dHR1dXV2dnZ3d3d4eHh5eXl6enp7e3t8fHx9fX1+fn5/f3+AgICBgYGCgoKDg4OEhISFhYWGhoaHh4eIiIiJiYmKioqLi4uMjIyNjY2Ojo6Pj4+QkJCRkZGSkpKTk5OUlJSVlZWWlpaXl5eYmJiZmZmampqbm5ucnJydnZ2enp6fn5+goKChoaGioqKjo6OkpKSlpaWmpqanp6eoqKipqamqqqqrq6usrKytra2urq6vr6+wsLCxsbGysrKzs7O0tLS1tbW2tra3t7e4uLi5ubm6urq7u7u8vLy9vb2+vr6/v7/AwMDBwcHCwsLDw8PExMTFxcXGxsbHx8fIyMjJycnKysrLy8vMzMzNzc3Ozs7Pz8/Q0NDR0dHS0tLT09PU1NTV1dXW1tbX19fY2NjZ2dna2trb29vc3Nzd3d3e3t7f39/g4ODh4eHi4uLj4+Pk5OTl5eXm5ubn5+fo6Ojp6enq6urr6+vs7Ozt7e3u7u7v7+/w8PDx8fHy8vLz8/P09PT19fX29vb39/f4+Pj5+fn6+vr7+/v8/Pz9/f3+/v7////isF19AAAAO0lEQVR4nGWNSQoAMAgD8/9PT6loIq0HMauShOR9DwphIkO5eALbEI1xp5DUoxYnNtf3t1rci9mGy30A2+0xz9q2+b0AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<PIL.Image.Image image mode=P size=50x4 at 0x1A20462668>"
      ]
     },
     "execution_count": 311,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "PIL.Image.fromarray(seq_enc[0].T.astype('uint8')*255).convert('P')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data setup for NN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# open sequence image function\n",
    "def open_seq_image(seq:str, cls:type=Image)->Image:\n",
    "    \"Return `Image` object created from sequence string `seq`.\"\n",
    "    \n",
    "    int_enc = LabelEncoder() # setup class instance to encode the four different bases to integer values (1D)\n",
    "    one_hot_enc = OneHotEncoder(categories=[range(4)]) # setup one hot encoder to encode integer encoded classes (1D) to one hot encoded array (4D)\n",
    "    \n",
    "    enc = int_enc.fit_transform(list(seq)) # bases (ACGT) to int (0,1,2,3)\n",
    "    enc = np.array(enc).reshape(-1,1) # reshape to get rank 2 array (from rank 1 array)\n",
    "    enc = one_hot_enc.fit_transform(enc) # encoded integer encoded bases to sparse matrix (sparse matrix dtype)\n",
    "    enc = enc.toarray().T # export sparse matrix to np array\n",
    "    #print('enc', enc, enc.shape)\n",
    "    \n",
    "    # https://stackoverflow.com/questions/22902040/convert-black-and-white-array-into-an-image-in-python\n",
    "    x = PIL.Image.fromarray(enc.astype('uint8')).convert('P')\n",
    "    x = pil2tensor(x,np.float32)\n",
    "    #x = x.view(4,-1) # remove first dimension\n",
    "    #x = x.expand(3, 4, 50) # expand to 3 channel image\n",
    "    #print('x', x, x.shape)\n",
    "    return cls(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/jpeg": "/9j/4AAQSkZJRgABAQEAZABkAAD/2wBDAAIBAQEBAQIBAQECAgICAgQDAgICAgUEBAMEBgUGBgYFBgYGBwkIBgcJBwYGCAsICQoKCgoKBggLDAsKDAkKCgr/2wBDAQICAgICAgUDAwUKBwYHCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgr/wAARCAAEADIDASIAAhEBAxEB/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAAAAAAAAECAwQFBgcICQoL/8QAtREAAgECBAQDBAcFBAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEIFEKRobHBCSMzUvAVYnLRChYkNOEl8RcYGRomJygpKjU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6goOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4uPk5ebn6Onq8vP09fb3+Pn6/9oADAMBAAIRAxEAPwD80/8AhrrUr/8A0rxT8B/h1q95B/pGltNotzaWGm6iflfUYdIsrmDSvPkjjsopUeze3nTTrfzYZGadpvuj4Jf8EwP2bf2gf2wL39kD4j6n4uuYLDwj4g1u38e/8JEz68IdG8WXnhax0rMqvZ/YlsbG3faLUTCVcJMkAWBSivm/HbGYzgrIq2IyKrLDTjhcTUThKSanTdBQkuZztbnleKXLK95xqOMXD9sPnT9kmOD9rf8A4Wbb6rcav4Ls/gx8Ita+Inwp0/wV4s1VYfCuqaf5LmKyF/dXRhguriVbqcg+eZreIxTQqHR+R+CXxp8V/tC+Mr34X/FXTdI1fwnpnhHxBrWlaFqWlx3s2nWmjaPearZaJaapd+bqtnpYkso4TbwXkbeTLOFkWSZ5SUV9lXwmGhmfENJRVsJDCOj3pSq0HOpKEvjUpz9+cudylJuTbbk2Hrn7DnwE+DH7cXg3xd4t1X4aaR8PovBvi7wX4fsNJ8DWguIbqPxRrA0fUbm4l1v+0Ll50tdptsTLHazJ58UayvI7/On/AA2R480r/ibfDvwF4R8F+IW+SXxP4N0uXT7mWE/vXiaCOb7H/wAfuL+OUW4mtLmK3+ySW0VpaQwFFHCuFoZnx5n2W4xOrQw06KpQnKUoxU6VKUk+ab51KUm2qjqpt+SSA/4ba+Mn/QmfCL/xH/wf/wDKqiiiv0n/AFT4Y/6A6f3S/wDlgH//2Q==\n",
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAADIAAAAECAYAAADMHGwBAAAABHNCSVQICAgIfAhkiAAAAHNJREFUKJFjdGEM+c8ABTufXWBwlzJgQAe4xNHVMDAwoKiD6cMmRwlANhdmJuPf5yr/cVlCjAdIdQAMELIP3bHobkEPJEZYjKBLYLMMV6gji+GykJDHcHkOm9uwqWNETlqEDEF2LD7LiUlK+JIxTC8pKQIA7jhk1J8wofAAAAAASUVORK5CYII=\n",
      "text/plain": [
       "Image (1, 4, 50)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# test open sequence image function\n",
    "open_seq_image('CCGAGGGCTATGGTTTGGAAGTTAGAACCCTGGGGCTTCTCGCGGACACC')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "class SeqItemList(ImageItemList):\n",
    "    _bunch,_square_show = ImageDataBunch,True\n",
    "    def __post_init__(self):\n",
    "        super().__post_init__()\n",
    "        self.sizes={}\n",
    "    \n",
    "    def open(self, seq): return open_seq_image(seq)\n",
    "    \n",
    "    def get(self, i):\n",
    "        seq = self.items[i][0]\n",
    "        res = self.open(seq)\n",
    "        return res\n",
    "    \n",
    "    @classmethod\n",
    "    def import_from_df(cls, df:DataFrame, cols:IntsOrStrs=0, **kwargs)->'ItemList':\n",
    "        \"Get the sequences in `col` of `df` and will had `path/folder` in front of them, `suffix` at the end.\"\n",
    "        return cls(items=df[cols].values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = (SeqItemList.import_from_df(seq_df, ['Sequences'])\n",
    "        .random_split_by_pct(valid_pct=0.25)\n",
    "        #.split_by_idxs(range(1500), range(1500,2000))\n",
    "        .label_from_list(seq_df['Target'].values)\n",
    "        .databunch())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "ImageDataBunch;\n",
       "Train: LabelList\n",
       "y: CategoryList (2000 items)\n",
       "[Category 0, Category 0, Category 0, Category 1, Category 1]...\n",
       "Path: .\n",
       "x: SeqItemList (1500 items)\n",
       "[Image (1, 4, 50), Image (1, 4, 50), Image (1, 4, 50), Image (1, 4, 50), Image (1, 4, 50)]...\n",
       "Path: .;\n",
       "Valid: LabelList\n",
       "y: CategoryList (2000 items)\n",
       "[Category 0, Category 0, Category 0, Category 1, Category 1]...\n",
       "Path: .\n",
       "x: SeqItemList (500 items)\n",
       "[Image (1, 4, 50), Image (1, 4, 50), Image (1, 4, 50), Image (1, 4, 50), Image (1, 4, 50)]...\n",
       "Path: .;\n",
       "Test: None"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/jpeg": "/9j/4AAQSkZJRgABAQEAZABkAAD/2wBDAAIBAQEBAQIBAQECAgICAgQDAgICAgUEBAMEBgUGBgYFBgYGBwkIBgcJBwYGCAsICQoKCgoKBggLDAsKDAkKCgr/2wBDAQICAgICAgUDAwUKBwYHCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgr/wAARCAAEADIDASIAAhEBAxEB/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAAAAAAAAECAwQFBgcICQoL/8QAtREAAgECBAQDBAcFBAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEIFEKRobHBCSMzUvAVYnLRChYkNOEl8RcYGRomJygpKjU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6goOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4uPk5ebn6Onq8vP09fb3+Pn6/9oADAMBAAIRAxEAPwD4w8Ny6l4h/wCEMjg8RavpkXi34ReJviPYjS9ZuY5vDeteHv7ej0f7DcmRrmSC0tfD1taQR3kt0beG7vfIaKWcSp5F/wANpfGDwt/xKvgLN/wq/wAPP++uPB/g3XtWl0qe/wD4dTaDUr27xeptgMdwpV4JLS3li8uWJZAUV9NwtlOWZnisTDF0Y1Iw5bRklKOtXGxu4tOLfLRp2lKMpJx5oyjKU5VP2wP+G0vjB4k/dfGub/hY8EnyX8HjDXtWX+0oT8zxXj2N7bSXm+WLTpDLMzzD+xtNiWRYLZYSf8NmfESD/QtL+HXw6s9Ktfm0bw/D8PtPawspF+SGeaGSNhq88Nu1xbxTar9ukjS8uJFYTyeeCivsf9U+GtlhIJfypOMV5qEZRhF9Lxgnb3bqOgHrvhC+8V/Ef9lTXP2vPiV471fXvEei/wBqxNaXc0cMN9aWV94XsZLOWe3SO9SC9i8X6018IbmJr2aZZ5mebfJJyOi/GDUvEHwG8SftK6t4U0ibxZoPi7RPDmoyuLk2HiGyv7XVbq3iv9OM32JoNOl0TTTZ2cMMNkqWyRz29xGqopRX5tl+EwtetjoygrU8wp0IJLlUaMnHmppR5VyO+1t7NNNRcQ/RX9n/AP4N/P2OP2kfgN4J/aJ8d/Ej4i2euePvCOm+I9Zs/D91o9jYQXd9ax3M0dtbR6bst4FeVgkS/KiBVHAooor/ADY4j8ePGTAcRY3C4bPcRCnTrVoQipQtGMK1WEYr909IxjGK1eiWr3Yf/9k=\n",
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAADIAAAAECAYAAADMHGwBAAAABHNCSVQICAgIfAhkiAAAAHBJREFUKJFjdGEM+b/z2QUGdykDBgYGBgZcbBgfBtylDLDykWlsAJuZ2PQQcge6+YwujCH/8SnABYgxHFkO2cPIYuieIMdcdykDBsa/z1X+oytENxjdEcR4EJ8+YgMMmztweRgeI6SEBK0AOUkTJgYA3Nhz1DR798MAAAAASUVORK5CYII=\n",
      "text/plain": [
       "Image (1, 4, 50)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "i = 2\n",
    "data.x[i]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Category 0"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.y[i]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/jpeg": "/9j/4AAQSkZJRgABAQEAZABkAAD/2wBDAAIBAQEBAQIBAQECAgICAgQDAgICAgUEBAMEBgUGBgYFBgYGBwkIBgcJBwYGCAsICQoKCgoKBggLDAsKDAkKCgr/2wBDAQICAgICAgUDAwUKBwYHCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgr/wAARCAAEADIDASIAAhEBAxEB/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAAAAAAAAECAwQFBgcICQoL/8QAtREAAgECBAQDBAcFBAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEIFEKRobHBCSMzUvAVYnLRChYkNOEl8RcYGRomJygpKjU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6goOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4uPk5ebn6Onq8vP09fb3+Pn6/9oADAMBAAIRAxEAPwD4C+Nvxp8V/s9eMrL4bfCrTdI0vw5P4R8P+JtP0y00uOCbTdU1PR7PWIrqK/h2X8k+n3d7J9huJ7maeCENA0ksM91HOfBL45fET46eMr34fa/caRpeh2/hHxB4pu/D+ieGNPj0e+1rRtHvNUtL9tJkgk06CdjYWlnM9tbQme0SWJ9xuLl5iivpv7Jyz/UX+0XRi6/subnavLm57czbbvK32mua/vcylq/2w679kmSf9t7/AIWbY/tCW+kX2lfCr4Ra18QfD2i6J4T0rSIZLvTfJ8vTGksbWKe30uU3E8k1paS26vNPLcApPI8zeRf8NrfGeT/iY3K6QuuTfNqfi3TbA6ZrGqSR/vbOe7vLB4Jbie3vcagJnYyXF3HBJeNdi2tkhKKMlynLMZxhnOArUYyo4f6p7KFvdh7XD1KlXlV7fvJxjOd+ZSlFNq6uB1/xJ+NPiv4I+Dfh/wCJPgbpukeFtN+IfhGbxHfeFk0uPVbDSbtdY1LSLiOwbVvtdzbwXlrpdut5EZmW8Rnhn8y28u3TkP8AhtH4rP8A6be+HPCOpah/Dd+IvC8Gr20Gfkf7Ppt+J9Nst0EOnWo+zWsXlW2kWUMXlIJhMUV7OQcPZHjssVbEYeM5udVOTTcmo168I80uZSlaMIxvKTbUVzOTu5B1/wC0B4y+BP7PXx58bfALRv2Kvh1r1n4H8Xal4ftdc8Qa14o+36jHZXUlstzc/ZdZgg8+QRh38mGKPezbI0XCgooo4cyHLsfw7gsViPaSqVKNGcn7fEaynRpTk9MQlrKUnoktdElZIP/Z\n",
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAADIAAAAECAYAAADMHGwBAAAABHNCSVQICAgIfAhkiAAAAHRJREFUKJFjdGEM+c+ABHY+u8DgLmWAwYbxGRgYUMSw6SMkDhPDZj42u3GpRwaMMI8gOxLdEHTHIxsMA8h8bA7HZjY2gK4Wn3nIcox/n6v8JyYkCTkAl6MIhTi+WMZmFq4YZnRhDPlPDUeiyxHjOGLNI0YfAGvwc9QnBcXaAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "Image (1, 4, 50)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "i = 3\n",
    "data.x[i]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Category 1"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.y[i]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(Image (1, 4, 50), Category 0)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.train_ds[0]#, data.train_ds.classes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjgAAAETCAYAAAA79nyeAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAACLtJREFUeJzt3TGPXFcZx+H3tbdwFEhBQxSlNQ1IuKCjjlxSpUEpaEH5AC5cIX8F+kgpKCKFiiLKV0iRFDRJRYEV0aAIoigo+FLESMh7Vxrv3DvnzH+ep7JGu3PPnZm9/mn2nbO9LEsBACS5M3oBAABbEzgAQByBAwDEETgAQByBAwDEETgAQByBAwDEETjsorvf7e5Puvvb7n5v9HqATK413ORq9AKI9bSqnlTVw6p6ZfBagFyuNawSOOxiWZYPq6q6+xdV9ebg5QChXGu4iV9RAQBxBA4AEEfgAABxBA4AEMeQMbvo7qv6/vV1t6rudve9qvpuWZbvxq4MSOJaw028g8NeHlfVN1X1qKreef7vx0NXBCRyrWFVL8syeg0AAJvyDg4AEEfgAABxBA4AEEfgAABxBA4AEGfYPjjPvrx/7eNbD994cO3rPnr66bXb1r5uzW2/99Dv2/rrjlnLqe/rZY6x5tzO/xSP3TFeXN8pnte1Y3z87IPe9MAbOMW1ZhZbn8PWP8/nZubzP8Xa9v4/9JjvPeRa4x0cACCOwAEA4ggcACDOsJ2M37rz9q0PvPXv9257X4fe/8xzLqPmDmaeh5lpBmtLh75u1hy63hlncI651qw55nnf0pbXvJe5v0OOcczjsfV5HXqMWa4rHMYMDgBwkQQOABBH4AAAcQQOABBnqiHjmYewRg0P7z34d8w6Znq+9h4eP2YdIzZ/HPXcXMKQ8ZoRj/cpNvWbeRg+4VzXjNpccMTGuGsO/d47r39hyBgAuDwCBwCII3AAgDgCBwCIM9WQ8TFmHhpbY71jjjHDMW8y818uXnPbwb9TO+YDDTO9Ps7JpT1ut/2Qw7ntfL92f6OeVzsZAwAXSeAAAHEEDgAQR+AAAHGGDRkDAOzFOzgAQByBAwDEETgAQByBAwDEETgAQByBAwDEETgAQByBAwDEETgAQByBAwDEETgAQByBAwDEETgAQByBAwDEETgAQByBAwDEETgAQByBAwDEETgAQByBAwDEETgAQByBAwDEETgAQByBAwDEETgAQByBAwDEETgAQByBAwDEETgAQByBAwDEETgAQByBAwDEETgAQByBAwDEETgAQByBAwDEETgAQByBAwDEETgAQByBAwDEETgAQByBAwDEETgAQByBAwDEETgAQByBAwDEETgAQByBAwDEETgAQByBAwDEETgAQByBAwDEETgAQByBAwDEETgAQByBAwDEETgAQByBAwDEETgAQByBAwDEETgAQByBAwDEETgAQByBAwDEETgAQByBAwDEETgAQByBAwDEETgAQByBAwDEETgAQByBAwDEETgAQByBAwDEETgAQByBAwDEETgAQByBAwDEETgAQByBAwDEETgAQByBAwDEETgAQByBAwDEETgAQByBAwDEETgAQByBAwDEETgAQByBAwDEETgAQByBAwDEETgAQByBAwDEETgAQByBAwDEETgAQByBAwDEETgAQByBAwDEETgAQByBAwDEETgAQByBAwDEETgAQByBAwDEETgAQByBAwDEETgAQByBAwDEETgAQByBAwDEETgAQByBAwDEETgAQByBAwDEETgAQByBAwDEETgAQByBAwDEETgAQByBAwDEETgAQByBAwDEETgAQByBAwDEETgAQByBAwDEETgAQByBAwDEETgAQByBAwDEETgAQByBAwDEETgAQByBAwDEETgAQByBAwDEETgAQByBAwDEETgAQByBAwDEETgAQByBAwDEETgAQByBAwDEETgAQByBAwDEETgAQByBAwDEETgAQByBAwDEETgAQByBAwDEETgAQByBAwDEETgAQByBAwDEETgAQByBAwDEETgAQByBAwDEETgAQByBAwDEETgAQByBAwDEETgAQByBAwDEETgAQByBAwDEETgAQByBAwDEETgAQByBAwDEETjsorvf7e5Puvvb7n5v9HqATN39o+7+U3d/3d1/7e5fj14Tc7gavQBiPa2qJ1X1sKpeGbwWINcfqurfVfXjqnpQVX/u7s+WZfnL2GUxWi/LMnoNBOvuJ1X15rIsvxm9FiBLd79aVf+oqp8ty/L589ver6q/LcvyaOjiGM6vqAA4Vz+pqv/8L26e+6yqfjpoPUxE4ABwrn5QVV+9cNtXVfXDAWthMgIHgHP1r6p67YXbXquqfw5YC5MROACcq8+r6qq77//fbT+vKgPGCBz20d1X3X2vqu5W1d3uvtfdPrUHbGZZlq+r6sOq+n13v9rdv6yqX1XV+2NXxgwEDnt5XFXfVNWjqnrn+b8fD10RkOh39f1WFH+vqj9W1W99RJwqHxMHAAJ5BwcAiCNwAIA4AgcAiCNwAIA4wz62++zL+wdNNz9848G12z56+ummX3dbW9//2v2t2fIctnbMY7L387r36+FlnOJxOvV9VVV9/OyDvvU37+StO29fu9ac4md3lmvNLNfLUzjFOWx5/Vlziv9DDj3u3teaQ9ex5pBrjXdwAIA4AgcAiCNwAIA4AgcAiDNsJ+O1wb9ZbD3Qt+YUA42HrOXchghfxovnP/u5bvnaGTWwfi5DxmtmHhQ+xf0dc9wR67gkMw9P731fNzFkDABcJIEDAMQROABAHIEDAMSZfsh4xG7Bp9jlc5ZdQ2ffIfXcdmFN2I16zaHndef1L6YbMl7bNX3EkP+InapHHeOYHXXXXNI1eXZ7D5lvea3xDg4AEEfgAABxBA4AEEfgAABxph8yPtSlD4jN/GftDz3GzEPBhxr1mtty1+Zjnptz3sl4zaXvbjzzz+kpdpcfsWv8uX2wYhQ7GQMAF0ngAABxBA4AEEfgAABxhg0ZAwDsxTs4AEAcgQMAxBE4AEAcgQMAxBE4AEAcgQMAxBE4AEAcgQMAxBE4AEAcgQMAxBE4AEAcgQMAxBE4AEAcgQMAxBE4AEAcgQMAxBE4AEAcgQMAxBE4AEAcgQMAxBE4AEAcgQMAxBE4AECc/wJLUH4q+JoU2gAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 576x576 with 4 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "data.show_batch(rows=2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model setup"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "## Lookup ResNet34 architecture (not needed, just for reference)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "test_learn = create_cnn(data, models.resnet18)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<function torch.nn.functional.cross_entropy(input, target, weight=None, size_average=None, ignore_index=-100, reduce=None, reduction='elementwise_mean')>"
      ]
     },
     "execution_count": 144,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_learn.loss_func"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "#test_learn.model[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Sequential(\n",
       "  (0): AdaptiveConcatPool2d(\n",
       "    (ap): AdaptiveAvgPool2d(output_size=1)\n",
       "    (mp): AdaptiveMaxPool2d(output_size=1)\n",
       "  )\n",
       "  (1): Lambda()\n",
       "  (2): BatchNorm1d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "  (3): Dropout(p=0.25)\n",
       "  (4): Linear(in_features=1024, out_features=512, bias=True)\n",
       "  (5): ReLU(inplace)\n",
       "  (6): BatchNorm1d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "  (7): Dropout(p=0.5)\n",
       "  (8): Linear(in_features=512, out_features=2, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 149,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_learn.model[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "def ExpandInput(): return Lambda(lambda x: x.expand(-1, 3, 4, 50))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "EI = ExpandInput()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# Test ExpandInput layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([64, 1, 4, 50])"
      ]
     },
     "execution_count": 166,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tt = torch.rand((64,1,4,50)); tt.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([64, 3, 4, 50])"
      ]
     },
     "execution_count": 167,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tt.expand(-1, 3, 4, 50).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "image/jpeg": "/9j/4AAQSkZJRgABAQEAZABkAAD/2wBDAAIBAQEBAQIBAQECAgICAgQDAgICAgUEBAMEBgUGBgYFBgYGBwkIBgcJBwYGCAsICQoKCgoKBggLDAsKDAkKCgr/2wBDAQICAgICAgUDAwUKBwYHCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgr/wAARCAAEADIDASIAAhEBAxEB/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAAAAAAAAECAwQFBgcICQoL/8QAtREAAgECBAQDBAcFBAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEIFEKRobHBCSMzUvAVYnLRChYkNOEl8RcYGRomJygpKjU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6goOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4uPk5ebn6Onq8vP09fb3+Pn6/9oADAMBAAIRAxEAPwD2D9jrwz4l/aM8P+IvBfif4x+N9CtPAWo+EjY/8Ib4jk0oatN4j0GDV7y51OOACLUblLq5cpcTo8sqZS6e6Es3m+IfEb4vax8Rb/xJ8evEWiWkGt6/4y8CwSQ6HfX2lWtqDDoesiRIbK4iEk63GuXkazzebMsCQwh9qvvKKAOH1vx1490T4laH8X/BHi0+Hn8afAW+8f8AijQNK0ewNhfX+hSeIr6xtMT28k0VkH0iCJoYpUPkXF1EjRpOwr2j9m39rr42/Gvw1+1X+0N8QdctL3Xfht8NPBniLw3BJYI9p9r1fSrhdQWaJwwmilgt4Ldo2ODFDGDlo0ZSigDY/Yk8G+Gv2t/jj4k8E/GrR7a+uvEvwH034v6p4ntrdItUuNXv4o4VsZpguLywtI5JVtY7pZpoWk8zzmljikj88+LNjdfEXWPh9rHjTxJreq6Qv7L2j/GufwfrWuXOo6Ve60G1LUBp92l48st7p3nWEqmO4kknaPUrpGnYJafZSigD0q0/4J6fCY2sR0n4q/FTTLUxr9m03TfiVqEdvaR4+WGJPMO2NRhVXJwABRRRQB//2Q==\n",
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAADIAAAAECAYAAADMHGwBAAAABHNCSVQICAgIfAhkiAAAAs5JREFUKJEFwU0o8wEAB+Dfei01lwlbm7GVjwNKajVMPg/aZeWwuAhlTXYQuzisWD6nrbXisuSwFNFqReKw2Z80X5G01bSY5bC1A5FR6Pc+jyiZTLKrqwuzs7Po6+uDVCrF5eUllEolbm5uUF1djbOzM1xcXODp6Qk+nw8HBwcIh8OYmZnB+vo6tFotDg8PsbOzg9PTUzw+PiKbzUIul8NoNEIikWBsbAwulwsrKyuoqanBwMAAnE4npqamIAgCLBYLSMLj8SCXyyEcDkOpVGJ6ehpXV1eoqqrC7e0t6uvrsb+/j2AwiN/fX0ilUgwODgLNzc1MpVL8/Pyk2+1mWVkZTSYTS0tL6fF4+PPzw1AoxEwmQ7FYTK1WS6fTycLCQo6OjvLh4YFWq5XDw8M0m810OBz8+vpiW1sbY7EYE4kEI5EIvV4v7XY7I5EI5+fnGYvFmM1m6ff7abfb6XK5GI/HGQgEWFxczEgkwsnJSep0Ora2tvL6+podHR20WCzUaDSUy+V8fX2lXq9nZWUl/5lMplmv1wuxWAy1Wg2FQoHn52csLy9jc3MTBoMBx8fH2N7eRnl5OXw+H1QqFYqKitDd3Y1sNguJRIK/vz9sbW0hFArB4/Egk8mgsbER6XQaLy8viMfjODo6gl6vR3t7O3Z3d5FOp6FWq2Gz2fD29oaRkRHc3d3BbrdDEASIxWI4HA7U1dXBYrFgbm4ONpsNZrMZ0WgU4+PjUCqVEIlEgNFoZCKRYElJCScmJpjP5xmNRrm3t8dEIsHe3l4KgkCZTMZAIMClpSUuLi6yoaGBCwsL7OzspNVqZSaTYVNTEw0GA3t6ephKpahSqajT6bi2tsba2loWFBQwl8vx/f2dGxsb1Ol01Gg0PD8/5+rqKhUKBfv7+/n9/c2Pjw+enJywoqKCyWSS+XyeMpmMQ0NDdLvdvL+/Z0tLC/1+P4PBIP8D9TaVeZbwiwgAAAAASUVORK5CYII=\n",
      "text/plain": [
       "Image (3, 4, 50)"
      ]
     },
     "execution_count": 168,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Image(EI(tt)[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "new_model = nn.Sequential(ExpandInput(), test_learn.model) # insert ExpandInput layer at the beginning of the network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([64, 3, 4, 50])"
      ]
     },
     "execution_count": 156,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_model[0](tt).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "new_learn = Learner(data, new_model, metrics=accuracy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "#[p.shape for p in new_model.parameters()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total time: 03:42\n",
      "epoch  train_loss  valid_loss  accuracy\n",
      "1      0.664468    0.764640    0.532000  (00:43)\n",
      "2      0.659713    0.819829    0.480000  (00:43)\n",
      "3      0.635178    0.888000    0.482000  (00:49)\n",
      "4      0.593556    0.875104    0.502000  (00:43)\n",
      "5      0.561881    0.893715    0.472000  (00:42)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "new_learn.fit_one_cycle(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEKCAYAAADjDHn2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xd8VfX9+PHXO5sMIIsZAmGPEBkxoiKIoIBVQEQFR918tY5WrYptf9VqbbVa66IqKmBrK7WKSl2oiILKClOGrLBCGCEQwgiZ798f5wQuISGB5HJzk/fz8biP3HvO59z7Prlw3vnMI6qKMcYYczIBvg7AGGNM3WfJwhhjTJUsWRhjjKmSJQtjjDFVsmRhjDGmSpYsjDHGVMmShTHGmCpZsjDGGFMlSxbGGGOqFOTrAGpLXFyctmvXztdhGGOMX1m8ePEeVY2vqly9SRbt2rUjPT3d12EYY4xfEZEt1SlnzVDGGGOq5NVkISLDRGStiGwQkQkV7G8rIrNEZIWIfCMiCR77bhSR9e7jRm/GaYwx5uS8lixEJBCYCAwHugPjRKR7uWLPAv9Q1RTgceDP7rExwKPAOUAa8KiIRHsrVmOMMSfnzT6LNGCDqmYAiMg0YCSw2qNMd+A+9/ls4EP3+VDgS1Xd6x77JTAMeOdUAigqKiIzM5MjR46c9kmYE4WFhZGQkEBwcLCvQzHGnCHeTBatgW0erzNxagqelgNXAi8AVwBRIhJbybGtTzWAzMxMoqKiaNeuHSJyqoebCqgqOTk5ZGZmkpSU5OtwjDFniDf7LCq6Ope/09KvgYEishQYCGwHiqt5LCIyXkTSRSQ9Ozv7hAOOHDlCbGysJYpaJCLExsZabc2YBsabySITaOPxOgHI8iygqlmqOlpVewO/dbftr86xbtlJqpqqqqnx8RUPE7ZEUfvsd2pMw+PNZLEI6CQiSSISAowFZngWEJE4ESmL4RFgsvt8JnCJiES7HduXuNuMMcYA5O+Dzd/DgtcgfXLV5WvIa30WqlosInfjXOQDgcmqukpEHgfSVXUGcCHwZxFRYA5wl3vsXhF5AifhADxe1tntT3Jychg8eDAAO3fuJDAwkLIa0MKFCwkJCanyPW6++WYmTJhAly5dvBqrMaaOKi2BvRmw80fYtQp2rXR+7vfo1k1Ig9RbvBqGqJ7QFeCXUlNTtfwM7jVr1tCtWzcfRXS8xx57jMjISH79618ft11VUVUCAvxrfmRd+t0aU2/k73MTwqpjyWH3GijOd/ZLIMR1hhbJ0LwHNO/p/IxqAafZPCwii1U1tapy9Wa5D3+yYcMGRo0aRf/+/VmwYAEff/wxf/jDH1iyZAn5+flcc801/P73vwegf//+vPzyyyQnJxMXF8cdd9zBZ599Rnh4OB999BHNmjXz8dmYOksVjuRCWNPTvpAYLyktgZyNbi3BrSnsXAl5mcfKNIpxkkLqLU5CaJEM8V0hKNQnITeYZPGH/61idVZerb5n91aNefTyHqd17OrVq5kyZQqvvvoqAE899RQxMTEUFxczaNAgxowZQ/fux89h3L9/PwMHDuSpp57i/vvvZ/LkyUyYcMLEeGNg3xaYcQ9s+haCwqBposej7fE/I+IsmXhTWW1h58pjyWH3Gih2RxSW1RbanltrtQVvaDDJoq7p0KEDZ5999tHX77zzDm+++SbFxcVkZWWxevXqE5JFo0aNGD58OAB9+/Zl7ty5ZzRm4wdKS2HxZPji9yABMOBBKMqH3K3OY/ti5+LlKTi8kmTiPg+PqVMXrTqrOrWF8Fhongyptx5rSvJhbeFUNJhkcbo1AG+JiIg4+nz9+vW88MILLFy4kKZNm3L99ddXOI/Bs0M8MDCQ4uLiMxKr8RP7NsNHd8PmudB+EIx4CZq2ObHckTynczR3q1MDyd0Kue7PbQvgyP7jy4dElksm5RJKo+iGl0yqU1uI7+LWFpKdR4tkiGzut7+rBpMs6rK8vDyioqJo3LgxO3bsYObMmQwbNszXYRl/UVoK6W/Cl486tYnLX4A+N1Z+UQprDGE9nL9qK5KfW0EycR9bfoCCcs25IVEQ3fYkyaRp7Z7vmXS0tvCjR3JYVXFt4ezb3GakZCdR+EFt4VRYsqgD+vTpQ/fu3UlOTqZ9+/acf/75vg7J+AvP2kSHi+DyFyuuTZyKRk2dR4ueJ+4r6zQvSx6eyWTfFtg0BwoPHn9MaBOIrqB5q+x5WOOaxVtbDu89NhJpl+dIJLe2EBDk0bdQP2oLp8KGzprTYr9bH/OsTQQEwtAnofcNvr9oqTpNNGXNWuVrJrlboOjw8ceENfWomZT/mQihkbUbY4W1hZWQt/1YmbLaQoue9bq2ADZ01pj667jaxGAY8SI0SajysDNCxOkQD4+BVr1P3K8Kh3MqTibZ62D9V8fmFJRpFFOumcszobSBkIgTP6dMtWsL5x8bntq84dQWToUlC2P8RfnaxIiX6kZt4lSIOEN1I+Kgdd8T96vCoezjO93LEsqu1bD2cygpOP6Y8DgneZQlFAlwyp5QW4hzksHZt7nNSD3qbW3BGyxZGOMP9m5y5k3UxdpEbRKByGbOI6GClpHSUji0+/hmrbJksmMF/PQJaOmx2oLnTOfIZv6VWOsYSxbG1GWlpbDoDfjqUafJZMTL0Pv6hnvRCwhwJqtFtYA2aSfuLy0FLYFAuzFXbbNkYUxdtTcDProHtnwHHYc4Q2LrY22iNgUE4N3FtBsuSxbG1DWlpbDodfjqMac2MXIi9Lqu4dYmTJ1gKdjLLrzwQmbOPP5WHM8//zy/+MUvKj0mMtIZKpiVlcWYMWMqfd/yQ4XLe/755zl8+NgwxUsvvZTc3Nzqhm58YW8GvHUZfPaQ0+b+i/kNu9nJ1BmWLLxs3LhxTJs27bht06ZNY9y4cVUe26pVK957773T/uzyyeLTTz+laVM/nk1bn5WWOjexeeV8Z9z/yIlw3X+hySnfet4Yr7Bk4WVjxozh448/pqDAGe63efNmsrKy6NWrF4MHD6ZPnz707NmTjz766IRjN2/eTHJyMgD5+fmMHTuWlJQUrrnmGvLzj41Fv/POO0lNTaVHjx48+uijALz44otkZWUxaNAgBg0aBEC7du3Ys2cPAM899xzJyckkJyfz/PPPH/28bt26cfvtt9OjRw8uueSS4z7HeEnOxmO1iXb94RfzrDZh6pyG02fx2QTnZiK1qUVPGP7USYvExsaSlpbG559/zsiRI5k2bRrXXHMNjRo14oMPPqBx48bs2bOHfv36MWLEiErvb/3KK68QHh7OihUrWLFiBX369Dm678knnyQmJoaSkhIGDx7MihUruPfee3nuueeYPXs2cXFxx73X4sWLmTJlCgsWLEBVOeeccxg4cCDR0dGsX7+ed955h9dff52rr76a999/n+uvv77mvytzotJSWDjJ6ZsIDIGRf4de11qSMHWS1SzOAM+mqLImKFXlN7/5DSkpKQwZMoTt27eza9euSt9jzpw5Ry/aKSkppKSkHN337rvv0qdPH3r37s2qVatYvXr1SeP57rvvuOKKK4iIiCAyMpLRo0cfXe48KSmJXr16Ac4y6Js3b67JqZvK5GyEqT+Dzx+GpAvgrvnQ2zqxTd3VcGoWVdQAvGnUqFHcf//9R++E16dPH6ZOnUp2djaLFy8mODiYdu3aVbgsuaeKah2bNm3i2WefZdGiRURHR3PTTTdV+T4nWw8sNPTYbNbAwEBrhqptpaWw8DX46g9ObWLUK3DWOEsSps6zmsUZEBkZyYUXXsgtt9xytGN7//79NGvWjODgYGbPns2WLVtO+h4DBgzgX//6FwArV65kxYoVgLO8eUREBE2aNGHXrl189tlnR4+JioriwIEDFb7Xhx9+yOHDhzl06BAffPABF1xwQW2drqnM0drEBEga4NQmrNnJ+ImGU7PwsXHjxjF69OijzVHXXXcdl19+OampqfTq1YuuXbue9Pg777yTm2++mZSUFHr16kVamjN79ayzzqJ379706NHjhOXNx48fz/Dhw2nZsiWzZ88+ur1Pnz7cdNNNR9/jtttuo3fv3tbk5C2lpbDgVZj1OASFwKhX4ayxliSMX/HqEuUiMgx4AQgE3lDVp8rtTwTeApq6ZSao6qci0g5YA6x1i85X1TtO9lm2RPmZZb/basrZCB/dBVvnQaehzizsxi19HZUxR/l8iXIRCQQmAhcDmcAiEZmhqp69r78D3lXVV0SkO/Ap0M7dt1FVe3krPmO8qrTEmTdRVpu44jVIucZqE8ZvebMZKg3YoKoZACIyDRgJeCYLBcpuk9UEyPJiPMacGTkb4cNfwLb50HkYXPa81SaM3/NmsmgNbPN4nQmcU67MY8AXInIPEAEM8diXJCJLgTzgd6o6t/wHiMh4YDxAYmJihUGoaqVzF8zpqS93V6x1pSUefROhVpsw9Yo3R0NV9D+k/FVmHDBVVROAS4F/ikgAsANIVNXewP3Av0XkhBv1quokVU1V1dT4+PgTPiwsLIycnBy7uNUiVSUnJ4ewsDBfh1K37NkAUy6Fmb+B9oPgroXWiW3qFW/WLDIBzzvHJ3BiM9OtwDAAVZ0nImFAnKruBgrc7YtFZCPQGTj5ynnlJCQkkJmZSXZ29mmegqlIWFgYCQm2VDbg1CbmvwJfPwFBYXDFJEi52pKEqXe8mSwWAZ1EJAnYDowFri1XZiswGJgqIt2AMCBbROKBvapaIiLtgU5AxqkGEBwcTFJSUk3OwZjK7dkAH/0Cti2ALpfCZX9zbspjTD3ktWShqsUicjcwE2dY7GRVXSUijwPpqjoDeAB4XUTuw2miuklVVUQGAI+LSDFQAtyhqnu9Fasxp6R8bWL069DzKqtNmHrNq/MszqSK5lkYU+v2rHdGOmUutNqEqRd8Ps/CmHqltATm/x2+/qPVJkyDZMnCmKpkr3P6JjIXQZefubWJ5r6OypgzypKFMZUpLYF5E53aREg4jH4Deo6x2oRpkCxZGFMRz9pE18vgZ89ZbcI0aJYsjPFUWgLzXoavn3RqE1e+CclXWm3CNHiWLIwpU742cdnfILKZr6Mypk6wZGHMcbWJCKtNGFMBSxamYcte68yb2J5utQljTsKShWmYSoqd2sTsPzm1iTGTocdoq00YUwlLFqbhyV4LH94J2xdDt8udkU5WmzDmpCxZmIajpBjmvQSz/2y1CWNOkSUL0zBsXwyfPujWJka4tYkT74FijKmYJQtTv+VsdFaHXfUBhMfBmCmQPNrXURnjdyxZmPrpwC6Y8xdYPBUCQ2HgBDjvbgiN8nVkxvglSxamfjmS54xy+uFlKCmAvjfDwIesA9uYGrJkYeqH4kJYPAW+/Qsc3uN0XF/0O4jt4OvIjKkXLFkY/1ZaCqumw6zHIXcLJA2AIX+A1n18HZkx9YolC+O/Nn4NXz4KO1dAi55w/XTocJENhTXGCyxZGP+TtdRJEpu+haaJzn0mkq+EgABfR2ZMvWXJwviPnI3OjYhWTYfwWBj2NKTeDEGhvo7MmHrPq3+KicgwEVkrIhtEZEIF+xNFZLaILBWRFSJyqce+R9zj1orIUG/Gaeq4g7vhk1/DxDRY9zkMeAjuXQb97rBEYcwZ4rWahYgEAhOBi4FMYJGIzFDV1R7Ffge8q6qviEh34FOgnft8LNADaAV8JSKdVbXEW/GaOqjggDME9oeXoPgI9L0JBj5sd6wzxge82QyVBmxQ1QwAEZkGjAQ8k4UCjd3nTYAs9/lIYJqqFgCbRGSD+37zvBivqSuKC53JdN8+7QyD7T4KLvp/ENfR15EZ02B5M1m0BrZ5vM4EzilX5jHgCxG5B4gAhngcO7/csa29E6apM8qGwX79BOzbDO0ucIbBJvT1dWTGNHjeTBYVjV/Ucq/HAVNV9a8ici7wTxFJruaxiMh4YDxAYmJiDcM1PrVxNnz1KOxYDs2T4br3oeNgGwZrTB3hzWSRCbTxeJ3AsWamMrcCwwBUdZ6IhAFx1TwWVZ0ETAJITU09IZkYP5C1DL56DDJmQ5NEuGIS9LzKhsEaU8d483/kIqCTiCSJSAhOh/WMcmW2AoMBRKQbEAZku+XGikioiCQBnYCFXozVnGl7M+C9W2DSQKc2MfTPcE86nHWNJQpj6iCv1SxUtVhE7gZmAoHAZFVdJSKPA+mqOgN4AHhdRO7DaWa6SVUVWCUi7+J0hhcDd9lIqHriYLazGmz6ZAgIhgt+DeffC2FNfB2ZMeYkxLk2+7/U1FRNT0/3dRimMgUHYN5EZxhsUT70vdEdBtvC15EZ06CJyGJVTa2qnM3gNt5VXAhL3nKGwR7Khu4j3WGwnXwdmTHmFFiyMN5RWgqrP4BZT8C+TdC2P4ybBglV/gFjjKmDLFmY2pfxjbPQ345l0KwHXPcedBxiw2CN8WOWLEzt2bHcGQa78Wto0gaueM0dBhvo68iMMTVkycLU3N5NMPtJ+PG/0Cgahv4JUm+F4DBfR2aMqSWWLMzpO7QH5jwDi96EgCC44AE4/5c2DNaYesiShTl1BQfdYbAvOsNg+9wAAydA45a+jswY4yWWLEz1lRS5q8H+BQ7thm6Xw0W/h/jOvo7MGONllixM1VRh1QfOarB7M6Dt+TD239DmbF9HZow5QyxZmJPL+NZZDTZrKTTrDtf+FzpdbMNgjWlgLFmYiu1Y4Q6DneUMgx31KqRcbcNgjWmgLFmY4+3bDF8/CT++6wyDveRJOPs2GwZrTANnycI4Du2BOc/CojecYbD973eGwTZq6uvIjDF1gCWLhq7wEMz7O3z/AhQdgt43wIUToHErX0dmjKlDLFk0VCVFzmqw3zztDIPtehkM/j3Ed/F1ZMaYOsiSRUOUtcy5S93ejZB4Loz9F7RJ83VUxpg6zJJFQ9QkAcJjnTWcOg+1YbDGmCpZsmiIIuLgti99HYUxxo8E+DoAY4wxdZ8lC2OMMVWyZGGMMaZKXk0WIjJMRNaKyAYRmVDB/r+JyDL3sU5Ecj32lXjsm+HNOI0xxpyc1zq4RSQQmAhcDGQCi0RkhqquLiujqvd5lL8H6O3xFvmq2stb8RljjKk+b9Ys0oANqpqhqoXANGDkScqPA97xYjzGGGNOkzeTRWtgm8frTHfbCUSkLZAEfO2xOUxE0kVkvoiM8l6YxhhjquLNeRYVzfTSSsqOBd5T1RKPbYmqmiUi7YGvReRHVd143AeIjAfGAyQmJtZGzMYYYyrgzZpFJtDG43UCkFVJ2bGUa4JS1Sz3ZwbwDcf3Z5SVmaSqqaqaGh8fXxsxG2OMqYA3k8UioJOIJIlICE5COGFUk4h0AaKBeR7bokUk1H0eB5wPrC5/rDHGmDPDa81QqlosIncDM4FAYLKqrhKRx4F0VS1LHOOAaarq2UTVDXhNREpxEtpTnqOojDHGnFly/DW6kkIiHYBMVS0QkQuBFOAfqpp78iPPnNTUVE1PT/d1GMYY41dEZLGqplZVrrrNUO8DJSLSEXgTZ+TSv2sQnzHGGD9S3WRRqqrFwBXA8+5kupbeC8sYY0xdUt1kUSQi44AbgY/dbcHeCckYY0xdU91kcTNwLvCkqm4SkSTgbe+FZYwxpi6p1mgodyTSveAMawWiVPUpbwZmjDGm7qhWzUJEvhGRxiISAywHpojIc94NzRhjTF1R3WaoJqqaB4wGpqhqX2CI98IyxhhTl1Q3WQSJSEvgao51cBtjjGkgqpssHseZib1RVRe5i/ut915Yxhhj6pLqdnD/F/ivx+sM4EpvBWWMMaZuqW4Hd4KIfCAiu0Vkl4i8LyIJ3g7OGGNM3VDdZqgpOCvGtsK5gdH/3G3GGGMagOomi3hVnaKqxe5jKmA3kDDGmAaiuslij4hcLyKB7uN6IMebgRljjKk7qpssbsEZNrsT2AGMwVkCxBhjTANQrWShqltVdYSqxqtqM1UdhTNBzxhjTANQk9uq3l9rURhjjKnTapIspNaiMMYYU6fVJFlUfT9WY4wx9cJJZ3CLyAEqTgoCNPJKRMYYY+qckyYLVY06U4EYY4ypu2rSDFUlERkmImtFZIOITKhg/99EZJn7WCciuR77bhSR9e7jRm/GaYwx5uSqtZDg6RCRQGAicDGQCSwSkRnuXfcAUNX7PMrfA/R2n8cAjwKpOM1gi91j93krXmOMMZXzZs0iDdigqhmqWghMA0aepPw44B33+VDgS1Xd6yaIL4FhXozVGGPMSXgzWbQGtnm8znS3nUBE2gJJwNencqyIjBeRdBFJz87OrpWgjTHGnMibyaKieRiVDbcdC7ynqiWncqyqTlLVVFVNjY+3dQ2NMcZbvJksMoE2Hq8TgKxKyo7lWBPUqR5rjDHGy7yZLBYBnUQkSURCcBLCjPKFRKQLEA3M89g8E7hERKJFJBq4xN1mjDHGB7w2GkpVi0XkbpyLfCAwWVVXicjjQLqqliWOccA0VVWPY/eKyBM4CQfgcVXd661YjTHGnJx4XKP9Wmpqqqanp/s6DGOM8SsislhVU6sq59VJecYYY+oHSxbGGGOqZMnCGGNMlSxZGGOMqZIlC2OMMVWyZFEJVaW+jBQzxpiasmRRgdJSpf/Ts3l7/hZfh2KMMXWCJYsKHCgoZntuPm/N22K1C2OMwZJFhfLyiwDYsPsgq7LyfByNMcb4niWLCuQdKTr6fPqS7T6MpHZY7cgYU1OWLCpw4EgxAHGRIcxYnkVxSamPIzo9hwuLefx/q0l57AuWb8ut+gBjjKmEJYsKlDVDXZuWyJ6DBczdsMfHEZ0aVeXbddkMe34uk7/fRFFpKU9//pOvwzLG+DGvrTrrz/LcmsWIXq34x/wtTF+ynQs7xyNS0T2ZfC8j+yBb9h4mKEDIys1nyveb+WnnAdrGhjNtfD9WZ+Xx+Mer+W79Hvp3ivN1uMYYP2TJogJlNYvYiFAuS2nJ2/O38tmPO2gaHsKE4V0Z0zfBxxEek3ekiFETvz+a4AC6NI/i6St7MrJXa8KCA+nVpilvzM3gmZk/cX7H8+ts0jPG1F2WLCpQ1mcRFRbEfUM6kxQXyd5DBcxYnsVbP2yuUbLYmH2QO/65mEcv71Erf+W/PX8LeUeKmXhtH5o3DiUsOJAerRoflxDCggP51ZDOPPT+Cj5alsXIXq0sYRhjTon1WVQg70gRESGBBAUGEBsZyq39k3hwaFeuO6ctP27fT+a+w6f93k9+sob1uw/y8PsrOFRQXPUBJ5FfWMKbczcxsHM8P0tpSWq7GJJbN6kwEYzu05pOzSL51X+WcfaTs7jrX0vYtOdQjT7fGNNwWLKoQF5+EVFhwSdsH9qjBQBfrNp1Wu87d302X/+0m8vPakXW/nye/WJtjeJ8N30bOYcKuWtQxyrLBgUGMG18P/48uicDOsUxd302I17+jm/W7q5RDMaYhsGSRQUOHCmmcaMTW+iS4iLo0jyKz1ftPOX3LClVnvxkDW1iGvHsVSnc0K8tU3/YzNKt+04rxsLiUl77diNnt4smLSmmWsfERoYyLi2R567pxSf3XkBCdDg3T13EI9N/ZPJ3m/hq9S6/HSZcEVXl+w172JJjNShjasr6LCqQd6SIxhXULACG9mjOy7M3kHOwgNjI0Gq/578XbuWnnQf4+3V9CA0K5MGhXfhy9S7+75+LeeCSzlzZJwEFlm3LZc2OPPYdKuJQYTFX9U2gU/MoAIpLSnlh1noWbd7L1pzDZO0/wpOje57WObaJCef9O8/ldx+sZPqSTAqKnSRxdWoCT1+Zckb7NFSVI0WlNAoJBJzznLM+m+wDBVzYpRnNG4ed8ntuzTnMozNWMnttNvFRoUy/8zzaxITXdujGNBh2D+4KXPbSXOIjQ5lyc9oJ+1Zl7ednL37H01f25JqzE6t8L1XllW838uzMtaQlxfDO7f2OXohXbt/Pbz9cyfJtubRu2oi8/CIOePRjBAUIjRsFM218PzrGR/Lgeyt4f0kmvROb0jYmnD5to7mhX9saX9hVlX2Hi3htzkZe+zaDP45K5vp+bWv0ntW1be9hbnsrnQ3ZB0lJaEK3lo2Z/dNuduw/crRMcuvGhAcHsT+/iLCQQIb2aM7lKa0qvfjP/mk3d7y9mKAA4db+SUz9YTNxUaG8f8d5REeEnJHzMsZfVPce3F5NFiIyDHgBCATeUNWnKihzNfAYoMByVb3W3V4C/OgW26qqI072WbWZLAb8ZTa9E5vywtjeJ+xTVS74y2zax0dy/8WdWZCRQ2q7aPq2PbEpqKC4hF9NW8ZnK3dyWUpL/jImhfCQoBPe78vVu/jn/C0kRDdiQKd4+raNJiYihG378rnmtXmUKlzQKY4Plm7nviGd+eWQTrVynuWVlCq3vbWI7zbs4dmrzqKgqJTMfYe5sm8CbWMjav3zlm/L5da30iksLuHq1DYs2bqPlVl59Gsfy7VpibSLC2fWmt3MWZeNCDRpFMyuvAKWubPRL+7enEeGd6V9fOTR99xzsIChf5tDs8ZhTLnpbFo0CWPhpr1c/+YCerZuwtu3nnO0BmOMqQPJQkQCgXXAxUAmsAgYp6qrPcp0At4FLlLVfSLSTFV3u/sOqmpkBW9dodpMFr0f/4LLUlrxxKjkCvc/8fFq3vxu09HXUWFBzLi7P0lxx19Q356/hd99uJIJw7vyfwPan1YNYMPuA4ydNJ89Bwu5/YIkfnNpN682Ee3PL2Lky9+xOefYiK+2seF8dNf5NA2vnb/KDxYU88bcDF79diNxkaFMvflsOjZzmtpUtcrz27b3MO8vyeSNuZs4UlTCdeck8sshnYkOD+aOtxcz+6ds/ndPf7q0iDp6zKc/7uDufy/hvA5xvHFjKmHBljCMgbqRLM4FHlPVoe7rRwBU9c8eZf4CrFPVNyo43ifJQlXp+NvPuGNgex4c2rXCMttz85n6/SZSEprSLjaCn09eQFxkKB/cdT6RoUFH3+fSF79DgE/u7V+jC/zG7IMs3ryPq1ITzkhfQs7BApZszaVjs0hyDhZw7esLSEuKYerNZxMUWLMxEf9bnsVjM1aRc6iQYT1a8MSoZOKjqt/34yn7QAHPf7WOdxZuJSI0iIu7NWf60u385tKujB/Q4YTy7y/O5NfvLWdAp3heu6Hv0YRRVFLKu+nb2JVXQL/2MfRJjLZhRBdKAAAZ3UlEQVRkYhqM6iYLb3Zwtwa2ebzOBM4pV6YzgIh8j9NU9Ziqfu7uCxORdKAYeEpVP/RirEcdLiyhpFQrHDpbpnXTRvz2Z92Pvp54bR9umLyQ+/+zjFev70tAgLBkq9NR/acretb4At8hPpIO8dXOmzUWGxnKxd2bA84IsD9ekcxD763g9zNW8djlPQgJOj5h7DtUyO3/SKdNTDj/77LuxFTSL7Bu1wEeeHc53Vo15o0bU+mdGF2jOOOjQnnyip7cdF47/vTpGqYv3U5auxhu7d++wvJX9k2guLSUh9//keEvzGVYcgvax0XwyjcbydhzCBF4cRY0Cg7kmatSuCylVY3iM6Y+8WayqOgKWb4aEwR0Ai4EEoC5IpKsqrlAoqpmiUh74GsR+VFVNx73ASLjgfEAiYlVdzZXR9ny5JWNhqrIeR3jeGR4V/74yRpe+noDvxzSiX/N30JkaBAje/n/Befq1DZs2H2QSXMy+H7DHh4e1pXhyS0QEQ4XFnPLW4tYtT2P5Zm5zF2fzcPDutI6uhEBInRtEUXT8BCKS0p58L0VRIYF8eaNqcSdwkiyqnRqHsWUm9NYvi2XdrERBAZUnpyvOTuRxmHB/HP+FibNyaCkVOkQH8Hkm1I5u10MizbvZeLsjfxy2jKCAoRhyS1rLU5j/Jk3k0Um0MbjdQKQVUGZ+apaBGwSkbU4yWORqmYBqGqGiHwD9AaOSxaqOgmYBE4zVG0EXbbUR0XzLE7m1v5JrM7K429fraN541A+/nEH16S2ISK0foxO/s2l3Ti/Yxx/+mQNv/jXEtrFhjOqd2uWbs1l+bZc/n5dX9rGhvPr/y7nwfdWHD2ucVgQDw7twoGCYpZvy+XFcb1rNVF4OqtN02qVG96zJcN7tiT3cCFrdx6gT9togt3mtYu6NictKZafv7mAu/+9lInXydHJmMY0ZN7sswjC6eAeDGzH6eC+VlVXeZQZhtPpfaOIxAFLgV5AKXBYVQvc7fOAkZ6d4+XVVp9F+ua9jHl1Hv+4JY0BneNP6dgjRSVcM2n+0XtHzPzVgOM6WeuDklLlo2XbeW9xJvMyclCFp0b3ZGyaU7MrKillRWYuhcVKQXEJk+Zk8MPGHAAu6d6c127o6xfrUuUdKeKGNxawYvt+xl/Qngcu6XJC85sx9YHP+yxUtVhE7gZm4vRHTFbVVSLyOJCuqjPcfZeIyGqgBHhQVXNE5DzgNREpxZll/tTJEkVtKmuGigo79V9NWHAgk27oy4iXv6NDfGS9SxQAgQHC6D4JjO6TwI79+WTl5h83bDg4MOC41wM7x/PJjzv4ZMUO/jCyh18kCnCaId8Z348nPl7Da3My+H7jHsaenUi3lo0JDQpg0ea9rMrK42c9WzKoazNfh2uM19mkvHI+WradX05bxqwHBp52p/JBd2JdZD1pgmroPl+5k//30UqyDxQctz08JJDDhSWMH9CeB4d2OdqUZYw/8XnNwl+V3cviVDq4y7MkUb8MS27B0B7N2Z6bz5odBzhcWEzfttHERYby5CdrmDQng2/XZjOkezPOSYqlRZMwAgOEsOBA4iNDrfnK1At2VSsnz+NeFsaUERESosNJiD5+iZEnRiVzXodYXp2TwavfZjBx9sYTjo2LDGFM3zY8OLTLSUdqGVOX2RURZwXXUlXCggPJyy8iJCjAJmWZaisbXXWwoJhlW3PZn19EcWkp+YUl7MorYPWO/bz67UbW7szjxXG9TzqHpzKqyg8bc1iwaS9Lt+6joKiUczvE0r9THF1aRNWoJmxMdTT4ZJGVm88Ff5nNk6OSGZuWSN6RYvuPZ05LZGhQpXc/fHv+Fh6dsYpRE7/n5vOTuLRnS1SV2WuzWbxlL8GBAYSHBNGlRSSXdG9x3JDr5dtyefzj1Szeso8Agc7NowgJCuDFr9fzwqz1gLNuVnxUKAKIQO820Yzq3ZpzkmIIsNqMqQUNPlm0aBxGcKCwfvdBwF2e/BTnWBhTlev7taV9XAS/n7GK3324ksdmrKJEFVXnQg9wqKCY4lIlPGQlg7o2o6RE2ZF3hOXbcomLDOGp0T257KxWR/vEcg8XsmDTXjbvOcS2fYfJOViIiFNT/nhFFv9J30ZiTDh/HJV8ysPAjSmvwV8VAwKEjs0iWbfrAFD5XfKMqanzOsbx5X0DWL0jj09/3EFQQABDujUnubVzz3RVZdHmfUxfksk3a7OJDAuiReMw7r2oI7cPaH/Cv8um4SGVThjMLyzhyzW7eHHWen4+eSFj+iYwYXhXr02INPVfg08WAJ2aRTE/w5k45jRD2a/FeIeI0KNVE3q0alLhvrSkmGrf+fBkGoUEMuKsVlzSvTkvfb2eV7/N4KNl2xnctTnDe7YgMEDILywhPiqU3m2iaRJufyCZk7OrItCpeSQfLN1O3pEiDhwpIiG6ka9DMqZWhAUH8uDQrlzRO4FpC7cyfen2Cm8L3KlZJKN6t+aqvgk0O407E5r6z5IFTs0CYMPug+TlWwe3qX86Novkd5d156FhXVm36wCh7oi/bfsOs3RrLnPWZfPMzLU89+U6Lu7WnJ+f15Zz28f6zYx7432WLIDOzZ2Z2ht2HXTvv22/FlM/hQQFkNz6WBNYm5hwzusQx12DOrJpzyGmLdzKf9K38fmqnXRpHsWTVyST2q7mzWLG/9nUUiAhOpzQoABWZu2nsLiUxo2sZmEanqS4CB65tBvzHxnMM2NSyHcXxnxp1npKSuvHskDm9FmywFkcr2OzSBZv2QdgNQvToIUFB3JVahs+ubc/l6W05K9frmPspHmsd0cMmobJkoWrU7NI1uzIA7Chs8bg/D94/ppe/PWqs1i36yCXvjiXZ2euJedgQdUHm3rH/oR2dWoeRVlN2yblGeMQEa7sm8DALvH86ZM1vDx7A3//ZgP92sdy+VmtuNxjkqCp36xm4erc/Ni9J2w0lDHHi4sM5blrevH5ry7grkEd2Zl3hEem/0jak1/xyPQV/LQzz9chGi+zPwlcnZodu3eFdXAbU7GuLRrTtUVj7r+4M0u35fLOgq18sHQ77yzcxgWd4rhjYAfO71jx+ljGv1nNwtUmxhkRBbY8uTFVERH6JEbzzFVnMf+RwTw4tAtrdx7gujcWcNtb6WzNOezrEE0ts2ThCgyQo3fGs2YoY6qvaXgIdw3qyNyHB/HwsK78sHEPQ/72Lfe/u4xZa3ZRUFzi6xBNLbA/oT10bh7J2l0HCA+xe1kYc6pCgwK588IOXNG7NS/MWscnK3Ywfcl2YiJCeGhoF65ObWPLpfsxuwe3h3kbc/h+wx5+PbRLLUVlTMNVWFzK9xv38PfZG1i0eR9ntWnK7Rck0a99rK1+W4dU9x7cliyMMV6lqnywdDt//uwnsg84czSSWzfmgUu6MKhLMx9HZ6qbLLzaZyEiw0RkrYhsEJEJlZS5WkRWi8gqEfm3x/YbRWS9+7jRm3EaY7xHRBjdJ4F5Ey7ig1+cx0PDunCooISbpyzi5ikLbditn/BazUJEAoF1wMVAJrAIGKeqqz3KdALeBS5S1X0i0kxVd4tIDJAOpAIKLAb6quq+yj7PahbG+I/C4lLe+mEzL85az4GCYs7vGMtt/dtzYZd4W+n2DKsLNYs0YIOqZqhqITANGFmuzO3AxLIkoKq73e1DgS9Vda+770tgmBdjNcacQSFBAdw+oP3REVQZ2Ye4eeoiHn5/BUeKbPRUXeTNZNEa2ObxOtPd5qkz0FlEvheR+SIy7BSONcb4uabhIdx5YQfmPDSIey7qyLvpmYx59Qcysg/6OjRTjjeHzlZUlyzf5hUEdAIuBBKAuSKSXM1jEZHxwHiAxMTEmsRqjPGh4MAAHrikCykJTbn/P8u46K/fktYuhtF9WjOyV2sa2XB2n/NmzSITaOPxOgHIqqDMR6papKqbgLU4yaM6x6Kqk1Q1VVVT4+PjazV4Y8yZd3H35nz1wEAeHNqFPYcKmDD9R857ahZ//WLt0ZFUxje82cEdhNPBPRjYjtPBfa2qrvIoMwyn0/tGEYkDlgK9ONap3cctugSng3tvZZ9nHdzG1C+qyqLN+3h9bgZfrdlFSGAA49IS+b+B7WnZpNFxZfMLS6z2cZqq28HttWYoVS0WkbuBmUAgMFlVV4nI40C6qs5w910iIquBEuBBVc1xT+AJnAQD8PjJEoUxpv4REdKSYkhLiiEj+yCvfruRt+dv4e35W+jcPIrurRojwOIt+8jYc4gerRoz9uw2jOzd2pbs8QKblGeM8RuZ+w4zbeE2Vmzfz+qs/ZSUKn3bxtC1RRSzftrNmh15hAQFcFGXZozs1Yoh3ZsTHGhL4J2MzeA2xjQoqsqKzP1MX5LJJz/uYM/BQjrER/DYiB5c0Mn6NCtjycIY02AVl5Ty1Zrd/PmzNWzJOczw5BY8NTqFJuHWPFVeXZiUZ4wxPhEUGMCw5BZ8cd8AHhzaha/W7GLExO9Yu/OAr0PzW5YsjDH1VmhQIHcN6sg7t/fjcGEJV/z9e16atZ7MfXZzplNlycIYU++ltovh43v6c3a7GP765Tr6Pz2bG95cwJodtohhdVmfhTGmQdm29zDTl2znrXmb2Z9fxK39k7h3cCciQxvmveCsg9sYY05i36FCnv78J6Yt2kZUWBDj0hK5Ni2RNjHhBDagO/pZsjDGmGpYti2X1+dm8PnKnZSUKiLQtFEwzaLCaNEkjMSYcEb1bkWfxOh6uXy6JQtjjDkF23PzmbVmF3sOFrL3UAG78grYuf8IG7MPcriwhB6tGnPPRR0ZltzS16HWKp8v92GMMf6kddNG/PzcdidsP1RQzIfLtjP1+83c8fYSxvRN4A8jehDRwPo4GtbZGmPMKYoIDeK6c9pyTWobXpi1npdnbyB9815uOLcdl3RvTpuYcF+HeEZYM5QxxpyC+Rk5PDZjFT+5E/zObR/LM1elkBDtn0nD+iyMMcaLtuQc4rOVO3n56w2IwJ+u6MnlZ7XydVinzJb7MMYYL2obG8EdAzvw6b0X0LFZJPe8s5T/+2f6CbPDC4tL+Wr1Lt76YTOb9hzyUbQ1ZzULY4ypoaKSUl6fm8FLszagKJentCIkKID8whJmr93NvsNFR8t2iI/ggk7x9GsfS+/EpsREhPh0GXVrhjLGmDNse24+f/p0DT9s2ENggBAYIKQlxXJF71Z0iI/km7XZfLVmF4s27+VIUenR4yJCAjm3Qxy39k+iX/uYMzqfw5KFMcbUUYXFpazIzGX1jjxyDxex+8ARPlmxg32Hi+jesjHj0tow4qzWNAkPJvdwITvzjhAWFEhEaBBFJaXsPVTIvsOFpCQ0pUmjmi27bsnCGGP8yJGiEqYv2c4/5m3mp50HCAkKoFFwIPvziyo9Ztr4fvRrH1ujz7VJecYY40fCggO59pxExqW1YVVWHh8s3U5BcQntYiNo0SSMwuJSDhUUExgQQExECDERIXRtGXXG4rNkYYwxdYiIkNy6Ccmtm/g6lOPY0FljjDFV8mqyEJFhIrJWRDaIyIQK9t8kItkissx93Oaxr8Rj+wxvxmmMMebkvNYMJSKBwETgYiATWCQiM1R1dbmi/1HVuyt4i3xV7eWt+IwxxlSfN2sWacAGVc1Q1UJgGjDSi59njDHGS7yZLFoD2zxeZ7rbyrtSRFaIyHsi0sZje5iIpIvIfBEZ5cU4jTHGVMGbyaKiKYjlJ3X8D2inqinAV8BbHvsS3bG/1wLPi0iHEz5AZLybUNKzs7NrK25jjDHleDNZZAKeNYUEIMuzgKrmqGqB+/J1oK/Hviz3ZwbwDdC7/Aeo6iRVTVXV1Pj4+NqN3hhjzFHeTBaLgE4ikiQiIcBY4LhRTSLieX/CEcAad3u0iIS6z+OA84HyHePGGGPOEK+NhlLVYhG5G5gJBAKTVXWViDwOpKvqDOBeERkBFAN7gZvcw7sBr4lIKU5Ce6qCUVTHWbx48R4R2VKDkOOAPTU4vq6oL+cBdi51VX05l/pyHlCzc2lbnUL1Zm2omhKR9Oqsj1LX1ZfzADuXuqq+nEt9OQ84M+diM7iNMcZUyZKFMcaYKlmyOGaSrwOoJfXlPMDOpa6qL+dSX84DzsC5WJ+FMcaYKlnNwhhjTJUafLKoamXcukxE2ojIbBFZIyKrROSX7vYYEflSRNa7P6N9HWt1iEigiCwVkY/d10kissA9j/+483XqPBFp6i5f85P73Zzrx9/Jfe6/rZUi8o6IhPnL9yIik0Vkt4is9NhW4fcgjhfd68AKEenju8hPVMm5POP+G1shIh+ISFOPfY+457JWRIbWRgwNOll4rIw7HOgOjBOR7r6N6pQUAw+oajegH3CXG/8EYJaqdgJmua/9wS9xJ2a6ngb+5p7HPuBWn0R16l4APlfVrsBZOOfkd9+JiLQG7gVSVTUZZ77UWPzne5kKDCu3rbLvYTjQyX2MB145QzFW11ROPJcvgWR3uaR1wCMA7jVgLNDDPebv7rWuRhp0ssDPV8ZV1R2qusR9fgDnotQa5xzK1tl6C6jzCzGKSALwM+AN97UAFwHvuUX85TwaAwOANwFUtVBVc/HD78QVBDQSkSAgHNiBn3wvqjoHZ7Kvp8q+h5HAP9QxH2haboUJn6roXFT1C1Utdl/Ox1lSCZxzmaaqBaq6CdiAc62rkYaeLKq7Mm6dJyLtcNbPWgA0V9Ud4CQUoJnvIqu254GHgFL3dSyQ6/GfwV++m/ZANjDFbVJ7Q0Qi8MPvRFW3A88CW3GSxH5gMf75vZSp7Hvw92vBLcBn7nOvnEtDTxbVWRm3zhORSOB94FeqmufreE6ViFwG7FbVxZ6bKyjqD99NENAHeEVVewOH8IMmp4q47fkjgSSgFRCB01xTnj98L1Xx139viMhvcZqk/1W2qYJiNT6Xhp4sqlwZt64TkWCcRPEvVZ3ubt5VVoV2f+72VXzVdD4wQkQ24zQFXoRT02jqNn+A/3w3mUCmqi5wX7+Hkzz87TsBGAJsUtVsVS0CpgPn4Z/fS5nKvge/vBaIyI3AZcB1emwehFfOpaEniypXxq3L3Hb9N4E1qvqcx64ZwI3u8xuBj850bKdCVR9R1QRVbYfzHXytqtcBs4ExbrE6fx4AqroT2CYiXdxNg3FWTPar78S1FegnIuHuv7Wyc/G778VDZd/DDODn7qiofsD+suaqukpEhgEPAyNU9bDHrhnAWBEJFZEknE77hTX+QFVt0A/gUpyRBBuB3/o6nlOMvT9O9XIFsMx9XIrT3j8LWO/+jPF1rKdwThcCH7vP27v/yDcA/wVCfR1fNc+hF5Dufi8fAtH++p0AfwB+AlYC/wRC/eV7Ad7B6Wspwvlr+9bKvgecppuJ7nXgR5wRYD4/hyrOZQNO30TZ//1XPcr/1j2XtcDw2ojBZnAbY4ypUkNvhjLGGFMNliyMMcZUyZKFMcaYKlmyMMYYUyVLFsYYY6pkycL4FREpEZFlIrJcRJaIyHlVlG8qIr+oxvt+IyL14n7MtUVEporImKpLmobAkoXxN/mq2ktVz8JZZfPPVZRvClSZLHzFYya0MXWaJQvjzxrjLJGNiESKyCy3tvGjiJStHvwU0MGtjTzjln3ILbNcRJ7yeL+rRGShiKwTkQvcsoHufQMWufcN+D93e0sRmeO+78qy8p5EZLOIPO2+50IR6ehunyoiz4nIbOBp9x4LH7rvP19EUjzOaYob6woRudLdfomIzHPP9b/u2mCIyFMistot+6y77So3vuUiMqeKcxIRedl9j0/wg8UOzZljf9UYf9NIRJYBYUBLnHWkAI4AV6hqnojEAfNFZAbOIn7JqtoLQESG4yxLfY6qHhaRGI/3DlLVNBG5FHgUZ22kW3GWfjhbREKB70XkC2A0MFNVnxTnXgHhlcSb577nz3HWu7rM3d4ZGKKqJSLyErBUVUeJyEXAP3Bmgf8/97N7urFHu+f2O/fYQyLyMHC/iLwMXAF0VVWVYzfC+T0wVFW3e2yr7Jx6A12AnkBznKU9JlfrWzH1niUL42/yPS785wL/EJFknOUa/iQiA3CWOW+Nc8ErbwgwRd21dFTV8x4BZQsxLgbauc8vAVI82u6b4Ky1swiYLM5Cjh+q6rJK4n3H4+ffPLb/V1VL3Of9gSvdeL4WkVgRaeLGOrbsAFXdJ84Kvd1xLvAAIcA8IA8nYb7h1go+dg/7HpgqIu96nF9l5zQAeMeNK0tEvq7knEwDZMnC+C1Vnef+pR2PsyZWPNBXVYvEWcE2rILDhMqXay5wf5Zw7P+GAPeo6swT3shJTD8D/ikiz6jqPyoKs5Lnh8rFVNFxFcUqwJeqOq6CeNJwFvsbC9wNXKSqd4jIOW6cy0SkV2Xn5NaobP0fUyHrszB+S0S64tzqMwfnr+PdbqIYBLR1ix0AojwO+wK4RUTC3ffwbIaqyEzgTrcGgYh0FpEIEWnrft7rOCv/VnbP5ms8fs6rpMwc4Dr3/S8E9qhzX5IvcC76ZecbjXNHtPM9+j/C3ZgigSaq+inwK5xmLESkg6ouUNXfA3twlq6u8JzcOMa6fRotgUFV/G5MA2I1C+NvyvoswPkL+Ua33f9fwP9EJB1nBc6fAFQ1R0S+F+dG95+p6oPuX9fpIlIIfAr85iSf9wZOk9QScdp9snH6PC4EHhSRIuAg8PNKjg8VkQU4f5idUBtwPYZzZ70VwGGOLaH9R2CiG3sJ8AdVnS4iNwHvuP0N4PRhHAA+EpEw9/dyn7vvGRHp5G6bBSzHWQ23onP6AKcP6EeclZi/PcnvxTQwtuqsMV7iNoWlquoeX8diTE1ZM5QxxpgqWc3CGGNMlaxmYYwxpkqWLIwxxlTJkoUxxpgqWbIwxhhTJUsWxhhjqmTJwhhjTJX+P2CHtlL3cUE8AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "new_learn.recorder.plot_losses()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "del test_learn\n",
    "del new_model\n",
    "del new_learn"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Setup model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "def Flatten(): return Lambda(lambda x: x.view((x.size(0), -1)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "def ResizeInput(): return Lambda(lambda x: x.view((-1,)+x.size()[-2:]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "#def ResizeOutput(): return Lambda(lambda x: x.view(-1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "net = nn.Sequential(ResizeInput(),\n",
    "                    nn.Conv1d(in_channels=4, out_channels=32, kernel_size=12),\n",
    "                    nn.MaxPool1d(kernel_size=4),\n",
    "                    Flatten(),\n",
    "                    nn.Linear(in_features=288, out_features=16),\n",
    "                    nn.ReLU(),\n",
    "                    nn.Linear(in_features=16, out_features=2),\n",
    "                    #Debugger()\n",
    "                   )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Sequential(\n",
       "  (0): Lambda()\n",
       "  (1): Conv1d(4, 32, kernel_size=(12,), stride=(1,))\n",
       "  (2): MaxPool1d(kernel_size=4, stride=4, padding=0, dilation=1, ceil_mode=False)\n",
       "  (3): Lambda()\n",
       "  (4): Linear(in_features=288, out_features=16, bias=True)\n",
       "  (5): ReLU()\n",
       "  (6): Linear(in_features=16, out_features=2, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "net"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "apply_init(net, nn.init.kaiming_normal_)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Run sequence through network for testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((Image (1, 4, 50), Category 0), (Image (1, 4, 50), Category 1))"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.train_ds[0], data.train_ds[3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([[0.2031, 0.2126]], grad_fn=<ThAddmmBackward>),\n",
       " tensor([[-0.2022,  1.0126]], grad_fn=<ThAddmmBackward>))"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "net(data.train_ds[0][0].data), net(data.train_ds[3][0].data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Learner setup"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "## TBLogger"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "\n",
    "# From https://gist.github.com/gyglim/1f8dfb1b5c82627ae3efcfbbadb9f514\n",
    "\"\"\"Simple example on how to log scalars and images to tensorboard without tensor ops.\n",
    "License: Copyleft\n",
    "\"\"\"\n",
    "#__author__ = \"Michael Gygli\"\n",
    "\n",
    "#import tensorflow as tf\n",
    "#from StringIO import StringIO\n",
    "#import matplotlib.pyplot as plt\n",
    "#import numpy as np\n",
    "\n",
    "class Logger(object):\n",
    "    \"\"\"Logging in tensorboard without tensorflow ops.\"\"\"\n",
    "\n",
    "    def __init__(self, log_dir):\n",
    "        \"\"\"Creates a summary writer logging to log_dir.\"\"\"\n",
    "        self.writer = tf.summary.FileWriter(log_dir)\n",
    "\n",
    "    def log_scalar(self, tag, value, step):\n",
    "        \"\"\"Log a scalar variable.\n",
    "        Parameter\n",
    "        ----------\n",
    "        tag : basestring\n",
    "            Name of the scalar\n",
    "        value\n",
    "        step : int\n",
    "            training iteration\n",
    "        \"\"\"\n",
    "        summary = tf.Summary(value=[tf.Summary.Value(tag=tag,\n",
    "                                                     simple_value=value)])\n",
    "        self.writer.add_summary(summary, step)\n",
    "\n",
    "    def log_images(self, tag, images, step):\n",
    "        \"\"\"Logs a list of images.\"\"\"\n",
    "\n",
    "        im_summaries = []\n",
    "        for nr, img in enumerate(images):\n",
    "            # Write the image to a string\n",
    "            s = StringIO()\n",
    "            plt.imsave(s, img, format='png')\n",
    "\n",
    "            # Create an Image object\n",
    "            img_sum = tf.Summary.Image(encoded_image_string=s.getvalue(),\n",
    "                                       height=img.shape[0],\n",
    "                                       width=img.shape[1])\n",
    "            # Create a Summary value\n",
    "            im_summaries.append(tf.Summary.Value(tag='%s/%d' % (tag, nr),\n",
    "                                                 image=img_sum))\n",
    "\n",
    "        # Create and write Summary\n",
    "        summary = tf.Summary(value=im_summaries)\n",
    "        self.writer.add_summary(summary, step)\n",
    "        \n",
    "\n",
    "    def log_histogram(self, tag, values, step, bins=1000):\n",
    "        \"\"\"Logs the histogram of a list/vector of values.\"\"\"\n",
    "        # Convert to a numpy array\n",
    "        values = np.array(values)\n",
    "        \n",
    "        # Create histogram using numpy        \n",
    "        counts, bin_edges = np.histogram(values, bins=bins)\n",
    "\n",
    "        # Fill fields of histogram proto\n",
    "        hist = tf.HistogramProto()\n",
    "        hist.min = float(np.min(values))\n",
    "        hist.max = float(np.max(values))\n",
    "        hist.num = int(np.prod(values.shape))\n",
    "        hist.sum = float(np.sum(values))\n",
    "        hist.sum_squares = float(np.sum(values**2))\n",
    "\n",
    "        # Requires equal number as bins, where the first goes from -DBL_MAX to bin_edges[1]\n",
    "        # See https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/summary.proto#L30\n",
    "        # Thus, we drop the start of the first bin\n",
    "        bin_edges = bin_edges[1:]\n",
    "\n",
    "        # Add bin edges and counts\n",
    "        for edge in bin_edges:\n",
    "            hist.bucket_limit.append(edge)\n",
    "        for c in counts:\n",
    "            hist.bucket.append(c)\n",
    "\n",
    "        # Create and write Summary\n",
    "        summary = tf.Summary(value=[tf.Summary.Value(tag=tag, histo=hist)])\n",
    "        self.writer.add_summary(summary, step)\n",
    "        self.writer.flush()\n",
    "        \n",
    "\"A `Callback` that saves tracked metrics into a log file for Tensorboard.\"\n",
    "# Based on https://gist.github.com/gyglim/1f8dfb1b5c82627ae3efcfbbadb9f514\n",
    "# and devforfu: https://nbviewer.jupyter.org/gist/devforfu/ea0b3fcfe194dad323c3762492b05cae\n",
    "# Contribution from MicPie\n",
    "\n",
    "#from ..torch_core import *\n",
    "#from ..basic_data import DataBunch\n",
    "#from ..callback import *\n",
    "#from ..basic_train import Learner, LearnerCallback\n",
    "#import tensorflow as tf\n",
    "\n",
    "__all__ = ['TBLogger']\n",
    "\n",
    "@dataclass\n",
    "class TBLogger(LearnerCallback):\n",
    "    \"A `LearnerCallback` that saves history of metrics while training `learn` into log files for Tensorboard.\"\n",
    "    \n",
    "    log_dir:str = 'logs'\n",
    "    log_name:str = 'data'\n",
    "    log_scalar:bool = True # log scalar values for Tensorboard scalar summary\n",
    "    log_hist:bool = True # log values and gradients of the parameters for Tensorboard histogram summary\n",
    "    log_img:bool = False # log values for Tensorboard image summary\n",
    "\n",
    "    def __post_init__(self): \n",
    "        super().__post_init__()\n",
    "    #def __init__(self):\n",
    "    #    super().__init__()\n",
    "        self.path = self.learn.path\n",
    "        (self.path/self.log_dir).mkdir(parents=True, exist_ok=True) # setup logs directory\n",
    "        self.Log = Logger(str(self.path/self.log_dir/self.log_name))\n",
    "        self.epoch = 0\n",
    "        self.batch = 0\n",
    "        self.log_grads = {}\n",
    "    \n",
    "    def on_backward_end(self, **kwargs:Any):\n",
    "        self.batch = self.batch+1\n",
    "        #print('\\nBatch: ',self.batch)\n",
    "        \n",
    "        if self.log_hist:\n",
    "            for tag, value in learn.model.named_parameters():\n",
    "                tag_grad = tag.replace('.', '/')+'/grad'\n",
    "                \n",
    "                if tag_grad in self.log_grads:\n",
    "                    #self.log_grads[tag_grad] += value.grad.data.cpu().detach().numpy()\n",
    "                    self.log_grads[tag_grad] = self.log_grads[tag_grad] + value.grad.data.cpu().detach().numpy() # gradients are summed up from every batch\n",
    "                    #print('if')\n",
    "                else:\n",
    "                    self.log_grads[tag_grad] = value.grad.data.cpu().detach().numpy()\n",
    "                    #print('else')\n",
    "                \n",
    "                #print(tag_grad, self.log_grads[tag_grad].sum())\n",
    "        return self.log_grads\n",
    "    \n",
    "    #def on_step_end(self, **kwards:Any):\n",
    "        #print('Step end: ', self.log_grads)\n",
    "\n",
    "    def on_epoch_end(self, epoch:int, smooth_loss:Tensor, last_metrics:MetricsList, **kwargs:Any) -> bool:\n",
    "        last_metrics = ifnone(last_metrics, [])\n",
    "        tr_info = {name: stat for name, stat in zip(self.learn.recorder.names, [epoch, smooth_loss] + last_metrics)}\n",
    "        self.epoch = tr_info['epoch']\n",
    "        self.batch = 0 # reset batch count\n",
    "        #print('\\nEpoch: ',self.epoch)\n",
    "        \n",
    "        if self.log_scalar:\n",
    "            for tag, value in tr_info.items():\n",
    "                if tag == 'epoch': continue\n",
    "                self.Log.log_scalar(tag, value, self.epoch+1)\n",
    "                \n",
    "        if self.log_hist:\n",
    "            for tag, value in learn.model.named_parameters():\n",
    "                \n",
    "                tag = tag.replace('.', '/')\n",
    "                self.Log.log_histogram(tag, value.data.cpu().numpy(), self.epoch+1)\n",
    "                \n",
    "                tag_grad = tag.replace('.', '/')+'/grad'\n",
    "                self.Log.log_histogram(tag_grad, self.log_grads[tag_grad], self.epoch+1)\n",
    "                #print(tag_grad, self.log_grads[tag_grad].sum())\n",
    "                \n",
    "        #if self.log_img:\n",
    "        #    for tag, value in learn.model.named_parameters():\n",
    "        #        \n",
    "        #        tag = tag.replace('.', '/')\n",
    "        #        self.Log.log_images(tag, value.data.cpu().numpy(), self.epoch+1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Learner setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn = Learner(data, net, loss_func=nn.functional.cross_entropy, metrics=accuracy, callback_fns=[TBLogger])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================================================\n",
      "Layer (type)               Output Shape         Param #   \n",
      "================================================================================\n",
      "Lambda                    [64, 4, 50]          0                   \n",
      "________________________________________________________________________________\n",
      "Conv1d                    [64, 32, 39]         1568                \n",
      "________________________________________________________________________________\n",
      "MaxPool1d                 [64, 32, 9]          0                   \n",
      "________________________________________________________________________________\n",
      "Lambda                    [64, 288]            0                   \n",
      "________________________________________________________________________________\n",
      "Linear                    [64, 16]             4624                \n",
      "________________________________________________________________________________\n",
      "ReLU                      [64, 16]             0                   \n",
      "________________________________________________________________________________\n",
      "Linear                    [64, 2]              34                  \n",
      "________________________________________________________________________________\n",
      "Total params:  6226\n"
     ]
    }
   ],
   "source": [
    "learn.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LR Finder is complete, type {learner_name}.recorder.plot() to see the graph.\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZIAAAEKCAYAAAA4t9PUAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzsvXl8XHd57/9+RqNttK+2LMn7Eq9xEmdPICEkOASSsgSSlhZKgdsF0sJtWvi1Fyi93AtdLpeWlDZNWVvCBQIkQELCloXEIXES27G82/Gizdp3jWb7/v4454xG0oxmRjNnJFnP+/Wal2fOnHPmfCX5fObZxRiDoiiKoswVz3xfgKIoirK4USFRFEVRMkKFRFEURckIFRJFURQlI1RIFEVRlIxQIVEURVEyQoVEURRFyQgVEkVRFCUjVEgURVGUjPDO9wXkgtraWrN69er5vgxFUZRFxUsvvdRjjKlLtt+SEJLVq1ezd+/e+b4MRVGURYWInEllP3VtKYqiKBmhQqIoiqJkhAqJoiiKkhEqJIqiKEpGqJAoiqIoGaFCoiiKomSEComiKIqSESokSk5oaR9k7+m++b4MRVFcQIVEyQl/99Oj/OVDB9I+rrV/jIGxgAtXpChKtlAhSQN/MMxt//QML+o367TpHZ3gXN844YhJ67i7//15PvOjQy5dlaIo2cBVIRGR3SJyVEROiMjH47z/BRHZZz+OichAzHvvFZHj9uO9MdsvE5FX7XP+k4iIm2uIpXPQT0v7EPvPDSTfWZlC/2iQQDhCx+B4ysd0Dfs51zfO/lb9eSvKQsY1IRGRPOA+4FZgC3C3iGyJ3ccY81FjzE5jzE7gn4Hv28dWA58CrgSuAD4lIlX2YV8GPgRssB+73VrDdIb9IQCG7H+V1OkbtdxTZ3rHUj6mpX0IgNd6RvEHw65cl6IomeOmRXIFcMIYc8oYEwC+Ddwxy/53Aw/az98E/MwY02eM6Qd+BuwWkQag3BizxxhjgG8Av+XeEqYyPBEEYGg8mKuPvCAYD4QZt4XgdO9oyse1tA0CEDFwtHPYlWtTFCVz3BSSRuBczOtWe9sMRGQVsAb4ZZJjG+3nSc/pBo5FMqwWSVr0xwTL07FIDrYNUVKQB8DhjqGsX5eiKNnBTSGJF7tIFGm9C/ieMcbxXyQ6NuVzisiHRGSviOzt7u5OerGpMBIVErVI0sFxawGcScMiOdg+yA2b6ikpyFMhUZQFjJtC0go0x7xuAtoT7HsXk26t2Y5ttZ8nPacx5n5jzC5jzK66uqRzWVLCERC1SNLDEZJKX37KFsnAWIDW/nG2N1WwaXkZh9W1pSgLFjeF5EVgg4isEZECLLF4ZPpOIrIJqAL2xGx+HLhFRKrsIPstwOPGmA5gWESusrO1fg942MU1TCHq2ppQiyQdHNfWJc2VnO4dxQpvzY4TaN+2ooLNDeUc7hhK6ThFUXKPa0JijAkBH8YShcPAd4wxLSLyGRG5PWbXu4Fvm5i7hDGmD/hbLDF6EfiMvQ3gj4AHgBPASeAxt9YwnZEJjZHMBcciuWRlFf5ghK7hiaTHHLQD7VtXlLO5oZxhf4i2gdRThxVFyR2ujto1xjwKPDpt2yenvf50gmO/Anwlzva9wLbsXWXqDGmwfU70jwbwCGxvqgDgdM8oy8qLZj3m1bZBGiuLqSopYHNDGQCHO4ZpqvK5fr2KoqSHVranwWSMJKhuljToGwtQ6StgXW0pAGf6ksdJWtqH2NZYDsCm5da/GnBXlIWJCkkaOK6tYNgwEYrM89UsHvpHg1T58llRWYTXI0kzt4b9QV7rGWXbCsuCKS30sqrGx5FOFRJFWYiokKRBrEtrKMMU4Mde7eDLT57M9JIWBX2jAapLCvDmeWiu9nE6SebWISfQbrvCADYvL+dwh2ZuKcpCRIUkDUZihCTTOMkj+9v55p7TmV3QIqF/LECVrwCAldW+pBbJwZiMLYfNDeWc7h1lLKDxKUVZaKiQpMGwP0hNSYH9PLMbmj8YzmnPrnu/u58nj3bl7PNicSwSgNU1Ps70jM0aY2ppG2RZeSF1ZYXRbZsbyjAGjmg9iaIsOFRI0mDYH6Khssh+nplryx+MMDIRIhR2P9Yy5A/y3Zda+eaeM65/1nSMMZZFYgvJqpoShidC9I8l/vkdbB+cYo2AZZGABtwVZSGiQpIikYhhJBBiRUUxkAWLJBTOynlSoa3fqr/Yc6qXQI6TBIYnQgTDhmrbtbW61krfTdS8cTwQ5kTXCFsbpwpJU1UxZYVejmicRFEWHCokKTIaCGEMNFY5QpK5RQKZB+1TodUWkrFAmJfP9rv+ebH028WIsRYJJO65dahjiIiBbSvKp2wXES5qKItrkZzqHtExvooyj6iQpIiT+utYJEPjmVkSE3Zb9UzPkwpt/ZNZUs8cz04Dy1Rxqtqd2FJTVTEicLonfuZWS7tV0b5tmkUClnvrSOcwkZgpi+OBMO/96gvc8+Ar2b50RVFSRIUkRRwX1LKKIkQyt0ic+Ry5sEjaBsYpyvewa1UVzxzvcf3zYnH6bDkWSaE3jxUVxQktkoNtg1SXFNBQMbPyfXNDOSMToaiFBfDFXxznXN84HUP+nLvtFEWxUCFJEUc4KorzKS3wZpxx5Y9aJLlxba2oLOZ1G+t4tW0w6m7KBX2j1vqcGAlYcZJEtSQH24bYuqKceBOUnYD7Idu9daRziAeeOUVdWSHGkNYYX0VRsocKSYo4FklpoZeyIm8W0n+tb8+DORCStoFxmqp8XL+hFmPg2ZO5s0omYyT50W2rako4G6dNykQozLHzw2yP49YC2LSsDI9YAhKJGD7x/VcpL87nM7dvBZhiqSiKkjtUSFLEEY7yIi9lRfkZubaMMdGsrVwF2xsri9nRVEl5kZdnjuVOSPrGAuTnCaWFk/1BV9f46BsNzBDRY50jhCImbnwEoLggj9W1JRzuGOK/XjjLK2cH+OvbNkf3b+1PffqioijZw9XuvxcSjpCUFeVnbJEEwhGcejy3g+1jgRB9owGaqorJ8wjXrq/lmePdGGPiuo+yTf+oVdUe+1krq63MrbO9Y9GOwAD7WgcAZtSQxLJ5eTm/ea2P5070cu36Gt52SSOhiMEjapEoynyhFkmKjNjDrEqLbNdWBsOtHLcWuG+RtNszPJrstOXrN9TRPujnZHfqI28zIbaq3SFeLUkgFOE/njnFhvpSmquLE55vc0MZPSMTTIQj/M/f2o6IkJ/noaGiWIVEUeYJFZIUGfaH8AiUFOTZrq25WxJO6i+4HyM51z9dSGqB3KUBx/bZclhZbQlJbObWfz5/htO9Y/x/b948q6XkFCre84b1rKktiW5vrCpW15aizBMqJCky7A9RWuhFRDJ2bU2xSFwWEqeqvbHSunk3V/tYU1uSszTg3jgWia/Ay7Lywuj89oGxAF/8xXGu31DLDZvqZj3f6zbUcf/vXsYfvn7dlO1NVWqRKMp8oUKSIsP+EGVFVuaRE2yf63ArJ9AOuN64sbV/nPw8oT6mAeL1G2p5PkftUvpHA1MythxW1ZREheSff3mCIX8wqTUCkOcRbtm6HG/e1D/dpiof57WWRFHmBRWSFBn2BykrsnITyoq8GQ23cmpICrwe9y2SAauGxOOZvEFft742J+1SwhHDwHiQ6pLCGe+tqvZxuneU0z2jfGPPad69qzlaJzIXmqqKiRjoHPRncMWKoswFFZIUGZkIRVNYy21BmWug3HFt1ZcVuh5sb+0fo7FyavD66nU15HnE9TjJ4HgQY6DaN9MiWV1bQtfwBJ96pIX8PA8fu2VjRp/lxIA0TqIouUeFJEUs15YtJMX50W1zwWmPUl9W6Hqwva1/PHqTdSgryufSlZWux0n6pjVsjGVVjRWzeepYN3/0+nXUl81siZIOTXYMSOMkipJ7VEhSxHJtOTESr71tbkLiuLaWlRfhD0aYiImZZJOJUJiu4YlooD2W6zdY7VIGxtxrl+L02ZoebAdYbXcBXl5exAeuX5vxZy2vKLJrSdQiUZRco0KSIiMTIUqjMRJLUOYa3/DHWCTg3kyS9gErXjDdIgHYuMyaONg24N43+KhF4pspJOvqStlQX8qnb99CcUFexp9V4PWwvLxILRJFmQe0sj1FhmJcW5laJBNOjKTccucMjQepLZ0ZkM6UaOpvHCGpLbVu7r0jLloko4ktkuKCPH72sddn9fOaqny0uiiMiqLERy2SFJgIhQmEIpQVTrVI5tpvy0n/dSwSt+IkjptnerAdJm/uvaMTrnw2WH22IL5F4gZNVcVR8VQUJXeokKTASEyfLevf7MRIohaJS66ttoFx8jwSd7ZHjW0BuW2RFOfnZcV1lQpNVcV0DI4TDGstiaLkEhWSFJhs2GgJSGmBN6PhVrHpv+BedXtr/zjLy4tmFO+BlcKcnyf0ujibJF5Vu5s0Vfm0lkRR5gEVkhRwxuw6dSQej2Q03MofDOP1SHT8rFu1JG3943HjI2DNQK8uKaB3JHPX1hd+doynj82sSUlU1e4WzlrPaeaWouQUFZIUcG70jmvLej73flv+YISi/LxoPYpbreTbBsZpihMfcagpKYxmVs2VSMRw369O8JVnX5vxXt9Y/Kp2t5gsStQ4iaLkEhWSFJju2rKez324lT8UpijfQ6HXQ0Gex5VgezAcoWNwZjFiLDWlBfRkGCPpGZ0gFDG8fKafSGRq77H+0UDcqna3aKgoRgQNuCtKjlEhSYGRuEKSgUUSCFPozUNEKC/2uuLa6hz0EzHxU38dakoKMs7acuIRQ/4Qx7tGprxnubZyFyPRWhJFmR9USFJgOJFra47DrRyLBKC8KN+VYHvrtPbx8agpLaQvQ4ukIyawvfdMX/R5IBRheCJEdY5Sfx2adC6JouQcFZIUmB5sBzIabuXESMDq2+VG+m/btMmI8agpLWA0EGY8MPcWLY5FUpyfx97Tk92EndYrubRIwC5KVItEUXKKq0IiIrtF5KiInBCRjyfY510ickhEWkTkWzHbPy8iB+3Hu2O2v0FEXra3f11EXK/OH/aHrHiGd/LHVV6cSbA9THGMkLgRI3G+lTdUJm6GWJOFosSOQT8FeR6u31A7xSLpm6XPlps0VRXTOeQnpLUkipIzXBMSEckD7gNuBbYAd4vIlmn7bAA+AVxrjNkK/Jm9/TbgUmAncCVwr4iUi4gH+DpwlzFmG3AGeK9ba3CIbY/ikMlwK38wPGmRFHkZdkFI2vrHWVZeSKE3cTFgjZ1RlUnmVufgOMsqCrl8dTXn+sbpGvJPOWeuqtodGiuLCUfMFJeboiju4qZFcgVwwhhzyhgTAL4N3DFtnw8C9xlj+gGMMV329i3AU8aYkDFmFNgP7AZqgAljzDF7v58B73BxDYDl2oqNj0Bmw60s15YdIynOdyXY3jYwHrc1SizVWei31THop6G8mF2rqwDYe8Zyb/WPWmvKvUVixYTcbEapKMpU3BSSRuBczOtWe1ssG4GNIvKsiDwvIrvt7fuBW0XEJyK1wI1AM9AD5IvILnu/d9rbZyAiHxKRvSKyt7s7swFOsdMRHTLpAOwPhSm0LZKK4nyGxkNzHtubiNb+8ehNNRG1tkXSk0FRYueQn+UVRWxdUUGh1xONk/TZ7rJcFiSC1pIoynzgppDEG749/W7pBTYANwB3Aw+ISKUx5gngUeA54EFgDxAy1t32LuALIvICMAzEDVQYY+43xuwyxuyqq6vLaCHD/tCUQDvETklMP04yEYxQ5HVcW/kEwpE5j+2Nh+XaSVzV7lBjWyRzdW0ZY7mQGiqKKPB6uLi5kpfsOEmfbZHk2rXVUFmE6FwSRckpbgpJK1OthSagPc4+DxtjgsaY14CjWMKCMeazxpidxpibsUTpuL19jzHmemPMFcDTznY3GYkbI3EaN87BIgnGpP8WW+fJZsC9a9hPMGySurZ8BXkUej1z7rfVNxogEIqw3G4KuWtVFS3tQ4wHwvSPBSgr8pIfp8+XmxR681hWprUkipJL3Pxf/iKwQUTWiEgBliXxyLR9fojltsJ2YW0ETolInojU2Nt3ADuAJ+zX9fa/hcBfAv/q4hoASyxKC6fHSOY+bndqsD2zIVnxcCq7Z0v9BavfVm1p4ZxdW05A2+kuvGt1FaGIYd+5AfpGA9GssFyT61qSF17r4+9+ekS7DitLFtdSZ40xIRH5MPA4kAd8xRjTIiKfAfYaYx6x37tFRA4BYeBeY0yviBQBz4gIwBDwHmOMc8e+V0TegiWCXzbG/NKtNTgMT8xmkaQnJMYY/KGpwXZIvXHjeCCMCFEhikcqNSQONaUFc3ZtdUaFxPqcS1daAfeXzvTRP5bbqvZYmqqKo0F/NxkcD/K5x47w4AtnAbh2fS3Xrq91/XMVZaHhag2GMeZRrFhH7LZPxjw3wMfsR+w+fqzMrXjnvBe4N+sXm4BIxDAyEYrGRBzmOtwqGDaEIyYaI6lIs3Hj+7/2Io1VxfzDnRcn3CeVqnYHqwPw3ISkY2iqRVLpK2BDfSl7z/TTNxpgeXniGhY3aawq5kcHOgiFI3Fb6GeKMYZHX+3k0z9qoW80wO9fu5pv7DnD86d6VUiUJYmO2k3CaCCEMUTntTvM1SJxpiPG1pFA6jGSo+eH6U7iijrXN0ZtaUFKA6VqSgo51jmc0mdPp3Nw3GqHHzMmeNfqKn5yoANfgZfNDeVzOm+mNFX5CEcM54cnksaJ0sEYw55TvfzbU6d46lg32xsr+Or7LmdbYwUvnx3g+VO9WfssRVlMqJAkwWmPMr2OZK7DrZzpiHNxbfmDYfpGAwyNBwmGIwkD2Se6RlhbV5rS9dSWFtA7GsAYg+1KTJmOQT/LyovI80wet2tVNQ++cI4hfyjnNSQO0RTgvrGsCIk/GObhfW189dnTHOkcpsqXz1/ftpn3XbM6avFctbaar/z6NcYD4ZxNhFSUhYL22kpCvBbyMPfhVhP2dETHInHOm0qwvWvIskRCEcPZvvjBZGMMx7tGWF+fmpBUlxQwEYowOod+W52D/mjGloNTmAi5T/11cOpnspG59czxbq7+37/gLx96FYC/e8cO9nziJj5w/dopbrOr1tYQDBtePut+bEZRFhoqJElwLI7pdSQwt1bykxaJJSSF3jyK8j0pCVLn0GTbj5PTWrY79IwEGBwPsiFFIZmc3Z5+5lY8IVlZ7aPWPmd1josRHVZUFlFa6OWbz5+J/rznyoMvnCXP4+HBD17FY396Pe+6vDluosOuVVXkeUTdW8qSRIUkCZMWycyb4lyGW/mnWSTgVLcnP0/H4OQ37BPd8YXkeJcV70jVInGKEtOtJYkWI04LqIsIu1ZZVsl8WSSF3jz+4c6L2XdugI8/dCCjrgGnuke5uKmCq9fVzOr6KyvKZ1tjhQqJsiRRIUlCItcWzK0D8GSwPaaTcFFqHYDP2xZJWaGXk12jcfdxLJUN9WUpXU+0A3CamVtD4yHGg+EZFglMurfmK0YCsHvbcv78lo38cF87//LkyTmdIxIxvNYzytq6kpT2v2ptNfvODWTUll9RFiMqJEmYDLbHc23lpz3carprC1Jv3Ng5OEFJQR7bmyo4mdAiGaGs0Muy8tRmpc/VtdUxZFlHTg1JLG+9eAXvvKyJrSsq0jpntvmTG9dzx84V/P3jR3m8pTPt49sGxpkIRVhTm5p1p3ESZamiQpKEeNMRHeYWI7FdWzHt3cuLvCnVkXQOjbOsooj19aWc7B6J67I5fn6EdfWlKWdgTc4kSc8icara41kky8qL+Ic7L5737CUR4fPv2MHFzZV89P/t41D7UFrHn+qxrL5ULRKNkyhLFRWSJAz7Q4iAL06ANbNge+yQrFQtEj/Ly4tYV1fKsD9E9/BMK+JE90jKgXbrOvIoKchL27XVOa09ykKlKD+Pf//dyygvyueD39jLWCD139cp2+pLVUg0TqIsVVRIkuB0/vV4Zn7DL7PnracTzI3n2qpIcUri+aEJlldYQgIzA+6DY0G6hydSDrQ71JQWpj0lsWNgHI9AXVlqLrT5pL68iC/etZO2gXG+8+K55AfYnOoepazQS11p6mvUOImyFFEhScKwP0RZnNRfsCySUMRE3VWp4AhJ4bRgezJBikQM54dsi6Te+oZ8sntqwP1Et5WxtWFZukKSfr+tjkE/dWWFOe/uO1euXFvDZauqeODXr6U8htcJtKdTqKlxEmUpsjjuAvOINdQqfj3EXPptxUv/LS/2EjHMWhTYMzpBKGJYXlHE8vIiSgryZtSSHD9vvV5fl1rGlkNNSQE96bq2hvwsjxNoX8h86HVrae0f57GDqQXeT3Wn3iHAQeMkylJEhSQJI3E6/zrMZbiVY5EUxwpJCq3kzw9arqfl5UWICOvsgHssx7tGKMr3JB1oNZ2aksL0s7bi1JAsdG7evIw1tSXc//SppO7IsUCI9kE/a2tTi484aJxEWYqokCRh2B+a0bDRYS7DrfyhMHkemeISSqXfllOM6GRJrasrnWGRnOgaYV1d6ZTeV6nguLbSifV0DvppqFxcQuLxCB+4fg2vtg2yJ8mN/jU7Y2tNioH2WDROoiw1VEiSYFkkyVxb6VgkEYq8U3/sTiv5wbFZLBK7GNFpzb6uroT2QT+jE5OffSKNHluxVJcUEIqYlFvZD/uDjEyEFnzGVjzecWkTNSUF/PvTp2bd75Qdf1qbYg1JLBonUZYaKiRJsKYjJrNI0nNtTe/VFHVtzXKeziH/lJbtjmA4N7zRiRBtA+Nppf46OL2xelLM3OqM1pAsrhgJWLGp916zml8d7ebY+cTt852f65o0XVugcRJl6aFCkoQh/8yhVg5zDbbPEJLi5B2AOwb91JcVRt1WTgqwEydx/p2LReL020o1c2v6iN3Fxu9etYri/Dzun8UqOdUzQmNl8ZyKKsuK8tm6opy9p9UiUZYGKiSzMBEKEwhFEgbb52SRhMJTUn8h1iKZ3bW1LObGvbLGR55HogJyossRkvQytmCyJ1aqAfeoRbLIgu0OVSUFvGtXEw/va4uuZTrp9NiKx/r6Us70xu+HpigXGiokszBiC0Qi19ZchltNBMNT2qPApCDNVpTYOeifYgEUevNYWe2LCsnxrhG8HmFVTfLxutOJurZSTAF2LJJli1RIAD5w/VrCEcPXnjs94z1jDKe6R9PO2IplZbWPjiE/EyENuCsXPiokszBbC3mwh1sVpjfcynJtTf2xe/M81nlmCXZ32tMIY1lXVxq1RI6fH2FNbcmcCgSddu+purY6h8apLS2kwLt4/3yaq3284aJ6Ht7XNiNbrXt4gpGJUNo1JLGsrPZhDLRlYbiWoix0Fu+dIAc4nX8Tpf+C5ZbKNNhunceb0LU17A8yGgjPcCWtqy/hdM8YoXCEk90jaVe0OxR4PZQXeVN2bXVMs44WK7u3NdAx6OdA6+CU7U7HgExcWyurLcvwTIJJlopyIaFCMgtD0c6/iYXEatyYumtrPJGQzDLcKpr6WzHTIgmEI5zsHuVM7yjrM/gGXVtaSE+qFkmcyYiLkTdursfrEX46rcX8qR7LyptLxpaDIyTnVEiUJYAKySw4MZLyBK4tSL8DsD8YnlLV7lBelLgDcEeC4LaTufXzw+eJGFi/LP1Au0NNaQF9KcZI2gfGLwiLpNJXwNXravjpwc4p7q1T3aMU5XtYkUF6c11ZIUX5Hs72qpAoFz4qJLMwnCTYDnYH4DTTf6dnbYFlkQwmiJF0Jpj94VggP7V7R2VikVSXFKTUAXh0IsSQP3RBWCQAb9q6nNd6Rjke0yXgVPcIq2tK4nZ8ThURYWW1j7NqkShLABWSWRhO2bWVukUyEUrk2vImdW1ND7ZX+PKpLS3k1bZBPJKZT7+mtDClmSSdQ4u7hmQ6t2xZhsikGIOV+rsuA1F2UCFRlgoqJLOQSrA93RiJ1SIlfddWlS8/rgCts8WjudoX9/1UqS0poH8sQDgye7+tyRqSxVfVHo/68iIuW1kVFZJAKMK5/vGMRNmhudrHub6xtHqYKcpiRIVkFob9IQq8Hgrj3PgdyuysrVRvFlbWVnzX1shEiEicG/n5oZmpvw7r7Er2ubRGiaW6pICIgYGx2a2SxV7VHo/d25ZzqGOIs71jnO0bJRwxWRGSldU+RgPhhGnVP3illdu/9Ou4ky4VZTGhQjILs7VHcagszicUMVHrZTZC4QihiEmY/mtM/Cp5a/ZH/Bu3ExdZl6GQOD28ks1u75zWhfhC4E1blwPweEvnZOrvHJo1TsfJ3Erk3nqi5TwHWgf54Df2RscLKMpiRIVkFkYmQrMG2mFy1Gwq3yr9IWeo1cwfe8UsreSnV7XHMmmRzD1jCyb7bSWLk5zrG6e2tCAjN9pCo7nax9YV5fy0pXOyWWOWLBJILCQt7UOsrPaxv3WAj/6/fXGtUUVZDKiQzMJs0xEd6susG3xKQhJnXruDM5NkepuUQChCz0ggoWvryjXV/PEN67h5y7Kknz8bNSWORTL7Ok73jrK6JvOb7EJj99blvHSmn+dP9VJbWjhryneqNFXZQhInBXjIH+Rs3xjvvryZv3rzZh472MnnHz+S8Wcqynww+9ftJc5H37gxqcvBsUi60hGSBMF2mGmRdA3P3iCxKD+Pv9h9UdLPTkaqFsnp3lGuW1+X8ectNHZvW84//uwYTx3r5oo11Vk5Z3FBHvVlhXEtksPtQwBsWVHODRvrON07yr89dYpV1SX89pUrs/L5ipIrVEhm4eLmyqT71Kfj2rKFJH4didNKfmqMJFENSbap8hUgMnuMZCwQ4vzQBGtq028MudBZX1/K2roSTnWPRjPhssGqmvgpwIc6LCHZ2lCOiPDpt27lXN84/+Phg6yu8XHN+tqsXYOiuI2rri0R2S0iR0XkhIh8PME+7xKRQyLSIiLfitn+eRE5aD/eHbP9JhF5WUT2icivRWS9m2tIRqUvn/w8SdEisWIkiSrbYeZMks4E7VGyTZ5HqPIVzNpv64ztoll1Abq2RITddtA9G4F2BycFeDot7UPUlhZELVpvnof7fudS6ssK+cqzr2Xt8xUlF7gmJCKSB9wH3ApsAe4WkS3T9tkAfAK41hizFfgze/ttwKXATuBK4F4RKbcP+zLwO8aYncC3gL/HMM+mAAAgAElEQVR2aw2pICLUlRZmHCOp8FlC4hQfOuRy9kdNScGsrq3TPXOfGrgYeOvFK/B6JCVLNFUStZM/1D7ElhUViExWz5cWerlhUx2/ea0vaT2Poiwk3LRIrgBOGGNOGWMCwLeBO6bt80HgPmNMP4AxpsvevgV4yhgTMsaMAvuB3fZ7BnBEpQJod3ENKVFXVhiNZcyGY5HET//N59KVlXzrhbNT4jKdg36K8j3RrC43qS8vpGMwcdvz01GL5MJzbQFsbijnlU/enLUYCcRvJx8IRTjeNcyWhvIZ+1+1toZhf4iW9sEZ7ynKQsVNIWkEzsW8brW3xbIR2Cgiz4rI8yLiiMV+4FYR8YlILXAj0Gy/9wHgURFpBX4X+JxrK0iRurKiNC2S+D/2j928iY5BP99+4Wx0W+eQn+XlRVO+ubrFhvoyjneNJExDPd0zSm1pQdJMtsVMttcWLwX4eNcwwbBh64qZQnL12hoA9pzUee/K4sFNIYl355t+h/ICG4AbgLuBB0Sk0hjzBPAo8BzwILAHcKLQHwXebIxpAr4K/J+4Hy7yIRHZKyJ7u7u7M13LrNSVpejaCiV2bQFcu76GK9dUc9+TJxkPWPvOVtWebTY3lDEWCHOuP37dw4Wa+usm8drJt8RkbE2nvryIdXUlPKdCoiwiUhISEVknIoX28xtE5B4RSeZIbmXSigBoYqYbqhV42BgTNMa8BhzFEhaMMZ81xuw0xtyMJUrHRaQOuNgY8xv7+P8HXBPvw40x9xtjdhljdtXVuZuuWl9WSN9YgGA4Mut+UddWgpYrIsJ/v2UT3cMTfPP500Buh0htWm7d2A53DMd9/3Tv6AUZaHeTurJCCr2eaKICWPERX0FeQlG+Zl0tL57uS/r3pCgLhVQtkoeAsJ0h9R/AGqxA92y8CGwQkTUiUgDcBTwybZ8fYrmtsF1YG4FTIpInIjX29h3ADuAJoB+oEJGN9vE3A4dTXINr1JUVYkzyGoxkri2AK9ZUc/2GWv71qVMM+4N0DU2wLEdCsnFZKSJwtHOmkFzIqb9uEq+d/KH2IS5aXkZegjb1V6+rYSwQnjG5UVEWKqkKScQYEwLeBvxfY8xHgYbZDrD3/zDwONbN/jvGmBYR+YyI3G7v9jjQKyKHgF8B9xpjeoF84Bl7+/3Ae+zAewgrQP+QiOzHipHcm86C3SDVWpLJOpLZ24v891s20Tca4P/87BiBcCQnGVsAvgIvq6p9HD0/NOO9Czn1121ihSQSMRzqGGLrioqE+19lx0mePxXfvdU56OdM72j2L1RR5kiqBYlBEbkbeC/wVntb0qikMeZRrFhH7LZPxjw3wMfsR+w+fqzMrXjn/AHwgxSvOydMVrf7sRLJ4jMxS6+tWHY2V/LGzfV8/bnTQG477W5aXsaROK4t58Z1oab+usnKGh/Pn+rFGMO5/jFGJkJx4yMO1SUFXLS8jD0ne/mTG6eWSRljeP/XXkQEfnLP9W5fuqKkRKoWye8DVwOfNca8JiJrgP9077IWF/XlqfXb8gfDiEBBXvIf+0dv3oiTPJWrYDtYcZLTvaMzWsO81nNhp/66SWw7+UN2oD1exlYsV62tYe+Zvhn1J08d6+ZQxxCnukd1zokyKx2D4/z4QHtaE1znSkpCYow5ZIy5xxjzoIhUAWXGmHlPu10o1Np9qpJVt48HwhR581JK5d26ooI3b7cqrRsymB2eLpuXlxExcPz8yJTtZ3ov/NRft4hNAW5pHyLPI2xcNnu35mvW1eAPRth/bmqc5MtPngRgPBhO2vJfmV++u/ccd9//PPc/fTJazJtLnj3Ry4e/9UpO5t2kmrX1pIiUi0g1Vo3HV0UkbtrtUqTQm0elLz+5RRIKU1yQevv1T9++lc+9fXtOZ39sWm7d4A53To2TvNajGVtzJVZIDnUMsb6uNGkb/ivX1CACz53siW57+Ww/v3mtL1pr0tqfuHhUmX+++1Ire8/08b8ePcIN//Akb/rC0/zjE0dzYiEAHDs/TIHXk5OU/VRdWxXGmCHg7cBXjTGXAW9077IWH3WlyavbrTG7qZfu1JcVcdcVue0Eu6qmhKJ8z4zMLa0hmTtOO/lzfWO0tA/OGh9xqPDls3VF+ZTCxH998iQVxfn8+Zs2Rc+nLEwiEcPh9iHuunwlz/zFjXzyLVuoKsnnvl+d4POP5WZcwNHOYTbUlybMDswmqd7VvCLSALwL+LGL17NoqS9PXpRojdld2AOhHLdLrJA4qb+rNT4yJ5x28q+cHeD80ETS+IjD1WtreOXsAP5gmBNdwzxx6Dy/d/WqqNWYqHBUmX/O9o0xPBFi64pymqt9vP+6NXz7Q1fzWzsbeWR/e04mYh47P8ymJC7UbJGqkHwGK1X3pDHmRRFZCxx377IWH5ZFkkxIIklTfxcCm5aVcSTGteWk/q7WjK05s7LaxzMnLDdVvB5b8bh6XQ2BcISXz/Tzb0+doijfw/uuWU1poZcqXz7n+tS1tVBpiSZVTM3ifOdlTQz7Qzze0unq5w+OBekY9LNx+QISEmPMd40xO4wxf2S/PmWMeYe7l7a4qC+3+m3NlkkzEQonTf1dCFzUUE7PSIAeu6W8k/qrrq25s7LaR8BO/07FtQVw+epq8jzCD15p44f72njXrmZqSq1U8+ZqH61qkSxYWtoH8XqEjcunjiS4am0NjZXFfO+lVlc//1iX5VFYUBaJiDSJyA9EpEtEzovIQyLS5PbFLSbqSguZCEUY8ocS7uMPhhO2R1lIXGR/i3HcW9HUX61qnzMrbbdgY2Uxlb6ClI4pK8pne2MF332plYiBD16/Nvpec5VPg+0LmJb2ITYsK6Nw2v93j0d4x2VN/PpED+0D7v3+nP+7C8oiwWqO+AiwAquD74/sbYpNfXny6nZ/MLIoLJJo5pY9xe9M7yg1JQVZmWO+VHEytzan6NZyuHqdlaH1lh0NNFdPCnlTdTFt/eMJOzUr84cxhpb2wYSxsDsva8IY+P7L7lklx84PU1boZUWOMj5TvavVGWO+6rQpMcZ8DbjwBndnQF1pbHV7fBZDsB2gtrSQ2tLCGItkVOMjGeIISaqBdoc3bV1ORXE+f3zD1Ar35iofgXCE8ynMwVFyS9fwBD0jgYS/6+ZqH1etreZ7L7W6VlR6tHOYjcvLcjJ+AlIXkh4ReY/dTDFPRN4DaJ/rGFKySEKLQ0jAcm8dPW8JyZneMa1oz5CLGsq5dGUlN29ZltZxO5sr2f+pW6JWokNTlVWkqgH3hYczlGy2fmrvvKyZ071j7D3Tn/XPN8Zw9Pxw0qLXbJKqkLwfK/W3E+gA3onVNkWxqStN3iZlsbi2wHJvHe0cZmQiROeQnzUaaM+I0kIv3//ja9nWmPjmkg6Om0sD7guPlrbE82Yc3rx9OSUFeXxvb/bdW93DEwyMBdm0rDT5zlki1ayts8aY240xdcaYemPMb2EVJyo25cVeCrye2YUksLgskolQhGeOWUPB1LW1sGisVItkodLSPsSa2hJKCxP3xPUVeHnz9gZ+fKCdsUDiBJ254HgSchVoh8wmJH4s+S5LBxGhrnT2osTF5dqyvk391M5319TfhUVRfh7Lygu1KHEB0tKRWveCO3c1MxoI89ir2a0pcWKbuUr9hcyEJDdRnEVEfXniosRwxBAMm0WR/guwYVkpHoFfHu4CNPV3IdJU5dM2KQuMwbEg5/rGU0qquHx1FatqfFmvKTnaOUxtaWG05igXZCIkmnc4jdksklSmIy4kivLzWF1bwvBESFN/FyjNVcVaS7LAaOlIHmh3EBHuvKyJPad6o7OHssGx88NsWp67+AgkERIRGRaRoTiPYayaEiUGyyKJn445KSSLwyKBycJEjY8sTJqrfXQMjuts9wVEqvNmHP7gurXcvGUZn3qkhU8/0kI4w7qgSMRw7PxITjO2IImQGGPKjDHlcR5lxphUpysuGepKi+gfC0ZbYcTiT3E64kLCiZNo6u/CpLnKR8RAx4DWkiwUWtqHWF5eRG2KbqXigjz+9T2X8QfXreFrz53mv31zL6MTcw++t/aPMx4M5zQ+Apm5tpRpOLUkTo+qWBajReLULmjq78IkWkuiAfcFw2wV7YnI8wj/4y1b+Ns7tvLLI12869/20Dk4ty8HTsbW9Lojt1EhySJOdXu8OIkjJNN77yxkdjZXUpTv4ZKVVfN9KUoctJZkYTEeCHOiayRtIXH43atX8x/vvZzTPaPc/IWn+I9fv5a22/Ko3bV7g1okixfHIomXueUPLj7X1rLyIvZ/6hau21A735eixKGhoog8j2gtyQLhSOcQEQNbUgi0J+LGi+p55CPXccnKKv72x4e49YvP8LRdy5UKR8+P0FRVPGsNixssnrvaIqCuLLFFMrEIXVuwuCyopYY3z0NDRZG6thYILWkG2hOxrq6Ur//+5Tzwe7sIhiP83lde4ANf38v5oeTurmOduRtmFYsKSRapnaVxoz9kCUnxIhMSZWHTVFWstSQLhJb2ISqK86Oxq0wQEd64ZRlPfPR1/OXui3j2RA9v+edfs/d0X8JjAqEIJ7tHclrR7qBCkkXy8zxUlxTEtUjGA45rS4VEyR7NVT7OaS3JguCQHWjPZsfdQm8ef3TDOn74J9dSUpDHXfc/zzf3nI7bNfi1nlFCERNN288lKiRZpr4sfnX7YitIVBYHzdU+uocncjIDXElMMBzhcOdwxm6tRGxaXsbDH76O6zfU8j8ebuHe7x2Y8TuP9thS19bip64sfnW749pSi0TJJs3VlhtFK9znl1PdowRCkZQq2udKRXE+//Hey7nnpg1876VW7vjSs/z0YGd0uNmxzmHyPMLautyn66uQZJmEQuJkbWnwWskiTVVWCrAG3OeX9kFLyGOnWLqBxyN87OaN/Md7d+EPhfnD/3yJ3V98mof3tXG4w+o6PB8JMlqdnmUcITHGTPGVRutI1LWlZJFmW0haNeA+rwyOBQGo8uWmJ91Nm5fx+o11/PhAB1/61Qn+9Nv7ALhte0NOPn86KiRZpr6siEA4wuB4kEpfQXT7RDCMCBR6VUiU7FFfVkiB16OurXmmfywAMOX/vNt48zz81iWN3H7xCh5v6eSbz5/hrRerkFwQxNaSxP5R+UMRCr2enM1QVpYGHo/QVFmsrq15ZsC2SCqKc98l2+MRbt3ewK3zZI2AxkiyTn1Z/Op2f3DxDLVSFheNVcVa3T7PDIwFKC/ykudZml8UVUiyTKLqdn8wrIF2xRWaq31qkcwzA+NBqkpy59ZaaLgqJCKyW0SOisgJEfl4gn3eJSKHRKRFRL4Vs/3zInLQfrw7ZvszIrLPfrSLyA/dXEO6TFokU6vb/cGI1pAortBc5WNgLMiwPzjfl7Jk6R8LUjkPbq2FgmsxEhHJA+4DbgZagRdF5BFjzKGYfTYAnwCuNcb0i0i9vf024FJgJ1AIPCUijxljhowx18cc/xDwsFtrmAulhV6K8j0zLJJxdW0pLhFbS7K5YenezOaTwbFATgPtCw03vyJfAZwwxpwyxgSAbwN3TNvng8B9xph+AGNMl719C/CUMSZkjBkF9gO7Yw8UkTLgDcCCskhEhKYqH8e7RqZs1xiJ4hZOLclZTQGeN/rHglTmKPV3IeKmkDQC52Jet9rbYtkIbBSRZ0XkeRFxxGI/cKuI+ESkFrgRaJ527NuAXxhjhly49oy4bn0te072Mh6YbGEwoa4txSXW1JZQnJ/Hxx86wAPPnNJ2KfPAwFiAKrVIXCFe+sL0TmNeYANwA3A38ICIVBpjngAeBZ4DHgT2ANPnT95tvxf/w0U+JCJ7RWRvd3fq/fyzwRs3L2MiFOHZEz3Rbf6QWiSKO1QU5/PdP7yabY0V/M+fHOYN//Ak39l7LuP530pqhMIRhvyheUn9XSi4KSStTLUimoD2OPs8bIwJGmNeA45iCQvGmM8aY3YaY27GEqXjzkEiUoPlOvtJog83xtxvjNlljNlVV1eXlQWlyhVrqikt9PKLI13RbZq1pbjJtsYKvvkHV/KtD1xJXVkhf/G9A7ztX54lEEpvwp6SPkN+6zturqraFyJuCsmLwAYRWSMiBcBdwCPT9vkhltsK24W1ETglInm2WCAiO4AdwBMxx90J/NgYM7fBxi5T4PXwuo21/PLI+Wi7Z83aUnLBNetr+eGfXMvf3L6VA62DPJXGdD1lbsxHVftCw7U7mzEmBHwYeBw4DHzHGNMiIp8Rkdvt3R4HekXkEPAr4F5jTC+QDzxjb78feI99Poe7mMWttRB4w0XLOD80EZ2apsF2JVeICL995UpqSwt46KXW+b6cCx6nqn0pB9tdbZFijHkUK9YRu+2TMc8N8DH7EbuPHytzK9F5b8jqhbrAjZvqEIGfHz7PtsYKFRIlp+TnebhjZyPf2HOagSWemuo2A2qRaGW7W9SUFnJJcyW/tOMk/lBEO/8qOeXtlzYSDBt+dKBjvi/lgmYgx51/FyJ6Z3ORmzYv40DrIJ2DfgKhiAbblZyypaGci5aXqXvLZaIxkmK1SBQXuGlzPQCPHbS+ERYXqJAouUNEeMelTew7N8DJ7pHkByhzYnA8iEegrGjpNlNXIXGRTcvKaKws5ie2a6FIZ5EoOeaOnSvwCPzg5bb5vpQLlv6xABXF+XiWaOdfUCFxFRHhps317D3TD+i8diX31JcXcf2GOn7wSlt0treSXQbGgku6qh1USFznDRfVR5+rkCjzwdsvbaRtYJznX+ud70u5IBkYC1KxhAPtoELiOletrcFnx0a0IFGZD960dTllhV6+r+4tVxgYX9p9tkCFxHWK8vO4bn0tAIVqkSjzQFF+Hm/e3sBjr3YwFpjesk7JlP7RpT2LBFRIcoKTveVTIVHmibdf2shoIMwTLefn+1IuOAbHg0u6GBFcrmxXLO7Y2ch4IMylq6rm+1KUJcrlq6tZXePjCz8/xo0X1afVqXY8EKYo34PI0s1KSkQgFGFkIrSk26OAWiQ5oSg/j/ddu4b8PP1xK/ODxyP8w50X09Y/zn//zv6UM7iG/UGu/twv+JcnT7p8hYuTwXGtagcVEkVZMuxaXc1f3baZnx8+z5efSk0Yfnmki4GxIF9+8mS0p5QyifMzqVjiri0VEkVZQrzvmtXcfvEK/vGJozxzPHmL+cde7aSsyMtoIMS/P3MqB1e4uBhQiwRQIVGUJYWI8Ll3bGdDfRn3PPgKbQPjCfcdnQjxq6NdvO2SRm7b3sBXnz1N78hEDq924dM/qn22QIVEUZYcvgIvX37PpYTChj/+z5eYCMWf8f7k0W4mQhFu3dbAn71xA/5gmPufVqskFsci0WC7oihLjrV1pfz9nTvY3zrIg785G3efRw92UFNSwBVrqllfX8YdOxv5+p7TdA0vyMGk88LkLBIVEkVRliC7tzVwxZpqvvzUSfzBqVbJeCDMr4508aZty8mzmxHec9MGgmHDvz6pVonDwFgQr0coLVzalRQqJIqyhPnTmzZwfmiC706bWfLUsW7GAmHevK0hum1NbQlvv6SR//zNGc4PqVUC0D8WpNKXv+RrbFRIFGUJc826Gi5bVcWXf3WCQCgS3f7Tgx1U+fK5cm31lP3vuWkDkYjhvl+dyPWlukIkYrAmfs+NwXEdYwwqJIqypBER7rlpA+2Dfh562bJKJkJhfn64i5u3LJtRRNtc7ePOXc186zdn+b8/PzbDJbbY+JsftXDrF5+he3hu2WjaZ8tChURRljiv21DLxc2V/MuTJwiGI/z6eA8jEyFu3d4Qd/+P776I3duW839/fpxbvvA0vzi8OPt3GWN47GAnRzqH+b2vvMCgPXs9HQa0zxagQqIoSx4R4U9vWs+5vnF++Eobj9pFiNeuq427f4Uvny/99qX81weupMDr4Q++vpc/+NqLs9akLERa+8fpGp7gth0NnOga5ve/9kLa3ZEHxgJLPmMLVEgURQFu3FTPtsZyvvSrE/zsUCc3b1lGQZLR0Neur+XRe67nE7dexJ5TvXz8oQM5utrssPdMHwAfvnE9/3TXJew7N8B/+2biupp4WNMRVUhUSBRFQUT4yBs2cKZ3jCF/aEq21mwUeD38t9ev43euXMlvTvUxHlg8MZMXT/dTVuhl47Iybt3ewOffsYNnjvdwz4OvTEk8SIQ/GGY8GFbXFiokiqLY3Lx5GRctL6O00Mt1G+K7tRJx7fpaAuFI9Fv+YuCl0/1csqoqWidz565mPvmWLTzecp7L/ufP+MiDr/DwvraEsZNBrWqPsrSraBRFieLxCP/yO5fSMxKgKM0hbFesqSY/T/j1iR6u31Dn0hVaGGP48YEOrlxTTX150ZzOMTge5FjXMLftmGp5vf+6NayrL+UnB9r55ZEufrS/nTyP8ObtDXzx3TvxeCbrRfrHtM+WgwqJoihR1taVsnYOOuAr8HLpyip+fbwHbs3+dcXy1LFuPvLgK9SWFvLPd1/C1etq0j7Hy2f7MQZ2xRk29/qNdbx+Yx2RiGFf6wDfeO40P9zXzp/etJ719WXR/QbGtPOvg7q2FEXJCtetr6WlfYi+UXfnljz4wlmqSwooL/byOw88z5efPJnyoC6Hl073k+cRdq6sTLiPxyNcurKKP75xPQAHWgenvD85i0SFRIVEUZSscK0dV3nuZE/Kx/SNBlIKbDt0Dfn5+eEu7tzVxCMfvs4Kkv/0CB/65t606kBePN3H1hXl+AqSO2XW1ZVSnJ8XR0gci0RdWyokiqJkhR2NFZQVeXn2RGpCEokYbvnCU2m1W/nO3nOEI4a7L19JaaGXL919CZ966xaePNrNnf/2XErtToLhCPtbB7gsjlsrHnkeYVtjOQdaB6Zs7x/TYLuDComiKFnBm+fh6rU1/DpFIekanqBnJMDzp3pT2j8SMTz4wjmuXV/D6toSwEpb/v1r1/DXt23m2PkRTveOJT1PS/sQ/mCEXauqk+7rsL2xkpb2IULhSetpYDxAgddDcZqJCRciKiSKomSN6zbUcq5vnLMp3NBP944CcLBtMKUYxzMnemgbGOfuK1bOeO+KNVbAfbrVEI+9p60U5V2rU7NIAHY0VTARinC8ayS6bcDus7XUO/+CComiKFnk2vVWnCQVq8QRm9FAmFM9I0n2hm/95gw1JQXcsmX5jPc2LiulKN/DvnPJheSlM/00VxezLI3U4e1NFQC8GhMnGRgPaHzExlUhEZHdInJURE6IyMcT7PMuETkkIi0i8q2Y7Z8XkYP2490x20VEPisix0TksIjc4+YaFEVJnbW1JTRUFKUUJznTNxp9Pj2QPR0nyP7OXU1xW7d48zxsW1GR9DzGGF483Z+WWwtgTU0JpYVeDrRNClX/WFAztmxcExIRyQPuw8oq3wLcLSJbpu2zAfgEcK0xZivwZ/b224BLgZ3AlcC9IlJuH/Y+oBm4yBizGfi2W2tQFCU9RIRr19fy7MkewkncVad7x2iqKo6bETWd777USjhiuOvymW4thx1NlRxsGyQYTpwFdrZvjJ6RiZQD7Q4eO+Aea5EMap+tKG5aJFcAJ4wxp4wxAawb/h3T9vkgcJ8xph/AGNNlb98CPGWMCRljRoH9wG77vT8CPmOMiUw7RlGUBcB162sZGAtyqH1o1v3O9o6xtq40bkZULFaQ/SzXrKthjR1kj8fFzVYc49j54YT77D3dD6QXH3HY0VTJ4Y7haLpy/1hAq9pt3BSSRuBczOtWe1ssG4GNIvKsiDwvIo5Y7AduFRGfiNQCN2JZIQDrgHeLyF4Recy2amYgIh+y99nb3d2dtUUpijI716y3At+zxUmMMZzuHWVVtS9uRlQsz5zoobV/nN++MrE1ArCz2Sou3H8usXWz90w/ZUVeNsZUqKfK9sYKAmFLqIwx1iySErVIwF0hiZfKMN3W9QIbgBuAu4EHRKTSGPME8CjwHPAgsAdwBgUUAn5jzC7g34GvxPtwY8z9xphdxphddXXu9v5RFGWS+rIiLlpeNmucZGAsyLA/xKoaX9SSiM2IiuXbL5xNGGSPZWW1j0pf/qzWzUtn+rhsVdWUnlmpssMOuB9oHWQ8GCYQiqhFYuOmkLQyaUUANAHtcfZ52BgTNMa8BhzFEhaMMZ81xuw0xtyMJUrHY455yH7+A2CHS9evKMocuXZ9LS+c7ks4ivdMn5WxtaqmhO2Nzg16pgCMBUL88kgXb9nRkHQ+ioiwo6kyYebWwFiAY+dH4vbXSoWV1T7Ki7y82jagfbam4aaQvAhsEJE1IlIA3AU8Mm2fH2K5rbBdWBuBUyKSJyI19vYdWGLxRMwxb7Cfvx445uIaFEWZA9etryUQivDymf6475+xa0hW1fhYXVNCWaE3bsD9yaPdTIQi7E5xPsrFTRUcOz8cd9KhU/h4WZoZWw6OUB1oHZzs/KtCArgoJMaYEPBh4HHgMPAdY0yLiHxGRG63d3sc6BWRQ8CvgHuNMb1APvCMvf1+4D32+QA+B7xDRF4F/jfwAbfWoCjK3Lh0pfWtf18CN9MZu4ZkZbUPj0fY3hQ/dfexg53UlBRwxZrUbv4XN1USMVb1+nS+/3IbtaWFXD6HQLvD9qYKjnYO0zU0AaBDrWxcbSNvjHkUK9YRu+2TMc8N8DH7EbuPHytzK945B4Dbsn6xiqJkjQpfPiurfVPSZWM50zvG8vKi6NyT7U0VfOXXrzERClPotbb5g2F+efg8t+9cER0+lYwdzZabbP+5AS5fPSk+PSMT/PJIF++/bg3evLl/f97RWEEoYthjWzdqkVhoZbuiKK6wvbGCV9viC8nZvlFW1viiry9uqiQYNhztnEzdffZED6OBMG/aOnuQPZb6siJWVBSxf5qAPbyvnVDE8M7LmtJcxVScCvenj1mZoFrZbqFCoiiKK2xrrKC1f5z+OPNJTveOsTpGSJyAe6wAPHawk7IiL9esS2/s746mSvZPC7h/76VWLm6qYOOy9NN+Y2msLKa6pIAjtuBVFKtFApNpLvEAAAsjSURBVCokiqK4hCMOB9unWgdjgRDdwxOsqpksLmyqKqbKl8+rdkwlGI7ws0PnuXnzsqTZWtO5uLmSs31jUQE72DbI4Y6hjK0RcALu1rqK8/PSHkl8oaJCoiiKK2xrtLoaTXdvOYH2VTEWSWxGFMBvTvUxOB7kTdtSd2s5XOzESWxR+t5LrRTkeXjrxSvSX0QcdtgCqfGRSVRIFEVxhUpfAc3VxRxMJCTVU9ud7Giq4HjXCOOBMI8d7MBXkMfrN6ZfTLy9sQIRq3AwEIrw8L42bt66LGsZVtubrAp6zdiaRIVEURTX2NFYOcMiOWt3/Y0NtoMV2whHDK+2DfJ4y3lu3FQ/J9dRWVE+6+pK2X9ugF8e6aJ/LJgVt9bkddoWicZHoria/qsoytJmW2MFP3m1g4GxQPQb/OneMap8+TMC1c4N+mvPvUbPyMSc3Fqx53r6WA8iUF9WyPXr0wvYz8ay8iKWlxdRV1aYtXMudtQiURTFNaIB97bJAsGzvWOsrJnZxXdZeRH1ZYU8+monBV4Pb7iofs6fu7O5kp6RCX5xpIu3X9qUUe1IPB547y7ufdOmrJ5zMaNCoiiKa8QLuJ/ps7r+xmOHHX943YZaSgvn7jBxzmMMvPOy6U3HM2dbYwXNCdawFFEhURTFNaYH3AOhCG3941NqSGJx3FvpFCHGY3NDGfl5ws7mStbPoWW8kh4aI1EUxVViK9zbBsaJGOK6tgDevH05r5ztzyg+AlDozeN/vW076+tLMzqPkhoqJIqiuMq2xgoefbWTwbHglK6/8VhfX8ZXf/+KrHzunbuak++kZAV1bSmK4iqxFe7xihGVxY9aJIqiuMq2FZOTBbuHJ/AV5FFXqqmzFxIqJIqiuEpVSQFNVVbAfSIUZmW1D5H0R90qCxcVEkVRXMcJuBd4Payrix9oVxYvGiNRFMV1tjVWcLZvjDO9o1O6/ioXBiokiqK4jhNwD4aNBtovQFRIFEVxHUdIYGbXX2Xxo0KiKIrrVJUU0FhZDGjq74WIComiKDlhe2MF+XlCQ0XRfF+KkmU0a0tRlJzwwdet5aq11VnvxKvMPyokiqLkhMtWVXHZqqr5vgzFBfSrgaIoipIRKiSKoihKRqiQKIqiKBmhQqIoiqJkhAqJoiiKkhEqJIqiKEpGqJAoiqIoGaFCoiiKomSEGGPm+xpcR0S6gTMxmyqAwWm7pbIt9nWi57VAT4aXHO9a0t0v0XuzrSnZa+d5NtY42zWms1821rnQf5eJ3te/2eSvF8vf7EL9Xa4yxtQlPcIYs+QewP1z2Rb7epbne924vnT3S/TebGtKdc3ZWONCWudC/10mel//Zi+cv9nF+LuMfSxV19aP5rjtRyk8zwapnm+2/RK9N9uakr2+UNe50NeY6H39m03+erGsczH+LqMsCddWLhGRvcaYXfN9HW6yFNYIus4LiaWwRpi/dS5Vi8RN7p/vC8gBS2GNoOu8kFgKa4R5WqdaJIqiKEpGqEWiKIqiZIQKSQJE5Csi0iUiB+dw7GUi8qqInBCRfxIRiXnvIyJyVERaROTvsnvV6ePGOkXk0yLSJiL77Mebs3/laV+rK79P+/0/FxEjIrXZu+L0cel3+bcicsD+PT4hIiuyf+VpX6sb6/x7ETlir/UHIlKZ/StP+1rdWOed9r0nIiLZi6VkIyXuQnwArwMuBQ7O4dgXgKsBAR4DbrW33wj8HCi0X9dfoOv8NPDn8702t9dpv9cMPI5Vp1R7oa0RKI/Z5x7gXy/E3yVwC+C1n38e+PwFus7NwCbgSWBXtq5VLZIEGGOeBvpit4nIOhH5qYi8JCLPiMhF048TkQas/3x7jPWb+wbwW/bbfwR8zhgzYX9Gl7urSI5L61xwuLjOLwB/Acx7sNGNNRpjhmJ2LeHCXecTxpiQvevzQJO7q0iOS+s8bIw5mu1rVSFJj/uBjxhjLgP+HPiXOPs0Aq0xr1vtbQAbgetF5Dci8pSIXO7q1c6dTNcJ8GHbTfAVEVmo81UzWqeI3A60GWP2u32hGZDx71JEPisi54DfAT7p4rVmQjb+Zh3ej/UtfiGSzXVmDZ3ZniIiUgpcA3w3xkVeGG/XONucb3FeoAq4Crgc+I6IrLW/NSwIsrTOLwN/a7/+W+Afsf5zLhgyXaeI+IC/wnKJLEiy9LvEGPNXwF+JyCeADwOfyvKlZkS21mmf66+AEPBf2bzGbJDNdWYbFZLU8QADxpidsRtFJA94yX75CNZNNNYsbgLa7eetwPdt4XhBRCJYvXG63bzwNMl4ncaY8zHH/TvwYzcveI5kus51wBpgv/2fugl4WUSuMMZ0unztqZKNv9lYvgX8hAUmJGRpnSLyXuAtwE0L6ctdDNn+fWaP+Q4oLeQHsJqYQBfwHHCn/VyAixMc9yKW1eEEut5sb/9D4DP2843AOexangtsnQ0x+3wU+PZ8r9GNdU7b5zTzHGx36Xe5IWafjwDfm+81urTO3cAhoG6+1+bmOmPef5IsBtvn/Qe1UB/Ag0AHEMSyJP4A6xvoT4H99h/dJxMcuws4CJwEvuSIBVAA/Kf93svAGy7QdX4TeBU4gPUNqSFX68nlOqftM+9C4tLv8iF7+wGsHkyNF+LvEjiB9cVun/1YCNlpbqzzbfa5JoDzwOPZuFatbFcURVEyQrO2FEVRlIxQIVEURVEyQoVEURRFyQgVEkVRFCUjVEgURVGUjFAhUZYkIjKS4897QES2ZOlcYbsb70ER+VGyTrUiUikif5yNz1aUeGj6r7IkEZERY0xpFs/nNZNN/1wl9tpF5OvAMWPMZ2fZfzXwY2PMtlxcn7L0UItEUWxEpE5EHhKRF+3Htfb2K0TkORF5xf53k739fSLyXRH5EfCEiNwgIk+KyPfs2Rb/FTMH4kln/oOIjNiNEPeLyPMisszevs5+/aKIfCZFq2kPk00kS0XkFyLyslizKO6w9/kcsM62Yv7e3vde+3MOiMjfZPHHqCxBVEgUZZIvAl8wxlwOvAN4wN5+BHidMeYSrO63/yvmmKuB9xpj3mC/vgT4M2ALsBa4Ns7nlADPG2MuBp4GPhjz+V+0Pz9pbyS7x9JNWN0DAPzA24wxl2LNvvnH/7+9O2aNIgjDOP5/lICKMZWKWFilsQokCGkEQdOK3dmJrY0EBT+BAbVSG0sbK7+ASaGIwRAMEQSxESxiIwgGkaBBH4sZuTXc5QhrMJDnBwu7ezM7e8Xdy84s71sD2Q3gve0x29clTQGjwClgDBiXdHrQeBH9JGljRNdZ4GQjs+ohScPACPBQ0igli+pQo8+c7WbNiEXbKwCSXlNyJb3YMM4Puoksl4BzdX+Sbq2TR8CdPve5v3HtJWCunhdwswaFX5QnlaM9+k/VbbkeH6QElud9xovYVAJJRNceYNL2WvOkpHvAU9sX6nrDs8bH3zZc43tj/ye9f2Pr7i5O9muzmTXbY5JGKAHpCnCXUi/kMDBue13SB2Bfj/4CZmw/2OK4ET1laiuia5ZSbwMASX/SdY8AH+v+pW0cf4EypQbQGdTY9iql/O01SUOU+/xUg8gZ4ERt+hUYbnR9Alyu9S2QdFzSkX/0HWIXSiCJ3eqApJXGNk35U56oC9BvKWn/AW4BM5Lmgb3beE9XgWlJi8AxYHVQB9vLlEywHUoxpglJryhPJ+9qm8/AfH1d+LbtWcrU2UtJb4DH/B1oIrYkr/9G7BC16uKabUvqABdtnx/UL+J/yxpJxM4xDtyvb1p9YYeVJ47oJ08kERHRStZIIiKilQSSiIhoJYEkIiJaSSCJiIhWEkgiIqKVBJKIiGjlN4fKpw8IabnkAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.lr_find()\n",
    "learn.recorder.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total time: 00:55\n",
      "epoch  train_loss  valid_loss  accuracy\n",
      "1      0.694043    0.694633    0.476000  (00:01)\n",
      "2      0.693227    0.693842    0.510000  (00:01)\n",
      "3      0.692341    0.693214    0.520000  (00:02)\n",
      "4      0.691532    0.693981    0.516000  (00:01)\n",
      "5      0.690240    0.693385    0.536000  (00:01)\n",
      "6      0.688261    0.694868    0.502000  (00:01)\n",
      "7      0.684323    0.696151    0.514000  (00:01)\n",
      "8      0.679574    0.703019    0.466000  (00:01)\n",
      "9      0.670428    0.706717    0.494000  (00:01)\n",
      "10     0.659853    0.717627    0.482000  (00:01)\n",
      "11     0.647399    0.731758    0.482000  (00:01)\n",
      "12     0.630531    0.734997    0.490000  (00:01)\n",
      "13     0.612728    0.781713    0.494000  (00:01)\n",
      "14     0.595601    0.755801    0.476000  (00:01)\n",
      "15     0.576510    0.771777    0.482000  (00:01)\n",
      "16     0.556412    0.786781    0.478000  (00:01)\n",
      "17     0.537086    0.798632    0.478000  (00:02)\n",
      "18     0.518724    0.815592    0.484000  (00:02)\n",
      "19     0.501062    0.828895    0.476000  (00:02)\n",
      "20     0.485658    0.859679    0.466000  (00:03)\n",
      "21     0.470815    0.864150    0.478000  (00:03)\n",
      "22     0.457828    0.868159    0.476000  (00:02)\n",
      "23     0.449460    0.869975    0.458000  (00:02)\n",
      "24     0.438471    0.874046    0.464000  (00:01)\n",
      "25     0.429300    0.882240    0.454000  (00:01)\n",
      "26     0.423787    0.885527    0.454000  (00:01)\n",
      "27     0.418790    0.883194    0.470000  (00:01)\n",
      "28     0.416018    0.886720    0.458000  (00:01)\n",
      "29     0.412148    0.888030    0.452000  (00:01)\n",
      "30     0.411357    0.888070    0.452000  (00:01)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "learn.fit_one_cycle(30, max_lr=1e-3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEKCAYAAAD9xUlFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xl8VNX5x/HPk8m+r6wBEnbZCZFFkEXUgm3RIipUq7jhWtu6tLa1Wm1ttVqLrf60bqhVwR0QF9zABZEdAkSWsIcA2SAb2SY5vz/uJYSQQIBM7kzyvF+vec2dO2fufJPAPHPPvfccMcaglFJKAfg5HUAppZT30KKglFKqhhYFpZRSNbQoKKWUqqFFQSmlVA0tCkoppWpoUVBKKVVDi4JSSqkaWhSUUkrV8Hc6wKmKj483SUlJTsdQSimfsmrVqlxjTMLJ2vlcUUhKSmLlypVOx1BKKZ8iIrsa086j3UciMkFENotIhojcW8/zXUTkCxFJE5HFIpLoyTxKKaVOzGNFQURcwNPARKAPME1E+tRp9jjwqjFmAPAQ8HdP5VFKKXVyntxTGApkGGO2G2MqgDnAxXXa9AG+sJcX1fO8UkqpZuTJYwodgT21HmcCw+q0WQdcCjwJ/AyIEJE4Y0xe7UYiMgOYAdC5c+fj3qiyspLMzEzKysqaLn0rFxwcTGJiIgEBAU5HUUo1I08WBalnXd3JG+4GnhKR6cDXwF7AfdyLjHkOeA4gNTX1uAkgMjMziYiIICkpCZH63ladCmMMeXl5ZGZmkpyc7HQcpVQz8mRRyAQ61XqcCGTVbmCMyQImA4hIOHCpMabgVN+orKxMC0ITEhHi4uLIyclxOopSqpl58pjCCqCHiCSLSCAwFZhfu4GIxIvIkQy/B1463TfTgtC09PepVOvksT0FY4xbRG4HFgIu4CVjzEYReQhYaYyZD4wF/i4iBqv76DZP5VFKqWZX5YaKIigvgvJi+77o2HUVJWCqqeldP2aK5Drrek2AjkM8GtmjF68ZYz4CPqqz7v5ay+8A73gyQ3PIy8tj/PjxAOzfvx+Xy0VCgnXh4PLlywkMDDzpNq699lruvfdeevXq5dGsSilbdRUU7oWDO61bZRmEt4GIdtZ9eFsIDDvJNqqtbeRvg7xtkL/duuVtg4I9UHm4CQOLlc2Xi0JrERcXx9q1awH485//THh4OHffffcxbYwxGGPw86u/x27WrFkez6lUq1NZCnkZRz/483ccXT60G6orT/z6wIijBSKirXXv529tJ387HNwB7lpnPfoHQ2xXiO8B3c+HkGgIDIegCAg6ch957LrAcBCX9foj3bYOdt9qUfCgjIwMLrnkEkaNGsWyZctYsGABDz74IKtXr6a0tJQrrriC+++3dpxGjRrFU089Rb9+/YiPj+fmm2/m448/JjQ0lHnz5tGmTRuHfxqlfERFCWz9FDbOte5rf1sPjoaYJGjXH/pMspaP3PxDoCQbig9A0QHrvuaWDfvSrPuqCohNhthu0ON86z6um1UMIjpAA1/8fEWLKwoPfrCR9KzCJt1mnw6RPPDTvqf12vT0dGbNmsWzzz4LwCOPPEJsbCxut5tx48YxZcoU+vQ59kLvgoICxowZwyOPPMKdd97JSy+9xL33HjdKiFLqiPoKQVgCDJwGSaOsD/GYJAiJOfF2ItoC/U/cxhhHv8l7WosrCt6mW7dunH322TWPZ8+ezYsvvojb7SYrK4v09PTjikJISAgTJ04EYMiQIXzzzTfNmlkpn1Bx2C4E7x9fCPpeAl1Ggp+r6d+3BRcEaIFF4XS/0XtKWNjRA1Vbt27lySefZPny5URHR3PVVVfVexV27QPTLpcLt/u46/mUal0qy6xjAzmbrNuBjbB9cfMVglakxRUFb1ZYWEhERASRkZHs27ePhQsXMmHCBKdjKeU9jhwYzt50tADkbLIO6ppqq434Wf33A6dCH7sQuPSjrKnob7IZpaSk0KdPH/r160fXrl0ZOXKk05GU8qzqaijYbX3Ql+RBaT4czofSg7WW8+Gw/bii+OhrxWV9+Lc5C/r+DBJ6W7e47hAQ7NzP1MKJMccNJeTVUlNTTd1Jdn744QfOOusshxK1XPp7VY1W5bZO88zZBLmbIWez/S1/C7hL6zQWCI6C0FgIiT16HxIDoXEQ1/Xoh79/kBM/TYskIquMMakna6d7CkqpxjMGCjLhwAb7lm4VgLyt1qmaR0QmQkJPSL0WEnpBXA/rfP+QWOvcfe3391paFJRS9asshex066Du/g1HC0FZrTEroztDwlnQffzR7p34HhAc6VxudUa0KCiljrVxLix62DoOcOTgbkAYtO0DfSdD277WxV9t+uiHfwukRUEpddTeVfDeDOvb/uh7rALQth/EJPv8lbqqcbQoKKUsxTnw5i+s8X2ung9hcU4nUg7QoqCUgqpKePsaOJwH13+qBaEV0/3BJjB27FgWLlx4zLqZM2dy6623Nvia8PBwALKyspgyZUqD2617+m1dM2fO5PDhowN+XXTRRRw6dKix0ZWyfHof7FoCk/4D7Qc6nUY5SItCE5g2bRpz5sw5Zt2cOXOYNm3aSV/boUMH3nnn9KeUqFsUPvroI6Kjo097e6oVWjsblj0Lw2+DAZc7nUY5TItCE5gyZQoLFiygvLwcgJ07d5KVlcWgQYMYP348KSkp9O/fn3nz5h332p07d9KvXz8ASktLmTp1KgMGDOCKK66gtPToRT+33HILqamp9O3blwceeACAf//732RlZTFu3DjGjRsHQFJSErm5uQA88cQT9OvXj379+jFz5sya9zvrrLO48cYb6du3LxdeeOEx76Namaw1sODXkHQuXPCQ02mUF2h5xxQ+vhf2r2/abbbrDxMfafDpuLg4hg4dyieffMLFF1/MnDlzuOKKKwgJCeH9998nMjKS3Nxchg8fzqRJkxqc//iZZ54hNDSUtLQ00tLSSElJqXnu4YcfJjY2lqqqKsaPH09aWhp33HEHTzzxBIsWLSI+Pv6Yba1atYpZs2axbNkyjDEMGzaMMWPGEBMTw9atW5k9ezbPP/88l19+Oe+++y5XXXVV0/yulO8oybUOLIfGw2Uv6/hBCtA9hSZTuwvpSNeRMYY//OEPDBgwgPPPP5+9e/dy4MCBBrfx9ddf13w4DxgwgAEDBtQ899Zbb5GSksLgwYPZuHEj6enpJ8zz7bff8rOf/YywsDDCw8OZPHlyzRDcycnJDBo0CLCG5t65c+eZ/OjKF1W54e3p1qQxU1+DsPiTvkS1Di3vq8EJvtF70iWXXMKdd95ZM6taSkoKL7/8Mjk5OaxatYqAgACSkpLqHSq7tvr2Inbs2MHjjz/OihUriImJYfr06SfdzonGtAoKOjqejMvl0u6j1ujzB2DnN3DJs9BhsNNplBfRPYUmEh4eztixY7nuuutqDjAXFBTQpk0bAgICWLRoEbt27TrhNkaPHs3rr78OwIYNG0hLSwOsIbfDwsKIioriwIEDfPzxxzWviYiIoKioqN5tzZ07l8OHD1NSUsL777/Pueee21Q/rvJlaW/D0qdg6E0w6OQnQ6jWpeXtKTho2rRpTJ48uaYb6corr+SnP/0pqampDBo0iN69e5/w9bfccgvXXnstAwYMYNCgQQwdOhSAgQMHMnjwYPr27XvckNszZsxg4sSJtG/fnkWLFtWsT0lJYfr06TXbuOGGGxg8eLB2FbV2+9Jg/i+h8znwo4edTqO8kA6drRqkv9cW5nA+PDfGOp5w01fWqKWq1Wjs0NnafaRUS2cMbPoQXrwQivbDFa9pQVAN0u4jpVqy3cvgs/thz/fWnAbT5kDiEKdTKS/WYoqCMabB8//VqfO1bkVVR84W+OJB2LTAGuDuJzNh8C/0WgR1Ui3iX0hwcDB5eXnExcVpYWgCxhjy8vIIDtZ5cH1O4T746hFY/T8ICIVx98GIWyEwzOlkyke0iKKQmJhIZmYmOTk5TkdpMYKDg0lMTHQ6hmqsskJY8iQsfRqq3TD0Rms+BL0oTZ2iFlEUAgICSE5OdjqGUs2vuhpWvQSL/mYNe93vUjjvPojt6nQy5aNaRFFQqlUqyIS5t8KOr44OaNcx5eSvU+oEtCgo5WuMgfVvw4d3W11FP5kJQ6aDHk9TTUCLglK+5HA+LPgNpM+FTsPgZ89qV5FqUh69eE1EJojIZhHJEJF763m+s4gsEpE1IpImIhd5Mo9SPm3rZ/B/w60L0cbfD9d+rAVBNTmP7SmIiAt4GrgAyARWiMh8Y0ztMZ/vA94yxjwjIn2Aj4AkT2VSyieVF1vTZa6aBQlnwZXvQPsBJ3+dUqfBk91HQ4EMY8x2ABGZA1wM1C4KBoi0l6OALA/mUcr37FkO782AgzvhnF9a1x0E6PUjynM8WRQ6AntqPc4EhtVp82fgUxH5JRAGnO/BPEr5jtKD8O2/4Lv/QGQiTF8ASaOcTqVaAU8WhfpOhag7dsI04GVjzD9FZATwPxHpZ4ypPmZDIjOAGQCdO3f2SFilvEJxDnz/NCx/ASqKYNBVMOHvEBx58tcq1QQ8WRQygU61HidyfPfQ9cAEAGPMUhEJBuKB7NqNjDHPAc+BNXS2pwIr5ZiCvdZewaqXwV0G/SbDuXdB275OJ1OtjCeLwgqgh4gkA3uBqcDP67TZDYwHXhaRs4BgQMeqUK1H/g5YMhPWvA4YGHAFjPoNxPdwOplqpTxWFIwxbhG5HVgIuICXjDEbReQhYKUxZj5wF/C8iPwGq2tputHhOVVrkLMZvnnCugjNzwUpV8PIX0FMF6eTqVbOoxevGWM+wjrNtPa6+2stpwMj675OqRbr4C74/AHYOBcCQmD4LTDidohs73QypQC9olmp5uGugKX/ga8eA/GzjhcMv0VHMVVeR4uCap1ytkDWaqsP39NjBu34Gj68C3K3wFmTrLOJonRYcuWdtCio1qe6Gt65Dg6sty4KG3vcCCxNo+iAdSXy+rcgJsm6ErnHBZ55L6WaiBYF1fpsfM8qCO0GwOK/Q2A4nHN7022/ugpWvgRf/AXcpTD6t3DundYxBKW8nBYF1bpUVcKih6FNX7jxS3j3evj0jxAUbg0/fab2roIFd8K+tZA8Bn78BMR3P/PtKtVMtCio1mXt65C/HabNAVcATH4BKg7DB7+29hj6Tzm97ZYVwOcPWnsI4W3g0hetWdB0jgPlY7QoqNajshQWPwqJQ6HnBGudfyBc8T94bYo18FxAKPQ+xRHcN39szXFQfACG3QTj/gDBUU2fX6lm4NH5FJTyKitehKIsay6C2t/gA0Jg2mxoPxDevga2LWrc9kpy4Z3rYfZUCImFGz6HiY9qQVA+TYuCah3KCuGbf0K38yD53OOfD46Eq96FuO4w5+ewe1nD2zIG1r8DTw+F9Hkw9g8wYzF0HOKp9Eo1Gy0KqnVY+jSU5sN5f2q4TWgs/GIuRLSH1y+DfeuOb1OYBbOnWQeoY5Lgpq9h7O+sbiilWgAtCqrlK8mDpU9ZF451TDlx24i2cPU8CIqA//3MGqMIrL2DVS/D08Ng+2K48GG4/jNo28fT6ZVqVloUVMv37RNQeRjOu69x7aM7wTXzQVzw6iWw/St4dRJ88CvruMMtS6zrGvxcns2tlAO0KKiWrWAvLH8eBk6DhF6Nf11cN7h6rlVMXp0Ee9fAT2bC1fOt55RqofSUVNWyffUomGoY87tTf23bvlZX0to3rGGtozo2fT6lvIwWBdVy5W2DNa/B2Tec/jwFHQZZN6VaCe0+Ui3XoofBPwhG3+10EqV8hhYF1TLtXw8b3rXmLAhv43QapXyGFgXVMn3xF+vK4nN+6XQSpXyKFgXV8uz+HrYutA4Oh8Q4nUYpn6JFQbUsxsAXD0FYGxh2s9NplPI5evaRajkqSqwRS3ctgYmPQWCY04mU8jlaFJTvqK6Con3WFJrH3HZZ9yXZVruYpKaZMEepVkiLgvJ+mSvhs/shcwVUVRxdL34QlWgVgV4TILqLtZw8RgeoU+o0aVFQ3qswCz7/M6S9CeHtrNNLY5KtD/6YJKsguAIcDqlUy6JFQXmfylL47ilrILvqKjj3Lhh1pzWPslLKo7QoKO9hDKTPhU/vh4Ld1lDXF/7F2itQSjULLQrKO+xbBx/fC7u/g7b94JIF9c+QppTyKC0KylnF2fDlX2D1/6yZz34yE1Ku1rkKlHKIFgXljLxtsOxZWPM6VJXDiNtg9D0QEu10MqVaNS0KqvkYA7uXWvMlb/oQ/Pyh/2Vw7p0Q38PpdEoptCio5lBVCenzrHmSs9ZASKx1RtHQGyGindPplFK1aFFQnlN6EFa9Asufg8K9ENcDfvIvGDAVAkOdTqeUqodHi4KITACeBFzAC8aYR+o8/y9gnP0wFGhjjNFOZV9kDJTkQt5WyMuAvash7S2oLIHk0VYx6H4B+OkYjEp5M48VBRFxAU8DFwCZwAoRmW+MST/Sxhjzm1rtfwkM9lQe1UQqDkP+Nsjdah0sPlIEcjOgvOBoO/9g6DsZRtwK7fo7l1cpdUo8uacwFMgwxmwHEJE5wMVAegPtpwEPeDCPOh3VVbB3FWR8ARmfQ9ZqMNVHn49MhPjuMOAyq3sorrv1OKqTnlaqlA/yZFHoCOyp9TgTGFZfQxHpAiQDX3owj2qswiyrCGz7ArYtgrJDgEBiqnWAuE0f62yh2K46PLVSLYwni4LUs8400HYq8I4xpqreDYnMAGYAdO7cuWnStXbGgLsMyouhvBAO7baKQMaXkL3RahPeDnr/BLqfB13HWReXKaVaNE8WhUygU63HiUBWA22nArc1tCFjzHPAcwCpqakNFZYTqqyq5u8fbSI00MXdP+p1ZLsUlFaSW1yOiBATGkju9rWEbJxDQFAIoaFhSEAw1a4gykwg4WFhhIaGIgEh4B9k9Zu7Aq1lV+Dxy65AaxRPEetDuNptfRC7y+vc11pGam0joM62aq2rdkNZgfWBXlYAZYV1HhccfVxeDOVFUFF8tAhUFFvbqM0vALqMgAsegm7joW1fK7tSqtXwZFFYAfQQkWRgL9YH/8/rNhKRXkAMsNSDWXjy8628tGQHAMEBfmzNLmZJRi65xRXHtBvvt4p/B7xGMBW45Gj9iTqD93ZLAH6mCj+qT964qQSEQlAkBEdCUAQEhkN4m6PLQRHWqKOBEdZyeAJ0Gq4jkSrVynmsKBhj3CJyO7AQ65TUl4wxG0XkIWClMWa+3XQaMMcYc1p7AI01fWQSTy3KAODxT7cQEezP6B4JDO4cTUJEENXGcOhwJQkRg0mPvJ3C0kqyD5WAuwy/6jKiAqooKi6hqLgYd9lhXKYCP3cZUlVJtbsMd2U5FeVlGHcF1ZXlVFWWUVVZQZh/FSFSRWm1UFIVQDm1biaAcgIpJ4AK/KkwARggUNzEB0N0EIT5VxPiV02In5tQVzVBflX4m0rEz5+AsGhCI2OIj29DSEQMIRFxtElIwARF4h8QiOi3fKXUKRIPfxY3udTUVLNy5crTfv2+glJyiyro0Tac4ADPnh1T4a4mwCU1H86VVdVUuK1bubuacncV2UXl+IlQWVWNu8pQWFbJtuxisgpKOVhSSZm7ivLKakorqzhc4cZdZagyhgp3NTlF5birj/37ufyEqmqDy8/qDosNC6BzbCidY8NITghjaFIs3duE4/LTgqFUayIiq4wxqSdr1+quaG4fFUL7qJBmea9A/2Mv1Apw+RHg8iMs6Oi6LnGnf/ZOdbUhu6ic7TnFFJa5OXS4gsyDpfi7hAp3NfklFeSXVLA7/zBLMvIorayqyRUa6GJYcixd4sLoEBXM+LPa0ilWrzJWqrVrdUWhJfHzE9pFBdMuKvikbY0x7M4/zIqdB9lyoIiDJRUs35nPos05VLir+fMH6fTrGEmf9pF0jA6lW5swRvdMIDJYp7tUqjXRotBKiAhd4sKO2zMxxpB5sJT567JYkpHLFz9kk1diHXwPcAkT+7XnxwPaU+GuJtDfj+T4MHq2jXDiR1BKNYNWd0xBndyO3BJ25x/mkw37WZCWRVHZsaeuTujbjlvGdmNgJx2mSilf0dhjCloU1AkVl7v5YV8hLj8h0OXHp+kHmLVkB0Vlbvp1jKRX20j6d4xk0qCOxIYFOh1XKdUALQrKY4rL3byxbBeLN+ewPaeE/YVlBLr8uLBvW6ae3ZlzusXhp2c3KeVVtCioZrNpfyFvrtjD+2v2cuhwJYkxIVye2onLUhOb7UwvpdSJNWlREJFuQKYxplxExgIDgFeNMYfOOOkp0qLgvcoqq/g0/QBvrtjNkow8/ATG9Ezg6nOSGNszQS+mU8pBTV0U1gKpQBLWFcrzgV7GmIvOMOcp06LgG3bnHebtVXt4c8UesovK6dU2gilDEpmc0pG48KCTb0Ap1aSauiisNsakiMg9QJkx5j8issYY0+yT4mhR8C0V7mo+WJfF899sZ9P+IrolhHHvxLMY2yuBAJfOwqZUc2nqK5orRWQacA3wU3udXtWkTirQ349L7T2ExZtz+NWcNdz46ko6Rofwu4m9GdE1joQI3XNQyls09qvatcAI4GFjzA575NPXPBdLtTQiwrjebfj23vN49qohVBvDHbPXcPbDn/Pa97ucjqeUsp3y2UciEgN0MsakeSbSiWn3UcuQX1LBh+v38fr3u9i0v4ibx3Tjzgt6HjdelFKqaTS2+6hR/wNFZLGIRIpILLAOmCUiT5xpSNV6xYYF8ovhXZh/+yimnt2JZ7/axpRnvyPrUKnT0ZRq1Rr7tSzKGFMITAZmGWOGAOd7LpZqLQL9/Xjk0gE8e1UK23NKGPf4YmYt2UFlVTNOSKSUqtHYouAvIu2By4EFHsyjWqkJ/doz//aRnJ0Uy4MfpDPkL5/x3NfbyC4qs+eR0CKhVHNo7CmplwF/ApYYY24Rka7AY8aYSz0dsC49ptCyGWP4clM2//hkM5sPFNWsDw108dTPB3Ne77YOplPKd+kwF8qnVVUb0jIPsXbPIcoqrWsdtucWM/vG4QzuHON0PKV8TlMfaE4UkfdFJFtEDojIuyKSeOYxlaqfy08Y3DmGa0cmc8vYbrxy3VDaRARz3csr2JFb4nQ8pVqsxh5TmIU1tEUHoCPwgb1OqWaREBHEK9cNRUSYPms52YVlTkdSqkVqbFFIMMbMMsa47dvLQIIHcyl1nOT4MJ6/OpXswnLGPr6Yu95aR2FZpdOxlGpRGlsUckXkKhFx2bergDxPBlOqPkO6xDDv9pF0jA7h3dWZjP/nV3y0fp/TsZRqMRp79lFn4CmsoS4M8B1whzFmt2fjHU8PNCuAgsOVfLHpAM8s3kZGTjEdo0MY3DmG0T3iuSy1k9PxlPI6TTognv3hP6nOG/wamHl68ZQ6M1GhAUxOSWR877bc+L+VLN+RT15xBR+sy6KwzM31o5KdjqiUTzrtU1JFZLcxpnMT5zkp3VNQdVVXG9L2FtC3QyRXPr+M5TvzuWFUMnf/qBfBAS6n4ynlFZr0lNSG3uMMXqtUk/HzEwZ1iibA5cer1w/litROvPDtDu55x5ExG5XyaWdSFHzrqjfVKgQHuHjk0v7cNLorH6zLYvXug05HUsqnnLAoiEiRiBTWcyvCumZBKa8jItw6rjsdo0P4zZtr+WTDPg4drnA6llI+4YQHmo0xEc0VRKmmFBUSwD8vH8g1Ly3n5tdWAzCwUzT3XNiLUT3iHU6nlPfSsY9Ui3agsIwlGbms2X2IjzfsI7e4gutGJvOzwR3pnxjldDylmo0OiKdUHWWVVfzhvfW8v3YvxsC0oZ35zfk9CAl0ERGsU46rlk2LglINyCkq5/55G/hk436MsSb6mX5OEneM70F4UKMu3VHK5zTpxWtKtSQJEUE8c9UQvt+ex+rdB/nih2ye+3o7SzJy+fvk/gxIjHY6olKO8egs6SIyQUQ2i0iGiNzbQJvLRSRdRDaKyBuezKNUbcO7xnHr2O68c/MIHpzUl41ZhUx6aglPL8pwOppSjvHYnoKIuICngQuATGCFiMw3xqTXatMD+D0w0hhzUETaeCqPUg0REa45J4nRPRP4zZtreWzhZqJDA7hyWBenoynV7Dy5pzAUyDDGbDfGVABzgIvrtLkReNoYcxDAGJPtwTxKnVByfBhv3TSCMT0T+OP7G3hg3ganIynV7DxZFDoCe2o9zrTX1dYT6CkiS0TkexGZUN+GRGSGiKwUkZU5OTkeiquUddD5xWtSuXJYZ15Zuou73lqHr52ModSZ8GRRqG9spLr/u/yBHsBYYBrwgogcd5TPGPOcMSbVGJOakKBz+yjP8nf5cecFPQl0+fHu6kxmLdnpdCSlmo0ni0ImUHtg+0Qgq54284wxlcaYHcBmrCKhlKPiwoPY9JcJ/KhvW/76YTqfbNjvdCSlmoUni8IKoIeIJItIIDAVa57n2uYC4wBEJB6rO2m7BzMp1Wh+fsITlw+iZ9sIbn5tFX9dkI67qtrpWEp5lMeKgjHGDdwOLAR+AN4yxmwUkYdE5MiEPQuBPBFJBxYB9xhjdJpP5TXCgvyZfeNwusSF8sK3O7jk/5ZwoLDM6VhKeYxe0axUIxSXu/l0437+NHcDIYH+3P/TPkwaqAMFK9/RHJPsKNVqhAf5MzklkTdvGkHHGGtI7s/TDzgdS6kmp0VBqVPQr2MUr98wjN7trOMMs5fvplKPM6gWRIuCUqcoPMif128YRlJ8GL9/bz13v63XMqiWQ4uCUqchOjSQT351LreP6868tVlMfPIbZn6+hepqLQ7Kt+koqUqdJn+XH3dd2JPVuw/y3bY8Nu0voqC0kgd+2tfpaEqdNi0KSp0BEeGZq4aw9UAR89ZmMWvJTs7tEc95vds6HU2p06LdR0qdoaiQAFKTYvn1+T2IDg1gxqur+NtHP5CWecjpaEqdMi0KSjWRuPAgPvnVaIZ0ieG5r7dz+X+X8vKSHRQcrnQ6mlKNpkVBqSbULiqYOTOGs/wP40mKC+PPH6Qz+ZkllJS7nY6mVKNoUVCqiYkIbSKD+fhX5zJr+tnsyC3ht++mUaVnJikfoEVBKQ8REcb1bsNvJ/Tmw7R9XPPScrIOlTodS6kT0qKglIfdPKYbj0zuz6pdBznnkS+ZtWSH05GUapAWBaWawdSNaSGSAAAUS0lEQVShnXn9xmGEBbp4aEE6H6btczqSUvXSoqBUM0npHMPK+y6gX4cobntjNffP26BXQCuvo0VBqWYUEujirZtGMGVIIq8u3cXctXudjqTUMbQoKNXMQgJd/OPSAfRqG8Gdb63jqS+3Oh1JqRpaFJRygJ+f8OS0QaR0jubxT7fw5ordTkdSCtCioJRjereLZPaM4QxLjuV3767nf9/v0mMMynFaFJRyUJC/i1nXns2QLjH8ae4GbnptFfklFU7HUq2YFgWlHBYa6M87N4/glrHdWLw5m5S/fMbwv33BC99sp6yyyul4qpURX5sxKjU11axcudLpGEp5RHpWIQvSsnh/zV72FZQRFxbI1SOSuGlMV4IDXE7HUz5MRFYZY1JP1k7nU1DKi/TpEEmfDpHcfWEvFm3O5sEP0vnX51vwdwm3jevudDzVCmj3kVJeyM9PGH9WW77+7TjG927Dvz7bwsKN+52OpVoBLQpKebm/XNKPXu0iuGP2Gt5dlYmvdfkq36JFQSkv1yE6hFevG0q/jlHc9fY6Hlu42elIqgXToqCUD4gLD+Ltm0Zw8aAO/N/ibcxbu1f3GJRHaFFQykf4+Ql3X9iLIH8/fjVnLde/spJVu/K1OKgmpWcfKeVDOsWGsuTe83h3VSZPfLaFLzdl0zE6hFevH0q3hHCn46kWQPcUlPIx8eFB3DSmG+/deg7ndItj76FSfvHCMn7/3nr25B92Op7ycVoUlPJRfTtE8caNw5l720jiwoN4b3Um17+yguJyt9PRlA/ToqCUjxvUKZoPfjmKF685m4zsYu56ay0V7mqnYykfpUVBqRZiVI947rqwFws3HuBPczfoAWh1WjxaFERkgohsFpEMEbm3nueni0iOiKy1bzd4Mo9SLd2tY7tx7cgk3ly5hwv/9TV/XZBO5kE9zqAaz2NFQURcwNPARKAPME1E+tTT9E1jzCD79oKn8ijVGogI9/24D/f9+CxKK6t4ZelORj26iAc/2Kh7DqpRPHlK6lAgwxizHUBE5gAXA+kefE+lWj2Xn3DDuV254dyu7Mk/zF8/TGfWkp1s3l/EvRN7MyAx2umIyot5svuoI7Cn1uNMe11dl4pImoi8IyKdPJhHqVanU2wo/5mWwl0X9GR9ZgEXP72E77blOh1LeTFPFgWpZ13d/dcPgCRjzADgc+CVejckMkNEVorIypycnCaOqVTLFujvxy/H92DJ78+jQ1QI97ydxhOfbWHN7oNOR1NeyJNFIROo/c0/Eciq3cAYk2eMKbcfPg8MqW9DxpjnjDGpxpjUhIQEj4RVqqWLDA7g/65MAeDfX2zl0me+4+P1+xxOpbyNJ4vCCqCHiCSLSCAwFZhfu4GItK/1cBLwgwfzKNXqDewUzbe/G8dX94ylS1wYd7+9Ts9OUsfwWFEwxriB24GFWB/2bxljNorIQyIyyW52h4hsFJF1wB3AdE/lUUpZRIQucWH87/qhVBsY9egirnt5Bd9ty6XcrXNCt3Y6R7NSrdjH6/fx+rLdfL89D3e1oUebcJ64fBBJ8aFEBAc4HU81ocbO0axFQSlFXnE589Zm8fBHP1BVbQgO8OPeCb2ZPjLZ6WiqiTS2KOjQ2Uop4sKDuG5UMj/q146l2/L4YF0WDy1IZ3tuCZcN6USPtuEEB7icjqmage4pKKWOU1RWyUMfpDN37V4qqwzx4YFcP6orN4/pikh9Z5srb6fdR0qpM5aRXcTSbXm8sXwPP+wrZGCnaF69dihRoXq8wdc0tijoKKlKqQZ1bxPBL0YkMe+2kfy4f3vW7TnEZf/9jr2HSp2OpjxEi4JS6qQC/f14+soUnvvFEPYVlDHlme949JNNFJZVOh1NNTEtCkqpRruwbzveuGE47aOCeWbxNgb8+VOu+O9ScorKT/5i5RO0KCilTkn/xCjeu3Ukr143FIBlO/I5++HPeW91JlXVvnWMUh1PDzQrpU6bMYb567J4fdlulu/IJyzQxeVnd2JE1zgu7NvO6XiqFj37SCnVbAoOV3LjqytZvjO/Zl23hDD+PnkAQ5NjHUymjtCioJRqduszCzhQWMb7a/fyYZo1Ausr1w1lTE8d3dhpWhSUUo7aX1DGRf/+hvySCmaM7srvJ/bWC98cpNcpKKUc1S4qmPm3j2TSwA489/V2/jh3A+6qaqdjqZPQsY+UUh6TGBPKk1MH4e8S3li2m837i3jxmlSiQwOdjqYaoHsKSimPEhGeuHwQ/7xsIGmZh3hg/kaqqw1llUfnbiirrOKbrTk64Y8X0D0FpVSzuHRIIpkHS/nX51v4fnsefiI8eukAnlqUwZYDRRw6bF0dfVH/dvxnWgouPz3+4AQtCkqpZnPbuG7klZQzd81eCsvcXP3ScgAigvy5ZWw3KtzVvPjtDrZlf8Mvx3enf8co9heU0adDpE7600z07COllCPW7jnES9/uYHf+YWZeMYik+DCMMfz2nTTeXpV5TNsOUcE8ftlAzuke71Ba36enpCqlfFJVteGHfYV8m5HLt1tzObdHPP/9ejv5JRV0bxNO94Rw/vjjs+gUG+p0VJ+iRUEp1WLkFpfz2ve7WLY9n7V7DlFaWcUtY7sR4CdcObwLbSODnY7o9bQoKKVapO+25fLH9zewI7cEgHaRwTx/dSr9E6McTubdtCgopVq0/QVl5BSVc/Nrq8grKefuC3sx/Zwk/F16pn19tCgopVqFI4Vh1a6DjOgax9lJMXyTkUtOUTnje7fhwYv7OR3RKzS2KOgpqUopn5YQEcQ7N4/gbx/9wPtrsli6Pa/muVeW7qKozM0jlw4g0F/3IBpD9xSUUi3K4Qo323NKiAsP5IlPt/D2qkwCXEJlleFHfdsybWhnFm48wJQhiXSJCyUmNLBVXCin3UdKKQXMXbOX177fxcpdB+t9vn/HKHq0CafMXcUFfdqSV1xB14QwwgL9Gdw5psXsYWhRUEqpOjKyi5m1ZAdDusSQXVTOyp0H+fyHAyd8zY8HtOc35/eke5vwZkrpGVoUlFKqEbKLyggL9Ce/pILsojL8/fxYs/sgIsKjn2yitLIKY+C83m0IDvDjjvE96N0u0unYp0yLglJKnaHqasOBojJe+GYHbyzbTak9smtCRBDje7ehb4dIQgP9mZzSkeJyN19uyiYkwMXBwxV8tSWHH/YVERkSwOFyN9tzS4gLC+SxywbSNT6MNXsO8aO+bQnydzXLz6JFQSmlmpAxhhU7D/Lz578nOMBFcbm75rkJfduRkVNMRnbxMa+JCgmgfVQwew+WEh7sz76CsmOen9ivHU//PAW/ZjjQrUVBKaU84GBJBdGhAaRlFrA7/zBbDhTx4rc7iAj257zebRnTM57EmFB6to3AT8Df5YcxBhEhp6iceWv3Uu6u5qvNOSzfmc/ongmEBrhI31fIlCGJBLj8WL37IGe1j2RcrwQKy9yEB/mzalc+M0Z3O+3cWhSUUqqZVFdbn6On8o3fXVXNs19t4/FPt1ivFag+wcdxdGgAX9w5hrjwoNPK6BUXr4nIBOBJwAW8YIx5pIF2U4C3gbONMfqJr5TyKafT/ePv8uP283pw8aCOrNyVz8UDO7K/sIyM7GIC/f0oq6zi0/QD9GwTTkRwAON6tyE2zPPTmHqsKIiIC3gauADIBFaIyHxjTHqddhHAHcAyT2VRSilv1Sk2tGYY8A7RIXSIDql5bmyvNs2ex5NXZQwFMowx240xFcAc4OJ62v0F+AdQVs9zSimlmpEni0JHYE+tx5n2uhoiMhjoZIxZ4MEcSimlGsmTRaG+Traawygi4gf8C7jrpBsSmSEiK0VkZU5OThNGVEopVZsni0Im0KnW40Qgq9bjCKAfsFhEdgLDgfkictzRcWPMc8aYVGNMakJCggcjK6VU6+bJorAC6CEiySISCEwF5h950hhTYIyJN8YkGWOSgO+BSXr2kVJKOcdjRcEY4wZuBxYCPwBvGWM2ishDIjLJU++rlFLq9Hn0OgVjzEfAR3XW3d9A27GezKKUUurkWsZA4UoppZqEzw1zISI5wK7TfHk8kNuEcTzJV7JqzqblKznBd7JqTksXY8xJz9TxuaJwJkRkZWPG/vAGvpJVczYtX8kJvpNVc54a7T5SSilVQ4uCUkqpGq2tKDzndIBT4CtZNWfT8pWc4DtZNecpaFXHFJRSSp1Ya9tTUEopdQKtpiiIyAQR2SwiGSJyr8NZXhKRbBHZUGtdrIh8JiJb7fsYe72IyL/t3GkiktKMOTuJyCIR+UFENorIr7wxq4gEi8hyEVln53zQXp8sIsvsnG/aw60gIkH24wz7+aTmyFkrr0tE1ojIAi/PuVNE1ovIWhFZaa/zqr+9/d7RIvKOiGyy/62O8NKcvezf5ZFboYj82uuyGmNa/A1r5rdtQFcgEFgH9HEwz2ggBdhQa90/gHvt5XuBR+3li4CPsUadHQ4sa8ac7YEUezkC2AL08bas9vuF28sBWBM2DQfeAqba658FbrGXbwWetZenAm8289//TuANYIH92Ftz7gTi66zzqr+9/d6vADfYy4FAtDfmrJPZBewHunhb1mb/ZTj0BxgBLKz1+PfA7x3OlFSnKGwG2tvL7YHN9vJ/gWn1tXMg8zysmfS8NisQCqwGhmFdCORf998A1nhcI+xlf7udNFO+ROAL4Dxggf0f3uty2u9ZX1Hwqr89EAnsqPt78bac9eS+EFjijVlbS/fRSSf88QJtjTH7AOz7I/PweUV2u+tiMNa3cK/LanfJrAWygc+w9gwPGWtgxrpZanLazxcAcc2RE5gJ/Baoth/HeWlOsOY/+VREVonIDHudt/3tuwI5wCy7S+4FEQnzwpx1TQVm28telbW1FIUTTvjj5RzPLiLhwLvAr40xhSdqWs+6ZslqjKkyxgzC+iY+FDjrBFkcySkiPwGyjTGraq8+QRan//YjjTEpwETgNhEZfYK2TmX1x+qKfcYYMxgoweqCaYjTv1PsY0aTgLdP1rSedR7P2lqKwskm/PEGB0SkPYB9n22vdzS7iARgFYTXjTHveXNWAGPMIWAxVh9stIgcGQm4dpaanPbzUUB+M8QbCUwSa1KpOVhdSDO9MCcAxpgs+z4beB+r2Hrb3z4TyDTGLLMfv4NVJLwtZ20TgdXGmAP2Y6/K2lqKwgkn/PES84Fr7OVrsPrvj6y/2j4TYThQcGRX09NERIAXgR+MMU94a1YRSRCRaHs5BDgfaw6PRcCUBnIeyT8F+NLYnbaeZIz5vTEm0ViTSk213/dKb8sJICJhIhJxZBmrD3wDXva3N8bsB/aISC971Xgg3dty1jGNo11HRzJ5T9bmPsDi1A3rSP4WrL7mPzqcZTawD6jE+jZwPVZf8RfAVvs+1m4rwNN27vVAajPmHIW1u5oGrLVvF3lbVmAAsMbOuQG4317fFVgOZGDtqgfZ64Ptxxn2810d+DcwlqNnH3ldTjvTOvu28cj/GW/729vvPQhYaf/95wIx3pjTfv9QIA+IqrXOq7LqFc1KKaVqtJbuI6WUUo2gRUEppVQNLQpKKaVqaFFQSilVQ4uCUkqpGloUlNcRkSp7FMl1IrJaRM45SftoEbm1EdtdLCKOz4HrTUTkZRGZcvKWqrXQoqC8UakxZpAxZiDW4IV/P0n7aKwRRb1SrauVlfJ6WhSUt4sEDoI1BpOIfGHvPawXkYvtNo8A3ey9i8fstr+126wTkUdqbe8yseZe2CIi59ptXSLymIissMetv8le315Evra3u+FI+9rEmnPgUXuby0Wku73+ZRF5QkQWAY/aY+bPtbf/vYgMqPUzzbKzponIpfb6C0Vkqf2zvm2PP4WIPCIi6Xbbx+11l9n51onI1yf5mUREnrK38SFHB19TytKcV/PpTW+NuQFVWFdPb8IaGXSIvd4fiLSX47Gu9BWOH4Z8IvAdEGo/PnKF6GLgn/byRcDn9vIM4D57OQjr6thk4C6OXsnrAiLqybqzVpurOXqV8stYQ2O77Mf/AR6wl88D1trLjwIza20vxv7ZvgbC7HW/A+4HYrGGTz5y0Wm0fb8e6FhnXUM/02SsUWRdQAfgEDDF6b+53rznpru1yhuVGmvEU0RkBPCqiPTDKgB/E2u0zmqsYYTb1vP684FZxpjDAMaY2oPIHRnUbxVWMQFrXJ8BtfrWo4AeWGNmvSTWoIBzjTFrG8g7u9b9v2qtf9sYU2UvjwIutfN8KSJxIhJlZ5165AXGmINijabaB1hiDT9FILAUKATKgBfsb/kL7JctAV4Wkbdq/XwN/Uyjgdl2riwR+bKBn0m1UloUlFczxiwVkXggAevbfQLWnkOlWKONBtfzMqHhIYbL7fsqjv77F+CXxpiFx23IKkA/Bv4nIo8ZY16tL2YDyyV1MtX3uvqyCvCZMWZaPXmGYg36NhW4HTjPGHOziAyzc64VkUEN/UwiclE976dUDT2moLyaiPTG6urIw/q2m20XhHFYUxkCFGFNF3rEp8B1IhJqbyP2JG+zELjF3iNARHrao4R2sd/veazRYhuaI/eKWvdLG2jzNXClvf2xQK6x5qb4FOvD/cjPGwN8D4ysdXwi1M4UjjWQ2kfAr7EGgkNEuhljlhlj7seana1TQz+TnWOqfcyhPTDuJL8b1cronoLyRiFizaIG1jfea4wxVSLyOvCBWJPIHznmgDEmT0SWiMgG4GNjzD32t+WVIlIBfAT84QTv9wJWV9JqsfprcoBLsEYyvUdEKoFirGMG9QkSkWVYX7KO+3Zv+zPW7GBpwGGODpX8V+BpO3sV8KAx5j0RmQ7MFpEgu919WMVvnogE27+X39jPPSYiPex1X2CNbJrWwM/0PtYxjfVYowZ/dYLfi2qFdJRUpc6A3YWVaozJdTqLUk1Bu4+UUkrV0D0FpZRSNXRPQSmlVA0tCkoppWpoUVBKKVVDi4JSSqkaWhSUUkrV0KKglFKqxv8DHRsuETgW/wUAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.recorder.plot_losses()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([[ 2.2482, -2.3293]], grad_fn=<ThAddmmBackward>),\n",
       " tensor([[-2.8670,  2.5273]], grad_fn=<ThAddmmBackward>))"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "net(data.train_ds[0][0].data), net(data.train_ds[3][0].data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "preds, targs = learn.get_preds()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 206,
   "metadata": {},
   "outputs": [],
   "source": [
    "#targs.view(-1,1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 207,
   "metadata": {},
   "outputs": [],
   "source": [
    "#preds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([0.0346, 0.9654]), tensor(0))"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "i = 0\n",
    "preds[i], targs[i]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((Image (1, 4, 50), Category 0), (Image (1, 4, 50), Category 1))"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.train_ds[0], data.train_ds[3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((Category 0, tensor(0), tensor([0.9898, 0.0102])),\n",
       " (Category 1, tensor(1), tensor([0.0045, 0.9955])))"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learn.predict(data.train_ds[0][0]), learn.predict(data.train_ds[3][0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.save('epoch15')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Learner(data=ImageDataBunch;\n",
       "Train: LabelList\n",
       "y: CategoryList (2000 items)\n",
       "[Category 0, Category 0, Category 0, Category 1, Category 1]...\n",
       "Path: .\n",
       "x: SeqItemList (1500 items)\n",
       "[Image (1, 4, 50), Image (1, 4, 50), Image (1, 4, 50), Image (1, 4, 50), Image (1, 4, 50)]...\n",
       "Path: .;\n",
       "Valid: LabelList\n",
       "y: CategoryList (2000 items)\n",
       "[Category 0, Category 0, Category 0, Category 1, Category 1]...\n",
       "Path: .\n",
       "x: SeqItemList (500 items)\n",
       "[Image (1, 4, 50), Image (1, 4, 50), Image (1, 4, 50), Image (1, 4, 50), Image (1, 4, 50)]...\n",
       "Path: .;\n",
       "Test: None, model=Sequential(\n",
       "  (0): Lambda()\n",
       "  (1): Conv1d(4, 32, kernel_size=(12,), stride=(1,))\n",
       "  (2): MaxPool1d(kernel_size=4, stride=4, padding=0, dilation=1, ceil_mode=False)\n",
       "  (3): Lambda()\n",
       "  (4): Linear(in_features=288, out_features=16, bias=True)\n",
       "  (5): ReLU()\n",
       "  (6): Linear(in_features=16, out_features=2, bias=True)\n",
       "), opt_func=functools.partial(<class 'torch.optim.adam.Adam'>, betas=(0.9, 0.99)), loss_func=<function cross_entropy at 0x1a1881ff28>, metrics=[<function accuracy at 0x1a18a44a60>], true_wd=True, bn_wd=True, wd=0.01, train_bn=True, path=PosixPath('.'), model_dir='models', callback_fns=[<class 'fastai.basic_train.Recorder'>, <class '__main__.TBLogger'>], callbacks=[], layer_groups=[Sequential(\n",
       "  (0): Lambda()\n",
       "  (1): Conv1d(4, 32, kernel_size=(12,), stride=(1,))\n",
       "  (2): MaxPool1d(kernel_size=4, stride=4, padding=0, dilation=1, ceil_mode=False)\n",
       "  (3): Lambda()\n",
       "  (4): Linear(in_features=288, out_features=16, bias=True)\n",
       "  (5): ReLU()\n",
       "  (6): Linear(in_features=16, out_features=2, bias=True)\n",
       ")])"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learn.load('epoch15')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "interpret = ClassificationInterpretation.from_learner(learn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAARoAAAEmCAYAAAC9C19sAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAE3tJREFUeJzt3XmUnXWd5/H3t7KQQEgqCxBJIOxEsO0ICijSIuLWItjN6VFEWhinGXXsaZrWFpczgtojgzqncVxaON2I0CqtfbRBnEZNiyIG2aRZmi2IGTAgISEh+1L1nT/uU6GyVFJZvnVreb/OuYf7/J7ffZ7vpao+9fv9nqduIjORpEod7S5A0vBn0EgqZ9BIKmfQSCpn0EgqZ9BIKmfQjFARMT4iboiIZRHx7V04ztkR8cPdWVu7RMRJEfFwu+sYjsL7aAa3iHgncCEwG1gO3AP8TWb+fBePew7w58CrMnPDLhc6yEVEAodn5vx21zISOaIZxCLiQuBvgf8J7AccCHwZOGM3HH4W8MhICJn+iIjR7a5hWMtMH4PwAUwCVgB/so0+e9AKooXN42+BPZp9JwNPAn8FPAM8BZzX7LsEWAesb87xHuBi4Npexz4ISGB0s30u8Gtao6rHgbN7tf+81+teBdwBLGv++6pe+24GPgXc2hznh8C0Pt5bT/1/3av+twF/CDwCLAE+2qv/ccA8YGnT94vA2Gbfz5r3srJ5v2/vdfwPA08D1/S0Na85tDnHMc32/sCzwMnt/t4Yio+2F+Cjjy8MvAnY0POD3kefTwK3AfsC+wC/AD7V7Du5ef0ngTHND+gqYHKzf/Ng6TNogL2A54Ejm30vAo5unm8MGmAK8BxwTvO6s5rtqc3+m4HHgCOA8c32pX28t576/0dT/58Bi4BvAHsDRwNrgEOa/scCJzTnPQh4ELig1/ESOGwrx/9ftAJ7fO+gafr8WXOcPYGbgM+1+/tiqD6cOg1eU4Fnc9tTm7OBT2bmM5m5iNZI5Zxe+9c3+9dn5g9o/TY/cifr6QZeEhHjM/OpzHxgK33eAjyamddk5obM/CbwEPDWXn2uysxHMnM18E/AnG2ccz2t9aj1wLeAacDlmbm8Of8DwEsBMvOuzLytOe9vgK8Cr+nHe/pEZq5t6tlEZl4JPAr8kla4fmw7x1MfDJrBazEwbTtrB/sDC3ptL2jaNh5js6BaBUzY0UIycyWt6cZ7gaci4saImN2PenpqmtFr++kdqGdxZnY1z3uC4He99q/ueX1EHBER34+IpyPieVrrWtO2cWyARZm5Zjt9rgReAvyfzFy7nb7qg0EzeM2jNTV42zb6LKS1qNvjwKZtZ6ykNUXoMb33zsy8KTNfT+s3+0O0fgC3V09PTb/dyZp2xFdo1XV4Zk4EPgrEdl6zzUuuETGB1rrX3wMXR8SU3VHoSGTQDFKZuYzW+sSXIuJtEbFnRIyJiDdHxGVNt28CH4+IfSJiWtP/2p085T3AH0TEgRExCfhIz46I2C8iTo+IvYC1tKZgXVs5xg+AIyLinRExOiLeDhwFfH8na9oRe9NaR1rRjLbet9n+3wGH7OAxLwfuysz/AtwI/N0uVzlCGTSDWGb+b1r30Hyc1kLoE8AHgO81XT4N3AncC9wH3N207cy5fgRc1xzrLjYNhw5aV68W0roS8xrg/Vs5xmLgtKbvYlpXjE7LzGd3pqYd9EHgnbSuZl1J6730djFwdUQsjYj/tL2DRcQZtBbk39s0XQgcExFn77aKRxBv2JNUzhGNpHIGjaRyBo2kcgaNpHKD6g/JJk2emtNnHNDuMlTg0V/v7O09Gsxy3XJyw+rt3a80uIJm+owD+PJ3ftzuMlTgtLMubncJKrD24X/qVz+nTpLKGTSSyhk0ksoZNJLKGTSSyhk0ksoZNJLKGTSSyhk0ksoZNJLKGTSSyhk0ksoZNJLKGTSSyhk0ksoZNJLKGTSSyhk0ksoZNJLKGTSSyhk0ksoZNJLKGTSSyhk0ksoZNJLKGTSSyhk0ksoZNJLKGTSSyhk0ksoZNJLKGTSSyhk0ksoZNJLKGTSSyhk0ksoZNJLKGTSSyhk0ksoZNJLKGTSSyhk0ksoZNJLKGTSSyhk0ksoZNJLKGTSSyhk0ksoZNJLKGTSSyhk0BV48fQInHTaF4w/u3Ni2795jOf7gTk45cip7jxu9sT2Ao140geMP6uSEgzuZNWV8GypWf/3dJ85mwdzPcOe3P7rFvgvOeR2rf/VFpnbutUn7sUcdyIo7v8AfnTpnoMocdAyaAk8tW8M9TyzbpG3F2i7u++1ylq7esEn7vhP3oCOCX/5mKbf/ZikzJo9j3Bi/LIPVNTfcxhn/7UtbtM/cr5NTTpjN/3tqySbtHR3Bp//iDH4078GBKnFQ8ju6wNLVG1jfnZu0rVrXxap1XVt2zqSjIwigI4JM2NCVW/bToHDr3Y+xZNmqLdov++CZfOzy75G56dfu/e94Dd+b++8sWrJ8oEoclAyaNntm+Tq6u5NXHzaFVx82hQWLV7Gh26AZSt7ymt9j4TNLue+R327Svv8+kzj9lN/nyu/c0qbKBo/SoImIN0XEwxExPyIuqjzXUDVx/Ggyk5/PX8Ktjy3hwCnjnToNIePHjeHD73kjn/zKjVvs++yHzuTjl/8L3f7iYPT2u+yciBgFfAl4PfAkcEdEXJ+Z/1F1zqFo+sQ9WLxyPQms70qWrd7AxHGjWbN+XbtLUz8cMnMfZs2Yyu3XfQSAGft2Mu8bH+akcz7LMUcdyNcvPQ+AqZ0TeOOrj2bDhm5uuPnedpbcFmVBAxwHzM/MXwNExLeAMwCDppc167uZvOcYnn5+LR0Bk8aP5onnVre7LPXTA/MXMut1H9m4/dCNl3Di2ZexeOlKXnzaxRvbr7jkXfzfW+4fkSEDtVOnGcATvbafbNqGvaP335uXz+pkz7GjOPHQybxo0h7sM2EsJx46mUnjRjNn5kTmzJwIwJPPrWZUR3D8wZ0cd1AnC5etZcXarSwaa1C4+jPncvPVf8URs/Zj/r9+ine/7ZXtLmlIqBzRxFbatpisRsT5wPkA++4/s7CcgfPAwq1fYVi0YsvpUFfC/X301+Dz7o98bZv7Z7/lE1ttP/8T1xZUM3RUjmieBA7otT0TWLh5p8y8IjNfnpkv75w8tbAcSe1SGTR3AIdHxMERMRZ4B3B94fkkDVJlU6fM3BARHwBuAkYB/5CZD1SdT9LgVblGQ2b+APhB5TkkDX7eGSapnEEjqZxBI6mcQSOpnEEjqZxBI6mcQSOpnEEjqZxBI6mcQSOpnEEjqZxBI6mcQSOpnEEjqZxBI6mcQSOpnEEjqZxBI6mcQSOpnEEjqZxBI6mcQSOpnEEjqZxBI6mcQSOpnEEjqZxBI6mcQSOpnEEjqZxBI6mcQSOpnEEjqZxBI6mcQSOpnEEjqZxBI6mcQSOpnEEjqZxBI6mcQSOpnEEjqZxBI6nc6L52RMQNQPa1PzNPL6lI0rDTZ9AAnxuwKiQNa30GTWb+dCALkTR8bWtEA0BEHA58BjgKGNfTnpmHFNYlaRjpz2LwVcBXgA3Aa4GvA9dUFiVpeOlP0IzPzLlAZOaCzLwYOKW2LEnDyXanTsCaiOgAHo2IDwC/BfatLUvScNKfEc0FwJ7AfweOBc4B3l1ZlKThZbsjmsy8o3m6AjivthxJw1F/rjr9hK3cuJeZrtNI6pf+rNF8sNfzccCZtK5ASVK/9GfqdNdmTbdGRMnNfE8/v5bL5s6vOLTa7Lk7vtjuElTgxONv61e//kydpvTa7KC1IDx958qSNBL1Z+p0F601mqA1ZXoceE9lUZKGl/4EzYszc03vhojYo6geScNQf+6j+cVW2ubt7kIkDV/b+jya6cAMYHxEvIzW1AlgIq0b+CSpX7Y1dXojcC4wE/g8LwTN88BHa8uSNJxs6/NorgaujogzM/OfB7AmScNMf9Zojo2Izp6NiJgcEZ8urEnSMNOfoHlzZi7t2cjM54A/rCtJ0nDTn6AZ1ftydkSMB7y8Lanf+nMfzbXA3Ii4qtk+D7i6riRJw01//tbpsoi4FziV1pWnfwVmVRcmafjo7z8g9zTQTesvt18HPFhWkaRhZ1s37B0BvAM4C1gMXEfrc4NfO0C1SRomtjV1egi4BXhrZs4HiIi/HJCqJA0r25o6nUlryvSTiLgyIl7HC3cHS1K/9Rk0mfndzHw7MBu4GfhLYL+I+EpEvGGA6pM0DGx3MTgzV2bmP2bmabT+7uke4KLyyiQNG/296gRAZi7JzK/6weSSdsQOBY0k7QyDRlI5g0ZSOYNGUjmDRlI5g0ZSOYNGUjmDRlI5g0ZSOYNGUjmDRlI5g0ZSOYNGUjmDRlI5g0ZSOYNGUjmDRlI5g0ZSOYNGUjmDRlI5g0ZSOYNGUjmDRlI5g0ZSOYNGUjmDRlI5g0ZSOYNGUjmDRlI5g0ZSOYNGUjmDRlI5g0ZSudHtLmC4uvCUQzh+1mSWrl7Pf/3WvQD86XEzeeXBk0lg6ar1fG7uYyxZtR6A9500i+NmTWbN+i4+P/cx5j+7qo3Vqy+jO2BUQALrul5oH9MBEZAJ67u3bAfo6oauHNByBw1HNEV++OAiPnbDg5u0fedXT/G+6+7j/dfdxy8XLOVdr5gJwCtmdTJj0njOu/YeLr/5cf785EPaUbL6oat704CBVvh0Z6u9O1vbsGkgret6oX0kGsFvvdb9Ty1n+dpNvyNXrX9he9zoDnp+ub3y4Mn8+OFFADz0uxXsNXYUU/YcM1ClagdsbUDSES+MVLqytb256OO1I4VTpwF27vEHcOqR01i5rou//t5/ADBtr7EsWrFuY59nV65j6l5jN06rNLhtnis9210JYwL2GNXa7j2lGmnKRjQR8Q8R8UxE3F91jqHoa798gnd9/Vf82yPPcvpLp/fZbyT/9hsuOpqp01qnTqVTp68Bbyo8/pD2k0ef5dWHTAFaI5h9JozduG/aXmNZsnJdXy/VILP5L4We7VHRWtPpacvccvQzUpQFTWb+DFhSdfyhaP9J4zY+P+GgyTzx3GoAbnv8OU49ch8AZu83gVXrupw2DSHd2QoVaP23u0maBEb1+gnrGeGMRG1fo4mI84HzAcZN7nsqMdRc9PrDeOmMiUwaN5pr3/0yrrn9SY6b1cnMzvF0Z/LM8nV84ae/BuD2BUt5xaxOrnrXHNZu6Obzcx9rc/Xqy5iOFxZ79xgFG7pbjzEdrVDpfXm7p32sazTtD5rMvAK4AmDSgS8eNoF/6Y/mb9F204OL+uz/pZ/9prAa7S59hcWOto80I3h5StJAMWgklau8vP1NYB5wZEQ8GRHvqTqXpMGtbI0mM8+qOrakocWpk6RyBo2kcgaNpHIGjaRyBo2kcgaNpHIGjaRyBo2kcgaNpHIGjaRyBo2kcgaNpHIGjaRyBo2kcgaNpHIGjaRyBo2kcgaNpHIGjaRyBo2kcgaNpHIGjaRyBo2kcgaNpHIGjaRyBo2kcgaNpHIGjaRyBo2kcgaNpHIGjaRyBo2kcgaNpHIGjaRyBo2kcgaNpHIGjaRyBo2kcgaNpHIGjaRyBo2kcgaNpHIGjaRyBo2kcgaNpHIGjaRyBo2kcgaNpHIGjaRyBo2kcgaNpHIGjaRyBo2kcpGZ7a5ho4hYBCxodx0DZBrwbLuL0G430r6uszJzn+11GlRBM5JExJ2Z+fJ216Hdy6/r1jl1klTOoJFUzqBpnyvaXYBK+HXdCtdoJJVzRCOpnEEjqZxBI6nc6HYXMBJExGzgDGAGkMBC4PrMfLCthUkDxBFNsYj4MPAtIIDbgTua59+MiIvaWZs0ULzqVCwiHgGOzsz1m7WPBR7IzMPbU5kqRcR5mXlVu+sYLBzR1OsG9t9K+4uafRqeLml3AYOJazT1LgDmRsSjwBNN24HAYcAH2laVdllE3NvXLmC/gaxlsHPqNAAiogM4jtZicABPAndkZldbC9MuiYjfAW8Entt8F/CLzNzaSHZEckQzADKzG7it3XVot/s+MCEz79l8R0TcPPDlDF6OaCSVczFYUjmDRlI5g0YARERXRNwTEfdHxLcjYs9dONbJEfH95vnp27oxMSI6I+L9O3GOiyPigztbowaWQaMeqzNzTma+BFgHvLf3zmjZ4e+XzLw+My/dRpdOYIeDRkOLQaOtuQU4LCIOiogHI+LLwN3AARHxhoiYFxF3NyOfCQAR8aaIeCgifg78cc+BIuLciPhi83y/iPhuRPx783gVcClwaDOa+mzT70MRcUdE3BsRl/Q61sci4uGI+DFw5ID939AuM2i0iYgYDbwZuK9pOhL4ema+DFgJfBw4NTOPAe4ELoyIccCVwFuBk4DpfRz+C8BPM/P3gWOAB4CLgMea0dSHIuINwOG07juaAxwbEX8QEccC7wBeRivIXrGb37oKeR+NeoyPiJ77QW4B/p7Wn04syMyee4BOAI4Cbo0IgLHAPGA28HhmPgoQEdcC52/lHKcAfwrQ3Ky4LCImb9bnDc3jV832BFrBszfw3cxc1Zzj+l16txpQBo16rM7MOb0bmjBZ2bsJ+FFmnrVZvzm0Pv5idwjgM5n51c3OccFuPIcGmFMn7YjbgBMj4jCAiNgzIo4AHgIOjohDm35n9fH6ucD7mteOioiJwHJao5UeNwH/udfaz4yI2Bf4GfBHETE+IvamNU3TEGHQqN8ycxFwLq3P0rmXVvDMzsw1tKZKNzaLwX39a6N/Abw2Iu4D7qL18RmLaU3F7o+Iz2bmD4FvAPOaft8B9s7Mu4HrgHuAf6Y1vdMQ4Z8gSCrniEZSOYNGUjmDRlI5g0ZSOYNGUjmDRlI5g0ZSuf8P4bKxNeJExWUAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "interpret.plot_confusion_matrix()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}