Datasets
Models
Downloads: 1
3364 lines (3363 with data), 178.2 kB
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Deep learning in genomics"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"This notebook is based on the [jupyter notebook](https://github.com/abidlabs/deep-learning-genomics-primer/blob/master/A_Primer_on_Deep_Learning_in_Genomics_Public.ipynb) from the publication [\"A primer on deep learning in genomics\"](https://www.nature.com/articles/s41588-018-0295-5) but uses the [fastai](https://www.fast.ai) library based on [PyTorch](https://pytorch.org)."
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Notebook setup"
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"%reload_ext autoreload\n",
"%autoreload 2\n",
"%matplotlib inline"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"from fastai import *\n",
"from fastai.vision import *"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
"from sklearn.preprocessing import LabelEncoder, OneHotEncoder"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"'1.0.35.dev0'"
]
},
"execution_count": 4,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"# fastai version\n",
"__version__"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Data setup"
]
},
{
"cell_type": "markdown",
"metadata": {
"heading_collapsed": true
},
"source": [
"## Load data from the web, generate dataframe, and save to disk"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {
"hidden": true
},
"outputs": [],
"source": [
"import requests"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {
"hidden": true
},
"outputs": [],
"source": [
"URL_seq = 'https://raw.githubusercontent.com/abidlabs/deep-learning-genomics-primer/master/sequences.txt'"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {
"hidden": true
},
"outputs": [],
"source": [
"# get data from URL\n",
"seq_raw = requests.get(URL_seq).text.split('\\n')\n",
"seq_raw = list(filter(None, seq_raw)) # Removes empty lists"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {
"hidden": true
},
"outputs": [
{
"data": {
"text/plain": [
"2000"
]
},
"execution_count": 8,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"# check length\n",
"len(seq_raw)"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {
"hidden": true
},
"outputs": [],
"source": [
"# setup df from list\n",
"seq_df = pd.DataFrame(seq_raw, columns=['Sequences'])"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {
"hidden": true
},
"outputs": [],
"source": [
"# show head of dataframe\n",
"#seq_df.head()"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {
"hidden": true
},
"outputs": [],
"source": [
"URL_labels = 'https://raw.githubusercontent.com/abidlabs/deep-learning-genomics-primer/master/labels.txt'"
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {
"hidden": true
},
"outputs": [],
"source": [
"seq_labels = requests.get(URL_labels).text.split('\\n')\n",
"seq_labels = list(filter(None, seq_labels)) # Removes empty entries"
]
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {
"hidden": true
},
"outputs": [
{
"data": {
"text/plain": [
"2000"
]
},
"execution_count": 13,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"len(seq_labels)"
]
},
{
"cell_type": "code",
"execution_count": 14,
"metadata": {
"hidden": true
},
"outputs": [],
"source": [
"seq_label_series = pd.Series(seq_labels)"
]
},
{
"cell_type": "code",
"execution_count": 15,
"metadata": {
"hidden": true
},
"outputs": [],
"source": [
"seq_df['Target'] = seq_label_series.astype('int')"
]
},
{
"cell_type": "code",
"execution_count": 16,
"metadata": {
"hidden": true
},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>Sequences</th>\n",
" <th>Target</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>CCGAGGGCTATGGTTTGGAAGTTAGAACCCTGGGGCTTCTCGCGGA...</td>\n",
" <td>0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>GAGTTTATATGGCGCGAGCCTAGTGGTTTTTGTACTTGTTTGTCGC...</td>\n",
" <td>0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>GATCAGTAGGGAAACAAACAGAGGGCCCAGCCACATCTAGCAGGTA...</td>\n",
" <td>0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>GTCCACGACCGAACTCCCACCTTGACCGCAGAGGTACCACCAGAGC...</td>\n",
" <td>1</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>GGCGACCGAACTCCAACTAGAACCTGCATAACTGGCCTGGGAGATA...</td>\n",
" <td>1</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" Sequences Target\n",
"0 CCGAGGGCTATGGTTTGGAAGTTAGAACCCTGGGGCTTCTCGCGGA... 0\n",
"1 GAGTTTATATGGCGCGAGCCTAGTGGTTTTTGTACTTGTTTGTCGC... 0\n",
"2 GATCAGTAGGGAAACAAACAGAGGGCCCAGCCACATCTAGCAGGTA... 0\n",
"3 GTCCACGACCGAACTCCCACCTTGACCGCAGAGGTACCACCAGAGC... 1\n",
"4 GGCGACCGAACTCCAACTAGAACCTGCATAACTGGCCTGGGAGATA... 1"
]
},
"execution_count": 16,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"seq_df.head()"
]
},
{
"cell_type": "code",
"execution_count": 67,
"metadata": {
"hidden": true
},
"outputs": [],
"source": [
"seq_df.to_csv('seq_df.csv')"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Data loading"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [],
"source": [
"seq_df = pd.read_csv('seq_df.csv', index_col=0)"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [],
"source": [
"#seq_df.drop('Unnamed: 0', axis=1, inplace=True)"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>Sequences</th>\n",
" <th>Target</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>CCGAGGGCTATGGTTTGGAAGTTAGAACCCTGGGGCTTCTCGCGGA...</td>\n",
" <td>0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>GAGTTTATATGGCGCGAGCCTAGTGGTTTTTGTACTTGTTTGTCGC...</td>\n",
" <td>0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>GATCAGTAGGGAAACAAACAGAGGGCCCAGCCACATCTAGCAGGTA...</td>\n",
" <td>0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>GTCCACGACCGAACTCCCACCTTGACCGCAGAGGTACCACCAGAGC...</td>\n",
" <td>1</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>GGCGACCGAACTCCAACTAGAACCTGCATAACTGGCCTGGGAGATA...</td>\n",
" <td>1</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" Sequences Target\n",
"0 CCGAGGGCTATGGTTTGGAAGTTAGAACCCTGGGGCTTCTCGCGGA... 0\n",
"1 GAGTTTATATGGCGCGAGCCTAGTGGTTTTTGTACTTGTTTGTCGC... 0\n",
"2 GATCAGTAGGGAAACAAACAGAGGGCCCAGCCACATCTAGCAGGTA... 0\n",
"3 GTCCACGACCGAACTCCCACCTTGACCGCAGAGGTACCACCAGAGC... 1\n",
"4 GGCGACCGAACTCCAACTAGAACCTGCATAACTGGCCTGGGAGATA... 1"
]
},
"execution_count": 7,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"seq_df.head()"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [],
"source": [
"# Preprocess y values"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"2000"
]
},
"execution_count": 9,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"len(seq_df['Target'])"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"array([0, 0, 0, 1, ..., 1, 0, 1, 1])"
]
},
"execution_count": 10,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"targA = seq_df['Target'].values; targA"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"array([1, 1, 1, 0, ..., 0, 1, 0, 0])"
]
},
"execution_count": 11,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"targB = np.logical_not(seq_df['Target'].values).astype(int); targB"
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"([[0, 1], [0, 1], [0, 1], [1, 0]], '...', [[1, 0], [0, 1], [1, 0], [1, 0]])"
]
},
"execution_count": 12,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"targ = [[a,b] for a, b in zip(targA, targB)]; targ[:4], '...', targ[-4:]#, len(targ)"
]
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {},
"outputs": [],
"source": [
"seq_df['MultiTarget'] = targ"
]
},
{
"cell_type": "code",
"execution_count": 14,
"metadata": {},
"outputs": [],
"source": [
"def joinarray(x):\n",
" y = ';'.join((str(x[0]),str(x[1])))\n",
" return y"
]
},
{
"cell_type": "code",
"execution_count": 15,
"metadata": {},
"outputs": [],
"source": [
"seq_df['MultiTarget'] = seq_df['MultiTarget'].apply(joinarray)"
]
},
{
"cell_type": "code",
"execution_count": 16,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>Sequences</th>\n",
" <th>Target</th>\n",
" <th>MultiTarget</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>CCGAGGGCTATGGTTTGGAAGTTAGAACCCTGGGGCTTCTCGCGGA...</td>\n",
" <td>0</td>\n",
" <td>0;1</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>GAGTTTATATGGCGCGAGCCTAGTGGTTTTTGTACTTGTTTGTCGC...</td>\n",
" <td>0</td>\n",
" <td>0;1</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>GATCAGTAGGGAAACAAACAGAGGGCCCAGCCACATCTAGCAGGTA...</td>\n",
" <td>0</td>\n",
" <td>0;1</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>GTCCACGACCGAACTCCCACCTTGACCGCAGAGGTACCACCAGAGC...</td>\n",
" <td>1</td>\n",
" <td>1;0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>GGCGACCGAACTCCAACTAGAACCTGCATAACTGGCCTGGGAGATA...</td>\n",
" <td>1</td>\n",
" <td>1;0</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" Sequences Target MultiTarget\n",
"0 CCGAGGGCTATGGTTTGGAAGTTAGAACCCTGGGGCTTCTCGCGGA... 0 0;1\n",
"1 GAGTTTATATGGCGCGAGCCTAGTGGTTTTTGTACTTGTTTGTCGC... 0 0;1\n",
"2 GATCAGTAGGGAAACAAACAGAGGGCCCAGCCACATCTAGCAGGTA... 0 0;1\n",
"3 GTCCACGACCGAACTCCCACCTTGACCGCAGAGGTACCACCAGAGC... 1 1;0\n",
"4 GGCGACCGAACTCCAACTAGAACCTGCATAACTGGCCTGGGAGATA... 1 1;0"
]
},
"execution_count": 16,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"seq_df.head()"
]
},
{
"cell_type": "markdown",
"metadata": {
"heading_collapsed": true
},
"source": [
"## Data encoding test (incorporated into \"open_seq_image\" function)"
]
},
{
"cell_type": "code",
"execution_count": 86,
"metadata": {
"hidden": true
},
"outputs": [],
"source": [
"# setup class instance to encode the four different bases to integer values (1D)\n",
"int_enc = LabelEncoder()"
]
},
{
"cell_type": "code",
"execution_count": 87,
"metadata": {
"hidden": true
},
"outputs": [],
"source": [
"# setup one hot encoder to encode integer encoded classes (1D) to one hot encoded array (4D)\n",
"one_hot_enc = OneHotEncoder(categories=[range(4)])"
]
},
{
"cell_type": "code",
"execution_count": 88,
"metadata": {
"hidden": true
},
"outputs": [],
"source": [
"seq_enc = []\n",
"\n",
"for s in seq:\n",
" enc = int_enc.fit_transform(list(s)) # bases (ACGT) to int (0,1,2,3)\n",
" enc = np.array(enc).reshape(-1,1) # reshape to get rank 2 array (from rank 1 array)\n",
" enc = one_hot_enc.fit_transform(enc) # encoded integer encoded bases to sparse matrix (sparse matrix dtype)\n",
" seq_enc.append(enc.toarray()) # export sparse matrix to np array"
]
},
{
"cell_type": "code",
"execution_count": 89,
"metadata": {
"hidden": true
},
"outputs": [
{
"data": {
"text/plain": [
"2000"
]
},
"execution_count": 89,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"len(seq_enc)"
]
},
{
"cell_type": "code",
"execution_count": 462,
"metadata": {
"hidden": true
},
"outputs": [
{
"data": {
"text/plain": [
"(array([[0., 0., 0., 1., ..., 0., 1., 0., 0.],\n",
" [1., 1., 0., 0., ..., 1., 0., 1., 1.],\n",
" [0., 0., 1., 0., ..., 0., 0., 0., 0.],\n",
" [0., 0., 0., 0., ..., 0., 0., 0., 0.]]), (4, 50))"
]
},
"execution_count": 462,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"seq_enc[0].T, seq_enc[0].T.shape"
]
},
{
"cell_type": "code",
"execution_count": 311,
"metadata": {
"hidden": true
},
"outputs": [
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAADIAAAAECAMAAAD7wpwzAAADAFBMVEUAAAABAQECAgIDAwMEBAQFBQUGBgYHBwcICAgJCQkKCgoLCwsMDAwNDQ0ODg4PDw8QEBARERESEhITExMUFBQVFRUWFhYXFxcYGBgZGRkaGhobGxscHBwdHR0eHh4fHx8gICAhISEiIiIjIyMkJCQlJSUmJiYnJycoKCgpKSkqKiorKyssLCwtLS0uLi4vLy8wMDAxMTEyMjIzMzM0NDQ1NTU2NjY3Nzc4ODg5OTk6Ojo7Ozs8PDw9PT0+Pj4/Pz9AQEBBQUFCQkJDQ0NERERFRUVGRkZHR0dISEhJSUlKSkpLS0tMTExNTU1OTk5PT09QUFBRUVFSUlJTU1NUVFRVVVVWVlZXV1dYWFhZWVlaWlpbW1tcXFxdXV1eXl5fX19gYGBhYWFiYmJjY2NkZGRlZWVmZmZnZ2doaGhpaWlqampra2tsbGxtbW1ubm5vb29wcHBxcXFycnJzc3N0dHR1dXV2dnZ3d3d4eHh5eXl6enp7e3t8fHx9fX1+fn5/f3+AgICBgYGCgoKDg4OEhISFhYWGhoaHh4eIiIiJiYmKioqLi4uMjIyNjY2Ojo6Pj4+QkJCRkZGSkpKTk5OUlJSVlZWWlpaXl5eYmJiZmZmampqbm5ucnJydnZ2enp6fn5+goKChoaGioqKjo6OkpKSlpaWmpqanp6eoqKipqamqqqqrq6usrKytra2urq6vr6+wsLCxsbGysrKzs7O0tLS1tbW2tra3t7e4uLi5ubm6urq7u7u8vLy9vb2+vr6/v7/AwMDBwcHCwsLDw8PExMTFxcXGxsbHx8fIyMjJycnKysrLy8vMzMzNzc3Ozs7Pz8/Q0NDR0dHS0tLT09PU1NTV1dXW1tbX19fY2NjZ2dna2trb29vc3Nzd3d3e3t7f39/g4ODh4eHi4uLj4+Pk5OTl5eXm5ubn5+fo6Ojp6enq6urr6+vs7Ozt7e3u7u7v7+/w8PDx8fHy8vLz8/P09PT19fX29vb39/f4+Pj5+fn6+vr7+/v8/Pz9/f3+/v7////isF19AAAAO0lEQVR4nGWNSQoAMAgD8/9PT6loIq0HMauShOR9DwphIkO5eALbEI1xp5DUoxYnNtf3t1rci9mGy30A2+0xz9q2+b0AAAAASUVORK5CYII=\n",
"text/plain": [
"<PIL.Image.Image image mode=P size=50x4 at 0x1A20462668>"
]
},
"execution_count": 311,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"PIL.Image.fromarray(seq_enc[0].T.astype('uint8')*255).convert('P')"
]
},
{
"cell_type": "markdown",
"metadata": {
"heading_collapsed": true
},
"source": [
"## fastai data setup for NN"
]
},
{
"cell_type": "code",
"execution_count": 18,
"metadata": {
"hidden": true
},
"outputs": [],
"source": [
"# open sequence image function\n",
"def open_seq_image(seq:str, cls:type=Image)->Image:\n",
" \"Return `Image` object created from sequence string `seq`.\"\n",
" \n",
" int_enc = LabelEncoder() # setup class instance to encode the four different bases to integer values (1D)\n",
" one_hot_enc = OneHotEncoder(categories=[range(4)]) # setup one hot encoder to encode integer encoded classes (1D) to one hot encoded array (4D)\n",
" \n",
" enc = int_enc.fit_transform(list(seq)) # bases (ACGT) to int (0,1,2,3)\n",
" enc = np.array(enc).reshape(-1,1) # reshape to get rank 2 array (from rank 1 array)\n",
" enc = one_hot_enc.fit_transform(enc) # encoded integer encoded bases to sparse matrix (sparse matrix dtype)\n",
" enc = enc.toarray().T # export sparse matrix to np array\n",
" #print('enc', enc, enc.shape)\n",
" \n",
" # https://stackoverflow.com/questions/22902040/convert-black-and-white-array-into-an-image-in-python\n",
" x = PIL.Image.fromarray(enc.astype('uint8')).convert('P')\n",
" x = pil2tensor(x,np.float32)\n",
" #x = x.view(4,-1) # remove first dimension\n",
" #x = x.expand(3, 4, 50) # expand to 3 channel image\n",
" #print('x', x, x.shape)\n",
" return cls(x)"
]
},
{
"cell_type": "code",
"execution_count": 19,
"metadata": {
"hidden": true
},
"outputs": [
{
"data": {
"image/jpeg": "/9j/4AAQSkZJRgABAQEAZABkAAD/2wBDAAIBAQEBAQIBAQECAgICAgQDAgICAgUEBAMEBgUGBgYFBgYGBwkIBgcJBwYGCAsICQoKCgoKBggLDAsKDAkKCgr/2wBDAQICAgICAgUDAwUKBwYHCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgr/wAARCAAEADIDASIAAhEBAxEB/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAAAAAAAAECAwQFBgcICQoL/8QAtREAAgECBAQDBAcFBAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEIFEKRobHBCSMzUvAVYnLRChYkNOEl8RcYGRomJygpKjU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6goOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4uPk5ebn6Onq8vP09fb3+Pn6/9oADAMBAAIRAxEAPwD80/8AhrrUr/8A0rxT8B/h1q95B/pGltNotzaWGm6iflfUYdIsrmDSvPkjjsopUeze3nTTrfzYZGadpvuj4Jf8EwP2bf2gf2wL39kD4j6n4uuYLDwj4g1u38e/8JEz68IdG8WXnhax0rMqvZ/YlsbG3faLUTCVcJMkAWBSivm/HbGYzgrIq2IyKrLDTjhcTUThKSanTdBQkuZztbnleKXLK95xqOMXD9sPnT9kmOD9rf8A4Wbb6rcav4Ls/gx8Ita+Inwp0/wV4s1VYfCuqaf5LmKyF/dXRhguriVbqcg+eZreIxTQqHR+R+CXxp8V/tC+Mr34X/FXTdI1fwnpnhHxBrWlaFqWlx3s2nWmjaPearZaJaapd+bqtnpYkso4TbwXkbeTLOFkWSZ5SUV9lXwmGhmfENJRVsJDCOj3pSq0HOpKEvjUpz9+cudylJuTbbk2Hrn7DnwE+DH7cXg3xd4t1X4aaR8PovBvi7wX4fsNJ8DWguIbqPxRrA0fUbm4l1v+0Ll50tdptsTLHazJ58UayvI7/On/AA2R480r/ibfDvwF4R8F+IW+SXxP4N0uXT7mWE/vXiaCOb7H/wAfuL+OUW4mtLmK3+ySW0VpaQwFFHCuFoZnx5n2W4xOrQw06KpQnKUoxU6VKUk+ab51KUm2qjqpt+SSA/4ba+Mn/QmfCL/xH/wf/wDKqiiiv0n/AFT4Y/6A6f3S/wDlgH//2Q==\n",
"image/png": "iVBORw0KGgoAAAANSUhEUgAAADIAAAAECAYAAADMHGwBAAAABHNCSVQICAgIfAhkiAAAAHNJREFUKJFjdGEM+c8ABTufXWBwlzJgQAe4xNHVMDAwoKiD6cMmRwlANhdmJuPf5yr/cVlCjAdIdQAMELIP3bHobkEPJEZYjKBLYLMMV6gji+GykJDHcHkOm9uwqWNETlqEDEF2LD7LiUlK+JIxTC8pKQIA7jhk1J8wofAAAAAASUVORK5CYII=\n",
"text/plain": [
"Image (1, 4, 50)"
]
},
"execution_count": 19,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"# test open sequence image function\n",
"open_seq_image('CCGAGGGCTATGGTTTGGAAGTTAGAACCCTGGGGCTTCTCGCGGACACC')"
]
},
{
"cell_type": "code",
"execution_count": 20,
"metadata": {
"hidden": true
},
"outputs": [],
"source": [
"class SeqItemList(ImageItemList):\n",
" _bunch,_square_show = ImageDataBunch,True\n",
" def __post_init__(self):\n",
" super().__post_init__()\n",
" self.sizes={}\n",
" \n",
" def open(self, seq): return open_seq_image(seq)\n",
" \n",
" def get(self, i):\n",
" seq = self.items[i][0]\n",
" res = self.open(seq)\n",
" return res\n",
" \n",
" @classmethod\n",
" def import_from_df(cls, df:DataFrame, cols:IntsOrStrs=0, **kwargs)->'ItemList':\n",
" \"Get the sequences in `col` of `df` and will had `path/folder` in front of them, `suffix` at the end.\"\n",
" return cls(items=df[cols].values)"
]
},
{
"cell_type": "code",
"execution_count": 21,
"metadata": {
"hidden": true
},
"outputs": [],
"source": [
"bs = 64"
]
},
{
"cell_type": "code",
"execution_count": 42,
"metadata": {
"hidden": true
},
"outputs": [],
"source": [
"data = (SeqItemList.import_from_df(seq_df, ['Sequences'])\n",
" .random_split_by_pct(valid_pct=0.25)\n",
" #.split_by_idxs(range(1500), range(1500,2000))\n",
" #.label_from_list(seq_df['Target'].values) # --> Two categories! AND WRONG LOSS FUNCTION!?\n",
" #.label_from_list(targ) # --> MultiCategory\n",
" #.label_from_list(seq_df['MultiTarget'], sep=';')\n",
" .label_from_list(seq_df['Target'].values, sep=';')\n",
" .databunch(bs=bs))"
]
},
{
"cell_type": "markdown",
"metadata": {
"heading_collapsed": true
},
"source": [
"## Verify data setup"
]
},
{
"cell_type": "markdown",
"metadata": {
"hidden": true
},
"source": [
"### Check data object"
]
},
{
"cell_type": "code",
"execution_count": 43,
"metadata": {
"hidden": true
},
"outputs": [
{
"data": {
"text/plain": [
"[tensor([[[[1., 0., 0., ..., 0., 1., 0.],\n",
" [0., 1., 1., ..., 0., 0., 0.],\n",
" [0., 0., 0., ..., 1., 0., 1.],\n",
" [0., 0., 0., ..., 0., 0., 0.]]],\n",
" \n",
" \n",
" [[[1., 0., 0., ..., 0., 0., 1.],\n",
" [0., 0., 0., ..., 0., 0., 0.],\n",
" [0., 1., 1., ..., 1., 1., 0.],\n",
" [0., 0., 0., ..., 0., 0., 0.]]],\n",
" \n",
" \n",
" [[[0., 0., 0., ..., 0., 0., 0.],\n",
" [1., 0., 0., ..., 1., 1., 1.],\n",
" [0., 0., 1., ..., 0., 0., 0.],\n",
" [0., 1., 0., ..., 0., 0., 0.]]],\n",
" \n",
" \n",
" ...,\n",
" \n",
" \n",
" [[[0., 0., 0., ..., 0., 0., 0.],\n",
" [0., 0., 0., ..., 0., 0., 0.],\n",
" [0., 0., 1., ..., 0., 1., 0.],\n",
" [1., 1., 0., ..., 1., 0., 1.]]],\n",
" \n",
" \n",
" [[[0., 0., 0., ..., 1., 0., 0.],\n",
" [0., 0., 0., ..., 0., 0., 0.],\n",
" [1., 1., 0., ..., 0., 0., 0.],\n",
" [0., 0., 1., ..., 0., 1., 1.]]],\n",
" \n",
" \n",
" [[[0., 0., 0., ..., 0., 0., 0.],\n",
" [0., 0., 1., ..., 1., 0., 0.],\n",
" [1., 1., 0., ..., 0., 0., 0.],\n",
" [0., 0., 0., ..., 0., 1., 1.]]]]), tensor([[1., 0.],\n",
" [1., 0.],\n",
" [0., 1.],\n",
" [0., 1.],\n",
" [0., 1.],\n",
" [0., 1.],\n",
" [0., 1.],\n",
" [0., 1.],\n",
" [0., 1.],\n",
" [0., 1.],\n",
" [1., 0.],\n",
" [1., 0.],\n",
" [1., 0.],\n",
" [0., 1.],\n",
" [0., 1.],\n",
" [1., 0.],\n",
" [0., 1.],\n",
" [0., 1.],\n",
" [1., 0.],\n",
" [0., 1.],\n",
" [0., 1.],\n",
" [0., 1.],\n",
" [0., 1.],\n",
" [1., 0.],\n",
" [0., 1.],\n",
" [1., 0.],\n",
" [0., 1.],\n",
" [0., 1.],\n",
" [1., 0.],\n",
" [1., 0.],\n",
" [1., 0.],\n",
" [0., 1.],\n",
" [0., 1.],\n",
" [0., 1.],\n",
" [0., 1.],\n",
" [1., 0.],\n",
" [0., 1.],\n",
" [0., 1.],\n",
" [0., 1.],\n",
" [0., 1.],\n",
" [0., 1.],\n",
" [0., 1.],\n",
" [1., 0.],\n",
" [0., 1.],\n",
" [0., 1.],\n",
" [1., 0.],\n",
" [1., 0.],\n",
" [0., 1.],\n",
" [0., 1.],\n",
" [0., 1.],\n",
" [0., 1.],\n",
" [1., 0.],\n",
" [1., 0.],\n",
" [0., 1.],\n",
" [1., 0.],\n",
" [1., 0.],\n",
" [1., 0.],\n",
" [1., 0.],\n",
" [1., 0.],\n",
" [0., 1.],\n",
" [1., 0.],\n",
" [1., 0.],\n",
" [1., 0.],\n",
" [1., 0.]])]"
]
},
"execution_count": 43,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"next(iter(data.train_dl))"
]
},
{
"cell_type": "code",
"execution_count": 23,
"metadata": {
"hidden": true
},
"outputs": [
{
"data": {
"text/plain": [
"ImageDataBunch;\n",
"\n",
"Train: LabelList\n",
"y: MultiCategoryList (2000 items)\n",
"[MultiCategory 0, MultiCategory 0, MultiCategory 0, MultiCategory 1, MultiCategory 1]...\n",
"Path: .\n",
"x: SeqItemList (1500 items)\n",
"[Image (1, 4, 50), Image (1, 4, 50), Image (1, 4, 50), Image (1, 4, 50), Image (1, 4, 50)]...\n",
"Path: .;\n",
"\n",
"Valid: LabelList\n",
"y: MultiCategoryList (2000 items)\n",
"[MultiCategory 0, MultiCategory 0, MultiCategory 0, MultiCategory 1, MultiCategory 1]...\n",
"Path: .\n",
"x: SeqItemList (500 items)\n",
"[Image (1, 4, 50), Image (1, 4, 50), Image (1, 4, 50), Image (1, 4, 50), Image (1, 4, 50)]...\n",
"Path: .;\n",
"\n",
"Test: None"
]
},
"execution_count": 23,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"data"
]
},
{
"cell_type": "code",
"execution_count": 24,
"metadata": {
"hidden": true
},
"outputs": [
{
"data": {
"text/plain": [
"(2, ['0', '1'])"
]
},
"execution_count": 24,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"# check classes\n",
"data.c, data.classes"
]
},
{
"cell_type": "code",
"execution_count": 25,
"metadata": {
"hidden": true
},
"outputs": [
{
"data": {
"text/plain": [
"64"
]
},
"execution_count": 25,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"data.train_dl.batch_size"
]
},
{
"cell_type": "markdown",
"metadata": {
"hidden": true
},
"source": [
"### Check data points"
]
},
{
"cell_type": "code",
"execution_count": 217,
"metadata": {
"hidden": true
},
"outputs": [
{
"data": {
"image/jpeg": "/9j/4AAQSkZJRgABAQEAZABkAAD/2wBDAAIBAQEBAQIBAQECAgICAgQDAgICAgUEBAMEBgUGBgYFBgYGBwkIBgcJBwYGCAsICQoKCgoKBggLDAsKDAkKCgr/2wBDAQICAgICAgUDAwUKBwYHCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgr/wAARCAAEADIDASIAAhEBAxEB/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAAAAAAAAECAwQFBgcICQoL/8QAtREAAgECBAQDBAcFBAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEIFEKRobHBCSMzUvAVYnLRChYkNOEl8RcYGRomJygpKjU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6goOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4uPk5ebn6Onq8vP09fb3+Pn6/9oADAMBAAIRAxEAPwD4w8Ny6l4h/wCEMjg8RavpkXi34ReJviPYjS9ZuY5vDeteHv7ej0f7DcmRrmSC0tfD1taQR3kt0beG7vfIaKWcSp5F/wANpfGDwt/xKvgLN/wq/wAPP++uPB/g3XtWl0qe/wD4dTaDUr27xeptgMdwpV4JLS3li8uWJZAUV9NwtlOWZnisTDF0Y1Iw5bRklKOtXGxu4tOLfLRp2lKMpJx5oyjKU5VP2wP+G0vjB4k/dfGub/hY8EnyX8HjDXtWX+0oT8zxXj2N7bSXm+WLTpDLMzzD+xtNiWRYLZYSf8NmfESD/QtL+HXw6s9Ktfm0bw/D8PtPawspF+SGeaGSNhq88Nu1xbxTar9ukjS8uJFYTyeeCivsf9U+GtlhIJfypOMV5qEZRhF9Lxgnb3bqOgHrvhC+8V/Ef9lTXP2vPiV471fXvEei/wBqxNaXc0cMN9aWV94XsZLOWe3SO9SC9i8X6018IbmJr2aZZ5mebfJJyOi/GDUvEHwG8SftK6t4U0ibxZoPi7RPDmoyuLk2HiGyv7XVbq3iv9OM32JoNOl0TTTZ2cMMNkqWyRz29xGqopRX5tl+EwtetjoygrU8wp0IJLlUaMnHmppR5VyO+1t7NNNRcQ/RX9n/AP4N/P2OP2kfgN4J/aJ8d/Ej4i2euePvCOm+I9Zs/D91o9jYQXd9ax3M0dtbR6bst4FeVgkS/KiBVHAooor/ADY4j8ePGTAcRY3C4bPcRCnTrVoQipQtGMK1WEYr909IxjGK1eiWr3Yf/9k=\n",
"image/png": "iVBORw0KGgoAAAANSUhEUgAAADIAAAAECAYAAADMHGwBAAAABHNCSVQICAgIfAhkiAAAAHBJREFUKJFjdGEM+b/z2QUGdykDBgYGBgZcbBgfBtylDLDykWlsAJuZ2PQQcge6+YwujCH/8SnABYgxHFkO2cPIYuieIMdcdykDBsa/z1X+oytENxjdEcR4EJ8+YgMMmztweRgeI6SEBK0AOUkTJgYA3Nhz1DR798MAAAAASUVORK5CYII=\n",
"text/plain": [
"Image (1, 4, 50)"
]
},
"execution_count": 217,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"i = 2\n",
"data.x[i]"
]
},
{
"cell_type": "code",
"execution_count": 218,
"metadata": {
"hidden": true
},
"outputs": [
{
"data": {
"text/plain": [
"MultiCategory 0"
]
},
"execution_count": 218,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"data.y[i]"
]
},
{
"cell_type": "code",
"execution_count": 219,
"metadata": {
"hidden": true
},
"outputs": [
{
"data": {
"image/jpeg": "/9j/4AAQSkZJRgABAQEAZABkAAD/2wBDAAIBAQEBAQIBAQECAgICAgQDAgICAgUEBAMEBgUGBgYFBgYGBwkIBgcJBwYGCAsICQoKCgoKBggLDAsKDAkKCgr/2wBDAQICAgICAgUDAwUKBwYHCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgr/wAARCAAEADIDASIAAhEBAxEB/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAAAAAAAAECAwQFBgcICQoL/8QAtREAAgECBAQDBAcFBAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEIFEKRobHBCSMzUvAVYnLRChYkNOEl8RcYGRomJygpKjU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6goOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4uPk5ebn6Onq8vP09fb3+Pn6/9oADAMBAAIRAxEAPwD4C+Nvxp8V/s9eMrL4bfCrTdI0vw5P4R8P+JtP0y00uOCbTdU1PR7PWIrqK/h2X8k+n3d7J9huJ7maeCENA0ksM91HOfBL45fET46eMr34fa/caRpeh2/hHxB4pu/D+ieGNPj0e+1rRtHvNUtL9tJkgk06CdjYWlnM9tbQme0SWJ9xuLl5iivpv7Jyz/UX+0XRi6/subnavLm57czbbvK32mua/vcylq/2w679kmSf9t7/AIWbY/tCW+kX2lfCr4Ra18QfD2i6J4T0rSIZLvTfJ8vTGksbWKe30uU3E8k1paS26vNPLcApPI8zeRf8NrfGeT/iY3K6QuuTfNqfi3TbA6ZrGqSR/vbOe7vLB4Jbie3vcagJnYyXF3HBJeNdi2tkhKKMlynLMZxhnOArUYyo4f6p7KFvdh7XD1KlXlV7fvJxjOd+ZSlFNq6uB1/xJ+NPiv4I+Dfh/wCJPgbpukeFtN+IfhGbxHfeFk0uPVbDSbtdY1LSLiOwbVvtdzbwXlrpdut5EZmW8Rnhn8y28u3TkP8AhtH4rP8A6be+HPCOpah/Dd+IvC8Gr20Gfkf7Ppt+J9Nst0EOnWo+zWsXlW2kWUMXlIJhMUV7OQcPZHjssVbEYeM5udVOTTcmo168I80uZSlaMIxvKTbUVzOTu5B1/wC0B4y+BP7PXx58bfALRv2Kvh1r1n4H8Xal4ftdc8Qa14o+36jHZXUlstzc/ZdZgg8+QRh38mGKPezbI0XCgooo4cyHLsfw7gsViPaSqVKNGcn7fEaynRpTk9MQlrKUnoktdElZIP/Z\n",
"image/png": "iVBORw0KGgoAAAANSUhEUgAAADIAAAAECAYAAADMHGwBAAAABHNCSVQICAgIfAhkiAAAAHRJREFUKJFjdGEM+c+ABHY+u8DgLmWAwYbxGRgYUMSw6SMkDhPDZj42u3GpRwaMMI8gOxLdEHTHIxsMA8h8bA7HZjY2gK4Wn3nIcox/n6v8JyYkCTkAl6MIhTi+WMZmFq4YZnRhDPlPDUeiyxHjOGLNI0YfAGvwc9QnBcXaAAAAAElFTkSuQmCC\n",
"text/plain": [
"Image (1, 4, 50)"
]
},
"execution_count": 219,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"i = 3\n",
"data.x[i]"
]
},
{
"cell_type": "code",
"execution_count": 220,
"metadata": {
"hidden": true
},
"outputs": [
{
"data": {
"text/plain": [
"MultiCategory 1"
]
},
"execution_count": 220,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"data.y[i]"
]
},
{
"cell_type": "code",
"execution_count": 221,
"metadata": {
"hidden": true
},
"outputs": [
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAjgAAAETCAYAAAA79nyeAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAACIFJREFUeJzt3TGrHFUYx+H3XFMY1BQ2SrCNjYIprOxDPoGNWNoofoAUKf0K9oKFgqCVQshXsEgKm6SyMIiNiIooMWNhBLl3Vsa7Z/bM/vM8VVju7pzduzn5MfvupE3TVAAASU5GLwAAoDeBAwDEETgAQByBAwDEETgAQByBAwDEETgAQByBw2paa8+31r5orf3aWvu2tfbW6DUBWewz7HJh9AKI9mFV/VFVL1TV1ar6srV2d5qmb8YuCwhin2FWcyVj1tBae6aqfqyqV6dpuvf4to+r6rtpmm4MXRwQwT7Df/ERFWt5uar+/GfTeexuVb0yaD1AHvsMOwkc1vJsVf106rafquq5AWsBMtln2EngsJZfqurSqdsuVdXPA9YCZLLPsJPAYS33qupCa+3Kv257raoM/gG92GfYyZAxq2mtfVpVU1W9U39/u+GrqnrDtxuAXuwz7OIMDmt6r6ouVtUPVfVJVb1r0wE6s88wyxkcACCOMzgAQByBAwDEETgAQByBAwDEETgAQJxh/5v4tZM3N/v1rVsP7py57frlqwNWMm/E+vY55tx958w93j73Xdsh1rb2MXq/l24/+qyd+84r2cpes6W/Q/scd0t74RKH2H96Pn7q72bEXuMMDgAQR+AAAHEEDgAQZ9iVjOc+F+/9Gd1WZlXm9F7HVj6P3co65ixd25afw1KjPts/lhmcQ7w+p39uS++1Q+y1Sx7fDMrxvSeWvK97m1vbyYv3zeAAAE8egQMAxBE4AEAcgQMAxBk2ZPzo+ytnDtz7QktbHjgbofeQ25aG/NZey5ae65wRg39zjmXIeM5WfsfHNmS66+cO/fjJtvzajboIpQv9AQBPJIEDAMQROABAHIEDAMTZ1JWMD6HnFTeX3ve86+h9jK0Mpe2y9tVVRz3/rQzKH+J/s15yddFDW/qFhjlbHu6cs5V19HaI38OT9NrN2cr7eukxDRkDAE8kgQMAxBE4AEAcgQMAxBk2ZAwAsBZncACAOAIHAIgjcACAOAIHAIgjcACAOAIHAIgjcACAOAIHAIgjcACAOAIHAIgjcACAOAIHAIgjcACAOAIHAIgjcACAOAIHAIgjcACAOAIHAIgjcACAOAIHAIgjcACAOAIHAIgjcACAOAIHAIgjcACAOAIHAIgjcACAOAIHAIgjcACAOAIHAIgjcACAOAIHAIgjcACAOAIHAIgjcACAOAIHAIgjcACAOAIHAIgjcACAOAIHAIgjcACAOAIHAIgjcACAOAIHAIgjcACAOAIHAIgjcACAOAIHAIgjcACAOAIHAIgjcACAOAIHAIgjcACAOAIHAIgjcACAOAIHAIgjcACAOAIHAIgjcACAOAIHAIgjcACAOAIHAIgjcACAOAIHAIgjcACAOAIHAIgjcACAOAIHAIgjcACAOAIHAIgjcACAOAIHAIgjcACAOAIHAIgjcACAOAIHAIgjcACAOAIHAIgjcACAOAIHAIgjcACAOAIHAIgjcACAOAIHAIgjcACAOAIHAIgjcACAOAIHAIgjcACAOAIHAIgjcACAOAIHAIgjcACAOAIHAIgjcACAOAIHAIgjcACAOAIHAIgjcACAOAIHAIgjcACAOAIHAIgjcACAOAIHAIgjcACAOAIHAIgjcACAOAIHAIgjcACAOAIHAIgjcACAOAIHAIgjcACAOAIHAIgjcACAOAIHAIgjcACAOAIHAIgjcACAOAIHAIgjcACAOAIHAIgjcACAOAIHAIgjcACAOAIHAIgjcACAOAIHAIgjcACAOAIHAIgjcACAOAIHAIgjcACAOAIHAIgjcACAOAIHAIgjcACAOAIHAIgjcACAOAIHAIgjcACAOAIHAIgjcACAOAIHAIgjcACAOAIHAIgjcACAOAIHAIgjcACAOAIHAIgjcACAOAIHAIgjcACAOAIHAIgjcACAOAIHAIgjcACAOAIHAIgjcACAOAIHAIgjcACAOAIHAIgjcACAOAIHAIgjcACAOAIHAIgjcACAOAIHAIgjcACAOAIHAIgjcACAOAIHAIgjcACAOAIHAIgjcFhFa+391trXrbXfW2sfjV4PkMlewy4XRi+AWA+q6oOqul5VFwevBchlr2GWwGEV0zR9XlXVWnu9ql4avBwglL2GXXxEBQDEETgAQByBAwDEETgAQBxDxqyitXah/n5/PVVVT7XWnq6qh9M0PRy7MiCJvYZdnMFhLTer6requlFVbz/+882hKwIS2WuY1aZpGr0GAICunMEBAOIIHAAgjsABAOIIHAAgzrCviV87efPMdPOtB3fO/Nz1y1dXXcfSY8793Jze993H6eMuffwRv4d9LVlz6vPa0jpuP/qsHWI9/8ej76+c2Wv2eQ23sk8tdWzP9RB6P68le+3axzzUcbdiyV7jDA4AEEfgAABxBA4AEEfgAABxhl3JuPfgX6qeA2LHOGy2z5rPO2S9zzoO8RovGTjdZ6B8n8fb4pDx3BcaettnCPi0Q/yd7P2liSVrNmS7bVt5nXp+ocEZHAAgjsABAOIIHAAgjsABAOIMGzJeOvg34oqT++g5FPt/7ntevQf1Rl0Zuufg9dJ19HZsz2vuGCcv3j/aIeOtDFkm6L0PzBl1teCla+l5zN7/hhzbgPZ59xpncACAOAIHAIgjcACAOAIHAIiz+SHjOb0H05Y8/paGDbcyGHtsg2q9HWKg+tgcy5WMt/zeHXXF332s/eWNQxhx1fgtvw936XnV7jk9r5ruDA4AEEfgAABxBA4AEEfgAABxhg0ZAwCsxRkcACCOwAEA4ggcACCOwAEA4ggcACCOwAEA4ggcACCOwAEA4ggcACCOwAEA4ggcACCOwAEA4ggcACCOwAEA4ggcACCOwAEA4ggcACCOwAEA4ggcACCOwAEA4ggcACCOwAEA4ggcACDOXxoNqLkuqPO9AAAAAElFTkSuQmCC\n",
"text/plain": [
"<Figure size 576x576 with 4 Axes>"
]
},
"metadata": {
"needs_background": "light"
},
"output_type": "display_data"
}
],
"source": [
"data.show_batch(rows=2)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Model setup"
]
},
{
"cell_type": "markdown",
"metadata": {
"heading_collapsed": true
},
"source": [
"## Setup fastai ResNet34 architecture"
]
},
{
"cell_type": "code",
"execution_count": 26,
"metadata": {
"hidden": true
},
"outputs": [],
"source": [
"test_learn = create_cnn(data, models.resnet18)"
]
},
{
"cell_type": "code",
"execution_count": 27,
"metadata": {
"hidden": true
},
"outputs": [
{
"data": {
"text/plain": [
"BCEWithLogitsLoss()"
]
},
"execution_count": 27,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"test_learn.loss_func.func"
]
},
{
"cell_type": "code",
"execution_count": 28,
"metadata": {
"hidden": true
},
"outputs": [],
"source": [
"#test_learn.model[0]"
]
},
{
"cell_type": "code",
"execution_count": 29,
"metadata": {
"hidden": true
},
"outputs": [
{
"data": {
"text/plain": [
"Sequential(\n",
" (0): AdaptiveConcatPool2d(\n",
" (ap): AdaptiveAvgPool2d(output_size=1)\n",
" (mp): AdaptiveMaxPool2d(output_size=1)\n",
" )\n",
" (1): Lambda()\n",
" (2): BatchNorm1d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
" (3): Dropout(p=0.25)\n",
" (4): Linear(in_features=1024, out_features=512, bias=True)\n",
" (5): ReLU(inplace)\n",
" (6): BatchNorm1d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
" (7): Dropout(p=0.5)\n",
" (8): Linear(in_features=512, out_features=2, bias=True)\n",
")"
]
},
"execution_count": 29,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"test_learn.model[1]"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"hidden": true
},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": 30,
"metadata": {
"hidden": true
},
"outputs": [],
"source": [
"def ExpandInput(): return Lambda(lambda x: x.expand(-1, 3, 4, 50))"
]
},
{
"cell_type": "code",
"execution_count": 31,
"metadata": {
"hidden": true
},
"outputs": [],
"source": [
"EI = ExpandInput()"
]
},
{
"cell_type": "code",
"execution_count": 32,
"metadata": {
"hidden": true
},
"outputs": [],
"source": [
"# Test ExpandInput layer"
]
},
{
"cell_type": "code",
"execution_count": 33,
"metadata": {
"hidden": true
},
"outputs": [
{
"data": {
"text/plain": [
"torch.Size([64, 1, 4, 50])"
]
},
"execution_count": 33,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"tt = torch.rand((64,1,4,50)); tt.shape"
]
},
{
"cell_type": "code",
"execution_count": 34,
"metadata": {
"hidden": true
},
"outputs": [
{
"data": {
"text/plain": [
"torch.Size([64, 3, 4, 50])"
]
},
"execution_count": 34,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"tt.expand(-1, 3, 4, 50).shape"
]
},
{
"cell_type": "code",
"execution_count": 35,
"metadata": {
"hidden": true
},
"outputs": [
{
"data": {
"image/jpeg": "/9j/4AAQSkZJRgABAQEAZABkAAD/2wBDAAIBAQEBAQIBAQECAgICAgQDAgICAgUEBAMEBgUGBgYFBgYGBwkIBgcJBwYGCAsICQoKCgoKBggLDAsKDAkKCgr/2wBDAQICAgICAgUDAwUKBwYHCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgr/wAARCAAEADIDASIAAhEBAxEB/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAAAAAAAAECAwQFBgcICQoL/8QAtREAAgECBAQDBAcFBAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEIFEKRobHBCSMzUvAVYnLRChYkNOEl8RcYGRomJygpKjU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6goOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4uPk5ebn6Onq8vP09fb3+Pn6/9oADAMBAAIRAxEAPwDzK9/bD+K3iz9rzwX8GPHmi+Etd0fVIvtGpm98I2cMt7a32lWy3thcG2jiW4hntWs7JpJVe4FrpdpEk6A3JuO81678XNomreKfFHj7U/EFtoPwv+JEejeHdbgtZNLgu9H0Sy1SLUXtkhRbq7uLy6uJrqS581Z5JWkKiTDgooA0fh5qcXhX4R/tDeJ00HSb7xJ8N/jVPZ6N4p1DSoXvbySz8SrJa3l2FVYZ7mI2+I5fLXylnlWMICoX0D9ohfBPhL9mNdT0n4U+HRYW3xTs/B0eh+TOlvHpc15b2TRRSJKtzbJ9lub+AW8E0dt5OpXcXkmOUoCigBPjQfiV8NP+CgHgn9jDRvjx4vudK+LmqeHdV8TeML65tW1+C81SDVvPmtrhbdYkaKTSbCW3LxObZraNYikSJGvjf7POn2HxR+Cul/tb/Euzj17WrHQ59Vj8P6ku7R2uYNB8HTuhtxhlsppr5XbTEddPjWytoYLaCASwylFAHs3gHxvrlt4E0W3jubgLHpFsqiLU7qJQBEo4SOVUQf7KqFHQADiiiigD/9k=\n",
"image/png": "iVBORw0KGgoAAAANSUhEUgAAADIAAAAECAYAAADMHGwBAAAABHNCSVQICAgIfAhkiAAAAsVJREFUKJEFwV8oAwEcB/Dv7U+0W1v+LPJAkUeipMyfyLU8SK3wsNWSrtTUrTzyciip5WWd2nlktFopYixKvJiNpfyZmbiIsYd1U+uk9PP5YH19ndLpNFmtVvr8/CSHw0E8z9P9/T2ZzWZKpVLU3NxM39/fdHZ2RqIo0urqKvE8TxzHUSQSoYmJCWptbaXOzk6KxWI0Pz9PLMtSNpuliooK2tjYoKurK3I6nRSNRqm2tpbe39+pvr6eVFWlh4cH6u7upng8Tjs7O/T09ERer5cKhQINDw9TLpej6elpkmWZLBYLLSwsEM/z5HA4yO120/7+PhlqamrAMAwKhQKKxSIMBgP6+vqQzWYhyzJmZ2dxeXmJk5MTRCIRqKoKnU6HyclJGI1G5PN5vL6+QhRF/P39IRwOQxAEMAwDVVURDoexubmJm5sbLC0toVgsYnt7G0NDQ+jv70c6nYbL5cLMzAx+f3/x/PwMTdMwMjICj8eD3d1dmM1mWK1WeDwe9Pb2IpVKIZPJ4O7uDn6/H9fX19C3t7eLo6OjsNls8Hq9ODo6gtvtxs/PDwRBQDAYBBFhZWUFFxcX+Pr6QiAQgNFoxOLiIs7Pz8EwDFiWRVdXF1iWRU9PD9bW1pDL5dDU1ASXywW9Xg+TyYTl5WUMDg6iqqoKPM+D53nY7Xbs7e2hVCqB4zgEg0FomobDw0PMzc1BVVUkEgmIoojx8XHk83nIsozb21scHBzAbrfDkEwmIUkSQqEQqqur0djYiLq6OpSXlyMUCmFsbAyKoqCtrQ1bW1uYmppCNBqFz+dDJpPB6ekpBgYGYDAYoNPpwHEckskkTCYTFEWBpmkIBAKorKxEQ0MDfD4fYrEYJElCqVSC3+9HS0sLXl5e4HQ6UVZWhkQigY6ODjw+PsJms0GSJLy9vUFRFAiCAEEQcHx8jHg8DovFgo+PD/wDZ+hmByIU2BsAAAAASUVORK5CYII=\n",
"text/plain": [
"Image (3, 4, 50)"
]
},
"execution_count": 35,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"Image(EI(tt)[0])"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"hidden": true
},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": 36,
"metadata": {
"hidden": true
},
"outputs": [],
"source": [
"new_model = nn.Sequential(ExpandInput(), test_learn.model) # insert ExpandInput layer at the beginning of the network"
]
},
{
"cell_type": "code",
"execution_count": 37,
"metadata": {
"hidden": true
},
"outputs": [
{
"data": {
"text/plain": [
"torch.Size([64, 3, 4, 50])"
]
},
"execution_count": 37,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"new_model[0](tt).shape"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"hidden": true
},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": 97,
"metadata": {
"hidden": true
},
"outputs": [],
"source": [
"new_learn = Learner(data, new_model)#, metrics=accuracy)"
]
},
{
"cell_type": "code",
"execution_count": 98,
"metadata": {
"hidden": true
},
"outputs": [],
"source": [
"#[p.shape for p in new_model.parameters()]"
]
},
{
"cell_type": "code",
"execution_count": 99,
"metadata": {
"hidden": true
},
"outputs": [
{
"data": {
"text/html": [
"\n",
" <div>\n",
" <style>\n",
" /* Turns off some styling */\n",
" progress {\n",
" /* gets rid of default border in Firefox and Opera. */\n",
" border: none;\n",
" /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
" background-size: auto;\n",
" }\n",
" .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
" background: #F44336;\n",
" }\n",
" </style>\n",
" <progress value='0' class='' max='5', style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
" 0.00% [0/5 00:00<00:00]\n",
" </div>\n",
" \n",
"<table style='width:300px; margin-bottom:10px'>\n",
" <tr>\n",
" <th>epoch</th>\n",
" <th>train_loss</th>\n",
" <th>valid_loss</th>\n",
" <th>accuracy</th>\n",
" </tr>\n",
"</table>\n",
"\n",
"\n",
" <div>\n",
" <style>\n",
" /* Turns off some styling */\n",
" progress {\n",
" /* gets rid of default border in Firefox and Opera. */\n",
" border: none;\n",
" /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
" background-size: auto;\n",
" }\n",
" .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
" background: #F44336;\n",
" }\n",
" </style>\n",
" <progress value='0' class='progress-bar-interrupted' max='8', style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
" Interrupted\n",
" </div>\n",
" "
],
"text/plain": [
"<IPython.core.display.HTML object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"ename": "RuntimeError",
"evalue": "Expected object of scalar type Long but got scalar type Float for argument #2 'other'",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mRuntimeError\u001b[0m Traceback (most recent call last)",
"\u001b[0;32m<ipython-input-99-c937c2aa1812>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mnew_learn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit_one_cycle\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
"\u001b[0;32m~/Downloads/fastai/fastai/train.py\u001b[0m in \u001b[0;36mfit_one_cycle\u001b[0;34m(learn, cyc_len, max_lr, moms, div_factor, pct_start, wd, callbacks, **kwargs)\u001b[0m\n\u001b[1;32m 19\u001b[0m callbacks.append(OneCycleScheduler(learn, max_lr, moms=moms, div_factor=div_factor,\n\u001b[1;32m 20\u001b[0m pct_start=pct_start, **kwargs))\n\u001b[0;32m---> 21\u001b[0;31m \u001b[0mlearn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcyc_len\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmax_lr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mwd\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mwd\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcallbacks\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 22\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 23\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mlr_find\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlearn\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mLearner\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstart_lr\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mFloats\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1e-7\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mend_lr\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mFloats\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m10\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnum_it\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mint\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m100\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstop_div\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mbool\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mAny\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/Downloads/fastai/fastai/basic_train.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, epochs, lr, wd, callbacks)\u001b[0m\n\u001b[1;32m 164\u001b[0m \u001b[0mcallbacks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mcb\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mcb\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcallback_fns\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mlistify\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 165\u001b[0m fit(epochs, self.model, self.loss_func, opt=self.opt, data=self.data, metrics=self.metrics,\n\u001b[0;32m--> 166\u001b[0;31m callbacks=self.callbacks+callbacks)\n\u001b[0m\u001b[1;32m 167\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 168\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mcreate_opt\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlr\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mFloats\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mwd\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mFloats\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0.\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m->\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/Downloads/fastai/fastai/basic_train.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(epochs, model, loss_func, opt, data, callbacks, metrics)\u001b[0m\n\u001b[1;32m 92\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 93\u001b[0m \u001b[0mexception\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 94\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 95\u001b[0m \u001b[0;32mfinally\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mcb_handler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_train_end\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mexception\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 96\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/Downloads/fastai/fastai/basic_train.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(epochs, model, loss_func, opt, data, callbacks, metrics)\u001b[0m\n\u001b[1;32m 87\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mhasattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m'valid_dl'\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalid_dl\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalid_ds\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 88\u001b[0m val_loss = validate(model, data.valid_dl, loss_func=loss_func,\n\u001b[0;32m---> 89\u001b[0;31m cb_handler=cb_handler, pbar=pbar)\n\u001b[0m\u001b[1;32m 90\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mval_loss\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 91\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mcb_handler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_epoch_end\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mval_loss\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;32mbreak\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/Downloads/fastai/fastai/basic_train.py\u001b[0m in \u001b[0;36mvalidate\u001b[0;34m(model, dl, loss_func, cb_handler, pbar, average, n_batch)\u001b[0m\n\u001b[1;32m 52\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mis_listy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0myb\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0myb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0myb\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 53\u001b[0m \u001b[0mnums\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0myb\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 54\u001b[0;31m \u001b[0;32mif\u001b[0m \u001b[0mcb_handler\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mcb_handler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_batch_end\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mval_losses\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;32mbreak\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 55\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mn_batch\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnums\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m>=\u001b[0m\u001b[0mn_batch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;32mbreak\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 56\u001b[0m \u001b[0mnums\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0marray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnums\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfloat32\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/Downloads/fastai/fastai/callback.py\u001b[0m in \u001b[0;36mon_batch_end\u001b[0;34m(self, loss)\u001b[0m\n\u001b[1;32m 237\u001b[0m \u001b[0;34m\"Handle end of processing one batch with `loss`.\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 238\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstate_dict\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'last_loss'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mloss\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 239\u001b[0;31m \u001b[0mstop\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0many\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'batch_end'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstate_dict\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'train'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 240\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstate_dict\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'train'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 241\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstate_dict\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'iteration'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/Downloads/fastai/fastai/callback.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, cb_name, call_mets, **kwargs)\u001b[0m\n\u001b[1;32m 185\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m__call__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcb_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcall_mets\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m->\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 186\u001b[0m \u001b[0;34m\"Call through to all of the `CallbakHandler` functions.\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 187\u001b[0;31m \u001b[0;32mif\u001b[0m \u001b[0mcall_mets\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mgetattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmet\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34mf'on_{cb_name}'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstate_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mmet\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmetrics\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 188\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mgetattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcb\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34mf'on_{cb_name}'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstate_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mcb\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 189\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/Downloads/fastai/fastai/callback.py\u001b[0m in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m 185\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m__call__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcb_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcall_mets\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m->\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 186\u001b[0m \u001b[0;34m\"Call through to all of the `CallbakHandler` functions.\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 187\u001b[0;31m \u001b[0;32mif\u001b[0m \u001b[0mcall_mets\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mgetattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmet\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34mf'on_{cb_name}'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstate_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mmet\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmetrics\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 188\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mgetattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcb\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34mf'on_{cb_name}'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstate_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mcb\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 189\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/Downloads/fastai/fastai/callback.py\u001b[0m in \u001b[0;36mon_batch_end\u001b[0;34m(self, last_output, last_target, **kwargs)\u001b[0m\n\u001b[1;32m 272\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mis_listy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlast_target\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mlast_target\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mlast_target\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 273\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcount\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mlast_target\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 274\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mval\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mlast_target\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlast_output\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0mlast_target\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdetach\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcpu\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 275\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 276\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mon_epoch_end\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/Downloads/fastai/fastai/metrics.py\u001b[0m in \u001b[0;36maccuracy\u001b[0;34m(input, targs)\u001b[0m\n\u001b[1;32m 37\u001b[0m \u001b[0minput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0margmax\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdim\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mview\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mn\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 38\u001b[0m \u001b[0mtargs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtargs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mview\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mn\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 39\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m==\u001b[0m\u001b[0mtargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfloat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmean\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 40\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 41\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0merror_rate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mTensor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtargs\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mTensor\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m->\u001b[0m\u001b[0mRank0Tensor\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;31mRuntimeError\u001b[0m: Expected object of scalar type Long but got scalar type Float for argument #2 'other'"
]
}
],
"source": [
"new_learn.fit_one_cycle(5)"
]
},
{
"cell_type": "code",
"execution_count": 41,
"metadata": {
"hidden": true
},
"outputs": [
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAZYAAAEKCAYAAAAxXHOuAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzs3Xd4VGX2wPHvSW+kJxAIEDqEFkLoSlFUQFFUFKKgWEDEsuKu3V133fW3urqirliwgBVElKJiFwtKSyD0FpqEUEJogUDq+/vjTnAICSnMZDLJ+TzPPJm55Z1zmTAn961ijEEppZRyFA9XB6CUUqpu0cSilFLKoTSxKKWUcihNLEoppRxKE4tSSimH0sSilFLKoTSxKKWUcihNLEoppRxKE4tSSimH8nJ1ADUhMjLSxMXFuToMpZRyK6mpqQeNMVFVPa9eJJa4uDhSUlJcHYZSSrkVEdlVnfO0KkwppZRDaWJRSinlUJpYlFJKOVS9aGNRStUNBQUFZGRkcOrUKVeHUqf4+fkRGxuLt7e3Q8rTxKKUchsZGRk0aNCAuLg4RMTV4dQJxhiys7PJyMigRYsWDilTq8KUUm7j1KlTREREaFJxIBEhIiLCoXeBmliUUm5Fk4rjOfrfVBOLAxhjmJ2ym6MnC1wdilJKuZwmFgf4/VAuD85Zw+s/bXN1KEopJ8rOziYhIYGEhAQaNWpEkyZNTr/Oz8+vVBm33HILmzdvdnKkruXUxCIiQ0Rks4iki8jDZexvJiKLRGSViKwRkWG27XEiclJE0myP1+zO6S4ia21lviS14L44KycPgPlpmRQXGxdHo5RyloiICNLS0khLS2PixIlMnjz59GsfHx/AqsEoLi4ut4zp06fTrl27mgrZJZyWWETEE5gKDAXigWQRiS912OPAbGNMN2A08Irdvm3GmATbY6Ld9leBCUAb22OIs66hsrJPWH+p7DlykmU7Drk4GqVUTUtPT6dTp05MnDiRxMRE9u7dy4QJE0hKSqJjx448+eSTp4+94IILSEtLo7CwkNDQUB5++GG6du1Knz59OHDggAuvwnGc2d24J5BujNkOICKzgKuADXbHGCDY9jwEyDxXgSISAwQbY5bYXr8LjAC+dGzoVXPIllg8PYS5qzLo0yrCleEoVS/847P1bMg85tAy4xsH88TwjtU6d8OGDUyfPp3XXrMqWJ5++mnCw8MpLCxk0KBBjBw5kvj4M/+2Pnr0KAMGDODpp5/m/vvv5+233+bhh8+q3HE7zqwKawLstnudYdtm7+/AGBHJABYC99jta2GrIvtJRC60KzOjgjJrXEliGdKpEV+u3cepgiIXR6SUqmmtWrWiR48ep1/PnDmTxMREEhMT2bhxIxs2bDjrHH9/f4YOHQpA9+7d2blzZ02F61TOvGMpq+2jdANEMjDDGPNfEekDvCcinYC9QDNjTLaIdAfmiUjHSpZpvbnIBKwqM5o1a1bda6iU7OP5BPh4ckPPZnyxZi/fbtjP8K6NnfqeStV31b2zcJbAwMDTz7du3cqLL77I8uXLCQ0NZcyYMWWOEylplwHw9PSksLCwRmJ1NmfesWQATe1ex3J2VddtwGwAW/WWHxBpjMkzxmTbtqcC24C2tjJjKygT23nTjDFJxpikqKgqLydQJYdO5BER5EPvlhE0CvZj7qo9Tn0/pVTtduzYMRo0aEBwcDB79+7l66+/dnVINcqZiWUF0EZEWoiID1bj/IJSx/wOXAwgIh2wEkuWiETZGv8RkZZYjfTbjTF7gRwR6W3rDXYTMN+J11Ap2SfyCQ/0xdNDGNGtCT9tyWLjXsfW/Zb4aUsW8zRxKVWrJSYmEh8fT6dOnRg/fjz9+vVzdUg1SoxxXvdYW/fhFwBP4G1jzFMi8iSQYoxZYOsl9gYQhFWl9aAx5hsRuRZ4EigEioAnjDGf2cpMAmYA/liN9veYCi4iKSnJOHOhr8tf+oWGwX68Pa4Hh07kM/j5n2ga5s+nk/rh6eG43tArdh7ihjeWIiKkPD6YYD/HTBinlLvYuHEjHTp0cHUYdVJZ/7YikmqMSapqWU6dhNIYsxCrUd5+29/snm8AzkrlxphPgE/KKTMF6OTYSM/PoRP5dIixOreFB/rwxPB4/jQrjem/7uD2C1s65D12H8rljvdSCfbzJvtEPt+u38+13WMrPlEppWqYjrw/T8YYsk/kExH4RyPclV0bc3H7aJ77ZjO/Z+dWqbxXf9zGqNeXUGQ30PJEXiHj302hsKiYj+7oQ5NQfz5bc86e2Uop5TKaWM7Tifwi8guLCbdLLCLCv67uhJeHB//4bH2ly8rNL+TVH9NZtuMQC9fuPb391R+3sWlfDi/fkEjr6CCGd23M4q0HT3dzVkqp2kQTy3k6dNz6crdPLAAxIf5MGtSK7zcdIHVX5Ubjz1uVybFThYQGePPyD+kUFxsOHDvFW4t3MLxrY/q3tXq3De8aQ2Gx4ct1eysoUSmlap4mlvOUfcKaJywiyOesfeP6xhEZ5Mt/vtpMWf0LThUUnd5ujOGd33YSHxPM366IZ/P+HL7duJ8Xv99KQVExf76k7enz4mOCaRUVyGertTpMKVX7aGI5TyXVUeGBvmftC/Dx4p6LWrNsxyEWpx88a//Vr/zGiFd+I/t4Hku3H2Lz/hzG9Y3jyq6NaR4RwNNfbmLWit3c0KsZcZF/DL4SEYZ3bcyyHYfYd1SXaFVK1S6aWM5TyQSUEYFn37EAjO7ZlCah/jz79Zl3LQdyTrFx7zFW7z7Cta/+xgvfbSEswJsrExrj5enBpIGt2HHwBL5eHtxzUZuzyh3etTHGwL++2MDxvMqN1i0oKuZPs1Yx+aM0nXZGqWoYOHDgWYMdX3jhBSZNmlTuOUFBQQBkZmYycuTIcsutaEjECy+8QG7uH52Bhg0bxpEjRyobeo3SxHKe/rhjKTux+Hp5ct/gNqzJOMovW/+4a1mz+ygAjw5rz5GTBSzbcYjRPZvh5+0JwNXdYunaNJTJg9sS1eDsu6FWUUHcN7gNX6zdy7AXf6mwHccYw6OfrmV+Wibz0vZw45vLOGyL/XheIQdy9M5HqYokJycza9asM7bNmjWL5OTkCs9t3Lgxc+bMqfZ7l04sCxcuJDQ0tNrlOZMmlvN06EQ+vl4eBPh4lnvM8K6N8fHy4JetWae3rc44gqeHMLZ3HJ/c2Zfknk257YIWp/f7eHkw/65+jO9f/jiY+wa3ZfYdfSg2htHTlrLnyMlyj53y3VY+Ts3g3ota88oNiazdc5QRr/zKVVN/pes/vmHQsz+Sc0pXwFTqXEaOHMnnn39OXp7Vtrpz504yMzNJSEjg4osvJjExkc6dOzN//tkTguzcuZNOnawheCdPnmT06NF06dKFUaNGcfLkH/9377zzztPT7T/xxBMAvPTSS2RmZjJo0CAGDRoEQFxcHAcPWn+sPv/883Tq1IlOnTrxwgsvnH6/Dh06MH78eDp27Mill156xvs4k1MHSNYH2cetMSznWm/Mz9uTbk1DWbr9j7uKtN1HaNuwAf4+nrSKCuLf13Sp1vv3iAtn+rgeXDLlZ37ZksXonmdPuPnZ6kxe+n4r13WPZfIlbRERIoJ8efjTNfh4CiMSmvDJygyWbMvm0o6NqhWHUjXuy4dh31rHltmoMwx9utzdERER9OzZk6+++oqrrrqKWbNmMWrUKPz9/Zk7dy7BwcEcPHiQ3r17c+WVV5b7vfDqq68SEBDAmjVrWLNmDYmJiaf3PfXUU4SHh1NUVMTFF1/MmjVruPfee3n++edZtGgRkZGRZ5SVmprK9OnTWbZsGcYYevXqxYABAwgLC2Pr1q3MnDmTN954g+uvv55PPvmEMWPGOObf6hz0juU8HTqRR3gZPcJK690ygvWZRzl6sgBjDGsyjpLQNMQhMbSODiKqgS+/bcs+a9/R3AL+8dl6ujYN5f+u6Xz6F71ni3B++PNAPp7Yl39f05lAH09+trujUkqVzb46rKQazBjDo48+SpcuXRg8eDB79uxh//795Zbx888/n/6C79KlC126/PGH5ezZs0lMTKRbt26sX7++zOn27S1evJirr76awMBAgoKCuOaaa/jll18AaNGiBQkJCUDNTsuvdyzn6ZBtAsqK9G4ZwYvfb2XFjkO0jg7i6MkCusQ6pn5UROjbKoJf07MxxpzxV9LTX23icG4B79zaCW/Psv+O8PHyoE+rCH7ecnbPNaVqrXPcWTjTiBEjuP/++1m5ciUnT54kMTGRGTNmkJWVRWpqKt7e3sTFxZU5Tb69su5mduzYwXPPPceKFSsICwtj3LhxFZZzrqkSfX3/+G7y9PSssaowvWM5T6WncylPt2ah+Hh5sHR7NqszrJ4cXR2UWAD6torg4PE80g8cP70tddchZi7/nVv6xtGx8bnvjvq3jeL3Q7nsPHjCYTEpVRcFBQUxcOBAbr311tON9kePHiU6Ohpvb28WLVrErl27zllG//79+eCDDwBYt24da9asAazp9gMDAwkJCWH//v18+eUfi+M2aNCAnJycMsuaN28eubm5nDhxgrlz53LhhReedVxN0juW82TdsVScWPy8PUlsFsrSHdkUG/Dz9qBtwyCHxdG3lVXv+tu2bNo0bEBhUTGPzV1H4xA/JtsNrixP/zbWqP6ft2adMWZGKXW25ORkrrnmmtNVYjfeeCPDhw8nKSmJhIQE2rdvf87z77zzTm655Ra6dOlCQkICPXv2BKBr165069aNjh070rJlyzOm258wYQJDhw4lJiaGRYsWnd6emJjIuHHjTpdx++23061bN5euRunUafNrC2dNm3+qoIj2f/2KB4e0Y9LA1hUe/8J3W3jx+620jgoiNMCbjyf2dWg8FzzzAx0bB/P62CQ+WLaLx+au47UxiQzpFFOp8/v/ZxFtGwbx5s09Kj5YKRfQafOdx5HT5mtV2HmoaHBkab1bRmAMbD1w3GHtK/b6topg6fZDHDtVwJRvt9AzLpzLqtDLq3/bSJZsyya/sNjhsSml6g9NLOch+7jVl70yjfcACU2tdhaArk2dkVgiOXqygPs/SuPg8XwevbzDObtBl3ZhmyhO5BeRuuuww2NTStUfmljOQ3YFo+5LK2lnAUhwwh1Ln1YRAHy38QBXdIkhoYrJq2+rCLw8hJ+2aLdjVXvVh+r7mubof1OnJhYRGSIim0UkXUQeLmN/MxFZJCKrRGSNbSljROQSEUkVkbW2nxfZnfOjrcw02yPamddwLiVT5le2Kgzg6m5NSGwWStNwf4fH0zDYj1ZRgfh4evDQkHM3HpalgZ83/VpHMic1Q+cSU7WSn58f2dnZmlwcyBhDdnY2fn5+DivTab3CRMQTmApcAmQAK0RkgW054hKPA7ONMa+KSDzWMsZxwEFguDEmU0Q6AV8DTezOu9G2RLFLnZ4nrBIDJEuM6tGMUT3OHh3vKI8O68DxvEKahgdU6/yJA1qR/MZSPk7NYGzv5mftLz1ORqmaFBsbS0ZGBllZelftSH5+fsTGOm6pc2d2N+4JpBtjtgOIyCzgKsA+sRgg2PY8BMgEMMassjtmPeAnIr7GmDwnxltl2Sfy8fYUGvjWnl7bF3doeF7n924ZTmKzUF77cRujezQ9Y1DlqYIirnttCV2bhvDPqzppglE1ztvbmxYtWlR8oHIpZ1aFNQF2273O4My7DoC/A2NEJAPrbuWeMsq5FlhVKqlMt1WD/VVc+O126EQe4RXME+ZuRIS7L2rNniMnWZB25kJi037ezto9R3l/6e98nJrhogiVUrWdMxNLWd+2pStGk4EZxphYYBjwnoicjklEOgLPAHfYnXOjMaYzcKHtMbbMNxeZICIpIpLirNvmyk7n4m4GtYumfaMGvPKjtTwywO5DuUxdlM6Qjo3o0zKCv81fx+Z9Z48CVkopZyaWDKCp3etYbFVddm4DZgMYY5YAfkAkgIjEAnOBm4wx20pOMMbssf3MAT7EqnI7izFmmjEmyRiTFBUV5ZALKi0rJ4/IKrSvuAsR4a5BrdmWdYIJ76Ww4+AJ/vn5BjxE+NvweF5MTiDI15tJH6SSm1+5RcaUUvWHMxPLCqCNiLQQER9gNLCg1DG/AxcDiEgHrMSSJSKhwBfAI8aYX0sOFhEvESlJPN7AFcA6J15DuYwxbMs6Qcs6Ov3JFV1ieHhoe5Zsy+aS53/imw37uefi1jQO9Se6gR9TRnVlW9YJZi3fXXFhSql6xWmJxRhTCNyN1aNrI1bvr/Ui8qSIXGk77M/AeBFZDcwExhmrH+HdQGvgr6W6FfsCX4vIGiAN2AO84axrOJf9x/I4nldI62jHzfdVm4gIEwe0YtEDA7kuKZb+baPOWIjswjZR9IgL4+1fd1BUrF0/lVJ/cGp3JmPMQqxGefttf7N7vgHoV8Z5/wL+VU6x3R0ZY3WVzCLcqo4mlhLRDfzKXYTstgtaMvH9VL5Zv4+hnSs3H5lSqu7TkffVlH7AarhuHVW3E8u5XBLfkGbhAby5eIerQ1FK1SKaWKppW9YJGvh5EdWg7vUKqyxPD+HWfnGk7jrMyt91fjGllEUTSzWlHzhO6+igOjWGpTquS2pKsJ8X//t+K3mFZU8D8/cF60metpSjuQU1HJ1SyhU0sVRTetbxel0NViLQ14s7B7Zm0eYshr34C0u2ZZ+x//M1mcz4bSdLtmdz41tLOZKb76JIlVI1RRNLNRw9WUBWTl6d7RFWVXcObMX0W3qQX1RM8htL+cvHqzmSm0/mkZM8+ulaEpqG8uZNSWzZf5wb3ljG4ROaXJSqyzSxVMPpHmF6x3LaoHbRfHPfACYNbMXcVXsY/PzPjH83haJiw4ujExgc35A3bkoiPes4932UdnpEv1Kq7tHEUg3bsqzEoncsZ/L38eTBIe1ZcHc/GoX4sj7zGE9c2ZHmEdYg0gFto/jrFfH8tCWL6b/tdG2wSimnqT3T8rqRbQeO4+PlUe2p6eu6jo1DmDepH+lZx2nfKPiMfWN6NeOnzVk88+Um+rSMIL5xcDmlnG3h2r08sWA9vl4exEUEEhPih5+3Jz5eHvSIC+Oyjo3O6kyxbs9RZqfs5qL20Qxs57Kle5SqV6Q+LJiTlJRkUlIct3zLbTNWsOfISb66r7/DyqxPDp3IZ8gLPxPs783cSX1p4Od9zuMLiop55stNvLl4B11iQ2gZGciO7Fz2HT1JfmExJwuKOFVQTOcmIdxzUWsAth44zg+bDpxeZjkmxI9FfxmIn7en069PqbpCRFKNMUlVPU/vWKohPes4nZqEuDoMtxUe6MMLoxK46e3l3PZOCu/c0hN/n7O/8E/kFTI/LZN3l+xk074cxvWN49FhHfDxOrMGt6jYMHfVHqZ8u4UJ76We3t4yKpDHL+9AbJg/E99fyazlvzOun67loZSzaWKpolMFRew+lMtVCaWXllFV0bd1JFNGJXDvrFVMfD+VV25MZNO+Y6TuOsy2AyfYdegE6/Yc43heoTWF/42JDCtn2hhPD2Fk91iGd41h8daDRAT50ioq8PSdkDGGXi3CmfrjNkb3bKZ3LUo5mSaWKtpx8ATFRhvuHWF418bk5hfy0Cdr6fz3rynpKBYZ5EPziECGd43h2sRYujcPq9RAVF8vzzJX0BQR7r+kLaOmLeX9pbu4/cKWjr4UpZQdTSxVdLpHmHY1dohRPZoR6OvF6t1H6N48nB5xYUQEOX6anF4tI7igdSSv/riNkd1jCQ2oe+voKFVbaHfjKtq8LwdPD6FlVN1ch8UVrujSmMcuj2dIp0ZOSSolHrisHcdOFTD85cWs23PUae+jVH2niaWKNmQeo1VUoNbTu6GuTUP56I4+FBYZrnn1Nz5O0UXKlHIGTSxVtHHvMeJjKj/2QtUuic3C+OLeC+kRF8aDn6zht/SDrg5JqTpHE8s57D6Ue8Z08IdP5JN59FSVBvWp2ic80IdpY5NoERnInz5K4+DxPFeHpFSd4tTEIiJDRGSziKSLyMNl7G8mIotEZJWIrBGRYXb7HrGdt1lELqtsmY704Jw1PPrp2tOvN+49BkAHvWNxe4G+Xky9IZGjJwuYrHOXKeVQTkssIuIJTAWGAvFAsojElzrscWC2MaYbMBp4xXZuvO11R2AI8IqIeFayTIe5sG0km/blcODYKQA2aGKpUzrEBPPE8Hh+2XqQab9sd3U4StUZzrxj6QmkG2O2G2PygVnAVaWOMUDJt3QIkGl7fhUwyxiTZ4zZAaTbyqtMmQ7Tv00UAItt9fAb9h6jYbAvkU7suaRq1g09mzG0UyP++81m1mdqTzGlHMGZiaUJYN/tJsO2zd7fgTEikgEsBO6p4NzKlOkw8THBhAf6sHirLbFkasN9XSMi/N/VnQkL8GHyR2mcKih7FUylVOU5M7GUNVS6dEV2MjDDGBMLDAPeExGPc5xbmTKtNxeZICIpIpKSlZVVhbD/4OEh9GsdyS/pBzlVUET6gePacF8HhQX68J+RXdiy/zjPfr3Z1eEo5facmVgygKZ2r2P5o6qrxG3AbABjzBLAD4g8x7mVKRNbedOMMUnGmKSoqKhqX8SFbSLJysnjizV7KSw22r5SRw1sF81NfZrz1uIdfL6mzF8ppVQlOTOxrADaiEgLEfHBaoxfUOqY34GLAUSkA1ZiybIdN1pEfEWkBdAGWF7JMh3qwjaRAEz72Wrc1aqwuuvRYR3oERfG5I/STld/KqWqzmmJxRhTCNwNfA1sxOr9tV5EnhSRK22H/RkYLyKrgZnAOGNZj3UnswH4CrjLGFNUXpnOugaAmBB/WkcHsXl/DgE+nqdXQ1R1j5+3J2/e3INWUUFMeC+F1buPuDokpdySLvRVCX9fsJ4Zv+0ksVkon07q58DIVG104Ngprn3tN/IKivlmcn+dsFLVW9Vd6EtH3ldC/7ZWdZi2r9QP0cF+vHpjdw6dyOeJBU69IVaqTtLEUgm9W0bQOjqIQbpmer3RqUkI91zUhvlpmXy1bq+rw1HKrWhiqYQAHy++u38Ag+PPXkRK1V2TBrWic5MQHpu7jmydT0ypStPEolQ5vD09+O/1Xck5VcjTX25ydThKuQ1NLEqdQ9uGDRjXL445KzNOT0KqlDo3TSxKVeCuga0J8ffm/xZupD70olTqfGliUaoCIQHe3HtRG37ZepCftlRveiCl6hNNLEpVwpjezWkeEcD/LdxIYVGxq8NRqlbTxKJUJfh4efDosA5s2X+c577Z4upwlKrVNLEoVUmXdWzEjb2a8dpP2/h+435Xh6NUraWJRakq+OsV8XRsHMz9s1eTcTjX1eEoVStpYlGqCvy8PXnlxkSKiw2jpy1l4dq92lNMqVI0sShVRc0jApl+Sw8CfbyY9MFKrnttCb9tO6gJRikbTSxKVUNSXDhf3HsB/76mM7sO5XLDG8sYMfVXvt1QfttLVk4eW/fn1GCUSrmGTpuv1Hk6VVDEnNQM3vhlO7uyc3lwSDsmDWwNQH5hMR+n7uaz1Zks33EIEWHOxD50axbm4qiVqlh1p83XxKKUgxQWFfPnj1czPy2TyYPbckGbCB75dC1b9h+ndXQQwzo14pOVe/D2FL6490ICfb1cHXL9UFwEW7+B9XOhuBA8fcHTGzx9wKvkeVnbfEpt97Fts3uUd6ynD3i4f4VQdROL/mYr5SBenh48f30CXh4eTPluC1O+g8Yhfrx1cxIXd7Bmxu7bOpLkN5byry828u9rOrs44jou9xCseh9WvAlHdkFAJPiHQlE+FOZbP4sKoCjPeu5oHl7nSEL228tKWGVsO2N7STK0PffyLf/YsDhrfw1yamIRkSHAi4An8KYx5ulS+6cAg2wvA4BoY0yoiAwCptgd2h4YbYyZJyIzgAHAUdu+ccaYNCdehlKV5ukhPDuyC9HBvhQbw70XtTnjzqR3ywgm9G/J6z9tp23DIEZ2j6WBn7cLI66D9q2D5a/Dmo+h8CQ0vwAu/Se0uxw8y/nKM8aWZPJLPQqgMO/s7YWljytvW17Z5dofW3ASTh2toMzzWLbhruUQ1a7651eD06rCRMQT2AJcAmQAK4BkY8yGco6/B+hmjLm11PZwIB2INcbk2hLL58aYOZWNRavCVG2SV1hE8rSlrPz9CD6eHlzQJpKusaG0bRhEYvMwGgb7uTpE91NUAJs+h2XT4PffwMsfulwPPSdAo06uju78GWNV41UnsbW9DHwbVOtta2NVWE8g3RizHUBEZgFXAWUmFiAZeKKM7SOBL40xOhpN1Qm+Xp7MmdiXVbsPs3DtPr7fuJ9Fmw9gDAT4ePLebb3o3lwb9yvleBakzoCUtyEnE0Kbw6X/gm5jwL8O/RuK2Kq/vIFAV0dTIWcmlibAbrvXGUCvsg4UkeZAC+CHMnaPBp4vte0pEfkb8D3wsDFGl/dTbsXDQ+jePJzuzcP56xXx5OYXsnlfDpM/SuOW6cuZNaEP8Y2DOVVQxO5DubSODkJEXB127ZGRCsunwfpPrb/MW10EV0yBNpeAh6ero6v3nJlYyvpfUF6922hgjjGm6IwCRGKAzsDXdpsfAfYBPsA04CHgybPeXGQCMAGgWbNmVY1dqRoV4ONFt2ZhvH97L657bQk3vb2MPq0i+WHjfk7kFzGscyP+M7IrQfW5J1lhntWza/k02JMKPg2g+y3QczxEtnF1dMqOM39LM4Cmdq9jgcxyjh0N3FXG9uuBucaYgpINxpi9tqd5IjId+EtZBRpjpmElHpKSkup+n2pVJ8SGBfDB7b24/vWlLN6axfCujYkI8uHVH7exZf9xXh/bnVZRQa4Os2Ydy7SqulJnwIksiGwLw56DrqOr3XagnMuZiWUF0EZEWgB7sJLHDaUPEpF2QBiwpIwykrHuUOyPjzHG7BWrXmAEsM7RgSvlSi2jglj80CC8PAQvT2ssRL9Wkdw9cxXXvPIbH47vRcfGIS6O0smMgd+XwLLXYeNnYIqh3VDr7qTlIKvNQdVaTkssxphCEbkbqxrLE3jbGLNeRJ4EUowxC2yHJgOzTKnuaSISh3XH81Opoj8QkSisqrY0YKKzrkEpV/HzPrOdoG/rSOZN6sfoaUsY+9ZyZo7vTbtGdfCv9fxcWPsxLH8D9q8Fv1DoMwl63G6Nx1BuQUfeK+VGdh48wahpSyi77VykAAAgAElEQVQqNvz50nY0DQugeUQATcMDXB3a+Tm80xrIuPI9OHUEGnayugp3vg583Pza3JhO6XIOmlhUXbIt6zhj31xG5tFTp7e1iQ5iaOcYBrSNonVUECEBbjDo0hjYvsi6O9n8JYgHdBgOve6AZn20uqsW0MRyDppYVF1TWFTMvmOnyDh8kk17j/Hlun0s33mIkv/OkUG+jOvbnDsHtsbTo5Z9QeflwOpZVu+ug1sgMAq6j4OkWyG4saujU3acmlhEpBWQYYzJE5GBQBfgXWPMkSpH6gKaWFR9kJWTx+rdR9h+8DjLth/i+00HSGoexpRRCbWjquzgVuvuJO1DyM+BJt2h5x3QcUSNz2WlKsfZiSUNSALisBrjFwDtjDHDqvqGrqCJRdVH81bt4a/zrE6Ts+7o7ZqeZMVFsPVba+6ubT9YkyJ2vMZqP4ntXvPxqCpx9pQuxbZeXlcDLxhj/iciq6r6ZkqpmjOiWxO6Nw/j+teXcNuMFObd1Y9GITU0D9nJw3/MLHx4JzSIgUGPQ/ebISi6ZmJQLlPZBQMKRCQZuBn43LbNDVoHlarfmoYH8NbNPcg5VcBt76zgRF6hc99w/3pYcC/8twN88zg0aAzXzYD71sKABzSp1BOVTSy3AH2Ap4wxO2yDHt93XlhKKUeJbxzMyzcksnHvMR76ZI3j36CoENbPg+mXw6t9Yc1s6HIdTFwMt34JHa+2TZ6o6otKVYXZprq/F0BEwoAGpddWUUrVXoPaR3Pf4LY8/+0Wbuh1kL6tIs+/0ONZsPIda7qVY3sgtBlc8k9rZuGA8PMvX7mtSiUWEfkRuNJ2fBqQJSI/GWPud2JsStWMjFTIOwbBTSA4ps7OPzWhf0tmp+zmyc828Pk9F5yeLqbK9qRavbvWfWLNLNxykDV3V9vLdGZhBVS+8T7EGHNMRG4HphtjnhARJ9xTK1WDcg/BVw/Dmo/O3O7TwBpPERxjtRGc9byxtcytm61p7uftyaPDOjDpg5XMWrGbMb2bV/7kwjzYMN+au2tPCvgEQeLNVu+uqLbOC1q5pcomFi/bFPbXA485MR6lasaGBfDFn+HkIej/ILQcAMf2WlU6OXutGXWPZcLBnyBnH5y5ogN4eEODRlaSaRDzR8Ip/byWjc8Y2qkRPVuE8/y3WxjetTEh/hW0fRzbazez8AGIaA1D/wNdk8EvuEZiVu6nsonlSazxK78aY1aISEtgq/PCUspJjmfBwr/AhnnQqAuM/RQadT73OcVFcPyAtULhMVvSOf18D+xfZ43VKDhx9rkBEbYkc447IL+QGpu+RER4Yng8V/xvMa8sSueRYR3OPsgY+H2pNfZk42fW9be9zLo7aTnI7e7UVM3TKV1U/WCM1Saw8AHIPw4DHoR+9zmut5IxVjtNyZ2O/V1Pji0BHdsLuQfPPtc7oIy7niZ2yScGgho6tP3i/tlpfLFmLz8+MJCYEH9rY8FJ28zC02DfWivhdRsLPW6D8JYOe2/lPpw6QFJEYoH/Af2wVoFcDPzJGJNR1TdUqsbl7IPP74fNX1jTiFw1FaLL+Ev9fIhYX8R+IecuuzDPiueMux6757uWWImouODM88TTSi7BMWfeAQU3+SMpNYip9EzAkwe35bPVmbz43VaevijEGsi46j1rYGN0PFzxAnS5Hnxq//rqqvapbFXYdOBD4Drb6zG2bZc4IyilHMIYWD3TaqAvzLO6wvaeBJ4uXN7XyxfCmluP8hQXQ272me09Z7T7bIXtP1l3SKX5hZbT7mN3BxQQTtMwfx7vkEVM2nOYdasQBDpcYVV3Ne+nMwur81LZ/2FRxpjpdq9niMh9zghIKYc4mgGf3Qfp30LT3tZdSmRrV0dVOR4eEBRlPUgo/7i842dWs+XYEk/J8/3r4fh+rEoGO56+4NuAm3MPcsijAV+HJjNk3KMQEuvMq1L1SGUTy0ERGQPMtL1OBrKdE5JS58EYa9De149bPbmGPGMtZ1sXx1f4BoFvG4hsU/4xRQVWcind4+3EQWjRnw8OdOK/P/zO6xleXFbHVztWNaeyieVW4GVgCtafP79hTfNyTiIyBHgRa2niN0uP1heRKcAg28sAINoYE2rbVwSste373RhzpW17C2AWEA6sBMYaY/IreR2qLju8CxbcAzt+grgL4cqXtNHZ09u6EwmJBXqctXtCYRHfbjnKAx+vJj4muHZMr6/cXqX6DRpjfjfGXGmMiTLGRBtjRgDXnOscEfEEpgJDgXggWUTiS5U72RiTYIxJwOoc8Knd7pMl+0qSis0zwBRjTBvgMHBbZa5B1WHFxdZI8Ff6WKPCL38eblqgSaUSfL08eTk5EQPc/eFK8guLXR2SqgPOp0N6RdO59ATSjTHbbXcUs4CrznF8Mn9UtZVJRAS4CJhj2/QOMKJy4ao6KXsbvHOFNTalWS+YtNTqHqtjLSqtWUQAz47swuqMozzz1SZXh6PqgPP531dRt5EmwG671xm2bWcXJNIcaAH8YLfZT0RSRGSpiJQkjwjgiDGmZO7vcstUdVxxESyZCq/2g33r4MqXYcynENrU1ZG5pSGdYhjbuzlvLd7B0u3afKrOz/kklopGVpaVeMo7ZzQwx5gz5s1oZhuYcwPwgm155EqXKSITbIkpJSsrq4JQlVvJ2gJvD4GvH7WmYrlrKSSO1S6y5+mRYe1pHhHAA3NWO3/dFlWnnTOxiEiOiBwr45EDNK6g7AzA/s/HWCCznGNHU6oazBiTafu5HfgR6AYcBEJFpKTTQbllGmOmGWOSjDFJUVFRFYSq3EJRIfzyPLx2AWRvhWvegORZ1lgNdd4CfLx4dmRXMg6f5OkvtUpMVd85e4UZY85n/vAVQBtbL649WMnjhtIHiUg7IAxYYrctDMg1xuSJSCTWiP//GGOMiCwCRmK12dwMzD+PGJW72L8e5k2CvWnQYTgM+y80aOjqqOqcni3CubVfC95avAN/H09uv6AF0cE1tJyxqjOcNgTZGFMoIndjTV7pCbxtjFkvIk8CKcaYBbZDk4FZ5sxJyzoAr4tIMdZd1dO2xcYAHgJmici/gFXAW866BlULFBVYdyk/P2tNl3LdDIgfodVeTvTAZe3IPp7Hm79sZ8ZvOxnTqzmPXd4BTw/9N1eVo5NQqtorMw3m3w3710KnkTD0GQh0wMqHqlJ2HjzB1EXpfJyawR39W5Y9E7Kq05w6CaVSNaowD376DyyeYiWS0R9C+8tdHVW9ExcZyLPXdcXfx5PXf95Oq+ggrk/SXneqYppYVO2SkQrzJ0HWJuh6Awz5P/APc3VU9drfrohne9YJHpu7lqZhAfRpFeHqkFQtp6PIVO1QcBK++Su8NRjycuCGj+HqVzWp1AJenh5MvTGR5hGB3Dx9OfPT9rg6JFXLaWJRrvf7UqsL8W8vWQtLTVoCbS91dVTKToi/N7Pv6EO3pqH8aVYaz369ieLiut8+q6pHE4tynfwT8OVD1mDHwnwYO8+aONJPp9mtjcIDfXjvtl6MSmrK1EXbeHTuWk0uqkzaxqJcY8fP1kzEh3dCj/Ew+O/WNPCqVvPx8uDpazsT1cCXlxelU1hseObaLtoVWZ1BE4uqWXk58O3fIOVtCGsB4xZCXD9XR6WqQET486Vt8fQQXvx+K4VFxTwzsgu+XnVwzRtVLZpYVM1J/x4++5O1umOfu2HQY5Veo13VLiLC5Eva4u0pPPfNFnZm5/LamO40CtFR+krbWFRNOHkE5t8F718D3v5w2zdw2VOaVOqAuy9qw2tjEtm6P4cr/reY1buPuDokVQtoYlHOtfkreKU3pH0IF0yGO36Bpj1dHZVyoCGdYph3Vz98vTyYPDuNgiJdLKy+08SinCP3EHw6AWaOssai3P691UDvrVUldVGbhg34x5Ud2Z51gveX7nJ1OMrFNLEox9uwAKb2gnWfwICHYMJP0CTR1VEpJ7u4QzQXtI7khe+2ciQ339XhKBfSxKIc53gWfDwOZo+FBo1g/CIY9Ch4+bg6MlUDRITHr+hAzqkCXvhuq6vDUS6kiUWdP2Ng7Rx4pRds+gIuehzG/wAxXVwdmaph7RsFM7pnM95buouUnYdcHY5yEU0s6vzk7INZN8Int0FYHNzxM/R/ADy9XR2ZcpG/XNqOZuEB3DJ9BWnaS6xe0sSiqscYSJtptaVs+x4u+Sfc+g1E65od9V14oA8fju9FWKAPN721jHV7jro6JFXDNLGoqju6Bz68HuZNhKj2MPFX6HcveOp4W2WJCfFn5oTeBPt7M/atZWzLOu7qkFQNcmpiEZEhIrJZRNJF5OEy9k8RkTTbY4uIHLFtTxCRJSKyXkTWiMgou3NmiMgOu/MSnHkNyo4xkDrDGpeyczEMeQZuWQiRrV0dmaqFmoT688HtvfD0EG56azkHjp1ydUiqhjgtsYiIJzAVGArEA8kiEm9/jDFmsjEmwRiTAPwP+NS2Kxe4yRjTERgCvCAioXanPlBynjEmzVnXoOwc3gXvjbCmZInpCnf+Cr0ngofOD6XK1zwikLfH9eBwbj43T1/B0dwCV4ekaoAz71h6AunGmO3GmHxgFnDVOY5PBmYCGGO2GGO22p5nAgeAKCfGqspTXAzL34BX+kBGClz+PNy0AMJbujoy5Sa6xIby6pjubN2fQ79nfuBv89eRfiDH1WEpJ3JmYmkC7LZ7nWHbdhYRaQ60AH4oY19PwAfYZrf5KVsV2RQR8XVcyOoM2dvgnStg4V+gWS9rAa4et4GHNs2pqhnQNoq5k/pxaXxDZi3fzSVTfuaT1AxXh6WcxJnfEGUt0FDeqkCjgTnGmKIzChCJAd4DbjHGlExA9AjQHugBhAMPlfnmIhNEJEVEUrKysqoTf/1VXARLpsKr/WDfOrjyZRjzKYQ2c3Vkyo11jg3h+VEJLHnkIvq2iuCBOat1meM6ypmJJQNoavc6Fsgs59jR2KrBSohIMPAF8LgxZmnJdmPMXmPJA6ZjVbmdxRgzzRiTZIxJiorSWrRKy9pirej49aPQcgDctRQSx4LoQk7KMSKCfHnzph70iAvn/tmrmZ2yW1eirGOcmVhWAG1EpIWI+GAljwWlDxKRdkAYsMRumw8wF3jXGPNxqeNjbD8FGAGsc9oV1DdFBdbU9ge3wNXTIHkWBDd2dVSqDvL38eTtcT3o3iyMB+esYciLP/PpygxyTmnjfl3gtIEHxphCEbkb+BrwBN42xqwXkSeBFGNMSZJJBmYZY+z/ZLke6A9EiMg427Zxth5gH4hIFFZVWxow0VnXUO94esO1b1orOzZo6OpoVB0X6OvFh+N78cXavbyyaBv3z14NQGyYPz1bhPPUiM74+2ivQ3ckZ36f101JSUkmJSXF1WEopcpRXGxYsj2btN1H2JB5jC/W7uX2C1rw+BXxFZ+snEZEUo0xSVU9T4dKK6VczsND6Nc6kn6tIwEInbuWt37dwbAuMSQ2C3NxdKqqtN+oUqrWeXhoe2KC/XhwzhryCosqPkHVKppYlFK1TgM/b566pjPpB47z74WbtNeYm9HEopSqlQa1i+amPs2Z8dtOJryXytGT2mPMXWhiUUrVWv+4siNPDI/nx80HuOrlxaTuOuzqkFQlaGJRStVaIsIt/Vowc0JvThUUc+2rvzH5ozT260zJtZomFqVUrdcjLpzv/zyAuwa14ou1exn03I+8tXgHRdr2UitpYlFKuYVAXy8euKw9300eQM8W4fzz8w2MmPqrrlBZC2liUUq5lWYRAUwf14OXb+jG3qOnuGrqr/z7y42cKtBuybWFJhallNsREa7o0pjv7x/AyMRYXv9pO0Nf/IX0A7oEcm2giUUp5bZCArx5ZmQXPri9FzmnCrh1xgoOnch3dVj1niYWpZTb69c6kmk3JbHv2CnueC9FR+u7mCYWpVSdkNgsjP9e15UVOw/zyCdrqQ8T7NZWOgmlUqrOGN61MTsOnuD5b7cQFezLI0M7uDqkekkTi1KqTrnnotYcyDnF6z9tJyzAh4kDWrk6pHpHE4tSqk4REf5xZSeO5Bbw9JebCPH3JrlnM1eHVa9oYlFK1TmeHsLz1ydwPK+QRz5dS2FRMWP7xLk6rHrDqY33IjJERDaLSLqIPFzG/ikikmZ7bBGRI3b7bhaRrbbHzXbbu4vIWluZL4mIOPMalFLuycfLg9fHdmdwh2j+On89037extGTBToFfw1w2tLEIuIJbAEuATKAFUCyMWZDOcffA3QzxtwqIuFACpAEGCAV6G6MOSwiy4E/AUuBhcBLxpgvzxWLLk2sVP2VX1jMfR+tYuHafae3dYgJZtrY7jQND3BhZLVfdZcmduYdS08g3Riz3RiTD8wCrjrH8cnATNvzy4BvjTGHjDGHgW+BISISAwQbY5YYKyO+C4xw3iUopdydj5cHL43uxv+Su/H45R2496LW7Dmcy3WvLWHr/hxXh1cnOTOxNAF2273OsG07i4g0B1oAP1RwbhPb88qUOUFEUkQkJSsrq1oXoJSqG7w8PRjetTG3X9iS+y9tx0d39KHIGK57fQkrdh5ydXh1jjMTS1ltH+XVu40G5hhjSobLlndupcs0xkwzxiQZY5KioqIqDFYpVX90iAlmzsQ+hPh7M+r1JTz95SYdre9AzkwsGUBTu9exQGY5x47mj2qwc52bYXtemTKVUqpczSMC+eLeCxnVoymv/bSNK//3qy4g5iDOTCwrgDYi0kJEfLCSx4LSB4lIOyAMWGK3+WvgUhEJE5Ew4FLga2PMXiBHRHrbeoPdBMx34jUopeqwIF8v/n1NF6bf0oPdh3OZ9MFK8guLXR2W23NaYjHGFAJ3YyWJjcBsY8x6EXlSRK60OzQZmGXsuqcZYw4B/8RKTiuAJ23bAO4E3gTSgW3AOXuEKaVURQa1i+aZa7uQuusw/7dwo6vDcXtO625cm2h3Y6VUZTz52Qbe/nUHL45O4KqEMvsF1Su1sbuxUkq5lUeGtadHXBgPzFnDb+kHXR2O29LEopRSNt6eHkwbm0RcRAC3v5vCyt8Puzokt6SJRSml7IQF+vD+bb2IDPJl3NvLWZ951NUhuR1NLEopVUp0sB8f3N6LQF8vRk9bqoMoq0gTi1JKlaFpeAAfT+xDVJAvY99axqJNB1wdktvQxKKUUuWIDQtg9sQ+tI4OYvy7Kby3ZKcueVwJmliUUuocIoN8mTm+N/3bRvHX+et55NO1Ov1LBXShL6WUqkADP2/euCmJKd9u4eVF6fy67SAD20bTr3Uk/dtGEuCjX6X29F9DKaUqwdND+Mtl7UhoGsqHy3/nk5UZvLd0F4E+ngzrHMPons3o3jzM1WHWCppYlFKqCgbHN2RwfEPyC4tJ3XWYeav28PmaTD5OzeC567oysntsxYXUcdrGopRS1eDj5UGfVhE8M7ILKx4fzAWtI3nokzV8t2G/q0NzOU0sSil1ngJ8vHh9bHc6NQ7mrg9Xsmx7tqtDcilNLEop5QCBvl5Mv6UnsWH+3Dx9Od9vrL93LppYlFLKQcIDfZg1oQ9tohsw/t0UZi7/3dUhuYQmFqWUcqCoBr7MmmCNe3nk07X8dd46cvMLXR1WjdLEopRSDhbo68UbNyVx+wUteH/ZLi5/aTGr6tFMyZpYlFLKCbw9PXj8ing+vL03+YXFjHp9Kev21I+Zkp2aWERkiIhsFpF0EXm4nGOuF5ENIrJeRD60bRskIml2j1MiMsK2b4aI7LDbl+DMa1BKqfPRp1UEn91zAaEB3kz+KI1TBXV/OhinJRYR8QSmAkOBeCBZROJLHdMGeAToZ4zpCNwHYIxZZIxJMMYkABcBucA3dqc+ULLfGJPmrGtQSilHCA/04dnrurL1wHGe+WqTq8NxOmfesfQE0o0x240x+cAs4KpSx4wHphpjDgMYY8qal3ok8KUxJteJsSqllFMNaBvFzX2aM/3Xnfy8JcvV4TiVMxNLE2C33esM2zZ7bYG2IvKriCwVkSFllDMamFlq21MiskZEpoiIb1lvLiITRCRFRFKysur2h6iUcg8PD+1A6+gg7vpgJWm7j7g6HKdxZmKRMraVXsjAC2gDDASSgTdFJPR0ASIxQGfga7tzHgHaAz2AcOChst7cGDPNGJNkjEmKioqq7jUopZTD+Pt48s6tPQkN9GbsW8tYk1E3k4szE0sG0NTudSyQWcYx840xBcaYHcBmrERT4npgrjGmoGSDMWavseQB07Gq3JRSyi00CfVn5vjehPh7M+bNZXVy+hdnJpYVQBsRaSEiPlhVWgtKHTMPGAQgIpFYVWPb7fYnU6oazHYXg4gIMAJY55TolVLKSWLDApg1oTeRQb7c8OYy3vxle51amdJpicUYUwjcjVWNtRGYbYxZLyJPisiVtsO+BrJFZAOwCKu3VzaAiMRh3fH8VKroD0RkLbAWiAT+5axrUEopZ4kNC2D+3f0Y3CGaf32xkVtnrGB+2h6yj+e5OrTzJnUpS5YnKSnJpKSkuDoMpZQ6izGGN37Zzis/buNIbgEi0K9VJLdd2IIBbaLw8CirubpmiEiqMSapyudpYlFKKdcrKjas23OURZsPMGv5bvYdO0Wb6CCmjEqgU5MQAPYdPcU9M1fSq0UEf760LVaLgPNoYjkHTSxKKXeSX1jMwrV7efbrzRzOzWfqDYm0igrixreWknnkFEXFhjsHtuLBy9o5NblUN7Ho0sRKKVXL+Hh5MKJbE/q2iuDWd1Zw+7sphPh7U2wMn9zZl49TdvPqj9vw9vRg8uA2Tr9zqSpNLEopVUtFB/vx0YQ+/GnWKtbtOcY7t/akXaMGdGkSQkFRMS99v5UNmcf454iOxIT4uzrc07QqTCml3EBRscHTriG/uNjw1uId/PfbzXh5eHDnwFYMahdN+0YNHNbgr20s56CJRSlVV/2enctj89byy9aDAIQFeHPvxW24pV+L8y5b21iUUqoeahYRwHu39WLv0ZMs2ZbNb9uyaRjs59KYNLEopVQdEBPizzWJsVyTGOvqUHQFSaWUUo6liUUppZRDaWJRSinlUJpYlFJKOZQmFqWUUg6liUUppZRDaWJRSinlUJpYlFJKOVS9mNJFRLKAXdU8PRI46MBwagO9Jveg1+Qe6vI1NTfGRFX15HqRWM6HiKRUZ66c2kyvyT3oNbkHvaazaVWYUkoph9LEopRSyqE0sVRsmqsDcAK9Jveg1+Qe9JpK0TYWpZRSDqV3LEoppRxKE8s5iMgQEdksIuki8rCr46kqEWkqIotEZKOIrBeRP9m2h4vItyKy1fYzzNWxVpWIeIrIKhH53Pa6hYgss13TRyLi4+oYq0JEQkVkjohssn1efdz9cxKRybbfu3UiMlNE/NztcxKRt0XkgIiss9tW5ucilpds3xdrRCTRdZGXr5xretb2u7dGROaKSKjdvkds17RZRC6rzHtoYimHiHgCU4GhQDyQLCLxro2qygqBPxtjOgC9gbts1/Aw8L0xpg3wve21u/kTsNHu9TPAFNs1HQZuc0lU1fci8JUxpj3QFeva3PZzEpEmwL1AkjGmE+AJjMb9PqcZwJBS28r7XIYCbWyPCcCrNRRjVc3g7Gv6FuhkjOkCbAEeAbB9X4wGOtrOecX23XhOmljK1xNIN8ZsN8bkA7OAq1wcU5UYY/YaY1banudgfVk1wbqOd2yHvQOMcE2E1SMiscDlwJu21wJcBMyxHeJW1yQiwUB/4C0AY0y+MeYIbv45Ya1Q6y8iXkAAsBc3+5yMMT8Dh0ptLu9zuQp411iWAqEiElMzkVZeWddkjPnGGFNoe7kUKFmG8ipgljEmzxizA0jH+m48J00s5WsC7LZ7nWHb5pZEJA7oBiwDGhpj9oKVfIBo10VWLS8ADwLFttcRwBG7/xju9lm1BLKA6bbqvTdFJBA3/pyMMXuA54DfsRLKUSAV9/6cSpT3udSV74xbgS9tz6t1TZpYyidlbHPLLnQiEgR8AtxnjDnm6njOh4hcARwwxqTaby7jUHf6rLyAROBVY0w34ARuVO1VFlu7w1VAC6AxEIhVVVSaO31OFXH330NE5DGsKvQPSjaVcViF16SJpXwZQFO717FApotiqTYR8cZKKh8YYz61bd5fcotu+3nAVfFVQz/gShHZiVU9eRHWHUyorcoF3O+zygAyjDHLbK/nYCUad/6cBgM7jDFZxpgC4FOgL+79OZUo73Nx6+8MEbkZuAK40fwxDqVa16SJpXwrgDa2Xiw+WA1YC1wcU5XY2h7eAjYaY56327UAuNn2/GZgfk3HVl3GmEeMMbHGmDisz+QHY8yNwCJgpO0wd7umfcBuEWln23QxsAE3/pywqsB6i0iA7few5Jrc9nOyU97nsgC4ydY7rDdwtKTKrLYTkSHAQ8CVxphcu10LgNEi4isiLbA6JiyvsEBjjD7KeQDDsHpIbAMec3U81Yj/Aqzb1jVAmu0xDKtN4ntgq+1nuKtjreb1DQQ+tz1vafuFTwc+BnxdHV8VryUBSLF9VvOAMHf/nIB/AJuAdcB7gK+7fU7ATKw2ogKsv95vK+9zwao2mmr7vliL1SPO5ddQyWtKx2pLKfmeeM3u+Mds17QZGFqZ99CR90oppRxKq8KUUko5lCYWpZRSDqWJRSmllENpYlFKKeVQmliUUko5lCYWVSeISJGIpInIahFZKSJ9Kzg+VEQmVaLcH0WkTq1nfr5EZIaIjKz4SFVfaWJRdcVJY0yCMaYr1sys/67g+FCgwsTiKnaj05VyO5pYVF0UjDUlOyISJCLf2+5i1opIyQzVTwOtbHc5z9qOfdB2zGoRedquvOtEZLmIbBGRC23HetrWsFhhW8PiDtv2GBH52VbuupLj7YnIThF5xlbmchFpbds+Q0SeF5FFwDO2dT/m2cpfKiJd7K5pui3WNSJyrW37pSKyxHatH9vmiENEnhaRDbZjn7Ntu84W32oR+bmCa/r/9s7uxaoyCuO/B0THyRisRKQLg8gkNIxKiUJUpM9gPHgAAANaSURBVOjjIuuiiUDFbgoKKpBAygzKGy+80K4Si4kYKOg7YyYSCoZJMp0+LvoH6qJGgjHtQ4ani7WG2ejZ5sUp6bh+N3vznv2+e619OGe977vYz5Kk/TnGx/yPxDCLi0PNiopeYb6kCaAPWEJoiAH8AWyyPSXpKuBLSR8QIo8rbK8CkHQ3IX++xvZpSVc0xp5je7Wke4AXCB2sRwnJjlslzQPGJI0CDwAjtl9W1K3ob7F3KsfcTGid3Zfty4CNtqcl7QOO275f0gZgiHhD//m898q0fWH69lz2PSXpWeAZSfuBTcBy29ZsAaedwF22f2y0tfl0E3A9sBJYTEizHLygb6W4JKnAUvQKvzeCxG3AkKQVhMzGbklrCZn9q4k/x7PZCLzm1Emy3axXMSPe+TVwTZ7fCdzYyDUMEDpKXwEHFeKf79meaLF3uHHc22h/2/Z0nt8BPJj2HJZ0paSBtHVwpoPtXxWqzzcQwQBgLjAOTBHB9UCuNj7KbmPA65LeavjX5tNaYDjt+knS4RafigKowFL0ILbHcwa/iNBGWwTcbPuMQhW5r0M30S4H/mcep5n9zQh40vbIOQNFELsXeEPSHttDncxsOT91lk2d+nWyVcCnth/uYM9qQgRyEHgC2GD7MUlr0s4JSavafMqVWmk/FRdM5ViKnkPScqIU7gli1v1zBpX1wNK87CRweaPbKLBNUn+O0dwK68QI8HiuTJC0TNJlkpbm/V4llKXb6p4/1DiOt1zzBfBIjr8OmHTU0xklAsSMvwuJqn+3N/I1/WnTAmDA9iHgKWIrDUnX2j5ieycwSUijd/Qp7RjMHMwSYP0/PJviEqdWLEWvMJNjgZh5b8k8xZvAh5KOEqqtPwDYPiFpTNL3wCe2t+es/aikv4BDwI7z3O8AsS12TLH39AuRo1kHbJd0BvgN2NzSf56kI8Tk7pxVRrKLqCr5LXCaWan2l4BX0vZp4EXb70jaCgxnfgQi53ISeF9SXz6Xp/OzPZKuy7bPgG8IZeVOPr1L5Ky+I9S+Pz/PcymKUjcuiv+a3I67xfbkxbalKP4NaiusKIqi6Cq1YimKoii6Sq1YiqIoiq5SgaUoiqLoKhVYiqIoiq5SgaUoiqLoKhVYiqIoiq5SgaUoiqLoKn8DaaB5ShCtXeAAAAAASUVORK5CYII=\n",
"text/plain": [
"<Figure size 432x288 with 1 Axes>"
]
},
"metadata": {
"needs_background": "light"
},
"output_type": "display_data"
}
],
"source": [
"new_learn.recorder.plot_losses()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"hidden": true
},
"outputs": [],
"source": [
"test_learn.get_predicts()"
]
},
{
"cell_type": "code",
"execution_count": 78,
"metadata": {
"hidden": true
},
"outputs": [],
"source": [
"del test_learn\n",
"del new_model\n",
"del new_learn"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Setup custom model"
]
},
{
"cell_type": "code",
"execution_count": 17,
"metadata": {},
"outputs": [],
"source": [
"def Flatten(): return Lambda(lambda x: x.view((x.size(0), -1)))"
]
},
{
"cell_type": "code",
"execution_count": 18,
"metadata": {},
"outputs": [],
"source": [
"def ResizeInput(): return Lambda(lambda x: x.view((-1,)+x.size()[-2:]))"
]
},
{
"cell_type": "code",
"execution_count": 19,
"metadata": {},
"outputs": [],
"source": [
"#def ResizeOutput(): return Lambda(lambda x: x.view(-1))"
]
},
{
"cell_type": "code",
"execution_count": 20,
"metadata": {},
"outputs": [],
"source": [
"drop_p = 0.2"
]
},
{
"cell_type": "code",
"execution_count": 21,
"metadata": {},
"outputs": [],
"source": [
"# inplace=True seems to generate problems?"
]
},
{
"cell_type": "code",
"execution_count": 22,
"metadata": {},
"outputs": [],
"source": [
"net = nn.Sequential(ResizeInput(),\n",
" nn.Conv1d(in_channels=4, out_channels=32, kernel_size=12),\n",
" nn.MaxPool1d(kernel_size=4),\n",
" Flatten(),\n",
" nn.Dropout(drop_p),\n",
" nn.Linear(in_features=288, out_features=16),\n",
" nn.ReLU(),\n",
" nn.Dropout(drop_p),\n",
" nn.Linear(in_features=16, out_features=2),\n",
" #nn.Dropout(drop_p), # not at the end?\n",
" #Debugger()\n",
" )"
]
},
{
"cell_type": "code",
"execution_count": 23,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"Sequential(\n",
" (0): Lambda()\n",
" (1): Conv1d(4, 32, kernel_size=(12,), stride=(1,))\n",
" (2): MaxPool1d(kernel_size=4, stride=4, padding=0, dilation=1, ceil_mode=False)\n",
" (3): Lambda()\n",
" (4): Dropout(p=0.2)\n",
" (5): Linear(in_features=288, out_features=16, bias=True)\n",
" (6): ReLU()\n",
" (7): Dropout(p=0.2)\n",
" (8): Linear(in_features=16, out_features=2, bias=True)\n",
")"
]
},
"execution_count": 23,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"net"
]
},
{
"cell_type": "code",
"execution_count": 24,
"metadata": {},
"outputs": [],
"source": [
"apply_init(net, nn.init.kaiming_normal_)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Run sequence through network for testing"
]
},
{
"cell_type": "code",
"execution_count": 25,
"metadata": {},
"outputs": [
{
"ename": "NameError",
"evalue": "name 'data' is not defined",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)",
"\u001b[0;32m<ipython-input-25-37a889cb3037>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain_ds\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain_ds\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m3\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
"\u001b[0;31mNameError\u001b[0m: name 'data' is not defined"
]
}
],
"source": [
"data.train_ds[0], data.train_ds[3]"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"net(data.train_ds[0][0].data), net(data.train_ds[3][0].data)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Learner setup"
]
},
{
"cell_type": "markdown",
"metadata": {
"heading_collapsed": true
},
"source": [
"## TBLogger"
]
},
{
"cell_type": "code",
"execution_count": 35,
"metadata": {
"hidden": true
},
"outputs": [],
"source": [
"import tensorflow as tf\n",
"\n",
"# From https://gist.github.com/gyglim/1f8dfb1b5c82627ae3efcfbbadb9f514\n",
"\"\"\"Simple example on how to log scalars and images to tensorboard without tensor ops.\n",
"License: Copyleft\n",
"\"\"\"\n",
"#__author__ = \"Michael Gygli\"\n",
"\n",
"#import tensorflow as tf\n",
"#from StringIO import StringIO\n",
"#import matplotlib.pyplot as plt\n",
"#import numpy as np\n",
"\n",
"class Logger(object):\n",
" \"\"\"Logging in tensorboard without tensorflow ops.\"\"\"\n",
"\n",
" def __init__(self, log_dir):\n",
" \"\"\"Creates a summary writer logging to log_dir.\"\"\"\n",
" self.writer = tf.summary.FileWriter(log_dir)\n",
"\n",
" def log_scalar(self, tag, value, step):\n",
" \"\"\"Log a scalar variable.\n",
" Parameter\n",
" ----------\n",
" tag : basestring\n",
" Name of the scalar\n",
" value\n",
" step : int\n",
" training iteration\n",
" \"\"\"\n",
" summary = tf.Summary(value=[tf.Summary.Value(tag=tag,\n",
" simple_value=value)])\n",
" self.writer.add_summary(summary, step)\n",
"\n",
" def log_images(self, tag, images, step):\n",
" \"\"\"Logs a list of images.\"\"\"\n",
"\n",
" im_summaries = []\n",
" for nr, img in enumerate(images):\n",
" # Write the image to a string\n",
" s = StringIO()\n",
" plt.imsave(s, img, format='png')\n",
"\n",
" # Create an Image object\n",
" img_sum = tf.Summary.Image(encoded_image_string=s.getvalue(),\n",
" height=img.shape[0],\n",
" width=img.shape[1])\n",
" # Create a Summary value\n",
" im_summaries.append(tf.Summary.Value(tag='%s/%d' % (tag, nr),\n",
" image=img_sum))\n",
"\n",
" # Create and write Summary\n",
" summary = tf.Summary(value=im_summaries)\n",
" self.writer.add_summary(summary, step)\n",
" \n",
"\n",
" def log_histogram(self, tag, values, step, bins=1000):\n",
" \"\"\"Logs the histogram of a list/vector of values.\"\"\"\n",
" # Convert to a numpy array\n",
" values = np.array(values)\n",
" \n",
" # Create histogram using numpy \n",
" counts, bin_edges = np.histogram(values, bins=bins)\n",
"\n",
" # Fill fields of histogram proto\n",
" hist = tf.HistogramProto()\n",
" hist.min = float(np.min(values))\n",
" hist.max = float(np.max(values))\n",
" hist.num = int(np.prod(values.shape))\n",
" hist.sum = float(np.sum(values))\n",
" hist.sum_squares = float(np.sum(values**2))\n",
"\n",
" # Requires equal number as bins, where the first goes from -DBL_MAX to bin_edges[1]\n",
" # See https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/summary.proto#L30\n",
" # Thus, we drop the start of the first bin\n",
" bin_edges = bin_edges[1:]\n",
"\n",
" # Add bin edges and counts\n",
" for edge in bin_edges:\n",
" hist.bucket_limit.append(edge)\n",
" for c in counts:\n",
" hist.bucket.append(c)\n",
"\n",
" # Create and write Summary\n",
" summary = tf.Summary(value=[tf.Summary.Value(tag=tag, histo=hist)])\n",
" self.writer.add_summary(summary, step)\n",
" self.writer.flush()\n",
" \n",
"\"A `Callback` that saves tracked metrics into a log file for Tensorboard.\"\n",
"# Based on https://gist.github.com/gyglim/1f8dfb1b5c82627ae3efcfbbadb9f514\n",
"# and devforfu: https://nbviewer.jupyter.org/gist/devforfu/ea0b3fcfe194dad323c3762492b05cae\n",
"# Contribution from MicPie\n",
"\n",
"#from ..torch_core import *\n",
"#from ..basic_data import DataBunch\n",
"#from ..callback import *\n",
"#from ..basic_train import Learner, LearnerCallback\n",
"#import tensorflow as tf\n",
"\n",
"__all__ = ['TBLogger']\n",
"\n",
"@dataclass\n",
"class TBLogger(LearnerCallback):\n",
" \"A `LearnerCallback` that saves history of metrics while training `learn` into log files for Tensorboard.\"\n",
" \n",
" log_dir:str = 'logs'\n",
" log_name:str = 'data'\n",
" log_scalar:bool = True # log scalar values for Tensorboard scalar summary\n",
" log_hist:bool = True # log values and gradients of the parameters for Tensorboard histogram summary\n",
" log_img:bool = False # log values for Tensorboard image summary\n",
"\n",
" def __post_init__(self): \n",
" super().__post_init__()\n",
" #def __init__(self):\n",
" # super().__init__()\n",
" self.path = self.learn.path\n",
" (self.path/self.log_dir).mkdir(parents=True, exist_ok=True) # setup logs directory\n",
" self.Log = Logger(str(self.path/self.log_dir/self.log_name))\n",
" self.epoch = 0\n",
" self.batch = 0\n",
" self.log_grads = {}\n",
" \n",
" def on_backward_end(self, **kwargs:Any):\n",
" self.batch = self.batch+1\n",
" #print('\\nBatch: ',self.batch)\n",
" \n",
" if self.log_hist:\n",
" for tag, value in learn.model.named_parameters():\n",
" tag_grad = tag.replace('.', '/')+'/grad'\n",
" \n",
" if tag_grad in self.log_grads:\n",
" #self.log_grads[tag_grad] += value.grad.data.cpu().detach().numpy()\n",
" self.log_grads[tag_grad] = self.log_grads[tag_grad] + value.grad.data.cpu().detach().numpy() # gradients are summed up from every batch\n",
" #print('if')\n",
" else:\n",
" self.log_grads[tag_grad] = value.grad.data.cpu().detach().numpy()\n",
" #print('else')\n",
" \n",
" #print(tag_grad, self.log_grads[tag_grad].sum())\n",
" return self.log_grads\n",
" \n",
" #def on_step_end(self, **kwards:Any):\n",
" #print('Step end: ', self.log_grads)\n",
"\n",
" def on_epoch_end(self, epoch:int, smooth_loss:Tensor, last_metrics:MetricsList, **kwargs:Any) -> bool:\n",
" last_metrics = ifnone(last_metrics, [])\n",
" tr_info = {name: stat for name, stat in zip(self.learn.recorder.names, [epoch, smooth_loss] + last_metrics)}\n",
" self.epoch = tr_info['epoch']\n",
" self.batch = 0 # reset batch count\n",
" #print('\\nEpoch: ',self.epoch)\n",
" \n",
" if self.log_scalar:\n",
" for tag, value in tr_info.items():\n",
" if tag == 'epoch': continue\n",
" self.Log.log_scalar(tag, value, self.epoch+1)\n",
" \n",
" if self.log_hist:\n",
" for tag, value in learn.model.named_parameters():\n",
" \n",
" tag = tag.replace('.', '/')\n",
" self.Log.log_histogram(tag, value.data.cpu().numpy(), self.epoch+1)\n",
" \n",
" tag_grad = tag.replace('.', '/')+'/grad'\n",
" self.Log.log_histogram(tag_grad, self.log_grads[tag_grad], self.epoch+1)\n",
" #print(tag_grad, self.log_grads[tag_grad].sum())\n",
" \n",
" #if self.log_img:\n",
" # for tag, value in learn.model.named_parameters():\n",
" # \n",
" # tag = tag.replace('.', '/')\n",
" # self.Log.log_images(tag, value.data.cpu().numpy(), self.epoch+1)"
]
},
{
"cell_type": "markdown",
"metadata": {
"heading_collapsed": true
},
"source": [
"## fastai learner setup"
]
},
{
"cell_type": "code",
"execution_count": 208,
"metadata": {
"hidden": true
},
"outputs": [],
"source": [
"#learn = Learner(data, net, loss_func=nn.functional.cross_entropy, metrics=accuracy)#, callback_fns=[TBLogger])"
]
},
{
"cell_type": "code",
"execution_count": 82,
"metadata": {
"hidden": true
},
"outputs": [],
"source": [
"def accuracy_float(input:Tensor, targs:LongTensor)->Rank0Tensor:\n",
" \"Compute accuracy with `targs` when `input` is bs * n_classes.\"\n",
" n = targs.shape[0]\n",
" input = input.argmax(dim=-1).view(n,-1)\n",
" targs = targs.type(torch.long) # convert to torch.long = int64 --> difference to accuracy metric!\n",
" targs = targs.view(n,-1)\n",
" return (input==targs).float().mean()"
]
},
{
"cell_type": "code",
"execution_count": 83,
"metadata": {
"hidden": true
},
"outputs": [],
"source": [
"learn = Learner(data, net, loss_func=BCEWithLogitsFlat(), metrics=accuracy_float)#, callback_fns=[TBLogger])"
]
},
{
"cell_type": "code",
"execution_count": 84,
"metadata": {
"hidden": true
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"======================================================================\n",
"Layer (type) Output Shape Param # Trainable \n",
"======================================================================\n",
"Lambda [64, 4, 50] 0 False \n",
"______________________________________________________________________\n",
"Conv1d [64, 32, 39] 1568 True \n",
"______________________________________________________________________\n",
"MaxPool1d [64, 32, 9] 0 False \n",
"______________________________________________________________________\n",
"Lambda [64, 288] 0 False \n",
"______________________________________________________________________\n",
"Dropout [64, 288] 0 False \n",
"______________________________________________________________________\n",
"Linear [64, 16] 4624 True \n",
"______________________________________________________________________\n",
"ReLU [64, 16] 0 False \n",
"______________________________________________________________________\n",
"Dropout [64, 16] 0 False \n",
"______________________________________________________________________\n",
"Linear [64, 2] 34 True \n",
"______________________________________________________________________\n",
"\n",
"Total params: 6226\n",
"Total trainable params: 6226\n",
"Total non-trainable params: 0\n"
]
}
],
"source": [
"learn.summary()"
]
},
{
"cell_type": "markdown",
"metadata": {
"heading_collapsed": true
},
"source": [
"## fastai training"
]
},
{
"cell_type": "code",
"execution_count": 85,
"metadata": {
"hidden": true
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"LR Finder is complete, type {learner_name}.recorder.plot() to see the graph.\n"
]
},
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEKCAYAAAD9xUlFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xl8XXWd//HXJ/u+J22aJt1LWyi0NAVRkAqCBRdgHFF03EceLuioo6Mzzk8dmBl13MZtVHQQHQRFUQRE2QTLDgFautOW0jbdkq5p2jS9y+f3x725vZRsbe/JvTd5Px+P++i553zvPZ/c3pxPvusxd0dERAQgJ90BiIhI5lBSEBGRBCUFERFJUFIQEZEEJQUREUlQUhARkQQlBRERSVBSEBGRBCUFERFJyEt3AMerrq7OJ0+enO4wRESyyjPPPLPL3euHKhdYUjCzG4A3AR3uftog5RYCTwBvd/ffDvW+kydPpq2tLXWBioiMAWa2aTjlgmw+uhFYPFgBM8sFvgbcE2AcIiIyTIElBXdfAuwZotjHgduAjqDiEBGR4UtbR7OZNQFXAD8aRtmrzazNzNo6OzuDD05EZIxK5+ij/wY+5+6RoQq6+/Xu3ururfX1Q/aTiIjICUrn6KNW4FdmBlAHXGpmYXe/PY0xiYiMaWlLCu4+pW/bzG4E7lJCEBFJryCHpN4CLALqzKwd+BKQD+DuQ/YjiIjIyAssKbj7VcdR9n1BxSEikik6ug7zyPpdXDG/iXjTecbRMhciIiPk989t5dO3LuOXT25OdygDUlIQERkhveEoANfdtYo1O7rSHE3/lBREREZIOBJLChXF+Vxz83P0HBlyRP6IU1IQERkhoaiTn2t8+8p5bOjs5tq7VqY7pFdQUhARGSHhSJS8nBzOnVHHR86fxi1PbeHOZdvSHdbLKCmIiIyQUMTJy42NOvrURTOZ31LFv/x+Oe17D6U5sqOUFERERkg4GiU/N3bZzc/N4Ttvn487fPrXy4hEPc3RxSgpiIiMkHDEycs5Oj+hpbaEay87lade2sMPH1qfxsiOUlIQERkhoYgnagp9rpjfxJvPmMC371/Hc5v3pimyo5QURERGSDgaTfQp9DEz/v3y0xhfUcQ//Gop3b3hNEUXo6QgIjJCjm0+6lNZnM9/v2Me7XsP8S+/W457+voXlBREREZIKBJ9RfNRn4WTa/jHi0/hjmXbuPmp9C2DoaQgIjJCwlF/RfNRso+cP43zZ9bzb3euYsXW/SMY2VFKCiIiIyQUn7w2kJwc49tvn0dNSQEfu/lZug6HRjC6eAwjfkYRkTEqHIktczGYmtICvv/O+bTv7eFzv31+xPsXlBREREZIODp4TaFP6+QaPrf4FP60Ygff+8vIzl9I5z2aRUTGlFDEKcof3s11PnTeVNbsOMC37nuBGQ1lXDK3MeDoYlRTEBEZIcnLXAzFzPjPK+ZyZksVn7p16Yh1PCspiIiMkIHmKQykKD+XH7+7lZqSAj70izY6ug4HGF2MkoKIyAgZbJ7CQOrLC/nJe1vZdyjEdx5YF1BkR6lPQURkhAw1T2Egp06o5OYPnc3sxooAono5JQURkRESaz46sQaa+S3VKY6mf2o+EhEZIbHmo+OvKYykwJKCmd1gZh1mtmKA45eZ2fNmttTM2szs3KBiERHJBCfafDSSgqwp3AgsHuT4A8AZ7j4P+ADw0wBjERFJu6GWucgEgUXn7kuAPYMc7/aj87dLgcy4F52ISECGs8xFuqU1ZZnZFWa2BvgjsdrCQOWujjcxtXV2do5cgCIiKRS7yc4YrSkMh7v/3t1nAZcD1w1S7np3b3X31vr6+pELUEQkRdw9djvO45i8lg4ZkbLiTU3TzKwu3bGIiAQhEo21kKumMAAzm25mFt8+EygAdqcrHhGRIIUTSSGzawqBTV4zs1uARUCdmbUDXwLyAdz9R8BbgfeYWQjoAd7u6bwxqYhIgEKRKAD5GT76KLCk4O5XDXH8a8DXgjq/iEgmCUeyo6aQ2SlLRGSUCEVjNQX1KYiISKKmoNFHIiKS1HyU2ZfdzI5ORGSU6Gs+0oxmERE5WlPI8NFHmR2diMgo0TckVaOPREQkMXlNzUciIkK4r6ag5iMREQlp8pqIiPQJJ0YfZfZlN7OjExEZJY6OPlJNQURkzEssiKeagoiIZMvS2UoKIiIjIKTRRyIi0iexIJ5qCiIiEtbS2SIi0iekpbNFRKRPYkazagoiIqLRRyIiknC0+SizL7uZHZ2IyCgR1tLZIiLSJxQd48tcmNkNZtZhZisGOP4uM3s+/njMzM4IKhYRkXQLR6Lk5RhmYzQpADcCiwc5vhE4391PB64Drg8wFhGRtApHPeObjgDygnpjd19iZpMHOf5Y0tMngIlBxSIikm6hSDTjO5khc/oUPgj8Kd1BiIgEJRwZ4zWF4TKz1xFLCucOUuZq4GqAlpaWEYpMRCR1wtEouaopDM7MTgd+Clzm7rsHKufu17t7q7u31tfXj1yAIiIpEop4xi+GB2lMCmbWAvwOeLe7v5CuOERERkI4Eh3bzUdmdguwCKgzs3bgS0A+gLv/CPgiUAv8T3yIVtjdW4OKR0QknUJRz4qO5iBHH101xPG/B/4+qPOLiGSSbKkpZH7aEhEZBcIRz/i7roGSgojIiAhF1dEsIiJxseajzL/kZn6EIiKjQKz5SDUFEREBQtEo+aopiIgIZM8yF0oKIiIjIBSJavSRiIjEhDX6SERE+mj0kYiIJIQiTr5GH4mICMSWzlZHs4iIAH2jjzL/kpv5EYqIjAKx23GqpiAiIsRGH6mmICIigCaviYhIknA0mhU32cn8CEVEslw06kQd1RRERCS2GB6gBfFERCTWnwBo6WwREUlKCqopiIjI0eYj1RRERMa8o81HmX/JzfwIRUSyXCgSqymM6dFHZnaDmXWY2YoBjs8ys8fNrNfMPhNUHCIi6RaOxmoKY7356EZg8SDH9wCfAL4RYAwiImkX7qspjOXmI3dfQuzCP9DxDnd/GggFFYOISCYIRVRTEBGRuHB0lNUUzGyamRXGtxeZ2SfMrCrY0F52/qvNrM3M2jo7O0fqtCIiKRFKzFMYPTWF24CImU0H/heYAtwcWFTHcPfr3b3V3Vvr6+tH6rQiIinR16cwmpa5iLp7GLgC+G93/xTQGFxYIiKjR9/oo2xY5iJvmOVCZnYV8F7gzfF9+YO9wMxuARYBdWbWDnyp7zXu/iMzGw+0ARVA1Mw+Ccxx967j/ilERDLY0XkKmV9TGG5SeD/wYeA/3H2jmU0BbhrsBe5+1RDHdwATh3l+EZGsFc6i0UfDSgruvorYnALMrBood/evBhmYiMhoMRpHHz1kZhVmVgMsA35mZt8KNjQRkdFhNM5TqIy39f8N8DN3XwC8PriwRERGj0RNIQv6FIYbYZ6ZNQJXAncFGI+IyKgTGoU32bkWuAfY4O5Pm9lUYF1wYYmIjB5HO5ozv6Yw3I7m3wC/SXr+IvDWoIISERlNjjYfjZKagplNNLPfx5fC3mlmt5mZhpOKiAxDoqN5tIw+An4G3AFMAJqAO+P7RERkCOFReJOdenf/mbuH448bAS1CJCIyDIllLkZRUthlZn9nZrnxx98Bu4MMTERktOhb5mI0NR99gNhw1B3AduBviS19ISIiQwhHnByDnNEyJNXdN7v7W9y93t0b3P1yYhPZRERkCKFoNCsmrsHJ3Xnt0ymLQkRkFAtHnPwsqCXAySWF7PgJRUTSLBwZGzUFT1kUIiKjWCjqWbEYHgwxo9nMDtD/xd+A4kAiEhEZZcKRaFYsmw1DJAV3Lx+pQERERqtwxLNijgKcXPORiIgMQ6z5KDsut9kRpYhIFos1H6mmICIixBbEGwujj0REZBjC0WjWjD5SUhARCVg44mo+EhGRmNAYmbw2KDO7IX5TnhUDHDcz+66ZrTez583szKBiERFJp3AWTV4LMnXdCCwe5PglwIz442rghwHGIiKSNtk0eS2wKN19CbBnkCKXAb/wmCeAKjNrDCoeEZF0CUVUUxiOJmBL0vP2+D4RkVElHFVNYTj6S5v9LrJnZlebWZuZtXV2dgYclohIammZi+FpB5qTnk8EtvVX0N2vd/dWd2+tr9etoUUku4SiUS1zMQx3AO+Jj0J6FbDf3benMR4RkUBk0zyFQVdJPRlmdguwCKgzs3bgS0A+gLv/CLgbuBRYDxxC93wWkVEqm5a5CCwpuPtVQxx34GNBnV9EJFNomQsREUmINR9lx+U2O6IUEclioYhqCiIiEheOakiqiIgA7k4kquYjEREhNvIIUPORiIjERh4BWTMkNTuiFBHJUn01hWyZvKakICISoHAkVlPQMhciIkI4Gq8pqE9BRERCfTUFjT4SEZFwRDUFERGJ0+gjERFJSMxT0OgjERE52nyUHZfb7IhSRCRLhRLNR6opiIiMeeFE81F2XG6zI0oRkSzVN3lNNQURESEU1YJ4IiISl6gpqPkosxw4HOIHD64nEs/aIiIjIaTJa5npvlU7+fo9a/nGvWvTHYqIjCGJyWuqKWSWK+Y38c6zW/jhQxu4/bmt6Q5HRMYILXORocyML7/5VM6aUsM/3fY8y7bsS3dIIjIGaEG8JGa22MzWmtl6M/t8P8cnmdkDZva8mT1kZhODjKcgL4cfvutMGsoL+dAv2tjZdTjI04mIaOnsPmaWC/wAuASYA1xlZnOOKfYN4BfufjpwLfCVoOLpU1tWyE/e00p3b5gLv/lXrvzx4/zbnSu57Zl2DvaGgz69iIwxmqdw1FnAend/0d2PAL8CLjumzBzggfj2g/0cD8Tsxgpu+vuzuXz+BEKRKL96agv/+JtlXPaDR1m388Aryq/a1qVahYickFCWzWjOC/C9m4AtSc/bgbOPKbMMeCvwHeAKoNzMat19d3IhM7sauBqgpaUlJcGd2VLNmS3VAESizsPrOvnMb5bxlu8/ylf+Zi6Xz2/iqY17+N5f1vHwul00VRVz+8deQ315YUrOLyJjQ1hrHyX09wkcO0ngM8D5ZvYccD6wFXhFG467X+/ure7eWl9fn/JAc3OMRac08MdPnMfcpko++eulvP5bsaal1du7+Oiiaew+2MvV/9fG4VAk5ecXkdErUVPIklVSg6wptAPNSc8nAtuSC7j7NuBvAMysDHiru+8PMKZBjaso4uYPnc037n2BP6/YzhffNIerzmqhuCCX0ydW8uGbnuWzv32e775jHmaZm/XDkShdh8PsO3SEfT0hJlQWM76yKN1hiYxJiSGpWXI/hSCTwtPADDObQqwG8A7gnckFzKwO2OPuUeCfgRsCjGdY8nJz+Pwls/j8JbNetn/xaY18bvEsvvbnNUypK+XTF81M2Tndnd0Hj3CoN0JzTfFxJ5wj4SjPbt7Lw+s6eWTdLpZv3U/yxO38XONvFzRzzQXTaaoqTlncIjK0vuaj3LGeFNw9bGbXAPcAucAN7r7SzK4F2tz9DmAR8BUzc2AJ8LGg4kmFD58/lY27uvnuA+v43bPtnNFcxbyJVSycUsMZEysHvZhv2XOIW9u2sKGzO7EvHHG27e9h065DHIiPfGqsLOK8GXWcO6OehvJCug+HOdAbors3QiQSJeoQdWffoRAbOrtZ39HNS7sPEoo4uTnGvOYqPnz+NBrKC6kqKaCiOI8H13Ty66e38NtntvC21mbmN1dRWZxPZXE+deWFTK4tzZovrEi2CUWc/FzL6NaFZOaeXWsBtba2eltbW9rOfyQc5ZdPbqLtpb0s3bKPrft6AJhUW8Ll85q4fH4T4yoK2XcoxP6eEOs6urn16S08sn4XOQZT6krJiX85zGB8ZTFTakuYXFdKXm4Oj2/YxSPrdtF1ePDhsbk5xqSaEqY1lDG1vpQzW6o5Z1otFUX5/Zbftq+HHzy4nlvbtiTaOPsU5uUwa3w5sxsrOLOlmnNn1DFBNQqRlPj3u1bxyyc3s/q6xWmNw8yecffWIcspKZyczgO9PLS2g9uXbuWxDbvp7+NsqirmytZmrlw4kcbKoS+2kaizYut+unvDlBflUV6UT2lhLvk5OeSYYTlQnJ97Qh1XB3vD7O4+QtfhWNLasf8wq7d3sXpHF6u2dbH3UAiAqXWlnDejjnefM4npDeXHfR4RifnyHSu57dl2ln/5DWmNY7hJIcg+hTGhvryQt7U287bWZnbsP8yfVmynNxxNNM80lBcyv6X6uJpncnOMM5qrAom3tDCP0sL+/9vdnXUd3Ty8bhePrOvk121b+MUTm7j0tEauuWA6sxsr2HfoCCu2drFmRxeVxfnMbqxgxrgyCvNyA4lXJNuFItGsGXkESgopNb6yiPe/Zkq6wzhhZsbMceXMHFfOB8+dwp6DR/jfR17k549t4o/LtzO+oogd/Uziy8sxWmpLqCjKp6Qgl5KCPPJzjXDUiUYdM5g1voKzptSwYFL1gElJZDQKRzxrRh6BkoIMoqa0gM++YRZXnzeNXzz+Eus6upkzoYLTJlQyu7GcfT2hWNPT9i427jpId2+EQ71h9hw8RNSdHDNyc4xI1HlwbSfff3A9eTnG1PpSSgryKMzLoTA/l7LCXCqL86koyqeyJJ+6skIaygupLy8kPzeHnV2H2bH/MJ3dvZQV5jGhspjGqiIayoswi3W843DoSIR9PSH2HjrCwd4wCyZVD6u5TiRIoahqCjLKVJbk8/ELZ7xif21ZIdPqy3jT6ROGfI/u3jDPbtrLkxt388LObnrDUXpDEfb3hNi2r4f9PbE+jiPhaMriNoOFk2t4yxkTuHjOOOrLC7NmBIiMHuGIZ81sZlBSkBFSVpjHa2fW89qZg89I7zkSofNAL53dh+k80MuRiDO+oohxFYU0lBdxoDfEtn2H2b6vh13dvbEXmWHEOt+rS/OpLC6gIDeHv6zp4A/LtvKvt6/gX29fQUlBLk1VxUysLqa5poSWmhKaa0qYXFvKzHFlShgSiHA0quYjkRNVXJBLS20JLbUlAx5vKC9i3jA64udOrOQTF05n5bYunty4h617e2jfe4j2vT20bdrLgaRhv42VRbzh1PEsPm08CyfXaN6GpExsnoKaj0QygplxWlMlpzVVvuLY/kMhNu85xOodXdy3aie3PLWZGx97idrSAi6ZO543zp3AWVOUIOTkhCNRNR+JZIPKknzmllQyd2IlV7Y2c+hImIfWdnL38u3c9sxWbnpiM/XlhbxmWi0LJlUzv6WaWePLycuiv/ok/cJRz5r7M4OSgkhCSUEel85t5NK5jRw6EubBNZ3cvWI7j27Yze1LY2s5lhXmcd6MOi6Y1cCiUxq0lLoMKTZPQTUFkaxWUpDHG09v5I2nN+LutO/t4dnNe3nixd38ZU0Hf1qxAzM4e0oN73v1ZF4/e5xqENKviGoKIqOLmdEcH6l02bwm3J1V27u4f1UHt7Zt4cM3PUtTVTHvPmcSV8xvYlyFlimXo0IRpyhfNQWRUcvMOHVCJadOqOSaC6Zz/+qd3PjoS3z1T2v42p/XsKClmkvnNrL4tPFaWFAIa/KayNiRm2O84dTxvOHU8azv6Obu5du5e/l2rr1rFdfetYq5TZVcNGccF80ZxynjysnRSKYxR8tciIxR0xvK+MSFM/jEhTPY0NnNPSt3cN+qnXz7/hf41n0vkJtjVJcUUFtaQG1ZAbVlhdSVFVBXVsiUulJeNbWWmtKCdP8YkmJaEE9EmFZfxkcXTeeji6bTceAwD63pZNOeg+w5eITd3UfYffAIy9v3sav7CN3xGyyZwZzGCs6dXsfi08Yzr7lKs6xHgXBUy1yISJKG8iKuXNg84PGeIxFW7+ji0XW7eGT9Lm54dCM/XvIiU+tLeeuZE7lifpP6JrJYrPlINQURGabiglzObKnmzJZqPn7hDA4cDiUm0H39nrV8/Z61zG6s4PyZ9Zw/s54Fk6opyMuei8xYp3kKInJSyovyefvCFt6+sIVNuw9y9/Id/PWFDn768Iv86K8bKC/K44JZDVw8Zzznn1JPme5PkdHUfCQiKTOptpSPLJrGRxZNo7s3zKPrd/HA6p3cv7qDPyzdRkFuDudMq+X1sxu4cPY4NTNloFAkquYjEUm9ssK8xPDXSNR5ZtNe7l25g/tX7+T//WEl/+8PK5nTWMGFsxu4YFYDZ0ys0hDYDBCOuJqPRCRYuTnGWVNqOGtKDV9442w2dB7k/tU7+cvqDn7w4Hq+95f11JYWxPohTqnnvBn1Gu6aJuFoNKuWQAk0KZjZYuA7QC7wU3f/6jHHW4CfA1XxMp9397uDjElktDEzpjeUMb2hjA+fP419h47w1xc6eWB1Bw+u7eB3z23FDE6fWMWFsxq4aM44Zo0v13DXEeDusfspZFGNLbCkYGa5wA+Ai4B24Gkzu8PdVyUV+1fgVnf/oZnNAe4GJgcVk8hYUFVSwGXzmrhsXhORqLN8637+uraTB9d28K37YhPpmqqKWXRKPQsn17BgUjUTq4uVJAIQiTqAagpxZwHr3f1FADP7FXAZkJwUHKiIb1cC2wKMR2TMyc0x5jVXMa+5in94/Qw6DhzmwTUd3LdqJ39Yuo1fPrkZgPEVRbx6Wi3nzazj3On1WhI8RcKJpJA9CTfIpNAEbEl63g6cfUyZLwP3mtnHgVLg9QHGIzLmNZQXJYa7RqLOmh1dPLNpL09t3MNDL3Tyu+e2AjC7sYJzptZy9tQazp5SQ1WJ+iP6uDvvueEpunvDnNlSzfyWKk5vqqKmrICS/NyXde6HIlEA8jX6CID+UqMf8/wq4EZ3/6aZnQP8n5md5u7Rl72R2dXA1QAtLS2BBCsy1uTmHF3t9T3nTCYadVZu62LJuk4eWbeLXz65iRse3YgZzGwoZ8Hkaha0VLNgUjWTakvGbHPTCzu7eXjdLibXlnDTE5v430c2Jo6ZxUaJ1ZYWMKGqmLqyWI1LNYWYdiB5bv9EXtk89EFgMYC7P25mRUAd0JFcyN2vB64HaG1tPTaxiEgK5OQYcyfGbk/6sddNpzccYdmW/Tz54m6e3rSXO5du4+Z4c1N9eSELJ1ezcHINCyfXjKnblC55oROAmz/0KurLC1m9vYvV27vY3xPiwOEwBw6H2dXdy7Z9PTy1cQ8FeTlMqStNc9TDF2RSeBqYYWZTgK3AO4B3HlNmM3AhcKOZzQaKgM4AYxKRYSrMy00MewWIRp11Hd20bdpD20uxJqe7l+8AoKQgl3nNVSyYVJ1oUhmtTU5L1nUyvaEsMVHw9IlVnD6xasDy7p5VtarAkoK7h83sGuAeYsNNb3D3lWZ2LdDm7ncA/wj8xMw+Raxp6X3urpqASAbKyTFOGV/OKePLedfZkwDYtq+Hp1/aw7Ob9vLM5r384MH1xPtWmVpXyryWKuY3VzGvuZpZjeVZtYR0f3qORHhy4x7+Lv7zD0c2JQQIeJ5CfM7B3cfs+2LS9irgNUHGICLBmVBVnBj+CnCwN8zz7ft5dvNentu8j7+u7eR3z8Y6rwvzcjh1QgWnT6zijOZKTp9YxZTa0qyadf3kxt0cCUd57cy6dIcSGM1oFpGUKS3M45xptZwzrRaINZ207+1h6ZZ9LN2yj+fb9/Hrp7dw42MvAVBemMecCRWc1lTJ3KZKTp1QwdT6MnIzNFE8vG4XBXk5nD2lNt2hBEZJQUQCY2Y015TQXFPCm8+YAEA4EmV9ZzfPb9nP8q2xx01PbKI3HBt0WJSfwynjKzh1QgVzGiuYM6GCWePLKSlI/+VqyQudnD2lhuKC3HSHEpj0f8oiMqbk5eYwa3wFs8ZXJG4+1JcoVm7tYuW2LlZu289dy46OdjKDKbWlzGosZ/b4CmY1xhLFSM7E3ravh3Ud3VzZOvANk0YDJQURSbvkRPHWBbF97s7WfT2s2tbFqviwzxVbuxIjniA2J2DmuDKm1JXRXFNMc3UJLbUlTK0rpbYstbOyH14XGxj52pn1KX3fTKOkICIZycyYWF3CxOoSLj51fGJ/d2+YtTsOxB9drNlxgEfX72LngcMkj12sLslPLBQ4rb4ssT2hsviEOreXvLCLcRWFzBxXloofL2MpKYhIVikrzGPBpNjM6mS94Qhb9/awac8hNnR0s6Gzm/Ud3fx5xQ72HgolyhXn5zKtoZTpiURRzoxxZUyqKRlwAl4k6jyyfhcXzRmXdUNMj5eSgoiMCoV5uUytL2NqfRmvO6XhZcd2d/eyvqOb9Z3dbOg4yPrObp5+aS+3Lz26yEJBbg4ttSVMqok1QU2uLWVqfSnTG8rYtu8w+3tCo77pCJQURGQMqC0rpLaskLOnvnwo6cHeMOs7ulnX0c26nQfYuOsgm/cc4rENu+kJRRLlcnMMMzhv+uidn9BHSUFExqzSwjzOaK7ijOaXL1Ph7nQe6I3VLDoPsqGjm/GVRVSPgbvXKSmIiBzDzGioKKKhoohXTxv9tYNk2b0QiYiIpJSSgoiIJCgpiIhIgpKCiIgkKCmIiEiCkoKIiCQoKYiISIKSgoiIJFi23RLZzDqBfcD+Yw5VDrFvqO2+f+uAXScQWn/nH87xY/cP9vzYWJP3nUjcIxlz8nY6Pmt9P/T9GOx4Nn4/jidmgBnuXjlkJO6edQ/g+uPdN9R20r9tqYppOMeP3T/Y82NjPdm4RzLmdH/W+n7o+zHavh/HE/NwztH3yNbmoztPYN9Q2/29/mRjGs7xY/cP9ry/WE8m7pGMOXk7HZ+1vh/HT9+P4W9neszDOQeQhc1HQTOzNndvTXccxysb41bMIycb41bM6ZGtNYUgXZ/uAE5QNsatmEdONsatmNNANQUREUlQTUFERBJGdVIwsxvMrMPMVpzAaxeY2XIzW29m37WkG7Oa2cfNbK2ZrTSz/0pt1MHEbWZfNrOtZrY0/rg002NOOv4ZM3MzS+nC9gF9zteZ2fPxz/heM5uQBTF/3czWxOP+vZlVDfVeGRL32+K/g1EzS1k7/snEOsD7vdfM1sUf703aP+j3Pm1OZPhUtjyA1wJnAitO4LVPAecABvwJuCS+/3XA/UBh/HlDlsT9ZeAz2fRZx481A/cAm4C6TI8ZqEgq8wngR1kQ88VAXnz7a8DXsuH7AcwGTgEeAlrTHWs8jsnH7KsBXoz/Wx3frh7s50r3Y1TXFNxWwIsrAAAGDUlEQVR9CbAneZ+ZTTOzP5vZM2b2sJnNOvZ1ZtZI7Jf7cY/97/0CuDx++CPAV929N36OjiyJO1ABxvxt4J+AlHd+BRGzu3clFS1NddwBxXyvu4fjRZ8AJqYy5gDjXu3uazMl1gG8AbjP3fe4+17gPmBxOn9XhzKqk8IArgc+7u4LgM8A/9NPmSagPel5e3wfwEzgPDN70sz+amYLA432qJONG+CaeBPBDWZWHVyoCScVs5m9Bdjq7suCDjTJSX/OZvYfZrYFeBfwxQBj7ZOK70afDxD7q3UkpDLuoA0n1v40AVuSnvfFnyk/1yuMqXs0m1kZ8GrgN0nNd4X9Fe1nX99ffHnEqoGvAhYCt5rZ1Hi2D0SK4v4hcF38+XXAN4ldAAJxsjGbWQnwBWJNGyMiRZ8z7v4F4Atm9s/ANcCXUhzq0UBSFHP8vb4AhIFfpjLG/qQy7qANFquZvR/4h/i+6cDdZnYE2OjuVzBw/Gn/uQYyppICsZrRPnefl7zTzHKBZ+JP7yB2AU2uQk8EtsW324HfxZPAU2YWJbbeSWcmx+3uO5Ne9xPgrgDjhZOPeRowBVgW/0WcCDxrZme5+44MjflYNwN/JMCkQIpijneAvgm4MMg/cJKk+rMOUr+xArj7z4CfAZjZQ8D73P2lpCLtwKKk5xOJ9T20k/6fq3/p7tQI+gFMJqnDCHgMeFt824AzBnjd08RqA32dQJfG938YuDa+PZNY1dCyIO7GpDKfAn6V6TEfU+YlUtzRHNDnPCOpzMeB32ZBzIuBVUB9qmMdie8HKe5oPtFYGbijeSOx1oXq+HbNcL/36XikPYBAfzi4BdgOhIhl5g8S++vzz8Cy+C/CFwd4bSuwAtgAfJ+jE/0KgJvix54FLsiSuP8PWA48T+wvsMZMj/mYMi+R+tFHQXzOt8X3P09srZmmLIh5PbE/bpbGHykdMRVg3FfE36sX2Anck85Y6ScpxPd/IP4Zrwfefzzf+3Q8NKNZREQSxuLoIxERGYCSgoiIJCgpiIhIgpKCiIgkKCmIiEiCkoKMCmbWPcLn+6mZzUnRe0UstqrqCjO7c6hVSs2sysw+mopzixxLQ1JlVDCzbncvS+H75fnRReIClRy7mf0ceMHd/2OQ8pOBu9z9tJGIT8YW1RRk1DKzejO7zcyejj9eE99/lpk9ZmbPxf89Jb7/fWb2GzO7E7jXzBaZ2UNm9luL3W/gl31r3sf3t8a3u+OL4C0zsyfMbFx8/7T486fN7Nph1mYe5+iCgGVm9oCZPWuxdfcvi5f5KjAtXrv4erzsZ+Pned7M/i2FH6OMMUoKMpp9B/i2uy8E3gr8NL5/DfBad59PbBXT/0x6zTnAe939gvjz+cAngTnAVOA1/ZynFHjC3c8AlgAfSjr/d+LnH3Jdm/i6PxcSm3EOcBi4wt3PJHYfj2/Gk9LngQ3uPs/dP2tmFwMzgLOAecACM3vtUOcT6c9YWxBPxpbXA3OSVrasMLNyoBL4uZnNILYyZX7Sa+5z9+S19J9y93YAM1tKbE2cR445zxGOLjD4DHBRfPscjq6RfzPwjQHiLE5672eIrbkPsTVx/jN+gY8Sq0GM6+f1F8cfz8WflxFLEksGOJ/IgJQUZDTLAc5x957knWb2PeBBd78i3j7/UNLhg8e8R2/SdoT+f2dCfrRzbqAyg+lx93lmVkksuXwM+C6x+zHUAwvcPWRmLwFF/bzegK+4+4+P87wir6DmIxnN7iV2PwMAzKxv6eNKYGt8+30Bnv8JYs1WAO8YqrC77yd2C8/PmFk+sTg74gnhdcCkeNEDQHnSS+8BPhBf9x8zazKzhhT9DDLGKCnIaFFiZu1Jj08Tu8C2xjtfVxFb9hzgv4CvmNmjQG6AMX0S+LSZPQU0AvuHeoG7P0dsJc53ELvZTauZtRGrNayJl9kNPBofwvp1d7+XWPPU42a2HPgtL08aIsOmIakiAYnfPa7H3d3M3gFc5e6XDfU6kXRSn4JIcBYA34+PGNpHgLc/FUkV1RRERCRBfQoiIpKgpCAiIglKCiIikqCkICIiCUoKIiKSoKQgIiIJ/x8fkKPXXkVuYQAAAABJRU5ErkJggg==\n",
"text/plain": [
"<Figure size 432x288 with 1 Axes>"
]
},
"metadata": {
"needs_background": "light"
},
"output_type": "display_data"
}
],
"source": [
"learn.lr_find()\n",
"learn.recorder.plot()"
]
},
{
"cell_type": "code",
"execution_count": 86,
"metadata": {
"hidden": true
},
"outputs": [
{
"data": {
"text/html": [
"Total time: 00:14 <p><table style='width:300px; margin-bottom:10px'>\n",
" <tr>\n",
" <th>epoch</th>\n",
" <th>train_loss</th>\n",
" <th>valid_loss</th>\n",
" <th>accuracy_float</th>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <th>0.766329</th>\n",
" <th>0.692531</th>\n",
" <th>0.500000</th>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <th>0.721194</th>\n",
" <th>0.693430</th>\n",
" <th>0.500000</th>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <th>0.707417</th>\n",
" <th>0.693202</th>\n",
" <th>0.500000</th>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <th>0.701203</th>\n",
" <th>0.692725</th>\n",
" <th>0.500000</th>\n",
" </tr>\n",
" <tr>\n",
" <th>5</th>\n",
" <th>0.697864</th>\n",
" <th>0.692572</th>\n",
" <th>0.500000</th>\n",
" </tr>\n",
" <tr>\n",
" <th>6</th>\n",
" <th>0.695997</th>\n",
" <th>0.692562</th>\n",
" <th>0.500000</th>\n",
" </tr>\n",
" <tr>\n",
" <th>7</th>\n",
" <th>0.694890</th>\n",
" <th>0.692712</th>\n",
" <th>0.500000</th>\n",
" </tr>\n",
" <tr>\n",
" <th>8</th>\n",
" <th>0.694252</th>\n",
" <th>0.692631</th>\n",
" <th>0.500000</th>\n",
" </tr>\n",
" <tr>\n",
" <th>9</th>\n",
" <th>0.693816</th>\n",
" <th>0.692724</th>\n",
" <th>0.500000</th>\n",
" </tr>\n",
" <tr>\n",
" <th>10</th>\n",
" <th>0.693551</th>\n",
" <th>0.692713</th>\n",
" <th>0.500000</th>\n",
" </tr>\n",
"</table>\n"
],
"text/plain": [
"<IPython.core.display.HTML object>"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"learn.fit_one_cycle(10, max_lr=1e-2)"
]
},
{
"cell_type": "code",
"execution_count": 87,
"metadata": {
"hidden": true
},
"outputs": [
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAZIAAAEKCAYAAAA4t9PUAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xt8VOWd+PHPdyb3CyE3BBIuAVHuBojgXVDXituKWi+kulVrl9ZWu6u9rO2vW11Xu24vanVt96ddofqzsFRrRYvFbsW6VVSCcqfIVQjXJEDIPZmZ7++PcxKGMJMMJJMZJt/36zWvOec5zznzPUPIN8/znPMcUVWMMcaYU+WJdQDGGGNOb5ZIjDHG9IglEmOMMT1iicQYY0yPWCIxxhjTI5ZIjDHG9EhUE4mIXCUim0Vkq4jcH2L7cBFZLiIfi8haEbnaLb9FRFYHvQIiUupue9s9Zvu2QdE8B2OMMV2TaN1HIiJe4BPgb4BKYCVQrqobg+o8A3ysqr8QkfHAUlUd2ek4k4BXVXWUu/428C1VrYhK4MYYY05KNFsk04GtqrpdVVuBRcCcTnUUGOAu5wB7QxynHFgYtSiNMcb0SFIUj10E7A5arwRmdKrzIPCmiNwDZAJXhDjOzZyYgOaLiB94GXhYQzSrRGQeMA8gMzNz2tixY0/lHIwxpt9atWpVtaoWdlcvmolEQpR1/oVfDixQ1Z+KyPnACyIyUVUDACIyA2hU1fVB+9yiqntEJBsnkfwd8PwJH6T6DPAMQFlZmVZUWE+YMcacDBH5NJJ60ezaqgSGBa0Xc2LX1Z3AYgBVXQGkAQVB2+fSqVtLVfe473XAr3G60IwxxsRINBPJSmCMiJSISApOUljSqc4u4HIAERmHk0iq3HUPcCPO2ApuWZKIFLjLycBngfUYY4yJmah1bamqT0TuBpYBXuA5Vd0gIg8BFaq6BPgm8KyI3IvT7XV70HjHJUClqm4POmwqsMxNIl7gf4Bno3UOxhhjuhe1y3/jiY2RGJMY2traqKyspLm5OdahJJS0tDSKi4tJTk4+rlxEVqlqWXf7R3Ow3RhjelVlZSXZ2dmMHDkSkVDX85iTparU1NRQWVlJSUnJKR3Dpkgxxpw2mpubyc/PtyTSi0SE/Pz8HrXyLJEYY04rlkR6X0+/U0skEfr92n0camiNdRjGGBN3LJFEoLaxja//+iNe+XhPrEMxxsRQTU0NpaWllJaWMnjwYIqKijrWW1sj+0PzjjvuYPPmzVGOtG/ZYHsEjja3AdDQ4otxJMaYWMrPz2f16tUAPPjgg2RlZfGtb33ruDqqiqri8YT+O33+/PlRj7OvWYskAvVuAmlq88c4EmNMPNq6dSsTJ07kq1/9KlOnTmXfvn3MmzePsrIyJkyYwEMPPdRR96KLLmL16tX4fD4GDhzI/fffzznnnMP555/PwYMHY3gWp85aJBFob4k0tVoiMSZe/MtrG9i492ivHnP80AE88LkJp7Tvxo0bmT9/Pv/5n/8JwKOPPkpeXh4+n49Zs2Zxww03MH78+OP2qa2t5dJLL+XRRx/lvvvu47nnnuP++094dFPcsxZJBNpbJM3WIjHGhDF69GjOPffcjvWFCxcydepUpk6dyqZNm9i4ceMJ+6SnpzN79mwApk2bxs6dO/sq3F5lLZIINLQ4CcS6toyJH6facoiWzMzMjuUtW7bws5/9jA8//JCBAwdy6623hrxPIyUlpWPZ6/Xi852e47DWIolAe9dWo3VtGWMicPToUbKzsxkwYAD79u1j2bJlsQ4pqqxFEgHr2jLGnIypU6cyfvx4Jk6cyKhRo7jwwgtjHVJU2aSNEXjqT1v46R8/oWxELi/ddUEvRmaMORmbNm1i3LhxsQ4jIYX6biOdtNG6tiJQ32qX/xpjTDiWSCLQYPeRGGNMWJZIItB+1VazDbYbY8wJLJFEoOOqLWuRGGPMCSyRRKCh1e5sN8aYcCyRRKDe7dpq8QUIBBL/KjdjjDkZlkgiEDzrb7PPWiXG9FczZ8484ebCJ554gq997Wth98nKygJg79693HDDDWGP290tCk888QSNjY0d61dffTVHjhyJNPSoimoiEZGrRGSziGwVkRNmIhOR4SKyXEQ+FpG1InK1Wz5SRJpEZLX7+s+gfaaJyDr3mE9KHzwuLTiRWPeWMf1XeXk5ixYtOq5s0aJFlJeXd7vv0KFDeemll075szsnkqVLlzJw4MBTPl5viloiEREv8DQwGxgPlIvI+E7Vvg8sVtUpwFzg50Hbtqlqqfv6alD5L4B5wBj3dVW0zqFdfYuPrFRnEgCbJsWY/uuGG27g9ddfp6WlBYCdO3eyd+9eSktLufzyy5k6dSqTJk3i1VdfPWHfnTt3MnHiRACampqYO3cukydP5uabb6apqamj3l133dUx/fwDDzwAwJNPPsnevXuZNWsWs2bNAmDkyJFUV1cD8NhjjzFx4kQmTpzIE0880fF548aN4+///u+ZMGECV1555XGf05uiOUXKdGCrqm4HEJFFwBwgeApMBQa4yznA3q4OKCJDgAGqusJdfx64Fnijd0MPClCVhhYfw/MyqG/xsXzzQRZ+uJvf3nUB6SneaH2sMaY7b9wP+9f17jEHT4LZj4bdnJ+fz/Tp0/nDH/7AnDlzWLRoETfffDPp6em88sorDBgwgOrqas477zyuueaasM9C/8UvfkFGRgZr165l7dq1TJ06tWPbI488Ql5eHn6/n8svv5y1a9fyjW98g8cee4zly5dTUFBw3LFWrVrF/Pnz+eCDD1BVZsyYwaWXXkpubi5btmxh4cKFPPvss9x00028/PLL3Hrrrb3zXQWJZtdWEbA7aL3SLQv2IHCriFQCS4F7graVuF1efxaRi4OOWdnNMQEQkXkiUiEiFVVVVad8Es1tAQIKBVmpAPzvlmo27TvKJwfqTvmYxpjTV3D3Vnu3lqryve99j8mTJ3PFFVewZ88eDhw4EPYY77zzTscv9MmTJzN58uSObYsXL2bq1KlMmTKFDRs2hJx+Pthf/vIXrrvuOjIzM8nKyuL666/nf//3fwEoKSmhtLQUiO409dFskYRKxZ0veSoHFqjqT0XkfOAFEZkI7AOGq2qNiEwDficiEyI8plOo+gzwDDhzbZ3qSbRP2FiY7SSS3YecPsrt1fWcMyw++ieN6Ze6aDlE07XXXst9993HRx99RFNTE1OnTmXBggVUVVWxatUqkpOTGTlyZMhp44OFaq3s2LGDn/zkJ6xcuZLc3Fxuv/32bo/T1XyJqampHcterzdqXVvRbJFUAsOC1os5sevqTmAxgNtdlQYUqGqLqta45auAbcBZ7jGLuzlmr2ofaG9vkVQedv4hth1siObHGmPiVFZWFjNnzuRLX/pSxyB7bW0tgwYNIjk5meXLl/Ppp592eYxLLrmEF198EYD169ezdu1awJl+PjMzk5ycHA4cOMAbbxzrtc/Ozqau7sSekEsuuYTf/e53NDY20tDQwCuvvMLFF198Qr1oimYiWQmMEZESEUnBGUxf0qnOLuByABEZh5NIqkSk0B2sR0RG4Qyqb1fVfUCdiJznXq31ReDEUa1eVN8pkbSvb6uqj+bHGmPiWHl5OWvWrGHu3LkA3HLLLVRUVFBWVsaLL77I2LFju9z/rrvuor6+nsmTJ/OjH/2I6dOnA3DOOecwZcoUJkyYwJe+9KXjpp+fN28es2fP7hhsbzd16lRuv/12pk+fzowZM/jyl7/MlClTevmMuxbVaeTdy3mfALzAc6r6iIg8BFSo6hL3Kq5ngSycLqrvqOqbIvJ54CHAB/iBB1T1NfeYZcACIB1nkP0e7eYkejKN/Afba7j5mff54XWT+N4rxwb2zj4jm2X3XnJKxzTGnBqbRj56ejKNfFQfbKWqS3EG0YPLfhC0vBE44Ykvqvoy8HKYY1YAE3s30vDap0cpyEo5rnxHdQP+gOL1RP02FmOMiWt2Z3s32qdHKcg+NmhVnJtOqz9A5eHGcLsZY0y/YYmkG0eb2gAYFJRIZpTkA7C9ygbcjelr/eGprn2tp9+pJZJuHDjajNcjDMlJJ9nrdGNdMDqftGQPv/zLdvw2iaMxfSYtLY2amhpLJr1IVampqSEtLe2UjxHVMZJEsK+2mUHZqXg9Qlqylza/j5LCTB66ZiLfeXktP/vTFu77m7NiHaYx/UJxcTGVlZX05CZjc6K0tDSKi4u7rxiGJZJu7K9tZnCOk6nTk73UNfsozEpl6vBcPtx5iKfe2kLZiFwuOaswxpEak/iSk5MpKSmJdRimE+va6sa+2iYGD3ATiTu3Vvs9Jf86ZyJnDcrm3v9eTYtNL2+M6acskXRBVdnXqUWSmeLtSCjpKV6+cukoahpa2X0oOlMPGGNMvLNE0oW6Fh+NrX6G5BxrkQRfBgwwIj8DgN12KbAxpp+yMZIu7K91JksbnJMOOF1a7c8laTcs100khyyRGGP6J0skXdjnJpL2Fsmj1086YarhwuxUUpM87KqxRGKM6Z8skXRhf60z7tE+2J6flXpCHRFheF6GdW0ZY/otGyPpQnuL5IwBXd+oMywvg1022G6M6acskXThwNFmCrJSSUnq+msanpfB7kONdretMaZfskTShX21zR3jI10Z5j7P/UhjWx9EZYwx8cXGSLpw/+yxNLR0f6PhsFznqq5dhxrJzUzpprYxxiQWSyRdGDt4QET1hrv3kuw61GjPcTfG9DvWtdULOu4lsSu3jDH9kCWSXpCZmkR+ZordlGiM6ZcskfQS5xJgSyTGmP4nqolERK4Skc0islVE7g+xfbiILBeRj0VkrYhc7Zb/jYisEpF17vtlQfu87R5ztfsaFM1ziNSwvAybuNEY0y9FLZGIiBd4GpgNjAfKRWR8p2rfBxar6hRgLvBzt7wa+JyqTgJuA17otN8tqlrqvg5G6xxOxvC8dPYcacLnD8Q6FGOM6VPRbJFMB7aq6nZVbQUWAXM61VGg/dKoHGAvgKp+rKp73fINQJqInDg/SRwZnpeBP6Add8MbY0x/Ec1EUgTsDlqvdMuCPQjcKiKVwFLgnhDH+Tzwsaq2BJXNd7u1/llEJNSHi8g8EakQkYq+eCynzQJsjOmvoplIQv2C7zyHSDmwQFWLgauBF0SkIyYRmQD8O/CVoH1ucbu8LnZffxfqw1X1GVUtU9WywsLoPwZ3WN6xe0mMMaY/iWYiqQSGBa0X43ZdBbkTWAygqiuANKAAQESKgVeAL6rqtvYdVHWP+14H/BqnCy3mhuSkkeQRSyTGmH4nmolkJTBGREpEJAVnMH1Jpzq7gMsBRGQcTiKpEpGBwO+B76rqu+2VRSRJRNoTTTLwWWB9FM8hYkleD0MHprP7sF25ZYzpX6KWSFTVB9wNLAM24VydtUFEHhKRa9xq3wT+XkTWAAuB29WZQvdu4Ezgnztd5psKLBORtcBqYA/wbLTO4WQNt3tJjDH9UFTn2lLVpTiD6MFlPwha3ghcGGK/h4GHwxx2Wm/G2JuG5WWwbMP+WIdhjDF9yu5s70XD8tI51NBKfYsv1qEYY0yfsUTSi4bn2SXAxpj+xxJJL2q/l8TGSYwx/Yklkl5kLRJjTH9kiaQXDcxIJjs1yRKJMaZfsUTSi0SEYrsE2BjTz1gi6WXD89JZvfsIVz7+Z9ZV1sY6HGOMiTpLJL1seF4Ghxvb+ORAPUvX74t1OMYYE3VRvSGxP/rbyUOpbWpjze5aVn16ONbhGGNM1FmLpJeVDhvIj244hwvOzGdt5RHa7EFXxpgEZ4kkSqaNyKW5LcCmfUdjHYoxxkSVJZIomTYiF8C6t4wxCc8SSZQMyUlnaE4aFTstkRhjEpslkig6b1Q+72+vwZkZ3xhjEpMlkig6f3Q+NQ2tfHKgPtahGGNM1FgiiaLzR+cD8O7W6hhHYowx0WOJJIqKczMYkZ/Be9tqYh2KMcZEjSWSKLtgdD4f2DiJMSaBWSKJsrPPyKauxUd1fWusQzHGmKiwRBJlRe7DrvYcaYpxJMYYEx1RTSQicpWIbBaRrSJyf4jtw0VkuYh8LCJrReTqoG3fdffbLCKfifSY8aZoYDoAew5bIjHGJKaoJRIR8QJPA7OB8UC5iIzvVO37wGJVnQLMBX7u7jveXZ8AXAX8XES8ER4zrhTluonkiD2jxBiTmKLZIpkObFXV7araCiwC5nSqo8AAdzkH2OsuzwEWqWqLqu4AtrrHi+SYcSUn3XlqorVIjDGJKpqJpAjYHbRe6ZYFexC4VUQqgaXAPd3sG8kxARCReSJSISIVVVVVp3oOvaIoN93GSIwxCSuaiURClHW+BrYcWKCqxcDVwAsi4uli30iO6RSqPqOqZapaVlhYeBJh976igelUWovEGJOgoplIKoFhQevFHOu6ancnsBhAVVcAaUBBF/tGcsy4Yy0SY0wii2YiWQmMEZESEUnBGTxf0qnOLuByABEZh5NIqtx6c0UkVURKgDHAhxEeM+4UDUynrtnH0ea2WIdijDG9LmqP2lVVn4jcDSwDvMBzqrpBRB4CKlR1CfBN4FkRuReni+p2dW4B3yAii4GNgA/4uqr6AUIdM1rn0Fs6rtw63MSAIckxjsYYY3pXVJ/ZrqpLcQbRg8t+ELS8EbgwzL6PAI9Ecsx4V+zelLijuoFxQwZ0U9sYY04vdmd7Hxg/ZAB5mSm8vjbuh3OMMeakWSLpAylJHuaUDuV/Nh7kcIPNuWWMSSyWSPrIjdOG0eoPsGSNtUqMMYnFEkkfGT90AJOKcljw3k78AZtS3hiTOCyR9KGvzxrNjuoGGysxxiQUSyR96Mrxgzn7jGx+vnxbrEMxxpheY4mkD3k8wrVTith8oI46uznRGJMgLJH0sZKCTAB2Vtu08saYxGCJpI+1J5IdNQ0xjsQYY3pHRIlEREaLSKq7PFNEviEiA6MbWmIake/e5V5licQYkxgibZG8DPhF5Ezgv4AS4NdRiyqBpSV7GZqTxk5rkRhjEkSkiSSgqj7gOuAJVb0XGBK9sBJbSWEmO6otkRhjEkOkiaRNRMqB24DX3TKbxvYUjczPtBaJMSZhRJpI7gDOBx5R1R3uM0L+X/TCSmwlBZkcaWyzebeMMQkhomnk3enevwEgIrlAtqo+Gs3AEtnIfOfKre3VDUzLTIlxNMYY0zORXrX1togMEJE8YA0wX0Qei25oieucYQNJ9gqv2QSOxpgEEGnXVo6qHgWuB+ar6jTgiuiFldgKs1P53DlDWVyxm9pGu8PdGHN6izSRJInIEOAmjg22mx748kWjaGz1s+C9nbEOxRhjeiTSRPIQznPSt6nqShEZBWyJXliJb/zQAcyeOJj/WL6FtZVHYh2OMcacsogSiar+RlUnq+pd7vp2Vf18d/uJyFUisllEtorI/SG2Py4iq93XJyJyxC2fFVS+WkSaReRad9sCEdkRtK305E45fvzwukkUZKXyj4tW2zNKjDGnrUgH24tF5BUROSgiB0TkZREp7mYfL/A0MBsYD5SLyPjgOqp6r6qWqmop8BTwW7d8eVD5ZUAj8GbQrt9u366qqyM92XiTm5nC964ex/bqBt75pCrW4RhjzCmJtGtrPrAEGAoUAa+5ZV2ZDmx1Wy+twCJgThf1y4GFIcpvAN5Q1YScLvczEwZTkJXCix/sinUoxhhzSiJNJIWqOl9Vfe5rAVDYzT5FwO6g9Uq37AQiMgJn/q63Qmyey4kJ5hERWet2jaWGOeY8EakQkYqqqvj9az8lycONZcN4668H2FfbFOtwjDHmpEWaSKpF5FYR8bqvW4GabvaREGXhBgLmAi+pqv+4AzhXik3CGehv911gLHAukAf8U6gDquozqlqmqmWFhd3lvNi6cVoxAYU/bjwQ61CMMeakRZpIvoRz6e9+YB9Od9Md3exTCQwLWi8Gwt2BF6rVgfuZr6hqx80WqrpPHS043WvTIzqDODaqMItRBZn8adPBWIdijDEnLdKrtnap6jWqWqiqg1T1WpybE7uyEhgjIiUikoKTLJZ0riQiZwO5wIoQxzhh3MRtpSAiAlwLrI/kHOLdZWMHsWJbDQ0tvliHYowxJ6UnT0i8r6uN7rTzd+N0S20CFqvqBhF5SESuCapaDixS1eO6vURkJE6L5s+dDv2iiKwD1gEFwMM9OIe4cdm4QbT6A/xla3WsQzHGmJMS0aSNYYQaAzmOqi4FlnYq+0Gn9QfD7LuTEIPzqnrZyQR5ujh3ZB7ZqUm8tekgn5kwONbhGGNMxHrSIrE76HpRstfDJWcX8tbmgwTs5kRjzGmky0QiInUicjTEqw7nnhLTi64YN4iquhbW762NdSjGGBOxLru2VDW7rwIxcOlZg/AI/GnTQSYXD4x1OMYYE5GedG2ZXpaXmcLU4bn8zya7n8QYc/qwRBJnrpo4mA17j/LJgbpYh2KMMRGxRBJnrptSRJJHWLxyd/eVjTEmDlgiiTP5WalcMe4MfvvxHlp9gViHY4wx3bJEEoduLCvmUEMr79rNicaY04Alkjh04ZkFpCV7+LM9o8QYcxqwRBKH0pK9nDcq3xKJMea0YIkkTs08q5Ad1Q18WtMQ61CMMaZLlkji1KVnDwKwVokxJu5ZIolTJQWZjCrIZNmG/bEOxRhjumSJJI5dPWkIK7bVUFPfEutQjDEmLEskcezqSUMIKCzbYFOmGGPilyWSODZuSDYlBZm8vjbcE4qNMSb2LJHEMRHh2tIi3ttWw/aq+liHY4wxIVkiiXPlM4aR4vWw4L2dsQ7FGGNCskQS5wZlp/G5c4by0qpKapvaYh2OMcacIKqJRESuEpHNIrJVRO4Psf1xEVntvj4RkSNB2/xB25YElZeIyAciskVE/ltEUqJ5DvHgtgtG0Njq57U1NlZijIk/UUskIuIFngZmA+OBchEZH1xHVe9V1VJVLQWeAn4btLmpfZuqXhNU/u/A46o6BjgM3Bmtc4gXk4pyGDs4m9+sqox1KMYYc4JotkimA1tVdbuqtgKLgDld1C8HFnZ1QBER4DLgJbfoV8C1vRBrXBMRbphWzJrdR9hiD7wyxsSZaCaSIiD46UyVbtkJRGQEUAK8FVScJiIVIvK+iLQni3zgiKr6ujtmoplT6jzwapE98MoYE2eimUgkRJmGqTsXeElV/UFlw1W1DPgC8ISIjD6ZY4rIPDcRVVRVnf7zVRVmpzJ70hAWr9xNXbMNuhtj4kc0E0klMCxovRgIN1o8l07dWqq6133fDrwNTAGqgYEiktTdMVX1GVUtU9WywsLCUz2HuHLnRSXUtfhYXGFjJcaY+BHNRLISGONeZZWCkyyWdK4kImcDucCKoLJcEUl1lwuAC4GNqqrAcuAGt+ptwKtRPIe4UjpsIGUjcpn/7g78gXCNO2OM6VtRSyTuOMbdwDJgE7BYVTeIyEMiEnwVVjmwyE0S7cYBFSKyBidxPKqqG91t/wTcJyJbccZM/ita5xCPvnxxCZWHm3jTZgU2xsQJOf73d2IqKyvTioqKWIfRK/wBZeZPlnNGdhov3XVBrMMxxiQwEVnljlV3ye5sP814PcIdF5RQ8elhVu48FOtwjDHGEsnpaO70YRRkpfLYm5/EOhRjjLFEcjrKSEniazNHs2J7De9trY51OMaYfs4SyWnqCzOGUzQwnYde34jPH4h1OMaYfswSyWkqLdnL9/92HH/dX8evP9wV63CMMf2YJZLT2FUTB3Phmfn8ZNlme667MSZmLJGcxkSEBz83gcZWPz9etjnW4Rhj+ilLJKe5MWdkc8eFI/nvit2s3n2k+x2MMaaXWSJJAN+4fAwFWak88Op6AjZ1ijGmj1kiSQDZacl8d/ZY1lTW8qINvBtj+pglkgRx3ZQiLh5TwCO/32gPvzLG9ClLJAlCRPjpjeeQmZLEPQs/prnN3/1OxhjTCyyRJJBBA9L4yU3n8Nf9dfxw6aZYh2OM6ScskSSYWWcP4ssXlfD8ik9tqnljTJ+wRJKAvn3V2UwsGsB3Xl7L3iNNsQ7HGJPgLJEkoNQkL0+VT6XNF+ArL6yiqdXGS4wx0WOJJEGVFGTyZPkU1u+t5Zu/WW33lxhjosYSSQK7fNwZ/J+rx7F03X4e/x97dokxJjqSYh2Aia47Lyphy4F6nnprK4MGpPF3542IdUjGmARjiSTBiQgPXzeRmoYWfvDqerJSvVw3pTjWYRljEkhUu7ZE5CoR2SwiW0Xk/hDbHxeR1e7rExE54paXisgKEdkgImtF5OagfRaIyI6g/UqjeQ6JINnr4T++MJXzSvL51m/W8of1dlmwMab3RC2RiIgXeBqYDYwHykVkfHAdVb1XVUtVtRR4Cvitu6kR+KKqTgCuAp4QkYFBu367fT9VXR2tc0gkaclenr2tjElFOdz96494Y92+WIdkjEkQ0WyRTAe2qup2VW0FFgFzuqhfDiwEUNVPVHWLu7wXOAgURjHWfiErNYnn75zO5OIc7l74MUvW7I11SMaYBBDNRFIE7A5ar3TLTiAiI4AS4K0Q26YDKcC2oOJH3C6vx0UkNcwx54lIhYhUVFVVneo5JJwBack8f+cMpo3I5R8Xfczilbu738kYY7oQzUQiIcrC3cwwF3hJVY+7c05EhgAvAHeoasAt/i4wFjgXyAP+KdQBVfUZVS1T1bLCQmvMBMtKTWLBHedy4ZkFfOfltfz87a2o2n0mxphTE81EUgkMC1ovBsL1pczF7dZqJyIDgN8D31fV99vLVXWfOlqA+ThdaOYkZaQk8V+3ncs15wzlR3/YzEOvb8TnD3S/ozHGdBLNy39XAmNEpATYg5MsvtC5koicDeQCK4LKUoBXgOdV9Ted6g9R1X0iIsC1wPronUJiS0ny8MTNpRRkpfLcuzvYvL+OJ8unUJAVsrfQGGNCilqLRFV9wN3AMmATsFhVN4jIQyJyTVDVcmCRHt+3chNwCXB7iMt8XxSRdcA6oAB4OFrn0B94PMIPPjeeH90wmVWfHuazT/6Fj3YdjnVYxpjTiPSHvvGysjKtqKiIdRhxb/2eWu56cRX7a5v5P1eP47YLRuI0/Iwx/ZGIrFLVsu7q2VxbpsPEohxev/tiLhlTyIOvbeSWX37ArprGWIdljIlzlkjMcXIykvnlbWX88LpJrK2s5TM08drEAAAR80lEQVRPvMOCd3fY7MHGmLAskZgTiAhfmDGcN++9hBmj8njwtY3c+H9XsGnf0ViHZoyJQ5ZITFhDB6Yz//Zzeeymc9heVc9nn/oLD7++kfoWX6xDM8bEEUskpksiwvVTi3nrmzO5qWwYv/zLDi7/6du8tmav3cRojAEskZgI5Wam8G/XT+K3X7uA/MxU7ln4Mdf+/D3e21Yd69CMMTFmicSclKnDc3ntnov48Q2TOXi0mS88+wG3PfchG/bWxjo0Y0yM2H0k5pQ1t/l5fsVOnl6+jdqmNi4fO4ivzTqTaSNyYx2aMaYXRHofiSUS02O1TW386r2dPPfuDo40tnH+qHzumjmai8cU2A2NxpzGLJEEsUTSNxpafCz8cBfPvLOdg3UtnDkoi9suGMn1U4rITLWnOhtzurFEEsQSSd9q8fl5fc0+Fry3k3V7aslOS+LzU4u5qWwY44cOiHV4xpgIWSIJYokkNlSVj3YdYcF7O1m2fj+t/gDjhwzghmnFzCkdSr7NMmxMXLNEEsQSSewdbmhlyZq9vLSqknV7akn2CrPOHsRnzxnKZWMHkWVdX8bEHUskQSyRxJe/7j/Ky6sqeXX1Xg7WtZCS5OHSswr520lDuGzcIAakJcc6RGMMlkiOY4kkPgUCyqpdh/n92n38Yf1+9h9tJsXrYcaoPGadPYhZYwdRUpAZ6zCN6bcskQSxRBL/AgHl491HeGPdPpZvPsi2qgYARuZnMPPsQVxyVgHTS/KtC8yYPmSJJIglktPP7kONvL35IMs3V/Hetmqa2wJ4PcLk4hwuGJ3PBaMLmDYil7Rkb6xDNSZhWSIJYonk9Nbc5uejXYdZsa2G97bVsGb3EXwBJcXroXT4QKYMG8jEohwmFuUwIi8Dj8dugjSmN1giCWKJJLHUt/hYufMQ72+r4f3tNWzaV0erPwBAdmoS44cOYJKbWCYW5VBSkInXkosxJy3SRBLVDmcRuQr4GeAFfqmqj3ba/jgwy13NAAap6kB3223A991tD6vqr9zyacACIB1YCvyD9odsaDpkpSY5g/FnDwKg1Rdgy8E61u+pZf2eo6zbU8sL739Ki89JLhkpXiYMHcCEoTkdCWZ0YSZJXpuz1JjeELUWiYh4gU+AvwEqgZVAuapuDFP/HmCKqn5JRPKACqAMUGAVME1VD4vIh8A/AO/jJJInVfWNrmKxFkn/4/MH2FpVz/o9R90EU8uGvUdpavMDkJbs4awzshmel8HwvAxGFWYxZlAWZw7KsulcjHHFQ4tkOrBVVbe7AS0C5gAhEwlQDjzgLn8G+KOqHnL3/SNwlYi8DQxQ1RVu+fPAtUCXicT0P0leD2MHD2DsYOdOegB/QNlRXc+6PbWsqzza0Yr5w/r9+IKeSV80MJ0xZ2RxZmEWowdlMbowi5H5GRRmp9oklMaEEM1EUgTsDlqvBGaEqigiI4AS4K0u9i1yX5UhykMdcx4wD2D48OEnH71JOF6PcOagbM4clM11U46V+/wBdh1qZMvBerYcqGPLwXo+OVDPim01Hd1jAKlJHoblZTAsN53heRkMHZhOQVYqhdmpDM5JY3BOGtmpSZZsTL8TzUQS6n9TuH60ucBLqurvZt+Ij6mqzwDPgNO11XWopj9L8noYVZjFqMIsPjNhcEd5IKDsOdLE1qp6dh9qZPehRnYdamT3oSYqdh6mLsSz6zNTvAzOSWNITrr77iSYwqxUCrJTnfesVNJT7LJlkziimUgqgWFB68XA3jB15wJf77TvzE77vu2WF0d4TGN6xOMRpwWSlxFye0OLj6q6FqrqW9hf28z+2mb21Taz/2gT+2qbeXdrNQeONhMI8WdMVmoSBVkpHS2aAjfBFGSndCwXZqWSk55MSpLHEo+Ja9FMJCuBMSJSAuzBSRZf6FxJRM4GcoEVQcXLgB+KSPuj9q4Evquqh0SkTkTOAz4Avgg8FcVzMCaszNQkMlOTGNnFNC4+f4Cq+haq61qprneSTlVdC9X1LVTXt1Jd18LWg/Ws2F7Dkca2sMfJSU8mLzOFjBSv+0oiPyuFoTnpnJGTRl5GCgMzkslJTyY3M4Wc9GQC7oU0mSlJdvmziaqoJRJV9YnI3ThJwQs8p6obROQhoEJVl7hVy4FFwZfwugnjX3GSEcBD7QPvwF0cu/z3DWyg3cSxJK+HITnpDMlJ77Zuqy9ATcPxSaeu2UeLz8/eI03UNvlobPHR2OrnSGMrWw7UcaCuBX+oJk8nackeslKTyEhxkl9WqtdJhClJZLrL7ds7trnb01O8pHg9JCcJyV4PyR5nOS3JS1qyl9Qkj90E2s/ZDYnGnMZ8/gA1Da0caWzjcKPzXtvkvLe3Qhpa/DS0+qhv8dHQ4nPWW3wdZY3uen2rj1P9dZCS5CE92Utasoe0ZK+TZFK8pCW568nt293kk+whLclL+nF1vB1125e9IqQmexiYkUxuRgrJdu9Pn4qHy3+NMVGW5PVwxoA0zhiQ1uNjqSpNbf6ORNOeeBrb/Pj8Sps/4L6UVl+AFp+f5rYAzW3+oFeAZp+fplY/zT5n25HGVprbAjQF1/MFaA26Ii7i8/U4raKUJPfl9eDxcEICbL9wTtzrc46tO2NfXhG8HueV1PHuIckrJHk9JLtlyV63zONxPjvJWU4OqufUccs8TnmKG2OyW57sxtq+3h57kteDVwQRJ0aPu9z+Lgie4PWg7R53+wn7uefbl1cPWiI5nQQC4Gt2Xm1NQcvuu68JfC1htrXv0+LUCy7vvI8GQDxBL6/7LsfKPN5OdcK8PCH2lRD7etrfk8CTDN6kY8uepOPXvcnOcTu2dV5P6mKb+955uSMWN04kxLoc+42UgESEjBSne6swu4dPr1R1fo7a3wlad/kVWvwBNxkFaPYFaPI5yy3uekCFpja/0+Jq8tHc5qfV5yS0Vn+AFl8AVfdyzvZ/Gj3uDVUNWoaAKv5A0EsVn1/xBZzk1tDqx+cP4A84ydMXUHy+AD6/H38ggC8QwO8POMvu1DzifoKg7uvY8kl/dSEvTj35eqMLM3nt7osgOcP5/xVFlkji0acr4LVvnJgI/K09O25SOiSnQZL7Sk6HpFSnPDUbsgY56+J1/9N38wr4j/1yCPhB29x1f6e62ql+523+Y9sDvmMvfxsEwg9Ax0znRBMy6XicX2xhk1Lwvj1JTj3ZV8P8og+VBLSbbUHHiJAXZ16k0NfEdaW9edH5+5NjZeG2E3QeEHROYd67Ct4LxPMz2OqAfwO+vhIKz4rqR1kiiUep2TBovPvLPswv/qRUdz0tsnrelNP3r+n2BNOeWAJ+d9nXad0t8/uCtrWvd7Mt3F/PYX95hvtlSgR1Ox37VPVofNO9LeuEhNddi8z9y7bLJNpVQm3/Ra9B8Xf+5U747WH3iWS7q6sEFPI9XB0iP07EIvw3PZl/+8yCk/j8U2OJJB4Nngg3/SrWUcQPj9d5JfWwu8UYExV2CYQxxpgesURijDGmRyyRGGOM6RFLJMYYY3rEEokxxpgesURijDGmRyyRGGOM6RFLJMYYY3qkX8z+KyJVwKenuHsBUN2L4Zyu7Htw2PdwjH0XjkT+HkaoamF3lfpFIukJEamIZBrlRGffg8O+h2Psu3DY92BdW8YYY3rIEokxxpgesUTSvWdiHUCcsO/BYd/DMfZdOPr992BjJMYYY3rEWiTGGGN6xBKJMcaYHrFE0gURuUpENovIVhG5P9bx9CUR2Ski60RktYhUuGV5IvJHEdnivufGOs7eJiLPichBEVkfVBbyvMXxpPvzsVZEpsYu8t4V5nt4UET2uD8Tq0Xk6qBt33W/h80i8pnYRN37RGSYiCwXkU0iskFE/sEt73c/E12xRBKGiHiBp4HZwHigXETGxzaqPjdLVUuDrpG/H/iTqo4B/uSuJ5oFwFWdysKd92xgjPuaB/yij2LsCws48XsAeNz9mShV1aUA7v+LucAEd5+fu/9/EoEP+KaqjgPOA77unm9//JkIyxJJeNOBraq6XVVbgUXAnBjHFGtzgPZnAP8KuDaGsUSFqr4DHOpUHO685wDPq+N9YKCIDOmbSKMrzPcQzhxgkaq2qOoOYCvO/5/TnqruU9WP3OU6YBNQRD/8meiKJZLwioDdQeuVbll/ocCbIrJKROa5ZWeo6j5w/oMBg2IWXd8Kd9798WfkbrfL5rmgrs1+8T2IyEhgCvAB9jNxHEsk4UmIsv50rfSFqjoVp6n+dRG5JNYBxaH+9jPyC2A0UArsA37qlif89yAiWcDLwD+q6tGuqoYoS6jvIhRLJOFVAsOC1ouBvTGKpc+p6l73/SDwCk5XxYH2Zrr7fjB2EfapcOfdr35GVPWAqvpVNQA8y7Huq4T+HkQkGSeJvKiqv3WL7WciiCWS8FYCY0SkRERScAYTl8Q4pj4hIpkikt2+DFwJrMc5/9vcarcBr8Ymwj4X7ryXAF90r9Q5D6ht7+5IRJ36+q/D+ZkA53uYKyKpIlKCM9D8YV/HFw0iIsB/AZtU9bGgTfYzESQp1gHEK1X1icjdwDLACzynqhtiHFZfOQN4xfk/RBLwa1X9g4isBBaLyJ3ALuDGGMYYFSKyEJgJFIhIJfAA8Cihz3spcDXO4HIjcEefBxwlYb6HmSJSitNVsxP4CoCqbhCRxcBGnKucvq6q/ljEHQUXAn8HrBOR1W7Z9+iHPxNdsSlSjDHG9Ih1bRljjOkRSyTGGGN6xBKJMcaYHrFEYowxpkcskRhjjOkRSyQmIYiI352Rdo2IfCQiF3RTf6CIfC2C474tImXd1etPRGSBiNwQ6zhM/LBEYhJFkzsj7TnAd4F/66b+QKDbRBIrImL3eJnThiUSk4gGAIfBmSNJRP7ktlLWiUj7DM6PAqPdVsyP3brfceusEZFHg453o4h8KCKfiMjFbl2viPxYRFa6kxh+xS0fIiLvuMdd314/mDjPevl395gfisiZbvkCEXlMRJYD/+4+8+J37vHfF5HJQec03411rYh83i2/UkRWuOf6G3d+KETkURHZ6Nb9iVt2oxvfGhF5p5tzEhH5D/cYv6f/TNZpImR/9ZhEke7eeZwGDAEuc8ubgetU9aiIFADvi8gSnOdHTFTVUgARmY0zFfgMVW0UkbygYyep6nRxHuT0AHAFcCfO9Bfnikgq8K6IvAlcDyxT1UfEeSZHRph4j7rH/CLwBPBZt/ws4ApV9YvIU8DHqnqtiFwGPI8zYeI/u589yY091z2377v7NojIPwH3ich/4ExnMlZVVUQGup/zA+AzqronqCzcOU0BzgYm4cx6sBF4LqJ/FdMvWCIxiaIpKCmcDzwvIhNxZmP9oTizFwdwpvQ+I8T+VwDzVbURQFWDn8XRPlHfKmCku3wlMDlorCAHZ46plcBz4kz09ztVXU1oC4PeHw8q/03Q9CIXAZ9343lLRPJFJMeNdW77Dqp6WEQ+i/MAtnfdqW1SgBXAUZxk+ku3NfG6u9u7wAJ3apP28wt3TpcAC9249orIW2HOyfRTlkhMwlHVFe5f6IU48x4VAtNUtU1EduK0WjoTwk/33eK++zn2f0aAe1R12QkHcpLW3wIviMiPVfX5UGGGWW7oFFOo/ULFKsAfVbU8RDzTgctxks/dwGWq+lURmeHGudqdQyvkObktMZtLyYRlYyQm4YjIWJyJNmtw/qo+6CaRWcAIt1odkB2025vAl0Qkwz1GcNdWKMuAu9yWByJyljizJo9wP+9ZnFljwz2z++ag9xVh6rwD3OIefyZQ7T4L402chNB+vrnA+8CFQeMtGW5MWUCO+1jcf8TpGkNERqvqB6r6A6AaZ+rzkOfkxjHXHUMZAszq5rsx/Yy1SEyiaB8jAecv69vccYYXgddEpAJYDfwVQFVrRORdEVkPvKGq33b/Kq8QkVacWVy/18Xn/RKnm+sjcfqSqnDGWGYC3xaRNqAe+GKY/VNF5AOcP+ZOaEW4HgTmi8hanJlk26ctfxh42o3dD/yLqv5WRG4HFrrjG+CMmdQBr4pImvu93Otu+7GIjHHL/gSsAdaGOadXcMac1gGfAH/u4nsx/ZDN/mtMH3O718pUtTrWsRjTG6xryxhjTI9Yi8QYY0yPWIvEGGNMj1giMcYY0yOWSIwxxvSIJRJjjDE9YonEGGNMj/x/3EG5ncvbPvEAAAAASUVORK5CYII=\n",
"text/plain": [
"<Figure size 432x288 with 1 Axes>"
]
},
"metadata": {
"needs_background": "light"
},
"output_type": "display_data"
}
],
"source": [
"learn.recorder.plot_losses()"
]
},
{
"cell_type": "code",
"execution_count": 88,
"metadata": {
"hidden": true
},
"outputs": [
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAD8CAYAAACb4nSYAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAADmlJREFUeJzt3H+s3fVdx/Hna+26OGAZGxcDpXg70+jYRmAcK4mGoAmsM7HVsGk1cXSK5Y81uIV/mBpRiH84f5AYcUmJ3ZiZlolOL8hWgUhcVLCnhgGlMirCelcCd1RgcRnY8faPe4qH09uec2/vvafd5/lIbnq/n+/ne8/nfHN43u/93nNJVSFJasObxr0ASdLyMfqS1BCjL0kNMfqS1BCjL0kNMfqS1BCjL0kNMfqS1BCjL0kNWTnuBQw666yzanJyctzLkKRTyp49e75ZVRPD5p100Z+cnKTb7Y57GZJ0SknyzCjzvL0jSQ0x+pLUEKMvSQ0x+pLUEKMvSQ0x+pLUEKMvSQ0x+pLUEKMvSQ0x+pLUEKMvSQ0x+pLUEKMvSQ0x+pLUEKMvSQ0x+pLUEKMvSQ0x+pLUEKMvSQ0x+pLUEKMvSQ0x+pLUEKMvSQ0x+pLUkJGin2RDkieS7E9ywxz7tySZSfJw7+Oa3vhFSf41yd4kjyT5+cV+ApKk0a0cNiHJCuBW4ApgGtidZKqqHh+YekdVbRsY+zbwkap6Msm5wJ4ku6rqxcVYvCRpfka50l8P7K+qp6rqVWAnsGmUL15VX6uqJ3ufHwSeByYWulhJ0okZJfqrgQN929O9sUFX9W7h3JlkzeDOJOuBVcB/LmilkqQTNkr0M8dYDWzfBUxW1YXAfcDtb/gCyTnAnwMfrarXjnqAZGuSbpLuzMzMaCuXJM3bKNGfBvqv3M8DDvZPqKoXquqV3uZtwCVH9iV5G/D3wG9W1YNzPUBVba+qTlV1Jia8+yNJS2WU6O8G1iVZm2QVsBmY6p/Qu5I/YiOwrze+Cvgi8Lmq+qvFWbIkaaGGvnunqg4n2QbsAlYAO6pqb5KbgG5VTQHXJdkIHAYOAVt6h/8ccBnwziRHxrZU1cOL+zQkSaNI1eDt+fHqdDrV7XbHvQxJOqUk2VNVnWHz/ItcSWqI0Zekhhh9SWqI0Zekhhh9SWqI0Zekhhh9SWqI0Zekhhh9SWqI0Zekhhh9SWqI0Zekhhh9SWqI0Zekhhh9SWqI0Zekhhh9SWqI0Zekhhh9SWqI0Zekhhh9SWqI0Zekhhh9SWqI0Zekhhh9SWqI0Zekhhh9SWqI0Zekhhh9SWqI0Zekhhh9SWrISNFPsiHJE0n2J7lhjv1bkswkebj3cU3fvi8neTHJ3Yu5cEnS/K0cNiHJCuBW4ApgGtidZKqqHh+YekdVbZvjS/w+8Fbg2hNdrCTpxIxypb8e2F9VT1XVq8BOYNOoD1BV9wPfWuD6JEmLaJTorwYO9G1P98YGXZXkkSR3JlmzKKuTJC2qUaKfOcZqYPsuYLKqLgTuA26fzyKSbE3STdKdmZmZz6GSpHkYJfrTQP+V+3nAwf4JVfVCVb3S27wNuGQ+i6iq7VXVqarOxMTEfA6VJM3DKNHfDaxLsjbJKmAzMNU/Ick5fZsbgX2Lt0RJ0mIZ+u6dqjqcZBuwC1gB7KiqvUluArpVNQVcl2QjcBg4BGw5cnySrwA/DJyeZBr4laratfhPRZI0TKoGb8+PV6fTqW63O+5lSNIpJcmequoMm+df5EpSQ4y+JDXE6EtSQ4y+JDXE6EtSQ4y+JDXE6EtSQ4y+JDXE6EtSQ4y+JDXE6EtSQ4y+JDXE6EtSQ4y+JDXE6EtSQ4y+JDXE6EtSQ4y+JDXE6EtSQ4y+JDXE6EtSQ4y+JDXE6EtSQ4y+JDXE6EtSQ4y+JDXE6EtSQ4y+JDXE6EtSQ4y+JDVkpOgn2ZDkiST7k9wwx/4tSWaSPNz7uKZv39VJnux9XL2Yi5ckzc/KYROSrABuBa4ApoHdSaaq6vGBqXdU1baBY98B3Ah0gAL29I7970VZvSRpXka50l8P7K+qp6rqVWAnsGnEr/8B4N6qOtQL/b3AhoUtVZJ0ooZe6QOrgQN929PAj84x76oklwFfAz5RVQeOcezqBa51qN+5ay+PH3x5qb68JC2pC859Gzf+9HuW9DFGudLPHGM1sH0XMFlVFwL3AbfP41iSbE3STdKdmZkZYUmSpIUY5Up/GljTt30ecLB/QlW90Ld5G/B7fcdePnDsA4MPUFXbge0AnU7nqG8Ko1rq75CSdKob5Up/N7Auydokq4DNwFT/hCTn9G1uBPb1Pt8FXJnkzCRnAlf2xiRJYzD0Sr+qDifZxmysVwA7qmpvkpuAblVNAdcl2QgcBg4BW3rHHkpyM7PfOABuqqpDS/A8JEkjSNWC76YsiU6nU91ud9zLkKRTSpI9VdUZNs+/yJWkhhh9SWqI0Zekhhh9SWqI0Zekhhh9SWqI0Zekhhh9SWqI0Zekhhh9SWqI0Zekhhh9SWqI0Zekhhh9SWqI0Zekhhh9SWqI0Zekhhh9SWqI0Zekhhh9SWqI0Zekhhh9SWqI0Zekhhh9SWqI0Zekhhh9SWqI0Zekhhh9SWqI0Zekhhh9SWqI0ZekhowU/SQbkjyRZH+SG44z70NJKkmnt70qyWeSPJrkq0kuX6R1S5IWYOWwCUlWALcCVwDTwO4kU1X1+MC8M4DrgIf6hn8VoKrel+Rs4EtJfqSqXlusJyBJGt0oV/rrgf1V9VRVvQrsBDbNMe9m4FPAd/rGLgDuB6iq54EXgc4JrViStGCjRH81cKBve7o39rokFwNrqurugWO/CmxKsjLJWuASYM0JrFeSdAKG3t4BMsdYvb4zeRNwC7Bljnk7gHcDXeAZ4F+Aw0c9QLIV2Apw/vnnj7AkSdJCjHKlP80br87PAw72bZ8BvBd4IMnTwKXAVJJOVR2uqk9U1UVVtQl4O/Dk4ANU1faq6lRVZ2JiYqHPRZI0xCjR3w2sS7I2ySpgMzB1ZGdVvVRVZ1XVZFVNAg8CG6uqm+StSU4DSHIFcHjwF8CSpOUz9PZOVR1Osg3YBawAdlTV3iQ3Ad2qmjrO4WcDu5K8BnwD+KXFWLQkaWFGuadPVd0D3DMw9lvHmHt53+dPAz+08OVJkhaTf5ErSQ0x+pLUEKMvSQ0x+pLUEKMvSQ0x+pLUEKMvSQ0x+pLUEKMvSQ0x+pLUEKMvSQ0x+pLUEKMvSQ0x+pLUEKMvSQ0x+pLUEKMvSQ0x+pLUEKMvSQ0x+pLUEKMvSQ0x+pLUEKMvSQ0x+pLUEKMvSQ0x+pLUEKMvSQ0x+pLUEKMvSQ0x+pLUkJGin2RDkieS7E9yw3HmfShJJen0tt+c5PYkjybZl+STi7VwSdL8DY1+khXArcAHgQuAX0hywRzzzgCuAx7qG/4w8Jaqeh9wCXBtkskTX7YkaSFGudJfD+yvqqeq6lVgJ7Bpjnk3A58CvtM3VsBpSVYC3we8Crx8YkuWJC3UKNFfDRzo257ujb0uycXAmqq6e+DYO4H/AZ4Fvg78QVUdWvhyJUknYpToZ46xen1n8ibgFuD6OeatB74LnAusBa5P8q6jHiDZmqSbpDszMzPSwiVJ8zdK9KeBNX3b5wEH+7bPAN4LPJDkaeBSYKr3y9xfBL5cVf9bVc8D/wx0Bh+gqrZXVaeqOhMTEwt7JpKkoUaJ/m5gXZK1SVYBm4GpIzur6qWqOquqJqtqEngQ2FhVXWZv6fxkZp3G7DeE/1j0ZyFJGsnQ6FfVYWAbsAvYB3yhqvYmuSnJxiGH3wqcDjzG7DePz1TVIye4ZknSAqWqhs9aRp1Op7rd7riXIUmnlCR7quqo2+eD/ItcSWqI0Zekhhh9SWqI0Zekhhh9SWqI0Zekhhh9SWqI0Zekhhh9SWqI0Zekhhh9SWqI0Zekhhh9SWqI0Zekhhh9SWqI0Zekhhh9SWqI0Zekhhh9SWqI0Zekhhh9SWqI0Zekhhh9SWqI0Zekhhh9SWpIqmrca3iDJDPAM+NexxI5C/jmuBdxCvA8Dec5Gk1L5+kHqmpi2KSTLvrfy5J0q6oz7nWc7DxPw3mORuN5Opq3dySpIUZfkhpi9JfX9nEv4BTheRrOczQaz9MA7+lLUkO80pekhhj9JZTk6SSPJnk4Sbc39o4k9yZ5svfvmeNe53JLsiPJ80ke6xub87xk1h8n2Z/kkSTvH9/Kl88xztFvJ/lG7/X0cJKf6tv3yd45eiLJB8az6uWVZE2Sf0yyL8neJL/WG/e1dBxGf+n9RFVd1Pe2sRuA+6tqHXB/b7s1nwU2DIwd67x8EFjX+9gKfHqZ1jhun+XocwRwS+/1dFFV3QOQ5AJgM/Ce3jF/mmTFsq10fA4D11fVu4FLgY/1zoWvpeMw+stvE3B77/PbgZ8Z41rGoqr+CTg0MHys87IJ+FzNehB4e5Jzlmel43OMc3Qsm4CdVfVKVf0XsB9Yv2SLO0lU1bNV9e+9z78F7ANW42vpuIz+0irgH5LsSbK1N/b9VfUszL5ogbPHtrqTy7HOy2rgQN+86d5Yq7b1bk3s6Ls12Pw5SjIJXAw8hK+l4zL6S+vHqur9zP5Y+bEkl417QaegzDHW6lvOPg38IHAR8Czwh73xps9RktOBvwY+XlUvH2/qHGPNnKcjjP4SqqqDvX+fB77I7I/czx35kbL37/PjW+FJ5VjnZRpY0zfvPODgMq/tpFBVz1XVd6vqNeA2/v8WTrPnKMmbmQ3+56vqb3rDvpaOw+gvkSSnJTnjyOfAlcBjwBRwdW/a1cDfjWeFJ51jnZcp4CO9d15cCrx05Ef31gzcf/5ZZl9PMHuONid5S5K1zP6i8t+We33LLUmAPwP2VdUf9e3ytXQc/nHWEknyLmav7gFWAn9RVb+b5J3AF4Dzga8DH66qUX9h9z0hyV8ClzP7f0B8DrgR+FvmOC+9/7D/hNl3pXwb+GhVdcex7uV0jHN0ObO3dgp4Grj2SLSS/Abwy8y+o+XjVfWlZV/0Mkvy48BXgEeB13rDv87sfX1fS8dg9CWpId7ekaSGGH1JaojRl6SGGH1JaojRl6SGGH1JaojRl6SGGH1Jasj/AQ7Y/tMWNxljAAAAAElFTkSuQmCC\n",
"text/plain": [
"<Figure size 432x288 with 1 Axes>"
]
},
"metadata": {
"needs_background": "light"
},
"output_type": "display_data"
}
],
"source": [
"learn.recorder.plot_metrics()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"hidden": true
},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": 89,
"metadata": {
"hidden": true
},
"outputs": [
{
"data": {
"text/plain": [
"[tensor([[0.5050, 0.4949],\n",
" [0.5050, 0.4949],\n",
" [0.5050, 0.4949],\n",
" ...,\n",
" [0.5050, 0.4949],\n",
" [0.5050, 0.4949],\n",
" [0.5050, 0.4949]]), tensor([[0., 1.],\n",
" [0., 1.],\n",
" [0., 1.],\n",
" ...,\n",
" [1., 0.],\n",
" [1., 0.],\n",
" [1., 0.]])]"
]
},
"execution_count": 89,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"learn.get_preds(DatasetType.Train)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"hidden": true
},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": 90,
"metadata": {
"hidden": true
},
"outputs": [
{
"data": {
"text/plain": [
"(MultiCategory 0, MultiCategory 1)"
]
},
"execution_count": 90,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"data.train_ds[0][1], data.train_ds[3][1]"
]
},
{
"cell_type": "code",
"execution_count": 91,
"metadata": {
"hidden": true
},
"outputs": [],
"source": [
"#net(data.train_ds[0][0].data), net(data.train_ds[3][0].data)"
]
},
{
"cell_type": "code",
"execution_count": 92,
"metadata": {
"hidden": true
},
"outputs": [
{
"data": {
"text/plain": [
"(tensor([[ 0.0202, -0.0202]], grad_fn=<ThAddmmBackward>),\n",
" tensor([[ 0.0202, -0.0202]], grad_fn=<ThAddmmBackward>))"
]
},
"execution_count": 92,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"net(data.train_ds[0][0].data), net(data.train_ds[3][0].data)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"hidden": true
},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": 65,
"metadata": {
"hidden": true
},
"outputs": [],
"source": [
"preds, targs = learn.get_preds()"
]
},
{
"cell_type": "code",
"execution_count": 66,
"metadata": {
"hidden": true
},
"outputs": [],
"source": [
"#targs.view(-1,1)"
]
},
{
"cell_type": "code",
"execution_count": 67,
"metadata": {
"hidden": true
},
"outputs": [],
"source": [
"#preds"
]
},
{
"cell_type": "code",
"execution_count": 68,
"metadata": {
"hidden": true
},
"outputs": [
{
"data": {
"text/plain": [
"(tensor([0.4612, 0.5234]), tensor([1., 0.]))"
]
},
"execution_count": 68,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"i = 0\n",
"preds[i], targs[i]"
]
},
{
"cell_type": "code",
"execution_count": 69,
"metadata": {
"hidden": true
},
"outputs": [
{
"data": {
"text/plain": [
"((Image (1, 4, 50), MultiCategory 0), (Image (1, 4, 50), MultiCategory 1))"
]
},
"execution_count": 69,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"data.train_ds[0], data.train_ds[3]"
]
},
{
"cell_type": "code",
"execution_count": 70,
"metadata": {
"hidden": true
},
"outputs": [
{
"data": {
"text/plain": [
"((MultiCategory 0, tensor([1., 0.]), tensor([0.5242, 0.4852])),\n",
" (MultiCategory 0, tensor([1., 0.]), tensor([0.5242, 0.4852])))"
]
},
"execution_count": 70,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"learn.predict(data.train_ds[0][0]), learn.predict(data.train_ds[3][0])"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"hidden": true
},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": 125,
"metadata": {
"hidden": true
},
"outputs": [],
"source": [
"#learn.save('v6')"
]
},
{
"cell_type": "code",
"execution_count": 126,
"metadata": {
"hidden": true
},
"outputs": [],
"source": [
"#learn.load('v6')"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"hidden": true
},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": 71,
"metadata": {
"hidden": true
},
"outputs": [],
"source": [
"interpret = ClassificationInterpretation.from_learner(learn)"
]
},
{
"cell_type": "code",
"execution_count": 72,
"metadata": {
"hidden": true
},
"outputs": [
{
"ename": "RuntimeError",
"evalue": "Expected object of scalar type Float but got scalar type Long for argument #2 'other'",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mRuntimeError\u001b[0m Traceback (most recent call last)",
"\u001b[0;32m<ipython-input-72-7cdf6701acab>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0minterpret\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconfusion_matrix\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
"\u001b[0;32m~/Downloads/fastai/fastai/vision/learner.py\u001b[0m in \u001b[0;36mconfusion_matrix\u001b[0;34m(self, slice_size)\u001b[0m\n\u001b[1;32m 107\u001b[0m \u001b[0;34m\"Confusion matrix as an `np.ndarray`.\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 108\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0marange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mc\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 109\u001b[0;31m \u001b[0;32mif\u001b[0m \u001b[0mslice_size\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mcm\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpred_class\u001b[0m\u001b[0;34m==\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m&\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0my_true\u001b[0m\u001b[0;34m==\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msum\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 110\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 111\u001b[0m \u001b[0mcm\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mzeros\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdtype\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;31mRuntimeError\u001b[0m: Expected object of scalar type Float but got scalar type Long for argument #2 'other'"
]
}
],
"source": [
"interpret.confusion_matrix()"
]
},
{
"cell_type": "code",
"execution_count": 130,
"metadata": {
"hidden": true
},
"outputs": [],
"source": [
"#interpret.plot_confusion_matrix()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Basic PyTorch"
]
},
{
"cell_type": "code",
"execution_count": 26,
"metadata": {},
"outputs": [],
"source": [
"def seq2array(seq:str)->List:\n",
" \"Return `List` object with np.array created from sequence string `seq`.\"\n",
" \n",
" int_enc = LabelEncoder() # setup class instance to encode the four different bases to integer values (1D)\n",
" one_hot_enc = OneHotEncoder(categories=[range(4)]) # setup one hot encoder to encode integer encoded classes (1D) to one hot encoded array (4D)\n",
" \n",
" enc = int_enc.fit_transform(list(seq)) # bases (ACGT) to int (0,1,2,3)\n",
" enc = np.array(enc).reshape(-1,1) # reshape to get rank 2 array (from rank 1 array)\n",
" enc = one_hot_enc.fit_transform(enc) # encoded integer encoded bases to sparse matrix (sparse matrix dtype)\n",
" enc = enc.toarray().T # export sparse matrix to np array\n",
" \n",
" return enc"
]
},
{
"cell_type": "code",
"execution_count": 27,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"(array([[0., 0., 0., 1., ..., 0., 1., 0., 0.],\n",
" [1., 1., 0., 0., ..., 1., 0., 1., 1.],\n",
" [0., 0., 1., 0., ..., 0., 0., 0., 0.],\n",
" [0., 0., 0., 0., ..., 0., 0., 0., 0.]]), (4, 50))"
]
},
"execution_count": 27,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"# test open sequence image function\n",
"test_arr = seq2array('CCGAGGGCTATGGTTTGGAAGTTAGAACCCTGGGGCTTCTCGCGGACACC'); test_arr, test_arr.shape"
]
},
{
"cell_type": "code",
"execution_count": 28,
"metadata": {},
"outputs": [
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAA9CAYAAABWdClAAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAAbhJREFUeJzt3cFNwzAYhmESMQTizp0lEBMwJRMgluDeO2KKhgmcguXEzpfnubZq3Ch5Val/3WlZljsAssy9FwBAe+IOEEjcAQKJO0AgcQcIJO4AgcQdIJC4AwQSd4BA970O/DK/Nf1p7Mf3V/Gx18fnloeqMvr6trD2nktan4ua816z7rXXI0PtPdz63v+8vk9/eZ5P7gCBxB0gkLgDBBJ3gEDiDhBI3AECTb3+rOP68/TvA48yanbGsUZuqx2hLBnhWtriWi+9Zutxwj3v0y3GJEvmh4tRSICzEneAQOIOEEjcAQKJO0CgbtMyNRuH7fmN9Jo9pxhG33yr9TpGmHAYYUrlltbXdMmRz0XrJoxyLmwcBnBi4g4QSNwBAok7QCBxBwgk7gCBDjUKeQR7bYpUs4Zao4yAHXl8rcboG9S1HpEd/f2OwigkwImJO0AgcQcIJO4AgcQdIJC4AwTqNgoJwHZ8cgcIJO4AgcQdIJC4AwQSd4BA4g4QSNwBAok7QCBxBwgk7gCBxB0gkLgDBBJ3gEDiDhBI3AECiTtAIHEHCCTuAIHEHSCQuAMEEneAQOIOEEjcAQL9Al/6fGlF2Jm2AAAAAElFTkSuQmCC\n",
"text/plain": [
"<Figure size 432x288 with 1 Axes>"
]
},
"metadata": {
"needs_background": "light"
},
"output_type": "display_data"
}
],
"source": [
"plt.imshow(test_arr)\n",
"plt.axis('off');"
]
},
{
"cell_type": "code",
"execution_count": 29,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"tensor([[[0., 0., 0., 1., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0., 0., 0., 0.,\n",
" 0., 1., 1., 0., 0., 0., 1., 0., 1., 1., 0., 0., 0., 0., 0., 0., 0.,\n",
" 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 1., 0., 0.],\n",
" [1., 1., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
" 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 1., 1., 0., 0., 0., 0.,\n",
" 0., 1., 0., 0., 1., 0., 1., 0., 1., 0., 0., 0., 1., 0., 1., 1.],\n",
" [0., 0., 1., 0., 1., 1., 1., 0., 0., 0., 0., 1., 1., 0., 0., 0., 1.,\n",
" 1., 0., 0., 1., 0., 0., 0., 1., 0., 0., 0., 0., 0., 0., 1., 1., 1.,\n",
" 1., 0., 0., 0., 0., 0., 0., 1., 0., 1., 1., 0., 0., 0., 0., 0.],\n",
" [0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 1., 0., 0., 1., 1., 1., 0.,\n",
" 0., 0., 0., 0., 1., 1., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0.,\n",
" 0., 0., 1., 1., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.]]])"
]
},
"execution_count": 29,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"test_ten = tensor(test_arr).view(1,4,50).type(torch.float); test_ten"
]
},
{
"cell_type": "code",
"execution_count": 30,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"tensor([[ 0.9148, -0.4416]], grad_fn=<ThAddmmBackward>)"
]
},
"execution_count": 30,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"net(test_ten)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": 31,
"metadata": {},
"outputs": [],
"source": [
"#x = [seq2array(s) for s in seq_df['Sequences'].values]\n",
"x = [tensor(seq2array(s)).view(1,4,50).type(torch.float) for s in seq_df['Sequences'].values]"
]
},
{
"cell_type": "code",
"execution_count": 32,
"metadata": {},
"outputs": [],
"source": [
"x = torch.stack(x) # convert list to tensor"
]
},
{
"cell_type": "code",
"execution_count": 33,
"metadata": {},
"outputs": [],
"source": [
"y = tensor(targ).type(torch.float)"
]
},
{
"cell_type": "code",
"execution_count": 34,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"(2000, 2000)"
]
},
"execution_count": 34,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"len(x), len(y)"
]
},
{
"cell_type": "code",
"execution_count": 35,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"(tensor([[[0., 1., 0., 0., 1., 0., 0., 1., 0., 0., 0., 1., 1., 1., 0., 1., 1.,\n",
" 1., 0., 1., 0., 1., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 1., 0.,\n",
" 1., 0., 0., 0., 1., 0., 0., 1., 0., 0., 0., 1., 0., 0., 0., 0.],\n",
" [0., 0., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0.,\n",
" 0., 1., 0., 0., 0., 0., 0., 0., 1., 1., 1., 0., 0., 1., 1., 0., 1.,\n",
" 0., 0., 1., 0., 0., 0., 1., 0., 0., 0., 0., 0., 0., 1., 1., 0.],\n",
" [1., 0., 0., 0., 0., 1., 0., 0., 1., 1., 1., 0., 0., 0., 0., 0., 0.,\n",
" 0., 0., 0., 1., 0., 1., 1., 1., 0., 0., 0., 0., 1., 0., 0., 0., 0.,\n",
" 0., 0., 0., 0., 0., 1., 0., 0., 1., 1., 0., 0., 1., 0., 0., 0.],\n",
" [0., 0., 1., 0., 0., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
" 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
" 0., 1., 0., 1., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0., 1.]]]),\n",
" tensor([0., 1.]))"
]
},
"execution_count": 35,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"i = 2\n",
"x[i], y[i]"
]
},
{
"cell_type": "code",
"execution_count": 36,
"metadata": {},
"outputs": [],
"source": [
"#train_ds = TensorDataset(*map(tensor, (x[:1500],y[:1500])))\n",
"#valid_ds = TensorDataset(*map(tensor, (x[-500:],y[-500:])))"
]
},
{
"cell_type": "code",
"execution_count": 37,
"metadata": {},
"outputs": [],
"source": [
"train_ds = TensorDataset(x[:1500],y[:1500])\n",
"valid_ds = TensorDataset(x[-500:],y[-500:])"
]
},
{
"cell_type": "code",
"execution_count": 38,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"(1500, 500)"
]
},
"execution_count": 38,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"len(train_ds), len(valid_ds)"
]
},
{
"cell_type": "code",
"execution_count": 39,
"metadata": {},
"outputs": [],
"source": [
"bs = 64"
]
},
{
"cell_type": "code",
"execution_count": 40,
"metadata": {},
"outputs": [],
"source": [
"train_dl = DataLoader(train_ds, batch_size=bs, shuffle=True)\n",
"valid_dl = DataLoader(valid_ds, batch_size=bs*2, shuffle=False)"
]
},
{
"cell_type": "code",
"execution_count": 41,
"metadata": {},
"outputs": [],
"source": [
"opt = optim.SGD(net.parameters(), lr=1e-3, momentum=0.9)"
]
},
{
"cell_type": "code",
"execution_count": 42,
"metadata": {},
"outputs": [],
"source": [
"# From https://github.com/fastai/fastai_docs/blob/master/dev_nb/mnist_sample.py\n",
"\n",
"def simple_loss_batch(model, loss_func, xb, yb, opt=None):\n",
" loss = loss_func(model(xb), yb)\n",
"\n",
" if opt is not None:\n",
" loss.backward()\n",
" opt.step()\n",
" opt.zero_grad()\n",
"\n",
" return loss.item(), len(xb)\n",
"\n",
"def simple_fit(epochs, model, loss_func, opt, train_dl, valid_dl):\n",
" for epoch in range(epochs):\n",
" model.train()\n",
" for xb,yb in train_dl: simple_loss_batch(model, loss_func, xb, yb, opt)\n",
"\n",
" model.eval()\n",
" with torch.no_grad():\n",
" losses,nums = zip(*[simple_loss_batch(model, loss_func, xb, yb)\n",
" for xb,yb in valid_dl])\n",
" \n",
" # Accuracy metric:\n",
" thr = 0.5\n",
" comparison = [np.array(torch.sigmoid(net(xb)).detach().numpy()>thr).astype(int)==yb.detach().numpy().astype(int)\n",
" for xb,yb in valid_dl]\n",
" accuracy = np.concatenate(comparison).mean()\n",
" \n",
" val_loss = np.sum(np.multiply(losses,nums)) / np.sum(nums)\n",
"\n",
" #if epoch % 10 == 0:\n",
" print(epoch, val_loss, accuracy)"
]
},
{
"cell_type": "code",
"execution_count": 43,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"0 0.68692982006073 0.547\n",
"1 0.6797177696228027 0.563\n",
"2 0.6759244503974915 0.571\n",
"3 0.6710091686248779 0.608\n",
"4 0.6701397824287415 0.606\n",
"5 0.6652094225883484 0.624\n",
"6 0.6608625617027283 0.626\n",
"7 0.6558821315765381 0.634\n",
"8 0.6506783146858215 0.645\n",
"9 0.6421114640235901 0.657\n",
"10 0.6383359127044678 0.653\n",
"11 0.6326445651054382 0.66\n",
"12 0.622108877658844 0.673\n",
"13 0.6152496490478515 0.684\n",
"14 0.609443440914154 0.683\n",
"15 0.6009174609184265 0.698\n",
"16 0.5920117449760437 0.703\n",
"17 0.5872178387641906 0.712\n",
"18 0.5786009964942932 0.717\n",
"19 0.5699475564956665 0.719\n",
"20 0.5632805652618408 0.724\n",
"21 0.5551681814193725 0.732\n",
"22 0.5455421586036682 0.734\n",
"23 0.5378458318710327 0.738\n",
"24 0.5294094376564026 0.742\n",
"25 0.5234851956367492 0.745\n",
"26 0.5149962892532348 0.751\n",
"27 0.5067254395484925 0.755\n",
"28 0.5028578705787659 0.75\n",
"29 0.4942864866256714 0.768\n",
"30 0.4868652968406677 0.767\n",
"31 0.4809345436096191 0.766\n",
"32 0.47337263011932373 0.778\n",
"33 0.46588316559791565 0.782\n",
"34 0.45987958240509036 0.786\n",
"35 0.453939590215683 0.793\n",
"36 0.44865147876739503 0.793\n",
"37 0.4399886100292206 0.796\n",
"38 0.4356886956691742 0.806\n",
"39 0.4297665309906006 0.81\n",
"40 0.4235528767108917 0.812\n",
"41 0.4227601671218872 0.811\n",
"42 0.4155415644645691 0.818\n",
"43 0.41043953800201416 0.82\n",
"44 0.4036995759010315 0.821\n",
"45 0.39897701191902163 0.826\n",
"46 0.3943213264942169 0.831\n",
"47 0.3920146129131317 0.827\n",
"48 0.3889892637729645 0.834\n",
"49 0.38026661682128904 0.836\n"
]
}
],
"source": [
"simple_fit(50, net, F.binary_cross_entropy_with_logits, opt, train_dl, valid_dl)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": 44,
"metadata": {},
"outputs": [],
"source": [
"valid_dl = DataLoader(valid_ds, batch_size=bs*2, shuffle=False)"
]
},
{
"cell_type": "code",
"execution_count": 50,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"(torch.Size([128, 1, 4, 50]), torch.Size([128, 2]))"
]
},
"execution_count": 50,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"xbatch, ybatch = next(iter(valid_dl)); xbatch.shape, ybatch.shape"
]
},
{
"cell_type": "code",
"execution_count": 51,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"array([[0, 1],\n",
" [1, 0],\n",
" [1, 0],\n",
" [1, 0],\n",
" [1, 0],\n",
" [1, 0],\n",
" [1, 0],\n",
" [1, 0],\n",
" [0, 1],\n",
" [1, 0]])"
]
},
"execution_count": 51,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"ybatch[:10].detach().numpy().astype(int)"
]
},
{
"cell_type": "code",
"execution_count": 52,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"array([[0, 1],\n",
" [1, 0],\n",
" [1, 0],\n",
" [1, 0],\n",
" [1, 0],\n",
" [1, 0],\n",
" [0, 1],\n",
" [1, 0],\n",
" [0, 1],\n",
" [1, 0]])"
]
},
"execution_count": 52,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"np.array(torch.sigmoid(net(xbatch[:10])).detach().numpy()>0.5).astype(int)"
]
},
{
"cell_type": "code",
"execution_count": 53,
"metadata": {},
"outputs": [],
"source": [
"#np.array(torch.sigmoid(net(xbatch)).detach().numpy()>0.5).astype(int)==ybatch.detach().numpy().astype(int)"
]
},
{
"cell_type": "code",
"execution_count": 54,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"0.8359375"
]
},
"execution_count": 54,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"np.array(np.array(torch.sigmoid(net(xbatch)).detach().numpy()>0.5).astype(int)==ybatch.detach().numpy().astype(int)).mean()"
]
},
{
"cell_type": "code",
"execution_count": 55,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"0.8359375"
]
},
"execution_count": 55,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"np.concatenate(np.array(torch.sigmoid(net(xbatch)).detach().numpy()>0.5).astype(int)==ybatch.detach().numpy().astype(int)).mean()"
]
},
{
"cell_type": "code",
"execution_count": 56,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"0.836"
]
},
"execution_count": 56,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"np.concatenate([np.array(torch.sigmoid(net(xb)).detach().numpy()>0.5).astype(int)==yb.detach().numpy().astype(int) for xb,yb in valid_dl]).mean()\n"
]
},
{
"cell_type": "code",
"execution_count": 57,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"[0.8359375, 0.86328125, 0.83203125, 0.8103448275862069]"
]
},
"execution_count": 57,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"[np.concatenate(np.array(torch.sigmoid(net(xb)).detach().numpy()>0.5).astype(int)==yb.detach().numpy().astype(int)).mean() for xb,yb in valid_dl]\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": 58,
"metadata": {},
"outputs": [
{
"ename": "NameError",
"evalue": "name 'data' is not defined",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)",
"\u001b[0;32m<ipython-input-58-6ff02be695bd>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain_ds\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain_ds\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m3\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
"\u001b[0;31mNameError\u001b[0m: name 'data' is not defined"
]
}
],
"source": [
"data.train_ds[0][1], data.train_ds[3][1]"
]
},
{
"cell_type": "code",
"execution_count": 780,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"(tensor([[-1.4258, 1.5087]], grad_fn=<ThAddmmBackward>),\n",
" tensor([[ 1.2086, -1.2096]], grad_fn=<ThAddmmBackward>))"
]
},
"execution_count": 780,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"net(data.train_ds[0][0].data), net(data.train_ds[3][0].data)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Check fastai dataloader"
]
},
{
"cell_type": "code",
"execution_count": 777,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"MultiCategory 0"
]
},
"execution_count": 777,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"data.train_ds.y[0]"
]
},
{
"cell_type": "code",
"execution_count": 778,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"[tensor([[[[0., 1., 0., ..., 0., 0., 0.],\n",
" [0., 0., 0., ..., 0., 0., 0.],\n",
" [1., 0., 0., ..., 1., 0., 0.],\n",
" [0., 0., 1., ..., 0., 1., 1.]]],\n",
" \n",
" \n",
" [[[0., 0., 0., ..., 1., 0., 0.],\n",
" [1., 0., 0., ..., 0., 1., 0.],\n",
" [0., 0., 0., ..., 0., 0., 0.],\n",
" [0., 1., 1., ..., 0., 0., 1.]]],\n",
" \n",
" \n",
" [[[0., 1., 1., ..., 0., 0., 1.],\n",
" [0., 0., 0., ..., 1., 0., 0.],\n",
" [1., 0., 0., ..., 0., 0., 0.],\n",
" [0., 0., 0., ..., 0., 1., 0.]]],\n",
" \n",
" \n",
" ...,\n",
" \n",
" \n",
" [[[0., 0., 1., ..., 0., 0., 0.],\n",
" [0., 0., 0., ..., 0., 0., 0.],\n",
" [0., 0., 0., ..., 0., 1., 0.],\n",
" [1., 1., 0., ..., 1., 0., 1.]]],\n",
" \n",
" \n",
" [[[0., 0., 0., ..., 1., 1., 1.],\n",
" [0., 1., 1., ..., 0., 0., 0.],\n",
" [1., 0., 0., ..., 0., 0., 0.],\n",
" [0., 0., 0., ..., 0., 0., 0.]]],\n",
" \n",
" \n",
" [[[1., 1., 1., ..., 0., 0., 1.],\n",
" [0., 0., 0., ..., 1., 0., 0.],\n",
" [0., 0., 0., ..., 0., 0., 0.],\n",
" [0., 0., 0., ..., 0., 1., 0.]]]]), tensor([[1., 0.],\n",
" [0., 1.],\n",
" [1., 0.],\n",
" [0., 1.],\n",
" [0., 1.],\n",
" [0., 1.],\n",
" [0., 1.],\n",
" [1., 0.],\n",
" [0., 1.],\n",
" [1., 0.],\n",
" [1., 0.],\n",
" [0., 1.],\n",
" [0., 1.],\n",
" [1., 0.],\n",
" [1., 0.],\n",
" [1., 0.],\n",
" [0., 1.],\n",
" [0., 1.],\n",
" [0., 1.],\n",
" [1., 0.],\n",
" [1., 0.],\n",
" [1., 0.],\n",
" [0., 1.],\n",
" [1., 0.],\n",
" [0., 1.],\n",
" [0., 1.],\n",
" [0., 1.],\n",
" [0., 1.],\n",
" [0., 1.],\n",
" [1., 0.],\n",
" [0., 1.],\n",
" [1., 0.],\n",
" [0., 1.],\n",
" [0., 1.],\n",
" [0., 1.],\n",
" [0., 1.],\n",
" [1., 0.],\n",
" [0., 1.],\n",
" [1., 0.],\n",
" [1., 0.],\n",
" [0., 1.],\n",
" [1., 0.],\n",
" [1., 0.],\n",
" [1., 0.],\n",
" [0., 1.],\n",
" [1., 0.],\n",
" [0., 1.],\n",
" [0., 1.],\n",
" [0., 1.],\n",
" [1., 0.],\n",
" [1., 0.],\n",
" [0., 1.],\n",
" [1., 0.],\n",
" [0., 1.],\n",
" [1., 0.],\n",
" [1., 0.],\n",
" [0., 1.],\n",
" [1., 0.],\n",
" [0., 1.],\n",
" [0., 1.],\n",
" [1., 0.],\n",
" [0., 1.],\n",
" [0., 1.],\n",
" [0., 1.]])]"
]
},
"execution_count": 778,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"next(iter(data.train_dl))"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.6"
}
},
"nbformat": 4,
"nbformat_minor": 2
}