diff options
25 files changed, 987 insertions, 954 deletions
@@ -142,6 +142,10 @@ src/training/experiments/* !src/training/experiments/line_ctc_experiment.yml !src/training/experiments/default_config_emnist.yml !src/training/experiments/embedding_experiment.yml +<<<<<<< HEAD +src/text_recognizer/weights/*Transformer*.pt +======= +>>>>>>> 6cb08a110620ee09fe9d8a5d008197a801d025df src/wandb/* !src/wandb/settings diff --git a/src/notebooks/00-testing-stuff-out.ipynb b/src/notebooks/00-testing-stuff-out.ipynb index 62e549c..3686dcd 100644 --- a/src/notebooks/00-testing-stuff-out.ipynb +++ b/src/notebooks/00-testing-stuff-out.ipynb @@ -2,18 +2,9 @@ "cells": [ { "cell_type": "code", - "execution_count": 11, + "execution_count": 6, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "The autoreload extension is already loaded. To reload it, use:\n", - " %reload_ext autoreload\n" - ] - } - ], + "outputs": [], "source": [ "%load_ext autoreload\n", "%autoreload 2\n", @@ -50,7 +41,56 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "from pathlib import Path" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "Path(\"/home/akternurra/Documents/projects/quest-for-general-artifical-intelligence/projects/text-recognizer/src/training/experiments/TransformerModel_EmnistLinesDataset_CNNTransformer/1112_081300/model/best.pt\").exists()" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "False" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "Path(\"/home/akternurra/Documents/projects/quest-for-general-artifical-intelligence/projects/text-recognizer/src/training/experiments/TransformerModel_EmnistLinesDataset_CNNTransformer/1112_201649/model/best.pt\").exists()" + ] + }, + { + "cell_type": "code", + "execution_count": 18, "metadata": {}, "outputs": [], "source": [ @@ -63,13 +103,13 @@ " width_factor=1,\n", " dropout_rate= 0.2,\n", " activation= \"SELU\",\n", - " use_decoder= True,\n", + " use_decoder= False,\n", ")" ] }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 19, "metadata": {}, "outputs": [], "source": [ @@ -78,7 +118,7 @@ }, { "cell_type": "code", - "execution_count": 39, + "execution_count": 20, "metadata": {}, "outputs": [], "source": [ @@ -89,7 +129,7 @@ }, { "cell_type": "code", - "execution_count": 40, + "execution_count": 21, "metadata": {}, "outputs": [ { @@ -97,78 +137,10 @@ "text/plain": [ "Sequential(\n", " (0): SELU(inplace=True)\n", - " (1): Sequential(\n", - " (0): Conv2d(1, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", - " (1): Sequential(\n", - " (0): WideBlock(\n", - " (activation): SELU(inplace=True)\n", - " (blocks): Sequential(\n", - " (0): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", - " (1): SELU(inplace=True)\n", - " (2): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", - " (3): Dropout(p=0.2, inplace=False)\n", - " (4): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", - " (5): SELU(inplace=True)\n", - " (6): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", - " )\n", - " )\n", - " )\n", - " (2): Sequential(\n", - " (0): WideBlock(\n", - " (activation): SELU(inplace=True)\n", - " (blocks): Sequential(\n", - " (0): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", - " (1): SELU(inplace=True)\n", - " (2): Conv2d(32, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", - " (3): Dropout(p=0.2, inplace=False)\n", - " (4): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", - " (5): SELU(inplace=True)\n", - " (6): Conv2d(64, 64, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n", - " )\n", - " (shortcut): Sequential(\n", - " (0): Conv2d(32, 64, kernel_size=(1, 1), stride=(2, 2), bias=False)\n", - " )\n", - " )\n", - " )\n", - " (3): Sequential(\n", - " (0): WideBlock(\n", - " (activation): SELU(inplace=True)\n", - " (blocks): Sequential(\n", - " (0): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", - " (1): SELU(inplace=True)\n", - " (2): Conv2d(64, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", - " (3): Dropout(p=0.2, inplace=False)\n", - " (4): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", - " (5): SELU(inplace=True)\n", - " (6): Conv2d(128, 128, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n", - " )\n", - " (shortcut): Sequential(\n", - " (0): Conv2d(64, 128, kernel_size=(1, 1), stride=(2, 2), bias=False)\n", - " )\n", - " )\n", - " )\n", - " (4): Sequential(\n", - " (0): WideBlock(\n", - " (activation): SELU(inplace=True)\n", - " (blocks): Sequential(\n", - " (0): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", - " (1): SELU(inplace=True)\n", - " (2): Conv2d(128, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", - " (3): Dropout(p=0.2, inplace=False)\n", - " (4): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", - " (5): SELU(inplace=True)\n", - " (6): Conv2d(256, 256, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n", - " )\n", - " (shortcut): Sequential(\n", - " (0): Conv2d(128, 256, kernel_size=(1, 1), stride=(2, 2), bias=False)\n", - " )\n", - " )\n", - " )\n", - " )\n", ")" ] }, - "execution_count": 40, + "execution_count": 21, "metadata": {}, "output_type": "execute_result" } @@ -179,94 +151,302 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 86, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "==========================================================================================\n", - "Layer (type:depth-idx) Output Shape Param #\n", - "==========================================================================================\n", - "├─Sequential: 1-1 [-1, 256, 4, 119] --\n", - "| └─Conv2d: 2-1 [-1, 32, 28, 952] 288\n", - "| └─Sequential: 2-2 [-1, 32, 28, 952] --\n", - "| | └─WideBlock: 3-1 [-1, 32, 28, 952] 18,560\n", - "| └─Sequential: 2-3 [-1, 64, 14, 476] --\n", - "| | └─WideBlock: 3-2 [-1, 64, 14, 476] 57,536\n", - "| └─Sequential: 2-4 [-1, 128, 7, 238] --\n", - "| | └─WideBlock: 3-3 [-1, 128, 7, 238] 229,760\n", - "| └─Sequential: 2-5 [-1, 256, 4, 119] --\n", - "| | └─WideBlock: 3-4 [-1, 256, 4, 119] 918,272\n", - "├─Sequential: 1-2 [-1, 80] --\n", - "| └─BatchNorm2d: 2-6 [-1, 256, 4, 119] 512\n", - "├─SELU: 1-3 [-1, 256, 4, 119] --\n", - "├─Sequential: 1 [] --\n", - "| └─SELU: 2-7 [-1, 256, 4, 119] --\n", - "| └─Reduce: 2-8 [-1, 256] --\n", - "| └─Linear: 2-9 [-1, 80] 20,560\n", - "==========================================================================================\n", - "Total params: 1,245,488\n", - "Trainable params: 1,245,488\n", + "===============================================================================================\n", + "Layer (type:depth-idx) Output Shape Param #\n", + "===============================================================================================\n", + "├─Sequential: 1-1 [-1, 256, 4, 119] --\n", + "| └─Conv2d: 2-1 [-1, 32, 28, 952] 288\n", + "| └─Sequential: 2-2 [-1, 32, 28, 952] --\n", + "| | └─WideBlock: 3-1 [-1, 32, 28, 952] --\n", + "| | | └─Sequential: 4-1 [-1, 32, 28, 952] --\n", + "| | | | └─BatchNorm2d: 5-1 [-1, 32, 28, 952] 64\n", + "| | | └─SELU: 4-2 [-1, 32, 28, 952] --\n", + "| | | └─Sequential: 4 [] --\n", + "| | | | └─SELU: 5-2 [-1, 32, 28, 952] --\n", + "| | | | └─Conv2d: 5-3 [-1, 32, 28, 952] 9,216\n", + "| | | | └─Dropout: 5-4 [-1, 32, 28, 952] --\n", + "| | | | └─BatchNorm2d: 5-5 [-1, 32, 28, 952] 64\n", + "| | | └─SELU: 4-3 [-1, 32, 28, 952] --\n", + "| | | └─Sequential: 4 [] --\n", + "| | | | └─SELU: 5-6 [-1, 32, 28, 952] --\n", + "| | | | └─Conv2d: 5-7 [-1, 32, 28, 952] 9,216\n", + "| └─Sequential: 2-3 [-1, 64, 14, 476] --\n", + "| | └─WideBlock: 3-2 [-1, 64, 14, 476] --\n", + "| | | └─Sequential: 4-4 [-1, 64, 14, 476] --\n", + "| | | | └─Conv2d: 5-8 [-1, 64, 14, 476] 2,048\n", + "| | | └─Sequential: 4-5 [-1, 64, 14, 476] --\n", + "| | | | └─BatchNorm2d: 5-9 [-1, 32, 28, 952] 64\n", + "| | | └─SELU: 4-6 [-1, 32, 28, 952] --\n", + "| | | └─Sequential: 4 [] --\n", + "| | | | └─SELU: 5-10 [-1, 32, 28, 952] --\n", + "| | | | └─Conv2d: 5-11 [-1, 64, 28, 952] 18,432\n", + "| | | | └─Dropout: 5-12 [-1, 64, 28, 952] --\n", + "| | | | └─BatchNorm2d: 5-13 [-1, 64, 28, 952] 128\n", + "| | | └─SELU: 4-7 [-1, 64, 28, 952] --\n", + "| | | └─Sequential: 4 [] --\n", + "| | | | └─SELU: 5-14 [-1, 64, 28, 952] --\n", + "| | | | └─Conv2d: 5-15 [-1, 64, 14, 476] 36,864\n", + "| └─Sequential: 2-4 [-1, 128, 7, 238] --\n", + "| | └─WideBlock: 3-3 [-1, 128, 7, 238] --\n", + "| | | └─Sequential: 4-8 [-1, 128, 7, 238] --\n", + "| | | | └─Conv2d: 5-16 [-1, 128, 7, 238] 8,192\n", + "| | | └─Sequential: 4-9 [-1, 128, 7, 238] --\n", + "| | | | └─BatchNorm2d: 5-17 [-1, 64, 14, 476] 128\n", + "| | | └─SELU: 4-10 [-1, 64, 14, 476] --\n", + "| | | └─Sequential: 4 [] --\n", + "| | | | └─SELU: 5-18 [-1, 64, 14, 476] --\n", + "| | | | └─Conv2d: 5-19 [-1, 128, 14, 476] 73,728\n", + "| | | | └─Dropout: 5-20 [-1, 128, 14, 476] --\n", + "| | | | └─BatchNorm2d: 5-21 [-1, 128, 14, 476] 256\n", + "| | | └─SELU: 4-11 [-1, 128, 14, 476] --\n", + "| | | └─Sequential: 4 [] --\n", + "| | | | └─SELU: 5-22 [-1, 128, 14, 476] --\n", + "| | | | └─Conv2d: 5-23 [-1, 128, 7, 238] 147,456\n", + "| └─Sequential: 2-5 [-1, 256, 4, 119] --\n", + "| | └─WideBlock: 3-4 [-1, 256, 4, 119] --\n", + "| | | └─Sequential: 4-12 [-1, 256, 4, 119] --\n", + "| | | | └─Conv2d: 5-24 [-1, 256, 4, 119] 32,768\n", + "| | | └─Sequential: 4-13 [-1, 256, 4, 119] --\n", + "| | | | └─BatchNorm2d: 5-25 [-1, 128, 7, 238] 256\n", + "| | | └─SELU: 4-14 [-1, 128, 7, 238] --\n", + "| | | └─Sequential: 4 [] --\n", + "| | | | └─SELU: 5-26 [-1, 128, 7, 238] --\n", + "| | | | └─Conv2d: 5-27 [-1, 256, 7, 238] 294,912\n", + "| | | | └─Dropout: 5-28 [-1, 256, 7, 238] --\n", + "| | | | └─BatchNorm2d: 5-29 [-1, 256, 7, 238] 512\n", + "| | | └─SELU: 4-15 [-1, 256, 7, 238] --\n", + "| | | └─Sequential: 4 [] --\n", + "| | | | └─SELU: 5-30 [-1, 256, 7, 238] --\n", + "| | | | └─Conv2d: 5-31 [-1, 256, 4, 119] 589,824\n", + "===============================================================================================\n", + "Total params: 1,224,416\n", + "Trainable params: 1,224,416\n", "Non-trainable params: 0\n", - "Total mult-adds (M): 12.61\n", - "==========================================================================================\n", + "Total mult-adds (G): 2.79\n", + "===============================================================================================\n", "Input size (MB): 0.10\n", - "Forward/backward pass size (MB): 7.44\n", - "Params size (MB): 4.75\n", - "Estimated Total Size (MB): 12.29\n", - "==========================================================================================\n" + "Forward/backward pass size (MB): 101.10\n", + "Params size (MB): 4.67\n", + "Estimated Total Size (MB): 105.88\n", + "===============================================================================================\n" ] }, { "data": { "text/plain": [ - "==========================================================================================\n", - "Layer (type:depth-idx) Output Shape Param #\n", - "==========================================================================================\n", - "├─Sequential: 1-1 [-1, 256, 4, 119] --\n", - "| └─Conv2d: 2-1 [-1, 32, 28, 952] 288\n", - "| └─Sequential: 2-2 [-1, 32, 28, 952] --\n", - "| | └─WideBlock: 3-1 [-1, 32, 28, 952] 18,560\n", - "| └─Sequential: 2-3 [-1, 64, 14, 476] --\n", - "| | └─WideBlock: 3-2 [-1, 64, 14, 476] 57,536\n", - "| └─Sequential: 2-4 [-1, 128, 7, 238] --\n", - "| | └─WideBlock: 3-3 [-1, 128, 7, 238] 229,760\n", - "| └─Sequential: 2-5 [-1, 256, 4, 119] --\n", - "| | └─WideBlock: 3-4 [-1, 256, 4, 119] 918,272\n", - "├─Sequential: 1-2 [-1, 80] --\n", - "| └─BatchNorm2d: 2-6 [-1, 256, 4, 119] 512\n", - "├─SELU: 1-3 [-1, 256, 4, 119] --\n", - "├─Sequential: 1 [] --\n", - "| └─SELU: 2-7 [-1, 256, 4, 119] --\n", - "| └─Reduce: 2-8 [-1, 256] --\n", - "| └─Linear: 2-9 [-1, 80] 20,560\n", - "==========================================================================================\n", - "Total params: 1,245,488\n", - "Trainable params: 1,245,488\n", + "===============================================================================================\n", + "Layer (type:depth-idx) Output Shape Param #\n", + "===============================================================================================\n", + "├─Sequential: 1-1 [-1, 256, 4, 119] --\n", + "| └─Conv2d: 2-1 [-1, 32, 28, 952] 288\n", + "| └─Sequential: 2-2 [-1, 32, 28, 952] --\n", + "| | └─WideBlock: 3-1 [-1, 32, 28, 952] --\n", + "| | | └─Sequential: 4-1 [-1, 32, 28, 952] --\n", + "| | | | └─BatchNorm2d: 5-1 [-1, 32, 28, 952] 64\n", + "| | | └─SELU: 4-2 [-1, 32, 28, 952] --\n", + "| | | └─Sequential: 4 [] --\n", + "| | | | └─SELU: 5-2 [-1, 32, 28, 952] --\n", + "| | | | └─Conv2d: 5-3 [-1, 32, 28, 952] 9,216\n", + "| | | | └─Dropout: 5-4 [-1, 32, 28, 952] --\n", + "| | | | └─BatchNorm2d: 5-5 [-1, 32, 28, 952] 64\n", + "| | | └─SELU: 4-3 [-1, 32, 28, 952] --\n", + "| | | └─Sequential: 4 [] --\n", + "| | | | └─SELU: 5-6 [-1, 32, 28, 952] --\n", + "| | | | └─Conv2d: 5-7 [-1, 32, 28, 952] 9,216\n", + "| └─Sequential: 2-3 [-1, 64, 14, 476] --\n", + "| | └─WideBlock: 3-2 [-1, 64, 14, 476] --\n", + "| | | └─Sequential: 4-4 [-1, 64, 14, 476] --\n", + "| | | | └─Conv2d: 5-8 [-1, 64, 14, 476] 2,048\n", + "| | | └─Sequential: 4-5 [-1, 64, 14, 476] --\n", + "| | | | └─BatchNorm2d: 5-9 [-1, 32, 28, 952] 64\n", + "| | | └─SELU: 4-6 [-1, 32, 28, 952] --\n", + "| | | └─Sequential: 4 [] --\n", + "| | | | └─SELU: 5-10 [-1, 32, 28, 952] --\n", + "| | | | └─Conv2d: 5-11 [-1, 64, 28, 952] 18,432\n", + "| | | | └─Dropout: 5-12 [-1, 64, 28, 952] --\n", + "| | | | └─BatchNorm2d: 5-13 [-1, 64, 28, 952] 128\n", + "| | | └─SELU: 4-7 [-1, 64, 28, 952] --\n", + "| | | └─Sequential: 4 [] --\n", + "| | | | └─SELU: 5-14 [-1, 64, 28, 952] --\n", + "| | | | └─Conv2d: 5-15 [-1, 64, 14, 476] 36,864\n", + "| └─Sequential: 2-4 [-1, 128, 7, 238] --\n", + "| | └─WideBlock: 3-3 [-1, 128, 7, 238] --\n", + "| | | └─Sequential: 4-8 [-1, 128, 7, 238] --\n", + "| | | | └─Conv2d: 5-16 [-1, 128, 7, 238] 8,192\n", + "| | | └─Sequential: 4-9 [-1, 128, 7, 238] --\n", + "| | | | └─BatchNorm2d: 5-17 [-1, 64, 14, 476] 128\n", + "| | | └─SELU: 4-10 [-1, 64, 14, 476] --\n", + "| | | └─Sequential: 4 [] --\n", + "| | | | └─SELU: 5-18 [-1, 64, 14, 476] --\n", + "| | | | └─Conv2d: 5-19 [-1, 128, 14, 476] 73,728\n", + "| | | | └─Dropout: 5-20 [-1, 128, 14, 476] --\n", + "| | | | └─BatchNorm2d: 5-21 [-1, 128, 14, 476] 256\n", + "| | | └─SELU: 4-11 [-1, 128, 14, 476] --\n", + "| | | └─Sequential: 4 [] --\n", + "| | | | └─SELU: 5-22 [-1, 128, 14, 476] --\n", + "| | | | └─Conv2d: 5-23 [-1, 128, 7, 238] 147,456\n", + "| └─Sequential: 2-5 [-1, 256, 4, 119] --\n", + "| | └─WideBlock: 3-4 [-1, 256, 4, 119] --\n", + "| | | └─Sequential: 4-12 [-1, 256, 4, 119] --\n", + "| | | | └─Conv2d: 5-24 [-1, 256, 4, 119] 32,768\n", + "| | | └─Sequential: 4-13 [-1, 256, 4, 119] --\n", + "| | | | └─BatchNorm2d: 5-25 [-1, 128, 7, 238] 256\n", + "| | | └─SELU: 4-14 [-1, 128, 7, 238] --\n", + "| | | └─Sequential: 4 [] --\n", + "| | | | └─SELU: 5-26 [-1, 128, 7, 238] --\n", + "| | | | └─Conv2d: 5-27 [-1, 256, 7, 238] 294,912\n", + "| | | | └─Dropout: 5-28 [-1, 256, 7, 238] --\n", + "| | | | └─BatchNorm2d: 5-29 [-1, 256, 7, 238] 512\n", + "| | | └─SELU: 4-15 [-1, 256, 7, 238] --\n", + "| | | └─Sequential: 4 [] --\n", + "| | | | └─SELU: 5-30 [-1, 256, 7, 238] --\n", + "| | | | └─Conv2d: 5-31 [-1, 256, 4, 119] 589,824\n", + "===============================================================================================\n", + "Total params: 1,224,416\n", + "Trainable params: 1,224,416\n", "Non-trainable params: 0\n", - "Total mult-adds (M): 12.61\n", - "==========================================================================================\n", + "Total mult-adds (G): 2.79\n", + "===============================================================================================\n", "Input size (MB): 0.10\n", - "Forward/backward pass size (MB): 7.44\n", - "Params size (MB): 4.75\n", - "Estimated Total Size (MB): 12.29\n", - "==========================================================================================" + "Forward/backward pass size (MB): 101.10\n", + "Params size (MB): 4.67\n", + "Estimated Total Size (MB): 105.88\n", + "===============================================================================================" + ] + }, + "execution_count": 86, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "summary(wr, (1, 28, 952), device=\"cpu\", depth=7)" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": {}, + "outputs": [], + "source": [ + "a = torch.rand(1, 1, 28, 952)" + ] + }, + { + "cell_type": "code", + "execution_count": 38, + "metadata": {}, + "outputs": [], + "source": [ + "b = wr(a)" + ] + }, + { + "cell_type": "code", + "execution_count": 39, + "metadata": {}, + "outputs": [], + "source": [ + "from einops import rearrange" + ] + }, + { + "cell_type": "code", + "execution_count": 40, + "metadata": {}, + "outputs": [], + "source": [ + "b = rearrange(b, \"b c h w -> b w c h\")" + ] + }, + { + "cell_type": "code", + "execution_count": 41, + "metadata": {}, + "outputs": [], + "source": [ + "c = nn.AdaptiveAvgPool2d((None, 1))" + ] + }, + { + "cell_type": "code", + "execution_count": 42, + "metadata": {}, + "outputs": [], + "source": [ + "d = c(b)" + ] + }, + { + "cell_type": "code", + "execution_count": 43, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "torch.Size([1, 119, 256, 1])" ] }, - "execution_count": 8, + "execution_count": 43, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "summary(wr, (1, 28, 952), device=\"cpu\", depth=3)" + "d.shape" ] }, { "cell_type": "code", - "execution_count": 64, + "execution_count": 44, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "torch.Size([1, 119, 256])" + ] + }, + "execution_count": 44, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "d.squeeze(3).shape" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "torch.Size([1, 256, 4, 119])" + ] + }, + "execution_count": 26, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "b.shape" + ] + }, + { + "cell_type": "code", + "execution_count": 15, "metadata": {}, "outputs": [], "source": [ @@ -533,7 +713,7 @@ }, { "cell_type": "code", - "execution_count": 74, + "execution_count": 45, "metadata": {}, "outputs": [], "source": [ @@ -542,16 +722,36 @@ }, { "cell_type": "code", - "execution_count": 113, + "execution_count": 57, "metadata": {}, "outputs": [], "source": [ - "dnet = DenseNet(12, (6, 8, 10, 6), 1, 24, 80, 4, 0, False)" + "dnet = DenseNet(12, (6, 12, 10), 1, 24, 80, 4, 0, True)" + ] + }, + { + "cell_type": "code", + "execution_count": 58, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "27.0" + ] + }, + "execution_count": 58, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "216 / 8" ] }, { "cell_type": "code", - "execution_count": 114, + "execution_count": 59, "metadata": {}, "outputs": [ { @@ -561,31 +761,31 @@ "==========================================================================================\n", "Layer (type:depth-idx) Output Shape Param #\n", "==========================================================================================\n", - "├─Sequential: 1-1 [-1, 168, 3, 119] --\n", + "├─Sequential: 1-1 [-1, 80] --\n", "| └─Conv2d: 2-1 [-1, 24, 28, 952] 216\n", "| └─BatchNorm2d: 2-2 [-1, 24, 28, 952] 48\n", "| └─ReLU: 2-3 [-1, 24, 28, 952] --\n", "| └─_DenseBlock: 2-4 [-1, 96, 28, 952] --\n", "| └─_Transition: 2-5 [-1, 48, 14, 476] --\n", "| | └─Sequential: 3-1 [-1, 48, 14, 476] 4,800\n", - "| └─_DenseBlock: 2-6 [-1, 144, 14, 476] --\n", - "| └─_Transition: 2-7 [-1, 72, 7, 238] --\n", - "| | └─Sequential: 3-2 [-1, 72, 7, 238] 10,656\n", - "| └─_DenseBlock: 2-8 [-1, 192, 7, 238] --\n", - "| └─_Transition: 2-9 [-1, 96, 3, 119] --\n", - "| | └─Sequential: 3-3 [-1, 96, 3, 119] 18,816\n", - "| └─_DenseBlock: 2-10 [-1, 168, 3, 119] --\n", - "| └─ReLU: 2-11 [-1, 168, 3, 119] --\n", + "| └─_DenseBlock: 2-6 [-1, 192, 14, 476] --\n", + "| └─_Transition: 2-7 [-1, 96, 7, 238] --\n", + "| | └─Sequential: 3-2 [-1, 96, 7, 238] 18,816\n", + "| └─_DenseBlock: 2-8 [-1, 216, 7, 238] --\n", + "| └─ReLU: 2-9 [-1, 216, 7, 238] --\n", + "| └─AdaptiveAvgPool2d: 2-10 [-1, 216, 1, 1] --\n", + "| └─Rearrange: 2-11 [-1, 216] --\n", + "| └─Linear: 2-12 [-1, 80] 17,360\n", "==========================================================================================\n", - "Total params: 34,536\n", - "Trainable params: 34,536\n", + "Total params: 41,240\n", + "Trainable params: 41,240\n", "Non-trainable params: 0\n", - "Total mult-adds (M): 229.41\n", + "Total mult-adds (M): 252.43\n", "==========================================================================================\n", "Input size (MB): 0.10\n", "Forward/backward pass size (MB): 53.69\n", - "Params size (MB): 0.13\n", - "Estimated Total Size (MB): 53.92\n", + "Params size (MB): 0.16\n", + "Estimated Total Size (MB): 53.95\n", "==========================================================================================\n" ] }, @@ -595,35 +795,35 @@ "==========================================================================================\n", "Layer (type:depth-idx) Output Shape Param #\n", "==========================================================================================\n", - "├─Sequential: 1-1 [-1, 168, 3, 119] --\n", + "├─Sequential: 1-1 [-1, 80] --\n", "| └─Conv2d: 2-1 [-1, 24, 28, 952] 216\n", "| └─BatchNorm2d: 2-2 [-1, 24, 28, 952] 48\n", "| └─ReLU: 2-3 [-1, 24, 28, 952] --\n", "| └─_DenseBlock: 2-4 [-1, 96, 28, 952] --\n", "| └─_Transition: 2-5 [-1, 48, 14, 476] --\n", "| | └─Sequential: 3-1 [-1, 48, 14, 476] 4,800\n", - "| └─_DenseBlock: 2-6 [-1, 144, 14, 476] --\n", - "| └─_Transition: 2-7 [-1, 72, 7, 238] --\n", - "| | └─Sequential: 3-2 [-1, 72, 7, 238] 10,656\n", - "| └─_DenseBlock: 2-8 [-1, 192, 7, 238] --\n", - "| └─_Transition: 2-9 [-1, 96, 3, 119] --\n", - "| | └─Sequential: 3-3 [-1, 96, 3, 119] 18,816\n", - "| └─_DenseBlock: 2-10 [-1, 168, 3, 119] --\n", - "| └─ReLU: 2-11 [-1, 168, 3, 119] --\n", + "| └─_DenseBlock: 2-6 [-1, 192, 14, 476] --\n", + "| └─_Transition: 2-7 [-1, 96, 7, 238] --\n", + "| | └─Sequential: 3-2 [-1, 96, 7, 238] 18,816\n", + "| └─_DenseBlock: 2-8 [-1, 216, 7, 238] --\n", + "| └─ReLU: 2-9 [-1, 216, 7, 238] --\n", + "| └─AdaptiveAvgPool2d: 2-10 [-1, 216, 1, 1] --\n", + "| └─Rearrange: 2-11 [-1, 216] --\n", + "| └─Linear: 2-12 [-1, 80] 17,360\n", "==========================================================================================\n", - "Total params: 34,536\n", - "Trainable params: 34,536\n", + "Total params: 41,240\n", + "Trainable params: 41,240\n", "Non-trainable params: 0\n", - "Total mult-adds (M): 229.41\n", + "Total mult-adds (M): 252.43\n", "==========================================================================================\n", "Input size (MB): 0.10\n", "Forward/backward pass size (MB): 53.69\n", - "Params size (MB): 0.13\n", - "Estimated Total Size (MB): 53.92\n", + "Params size (MB): 0.16\n", + "Estimated Total Size (MB): 53.95\n", "==========================================================================================" ] }, - "execution_count": 114, + "execution_count": 59, "metadata": {}, "output_type": "execute_result" } @@ -634,6 +834,37 @@ }, { "cell_type": "code", + "execution_count": 84, + "metadata": {}, + "outputs": [], + "source": [ + " backbone = nn.Sequential(\n", + " *list(dnet.children())[:][:-4]\n", + " )" + ] + }, + { + "cell_type": "code", + "execution_count": 85, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "Sequential()" + ] + }, + "execution_count": 85, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "backbone" + ] + }, + { + "cell_type": "code", "execution_count": 29, "metadata": {}, "outputs": [], @@ -821,166 +1052,280 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 59, + "metadata": {}, + "outputs": [], + "source": [ + "pred = torch.Tensor([1,1,1,1,1, 81, 1, 79, 79, 79, 2,1,1,1,1, 81, 1, 79, 79, 79, 1,1,1,1,1, 81, 79, 79, 79, 79]).long()\n", + "target = torch.Tensor([1,1,1,1,1, 81, 79, 79, 79, 79, 1,1,1,1,1, 81, 79, 79, 79, 79, 1,1,1,1,1, 81, 79, 79, 79, 79]).long()" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "metadata": {}, + "outputs": [], + "source": [ + "from text_recognizer.models.metrics import accuracy" + ] + }, + { + "cell_type": "code", + "execution_count": 61, + "metadata": {}, + "outputs": [], + "source": [ + "pad_indcies = torch.nonzero(target == 79, as_tuple=False)" + ] + }, + { + "cell_type": "code", + "execution_count": 76, + "metadata": {}, + "outputs": [], + "source": [ + "t1 = torch.nonzero(target == 81, as_tuple=False).squeeze(1)" + ] + }, + { + "cell_type": "code", + "execution_count": 66, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "15.0" + "30" ] }, - "execution_count": 8, + "execution_count": 66, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "120 / 8" + "target.shape[0]" ] }, { "cell_type": "code", - "execution_count": 27, + "execution_count": 84, + "metadata": {}, + "outputs": [], + "source": [ + "t2 = torch.arange(10, target.shape[0] + 1, 10)" + ] + }, + { + "cell_type": "code", + "execution_count": 85, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "120" + "tensor([10, 20, 30])" ] }, - "execution_count": 27, + "execution_count": 85, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "2 * 60" + "t2" ] }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 89, "metadata": {}, "outputs": [], "source": [ - "import yaml" + "for start, stop in zip(t1, t2):\n", + " pred[start+1:stop] = 79" ] }, { "cell_type": "code", - "execution_count": 22, + "execution_count": 90, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([ 1, 1, 1, 1, 1, 81, 79, 79, 79, 79, 2, 1, 1, 1, 1, 81, 79, 79,\n", + " 79, 79, 1, 1, 1, 1, 1, 81, 79, 79, 79, 79])" + ] + }, + "execution_count": 90, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "path = \"../training/experiments/cnn_transformer.yml\"" + "pred" ] }, { "cell_type": "code", - "execution_count": 26, + "execution_count": 88, + "metadata": {}, + "outputs": [ + { + "ename": "SyntaxError", + "evalue": "invalid syntax (<ipython-input-88-b8a4aef86401>, line 1)", + "output_type": "error", + "traceback": [ + "\u001b[0;36m File \u001b[0;32m\"<ipython-input-88-b8a4aef86401>\"\u001b[0;36m, line \u001b[0;32m1\u001b[0m\n\u001b[0;31m [pred[start+1:stop] = 79 for start, stop in zip(t1, t2)]\u001b[0m\n\u001b[0m ^\u001b[0m\n\u001b[0;31mSyntaxError\u001b[0m\u001b[0;31m:\u001b[0m invalid syntax\n" + ] + } + ], + "source": [ + "[pred[start+1:stop] = 79 for start, stop in zip(t1, t2)]" + ] + }, + { + "cell_type": "code", + "execution_count": 69, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([[ 6],\n", + " [ 7],\n", + " [ 8],\n", + " [ 9],\n", + " [16],\n", + " [17],\n", + " [18],\n", + " [19],\n", + " [26],\n", + " [27],\n", + " [28],\n", + " [29]])" + ] + }, + "execution_count": 69, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "pad_indcies" + ] + }, + { + "cell_type": "code", + "execution_count": 71, + "metadata": {}, + "outputs": [ + { + "ename": "TypeError", + "evalue": "only integer tensors of a single element can be converted to an index", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mTypeError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m<ipython-input-71-39b5cc3b1445>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mpred\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mpad_indcies\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mpad_indcies\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m79\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", + "\u001b[0;31mTypeError\u001b[0m: only integer tensors of a single element can be converted to an index" + ] + } + ], + "source": [ + "pred[pad_indcies:pad_indcies] = 79" + ] + }, + { + "cell_type": "code", + "execution_count": 50, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "torch.Size([20])" + ] + }, + "execution_count": 50, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "pred.shape" + ] + }, + { + "cell_type": "code", + "execution_count": 51, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "torch.Size([20])" + ] + }, + "execution_count": 51, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "target.shape" + ] + }, + { + "cell_type": "code", + "execution_count": 91, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "0.0" + ] + }, + "execution_count": 91, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "accuracy(pred, target)" + ] + }, + { + "cell_type": "code", + "execution_count": 92, "metadata": {}, "outputs": [], "source": [ - "with open(path, \"r\") as f:\n", - " f = yaml.safe_load(f)" - ] - }, - { - "cell_type": "code", - "execution_count": 27, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'experiment_group': 'Transformer Experiments',\n", - " 'experiments': [{'train_args': {'transformer_model': True,\n", - " 'batch_size': 16,\n", - " 'max_epochs': 128,\n", - " 'input_shape': [[1, 28, 952], [92]]},\n", - " 'dataset': {'type': 'EmnistLinesDataset',\n", - " 'args': {'subsample_fraction': None,\n", - " 'transform': [{'type': 'ToPILImage', 'args': None},\n", - " {'type': 'Resize', 'args': {'size': [28, 952]}},\n", - " {'type': 'ToTensor', 'args': None}],\n", - " 'max_length': 97,\n", - " 'min_overlap': 0.0,\n", - " 'max_overlap': 0.33,\n", - " 'num_samples': 1,\n", - " 'seed': 4711,\n", - " 'init_token': '<sos>',\n", - " 'pad_token': '_',\n", - " 'eos_token': '<eos>',\n", - " 'target_transform': [{'type': 'AddTokens',\n", - " 'args': {'init_token': '<sos>',\n", - " 'eos_token': '<eos>',\n", - " 'pad_token': '_'}}]},\n", - " 'train_args': {'num_workers': 8,\n", - " 'train_fraction': 0.85,\n", - " 'batch_size': 16}},\n", - " 'model': 'VisionTransformerModel',\n", - " 'metrics': ['accuracy'],\n", - " 'network': {'type': 'CNNTransformer',\n", - " 'args': {'backbone': 'DenseNet',\n", - " 'backbone_args': {'growth_rate': 8,\n", - " 'block_config': [4, 6, 8, 6],\n", - " 'in_channels': 1,\n", - " 'base_channels': 24,\n", - " 'num_classes': 256,\n", - " 'bn_size': 4,\n", - " 'dropout_rate': 0.1,\n", - " 'classifier': False,\n", - " 'activation': 'elu'},\n", - " 'num_encoder_layers': 3,\n", - " 'num_decoder_layers': 3,\n", - " 'hidden_dim': 256,\n", - " 'vocab_size': 82,\n", - " 'num_heads': 8,\n", - " 'max_len': 99,\n", - " 'expansion_dim': 512,\n", - " 'mlp_dim': 256,\n", - " 'spatial_dim': 357,\n", - " 'dropout_rate': 0.1,\n", - " 'trg_pad_index': 79,\n", - " 'activation': 'gelu'}},\n", - " 'criterion': {'type': 'CrossEntropyLoss', 'args': {'ignore_index': 79}},\n", - " 'optimizer': {'type': 'AdamW',\n", - " 'args': {'lr': 0.0003,\n", - " 'betas': [0.9, 0.999],\n", - " 'eps': 1e-08,\n", - " 'weight_decay': 3e-06,\n", - " 'amsgrad': False}},\n", - " 'lr_scheduler': {'type': 'OneCycleLR',\n", - " 'args': {'max_lr': 0.0007,\n", - " 'epochs': 128,\n", - " 'anneal_strategy': 'cos',\n", - " 'pct_start': 0.475,\n", - " 'cycle_momentum': True,\n", - " 'base_momentum': 0.85,\n", - " 'max_momentum': 0.9,\n", - " 'div_factor': 10,\n", - " 'final_div_factor': 10000,\n", - " 'interval': 'step'}},\n", - " 'callbacks': ['Checkpoint',\n", - " 'ProgressBar',\n", - " 'WandbCallback',\n", - " 'WandbImageLogger'],\n", - " 'callback_args': {'Checkpoint': {'monitor': 'val_loss', 'mode': 'min'},\n", - " 'ProgressBar': {'epochs': 128},\n", - " 'WandbCallback': {'log_batch_frequency': 10},\n", - " 'WandbImageLogger': {'num_examples': 6}},\n", - " 'test_metric': 'test_accuracy'}]}" - ] - }, - "execution_count": 27, + "acc = (pred == target).sum().float() / target.shape[0]" + ] + }, + { + "cell_type": "code", + "execution_count": 93, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor(0.9667)" + ] + }, + "execution_count": 93, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "f" + "acc" ] }, { diff --git a/src/notebooks/01-look-at-emnist.ipynb b/src/notebooks/01-look-at-emnist.ipynb index 0ef77b6..b70ce12 100644 --- a/src/notebooks/01-look-at-emnist.ipynb +++ b/src/notebooks/01-look-at-emnist.ipynb @@ -49,7 +49,7 @@ }, { "cell_type": "code", - "execution_count": 44, + "execution_count": 5, "metadata": {}, "outputs": [ { @@ -59,7 +59,7 @@ "EMNIST Dataset\n", "Num classes: 80\n", "Input shape: [28, 28]\n", - "Mapping: {0: '0', 1: '1', 2: '2', 3: '3', 4: '4', 5: '5', 6: '6', 7: '7', 8: '8', 9: '9', 10: 'A', 11: 'B', 12: 'C', 13: 'D', 14: 'E', 15: 'F', 16: 'G', 17: 'H', 18: 'I', 19: 'J', 20: 'K', 21: 'L', 22: 'M', 23: 'N', 24: 'O', 25: 'P', 26: 'Q', 27: 'R', 28: 'S', 29: 'T', 30: 'U', 31: 'V', 32: 'W', 33: 'X', 34: 'Y', 35: 'Z', 36: 'a', 37: 'b', 38: 'c', 39: 'd', 40: 'e', 41: 'f', 42: 'g', 43: 'h', 44: 'i', 45: 'j', 46: 'k', 47: 'l', 48: 'm', 49: 'n', 50: 'o', 51: 'p', 52: 'q', 53: 'r', 54: 's', 55: 't', 56: 'u', 57: 'v', 58: 'w', 59: 'x', 60: 'y', 61: 'z', 62: ' ', 63: '!', 64: '\"', 65: '#', 66: '&', 67: \"'\", 68: '(', 69: ')', 70: '*', 71: '+', 72: ',', 73: '-', 74: '.', 75: '/', 76: ':', 77: ';', 78: '?', 79: '_'}\n", + "Mapping: {0: '0', 1: '1', 2: '2', 3: '3', 4: '4', 5: '5', 6: '6', 7: '7', 8: '8', 9: '9', 10: 'A', 11: 'B', 12: 'C', 13: 'D', 14: 'E', 15: 'F', 16: 'G', 17: 'H', 18: 'I', 19: 'J', 20: 'K', 21: 'L', 22: 'M', 23: 'N', 24: 'O', 25: 'P', 26: 'Q', 27: 'R', 28: 'S', 29: 'T', 30: 'U', 31: 'V', 32: 'W', 33: 'X', 34: 'Y', 35: 'Z', 36: 'a', 37: 'b', 38: 'c', 39: 'd', 40: 'e', 41: 'f', 42: 'g', 43: 'h', 44: 'i', 45: 'j', 46: 'k', 47: 'l', 48: 'm', 49: 'n', 50: 'o', 51: 'p', 52: 'q', 53: 'r', 54: 's', 55: 't', 56: 'u', 57: 'v', 58: 'w', 59: 'x', 60: 'y', 61: 'z', 62: ' ', 63: '!', 64: '\"', 65: '#', 66: '&', 67: \"'\", 68: '(', 69: ')', 70: '*', 71: '+', 72: ',', 73: '-', 74: '.', 75: '/', 76: ':', 77: ';', 78: '?', 79: None}\n", "\n" ] } diff --git a/src/notebooks/02b-emnist-lines-dataset.ipynb b/src/notebooks/02b-emnist-lines-dataset.ipynb index fb856c5..0f2626f 100644 --- a/src/notebooks/02b-emnist-lines-dataset.ipynb +++ b/src/notebooks/02b-emnist-lines-dataset.ipynb @@ -2,18 +2,9 @@ "cells": [ { "cell_type": "code", - "execution_count": 7, + "execution_count": 1, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "The autoreload extension is already loaded. To reload it, use:\n", - " %reload_ext autoreload\n" - ] - } - ], + "outputs": [], "source": [ "%load_ext autoreload\n", "%autoreload 2\n", @@ -31,7 +22,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ @@ -40,27 +31,28 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 8, "metadata": {}, "outputs": [], "source": [ "emnist_lines = EmnistLinesDataset(train=False,\n", - " max_length = 97,\n", + " max_length = 34,\n", " min_overlap = 0.0,\n", " max_overlap = 0.33,\n", - " num_samples = 10_000,)" + " num_samples = 5_000,)" ] }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 9, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "2020-10-25 18:35:53.133 | DEBUG | text_recognizer.datasets.emnist_lines_dataset:_load_data:147 - EmnistLinesDataset loading data from HDF5...\n" + "2020-11-12 08:12:02.064 | DEBUG | text_recognizer.datasets.emnist_lines_dataset:_generate_data:154 - Generating data...\n", + "2020-11-12 08:12:05.917 | DEBUG | text_recognizer.datasets.emnist_lines_dataset:_load_data:147 - EmnistLinesDataset loading data from HDF5...\n" ] } ], @@ -70,7 +62,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 7, "metadata": {}, "outputs": [], "source": [ @@ -80,7 +72,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 8, "metadata": { "scrolled": false }, @@ -209,7 +201,7 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 10, "metadata": {}, "outputs": [ { @@ -222,8 +214,8 @@ "Max overlap: 0.33\n", "Num classes: 80\n", "Input shape: (28, 952)\n", - "Data: (1000, 28, 952)\n", - "Tagets: (1000, 34)\n", + "Data: (5000, 28, 952)\n", + "Tagets: (5000, 34)\n", "\n" ] } diff --git a/src/notebooks/04a-look-at-iam-lines.ipynb b/src/notebooks/04a-look-at-iam-lines.ipynb index 8132f44..576dd5e 100644 --- a/src/notebooks/04a-look-at-iam-lines.ipynb +++ b/src/notebooks/04a-look-at-iam-lines.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 1, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ @@ -24,7 +24,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ @@ -33,7 +33,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 4, "metadata": {}, "outputs": [ { @@ -57,7 +57,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 5, "metadata": {}, "outputs": [ { @@ -66,7 +66,7 @@ "(28, 952)" ] }, - "execution_count": 4, + "execution_count": 5, "metadata": {}, "output_type": "execute_result" } @@ -77,7 +77,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 6, "metadata": {}, "outputs": [ { @@ -86,7 +86,7 @@ "(97, 80)" ] }, - "execution_count": 5, + "execution_count": 6, "metadata": {}, "output_type": "execute_result" } @@ -97,7 +97,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 7, "metadata": {}, "outputs": [ { @@ -106,7 +106,7 @@ "'He rose from his breakfast-nook bench____________________________________________________________'" ] }, - "execution_count": 6, + "execution_count": 7, "metadata": {}, "output_type": "execute_result" } @@ -120,7 +120,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 8, "metadata": {}, "outputs": [ { @@ -254,7 +254,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 9, "metadata": {}, "outputs": [], "source": [ @@ -264,7 +264,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 10, "metadata": {}, "outputs": [ { @@ -273,7 +273,7 @@ "torch.Size([97])" ] }, - "execution_count": 9, + "execution_count": 10, "metadata": {}, "output_type": "execute_result" } @@ -284,16 +284,16 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 84, "metadata": {}, "outputs": [], "source": [ - "h, w, s = 28, 8, 2" + "h, w, s = 28, 5, 5" ] }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 85, "metadata": {}, "outputs": [], "source": [ @@ -303,7 +303,27 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 86, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "140" + ] + }, + "execution_count": 86, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "28 * 5" + ] + }, + { + "cell_type": "code", + "execution_count": 87, "metadata": {}, "outputs": [], "source": [ @@ -312,8 +332,10 @@ }, { "cell_type": "code", - "execution_count": 13, - "metadata": {}, + "execution_count": 88, + "metadata": { + "scrolled": true + }, "outputs": [ { "name": "stdout", @@ -324,7 +346,7 @@ }, { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAACMkAAACUCAYAAACDFexpAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAABD9ElEQVR4nO3deZgdVZ3/8VPdt7uTzg5ZSfgRQti3GNBHeBBhQJYBicAgDGsAIcCIyKAMgzwoi8MSAQUFHBwWBYSRzQEGhPAoDDAJ+6IJMJCV7J10Oun9LvX7I+Z5cpZb9a1zq253X9+v/87JqVOfe24tp+pWuoIwDBUAAAAAAAAAAAAAAABQy+r6OgAAAAAAAAAAAAAAAACQNR6SAQAAAAAAAAAAAAAAQM3jIRkAAAAAAAAAAAAAAADUPB6SAQAAAAAAAAAAAAAAQM3jIRkAAAAAAAAAAAAAAADUPB6SAQAAAAAAAAAAAAAAQM3LJWkcBEGYVZBKhGEYuOr7a16lVEsYhmNc/0Dm9JTbLpQic5rY/6piwGWupW1ZqYGXub/mVWzLVUHm6mD/y14tbRdKDbzM/TWvGoDbshqAmWtpW1Zq4GXur3nVANyW1QDMXEvbslIDL3N/zavYlquCzNXB/pe9WtoulBp4mftrXjUAt2U1ADPX0ras1MDL3F/zqgG4LasBmLmWtmWlBl7m/ppXldmW+UsyfWNJXwfwMBAzAy4DcVseiJkBF7ZloO+w/6FWDMRteSBmBlwG4rY8EDMDLmzLQN9h/0OtGIjb8kDMDLgMxG15IGYGXJzbMg/JAAAAAAAAAAAAAAAAoObxkAwAAAAAAAAAAAAAAABqHg/JAAAAAAAAAAAAAAAAoObxkAwAAAAAAAAAAAAAAABqHg/JAAAAAAAAAAAAAAAAoObxkAwAAAAAAAAAAAAAAABqHg/JAAAAAAAAAAAAAAAAoObxkAwAAAAAAAAAAAAAAABqHg/JAAAAAAAAAAAAAAAAoObxkAwAAAAAAAAAAAAAAABqHg/JAAAAAAAAAAAAAAAAoObxkAwAAAAAAAAAAAAAAABqHg/JAAAAAAAAAAAAAAAAoObxkAwAAAAAAAAAAAAAAABqHg/JAAAAAAAAAAAAAAAAoObxkAwAAAAAAAAAAAAAAABqHg/JAAAAAAAAAAAAAAAAoObxkAwAAAAAAAAAAAAAAABqHg/JAAAAAAAAAAAAAAAAoObxkAwAAAAAAAAAAAAAAABqHg/JAAAAAAAAAAAAAAAAoObxkAwAAAAAAAAAAAAAAABqHg/JAAAAAAAAAAAAAAAAoObxkAwAAAAAAAAAAAAAAABqHg/JAAAAAAAAAAAAAAAAoObl+joAAAAY2MaNG2fVTZ482aqbN29eFdIotd1221l1U6dO1cqvvPJKv8miVPXy5HI5NXr0aK3O/K7mzp1blSwAAAAAAACozOmnn27Vue7tfPrpp9WIo6655hqr7umnn9bKb731Vr/JolT18owbN06dddZZWt0LL7ygld97773U1nfcccdZdRs3btTK1frsSil18MEHW3WdnZ1auRp5JDmqlUUppYYOHaqmT58emaea3xPwt4K/JAMAAAAAAAAAAAAAAICax0MyAAAAAAAAAAAAAAAAqHk8JAMAAAAAAAAAAAAAAICax0MyAAAAAAAAAAAAAAAAqHm5vg4AAAAGtsbGRqvugAMOsOrmzZtXjTjOPH/3d3+nlV955ZV+k0Wp6uWpq6uzMn3lK1/RynPnzq1KFgAAAAAAAFRmxIgRVt3Xv/51q+62226rRhw1ZswYq+6iiy7Syuecc06/yaJU9fLkcjm17bbbanVnn322Vr7kkktSW99+++1n1ZljsnDhwtTWF2ePPfaw6kaNGqWV169fb7XZcccdtfIbb7yReQ5plpdeeqmiLEopNXjwYCuTz7iklUeisbFRTZo0KTJPtbIAvvhLMgAAAAAAAAAAAAAAAKh5PCQDAAAAAAAAAAAAAACAmsdDMgAAAAAAAAAAAAAAAKh5uUSNHe/LW716daqBAADAwNLS0mLV7bDDDn2QZLO1a9dadTvttFMfJOlfWZRSqlAoWN+X6/21AAAAAAAA6P82bdpk1U2fPr0Pkmy2bt06q+6oo47SynV1sv+/XyqVMs8izVNpFqWUKhaLasOGDVrdIYcckjiLNE9vb69Vd+ihh2rlX/ziF7H9xGWSjk0+n7fq9t9/f6389NNPW23mzJkTuUxSkhzSLIMHD7badHd3J8oThqGVyWdc0sojUVdXp5qbmyPzVCuLVBAEqrGxUavr6enJbH2TJ0+26jo6OrRya2trZus3SfK4fsvoixzVysJfkgEAAAAAAAAAAAAAAEDN4yEZAAAAAAAAAAAAAAAA1DwekgEAAAAAAAAAAAAAAEDN4yEZAAAAAAAAAAAAAAAA1LxcksbbbLONOvnkk7W6JUuWaOU33njDWm7lypUe0SrX0NCgRo8e3S+yAABQqwqFglU3adKkPkiyWT6ft+omT55c/SCqf2VRSqkwDK3va/vtt++jNAAAAAAAAKhEW1ubVTdlypQ+SLJZS0uLVbfvvvv2QZJ0s9TVxf/NgVKpFPnvhUJBrVu3Tqvba6+9Msvzhz/8war74Q9/qJWbmpq81p80i1JKzZ0716o7//zztfLy5cutNq+88opWbm9vryiPJIc0y/jx4602S5cuteqito329nYrk8+4SPPEbacS+XzeypTF2EQxv+e4fnK5nPXMwIoVK7RyGIZeWZRSKgiC2DynnHKKVn700Ue915ckizTPHXfckXkWSY6sspj4SzIAAAAAAAAAAAAAAACoeTwkAwAAAAAAAAAAAAAAgJrHQzIAAAAAAAAAAAAAAACoeTwkAwAAAAAAAAAAAAAAgJqXS9K4vr5ejRw5UqvbaaedtPL06dOt5X70ox9ZdWEYJlm1lzFjxqhZs2ZFZqlGDgCQqquzn10slUp9kGSzIAi0cl8eM80sSvWvPH/L5xPXNjpu3Lg+SLKZK8/EiRO18rBhw6w2mzZt6pMs1cwThqGVacKECVp56NCh1nLt7e2pZ5Gqq6tTzc3NWl1f5kHl+tu5TpLH1SZumayyVDNP3LnOdW52kZwj0zrPx2WS9tmf5h39KcsWzIMAAADwt8S8ButP17CueyS5XKKf/lLLopRSra2tVl1DQ0M14lh5+jKLS7FYVBs2bNDqssyzcOFCq85cn/S6Pg2ff/65VSf5/Ndff71WXrFiRZ/kcGVxbWNJjw/5fN7K5DMu0jxZ3ZfzHRvfPEkzF4tF1dbWptVJ7nNJ73eY7ZYtW2a12WWXXbRyU1NTbL8+919cbSR5suAzLtXCX5IBAAAAAAAAAAAAAABAzeMhGQAAAAAAAAAAAAAAANQ8HpIBAAAAAAAAAAAAAABAzUv0YsKGhgY1btw4re6OO+7Qyuedd561XGNjo1XX09Nj1Znv3urt7bXaJHnXeV1dnRo6dGhkFkmOSrJMnz7dqnvnnXdil4vKU8m4mHlWr15ttXG9zy+uf593ch500EFWnetdgEuXLk3ct48vf/nLWtk1Dq581Xr/aGNjoxo/fnxknjSzZPVeQh9/S1l8+vJ5J6FUmn1VyjdLJe+RjJJlnqwy+6xD0s61X/iIO5dI9z9JniFDhlh1HR0dsX3HrUuS2TePS9JjRhAEsZmam5utus7OTlH/WRyb6+rqrEySPP3p/eBZZpFuY9Uaj7SOB2nIOovPmGaZqS+3+a35nq98z0F9nSfLDNXKksb7raudxydz3P5XybvRq3Hcl6yjP103KVXd8yEAAMDfKtd9nNbW1j5Istnw4cOtOvN3qMGDB1ttisWiVdfd3Z15FmkeV776+nqt7Pq90Ww/bNiwyDyDBg2ylnPNoyV5XON3/PHHa+XPPvusfGC1+drL/F3VzCO9NnGNz2WXXRbb12uvvaaV47YLc91mWZJDmqVQKMSuv1xfW/+bmclnXKR50rouM/vJYmzSFIahNc7V/m1t5MiRWjmXi39MI61MkjzV+F1KkqNaWfrPXWwAAAAAAAAAAAAAAAAgIzwkAwAAAAAAAAAAAAAAgJrHQzIAAAAAAAAAAAAAAACoeTwkAwAAAAAAAAAAAAAAgJqXS9K4paVFPfDAA1rd8uXLtfKtt95qLXfUUUdZdZdccolVd+ihh2rll156yWqzYMECrXzzzTeXzbt69Worj5lFkkOa5f3337fa7LnnnlbdO++84w6slNpll13U3XffHZlHkkWaZ/Xq1Vabm266qWw+l7333lv993//t1b37rvvauX//d//tZbbf//9rbrPP//cqrv00ksT5ZEYO3asOu2007S6adOmaeUVK1ZYy/3gBz9IPYvUmDFj1IUXXqjVZZmnVCpl1ndSZIkWhmHV1hUEQWybgZDHXC6NzJIs0jx1dfHPsBaLxYrySHJIs7jadHV1xS5nMvcvs1/XeqR1HR0dWnnixIlWmx//+Mda+dprry0fVphRkkWa59hjj7XaHHPMMVp5/vz55cOWydTZ2amVJ0yYYC1z5ZVXWnVHHHGEVXfqqadq5ffeey82T5yGhgYrk5lHkiWtPHH23HNP9bvf/S4yS9Y5+tu5qj/lSTOLuS/59J1lnjTW53N+9D0fupar1vlZek6U5JF+/qhl0hwbnzw+/WY1NpUsZ4qbZ0iltVyl+79vjiyylOvXdxnfPLmcfYurUChUlKeSsXHlMSXN50uSRan+ladaWVzMfH2ZBQBQfZXMs9ImyTJq1CirbtmyZX2SRSmlxo0bZ9W5fr9KQ1ymNLPU19fH1sVdu+RyOTV27Fit7tNPP00tj+Q7Mn9fdN2nTEp6f9Zl3rx5Wlky74q6/gyCQHR/Ni6HNEtW80SfcZG2S+MarFQqxd7/l2ZO85owShiGsZnSvDfV0NBg1Q0ZMkQrS7bNtH7L8s0T9ztUFjmqlaX/nO0BAAAAAAAAAAAAAACAjPCQDAAAAAAAAAAAAAAAAGoeD8kAAAAAAAAAAAAAAACg5vGQDAAAAAAAAAAAAAAAAGpeLknjrq4u9f7772t1hUJBK/f29lrLffjhh1bdWWedZdV973vf08rr1q2z2ixatEgrd3R0lM1bKBRUS0tLZBZJDmmWUaNGWW3mzp1bNp/LokWLrEw+4yLNc+yxx1pt6ursZ6eKxaI7sFKqra1NPfvss1rdRx99pJXHjBljLTdnzhyr7swzz4zNUyqVymaRqq+vV0OGDNHq/vSnP2nlU045JTZLWnkkmpub1fTp0yPzVCtLf9SX342Lb54sPkcQBFZdGIZey0qXy6Jf389RX19v1UUd06R8x0aSx/Wd53K52DZJ8kjHRZLFxTz/Spj9mnMMyTLlrF69WiufeuqpVhtz/3NtcyZzfCR5zCzSPCeddJLV5s4779TKrvNonLVr12rlE088MTaLUkqdc845Vt1NN92klY8++ujY9cdty6NGjbIymXkkWaR5JMfhqMyLFy+28viMizSLi2s7jNufXOtKI0u5PHF9+XxO37HxzRM3ZhI+YyPNk9acOapP17nG93ztajNo0CCrrru7O1E/vudrSZ6enh7Rcj7rSpolzTwSZr9ZjU0ly5nS2keqvVwW/WV1LvHJ5HsuOffcc602559/vlW33377pZ7HNTaSPLvssovVZvbs2Vr5nnvuiV1/XB7Xfa6ZM2dada48P/3pT7Wyaz5l7ktx28Xo0aPVjBkzIvNIskjzTJ061WqzceNGrWzOf7e27777qhdeeCEyiySHNMuKFSvKZknCXL9Z3mmnnWKzKKXUypUrU8kTp6mpSU2ePFmra2tr08qrVq2qShYAMLnmA42NjVo5n8/3myxmWSmlNm3aVJUsruuS5uZmq279+vVa2XUP0nWvMmkmM48kizSPq41ZF3ftUldXZ2VsbW3VytL7D5J7BZJ7vZLrLXMZybWV770MF9fvzkmkeY9HkiXp/aIwDK05dZrjUq3fEX3Hppq/46V1z0jCdZ1k7v+S3x/SyiTJ47rWNesq3R8lOaRZXH0l+e2PvyQDAAAAAAAAAAAAAACAmsdDMgAAAAAAAAAAAAAAAKh5PCQDAAAAAAAAAAAAAACAmme/0CmC671oJte/r1692qpzvXfsjTfe0MrDhw+32jz11FNauaOjo2wWV14ziySHNIvr3Y9J3z1XLBbVhg0bIvNIskjz/OM//qNoua6uLldcpdTm79wc1/vvv18rNzU1xWZRSqlvf/vbVp1rWZPknYJbW7t2rfrlL3+p1ZnvmHS9z9w1Nmnkkaivr7e+e0ke6bsfTa6+zfeEf/rpp5F9mO+H8x0XSZaFCxdabTo7O0X9JyHJUs08QRBY42y+a3zJkiXWclH7dNaCILDeMZhlHtc7CLN812QcSR7JuzkrzSzJIc3iahM3X3CJe/eq9J2lkjxnn3221WavvfbSymvXri0ftsy6fMdGksdlxIgRWtnn/c1mnlNOOcVqc/DBB4v6MvO4jpnmGMW9x3ubbbaxMknymFmkea666iqrzRFHHKGVZ86cWXa9PT09avHixZFZJDmkWb773e9abd56662y+VyCILAy+YyLNM92221ntVmzZo1W7unpKZtXKaUmTJigZs2apdV97Wtf08qXXXZZbBZpnu7u7sg8EkOGDFF77713ZB5JFmke13Zm7u9Rc7GGhgY1evRorW7dunVaWfoe4oaGhtgsrmPPhRdeaNVJj0eVZJHmmTdvntXmiiuusOqi3ofsmsv5jo0kj+s7Tzqn2GabbdSRRx4ZmUc6NpI8rvlJ0sxTp05Vt912m1Z3/PHHx2aRzjOyeIe6zxzDN4err3/7t3+z6i6//PJE/WSZ5/PPP7faHHbYYV7rqzSLNI+rr8cff1wrP/LII4kz/fCHP9TKK1assJY59thjY/tRSqmHH35YK8+ePdtqc+2112rlX/ziF+XDKvc8w8wjySLN45pD77jjjlr55JNPLpv3z3/+s9ptt90is0hySLOcccYZVptly5ZZdeY52GSOoTl323333a1lJk+ebNWdd955Vp25fQ8bNsxq47rvEGXixInqmmuu0ep22GGH2Cyu7TuNPBKDBg1SU6dOjcxTrSxSjY2NasKECVpdX+YZCA466CCr7tVXX+2DJJuZeVzHH/P+7NVXX12VLNI8rn3ZvI7wmTP4jI00j4s5R4+aj+62227qvvvui8zim8OVxdzPlVLq448/FvW1RV1dnRo8eLBWJ7lf68ps3ptWSqm5c+dqZdc1UtR1Uznm+c/MI8mSZp44jY2N1vn37bffrmoWc4zirq3CMMzk964t4u4Hu+qi1u/K65MjjSyV8BmXLPOYXONcrbEZKFzfj3nv3nW/IyuSPHvuuafV5q677tLKZ511VuY5pFkWLFhgtYm6b29lEbcEAAAAAAAAAAAAAAAABigekgEAAAAAAAAAAAAAAEDN4yEZAAAAAAAAAAAAAAAA1DwekgEAAAAAAAAAAAAAAEDNyyVdoFQqRf57XZ393E13d7e94py96kWLFmnlI488MmG6eGYWSQ5pls7OTqvO1X+UMAxVb29vZB7puEjybL/99labq666yqr7wQ9+UHY9DQ0NatKkSZFZ2tvbncuZdtppJ6vurrvu0srf/OY3rTYXXHCBVn766acj85RKJdXR0RGZZ/LkydZyP/nJT6y6448/3qr753/+Z638+OOPW23MfaWnp6dsXqWUamxstDKZeSRZpHnMcVdKqaamJq0ctV3ss88+6g9/+ENkFkkOaZZisWi1Ofvss626QqHgDqyUCoLA2g58xkWax3U8jTvGmiZOnKguu+wyra6xsTG2z+9+97uidYdhmCiPxKRJk9Tll18emSfNLEEQeC0X10+WeVz7gSnptuKTQ5rF3OaUUmrs2LGJM8WtS5JFKfc+OX78eK380EMPWW3M40PcdxwEQWwmSRZpHtd3PnToUK1cX18fmScIAuv7Mr+rJ554IjZLuTzNzc1a2TUPOvbYY7Wyea4wtba2WpkkY2NmkeZx+da3vqWVFy9eXLatay7nMy7SLNdcc43V5rTTTrPq4s5/ZiafcZHmcS03fPhwrXzLLbfErt80a9YsrXz11Vdbbc4880yr7pxzzonNY563lFJq2rRpWnn+/PmR+caNG2fNhcw8kizSPK7jinkdsWLFirJ5R44cqU488UStzjzmzJ4921pu9913t+pc7czt4LDDDrPa/MM//EPZfOWYx0EzjySLNM/tt99utXGd/1auXOkOq5SaOnWq1Y/v2EjyrFq1qmwWqWHDhlmZfMdGkieN+eigQYPUrrvuqtWZ17BLly4V9SWZg7nmB5XO3Xz7k2Q56aSTrDYjR45MPZN0XCR5XPctfJiZfMdGksc1LzzuuOO0snmfwrVuc/8z81x33XWxWcrlMe+37LXXXlab3XbbTSsPGjQocj0tLS3qP/7jPyrOIs1zxhlnxLZZsmRJ2SxhGFrzJp9xkWZ57LHHrDaHH364Vbdu3Tp3YKXU4MGD1c4776zVmeXzzjvPWm6XXXax6n7zm99YdeZ26treL774Yq0cd49r8eLF6vzzz4/M8+CDD1rLzZgxw6pz3Zv6zne+o5V//etfW21uvvlmrRw3l5syZYqVycwjySLN47pm+NnPfqaVzzrrrPKBlVITJkywMvmMjTTPhx9+aLUx79lGXZdMmTJF3XjjjZFZJDmkWcz5s1Lua6lXX33VmVcppYYMGaL23ntvre6OO+7Qyh988IG1nHntUi6PeRxxHRvNOcxPf/rTcnGVUkrtsMMO1rWSmUeSRZrniiuusNqsXr1aK8fNIZqbm9Uee+wRmUeSpZI8cXOGrX322WdWHp9xkWYZM2aM1ea3v/1t2XwuDQ0NauLEiVqd+buU63636z6U6zen66+/PnY5l3w+L2pXrl9JFmke816PS9y1S2NjozXOt912W+Is0jxpkNz/jDq2J5XGbyZpXXv5ZvFZf9wylYxLVtew5r+nOTaSY26S43I5kvsNrt9RXMuZ7Vxjav5W7pNZ8juVK7Mkj2t+cu6552rlqPviLj7jIs1yww03WG0mTJhg1ZW7L8dfkgEAAAAAAAAAAAAAAEDN4yEZAAAAAAAAAAAAAAAA1DwekgEAAAAAAAAAAAAAAEDN4yEZAAAAAAAAAAAAAAAA1Lxc0gXq6tJ5rqZUKll17e3tWnm77bZLZV2V5pBmcY1NLpd4iK1MvuMiydPY2Gi12XnnnUX9b93H//t//y+yTX19vVXX0NBg1TU1NVl1e+21l1b+0pe+ZLV54403tPIHH3wQmceVyRwbV5Y99tjDqjv88MOtujlz5mjl1157zWqz5557xrbZWl1dnZXJzCPJIs3j+vyzZs3Syh0dHWXzfvLJJ1Yen3GRZpk/f77VZvz48Vbd559/7g6sNm+T5jI+4yLNE5VFqq6uztqPL7/8cq388ssvW8uNHTvWqlu5cqVXhqOPPlorx23LQ4cOVQcccEBkHt8srjwLFy602nz66adauVgsxvYbhmHFWaR5suAzLlKuc93uu++eqI9BgwapqVOnanUfffSRVx7X+WXffffVyt///vetNl1dXVrZNUfIIos0j+v4k3ReFgSB9X2Z59lrrrnGWq67u9uqc80hgiDQyuPGjbPanHHGGVr5rbfeKh9YKdXW1qaef/75yDySLNI8F154odXGXF+hUCgf2MFnXKRZXPufazxc3+EWDQ0NViafcZHmufPOO60277//vla+7777yuZVSql169ape++9NzKPdGzuvvtuq+7tt9/Wyg8//LDV5swzz9TKs2fPLh9Yufc/M48kizTPrbfearU58sgjtfLjjz9eNm9ra6t65JFHtLonnnhCKz/zzDPWcjNmzLDq7r//fqvuoIMO0sqLFi2y2owePdqqW7NmjTOvUu5ziZlHkkWaZ/vtt7fabNq0qWw+l5aWFiuT79j45jGPUXFznp6eHitTmmOTNI9EU1OTdZ3b2toau5zrPCuZH7jamPt/0nOJmUU6T5FkueSSS6w23/72txOkk5FkqSSPz9zNXCbNsTH7Nud2Stnfa9z2PnbsWHXRRRdpdZdddlniLNI8++23n9Xmueee08ptbW2x64/LI8kizTNixAirzfLly7Vyb29vZDYzj8+4SLM88MADVpuTTz7ZqrvpppvcgZVSzc3Natq0aVrdiy++GJvFdd3tmmOccMIJWvnQQw+12px++ula+amnnioXVym1+Z6cmcnM48py/PHHW3WHHHKIVXfaaadp5d12281q8+yzz2rlI444omxepTbPP81MZh5JFmke1741c+ZMrbxs2bJycZVSSg0bNszK5DM20jyu+fB5552nlR977LFycdWgQYOsPD7jIs3i2pZ//OMfl83nsuOOO6oHH3xQqzPn5bfccou1nOteoivP9ddfr5Vd+4W5/vXr15cPrJQaPHiwdU/ZzCPJIs2zbt06q03Sc/b48ePVFVdcEZlHkkWax3UOMutc9xy2mDJlivrVr34VmUU6LpIsr7/+utVm1apVZfO55PN56xxl3i91/cbjqjOvG5VS6rPPPovs20cQBNb6zbIkS1p5JDZs2GAd1xYvXlzVLEn3vzAMU7lHWo7P/pdFHt/9L6ux8RkXaR7f5ZL2K+0zrc+RxmdwHdul9yTMPK5r3cGDB8euLy6TJI/0eQUzj+uYbh4ze3p6YtcflUWSQ5rlhRdesNrssMMOVl253xv5SzIAAAAAAAAAAAAAAACoeTwkAwAAAAAAAAAAAAAAgJrHQzIAAAAAAAAAAAAAAACoeTwkAwAAAAAAAAAAAAAAgJqXS7pAqVTSynV18c/Z5HL2alzLNTY2auVCoRDbVxAEseuPWl6SQ5qlt7fXamOOl4TZr8+4SPO42jz55JOinFGZGhoatHJPT4+1TLFYtOpc7e69916tvHr1aqvNf/7nf2rl9evXlw9bhjk2riwPPvigVbdmzRqrzhzDU0891Wqz0047aeX58+fH5jMzmXkkWaR5nn32WavNww8/rJUvvfTSsnkLhYKVx2dcpFlGjBhhtWlrayubz2WbbbaxMvmMizSPa791HTOijiMbN25Uc+bM0eruvPNOrTx8+HDncqb6+vrYdbuOmd/85je18oIFC8rm3bIeM5OZR5JFmuehhx6y2ixcuDC2b5OZyWds0swThmHkv5t5fcZFmsV1XnLVRWlqalJTpkzR6j755JPEWZRSKp/Px+ZZvnx5onwuYRjGZpJkkeZxnZf+8pe/aOXu7u7IPsIwtNZvlleuXBmbRSn3HML8znbZZRerjTmviZvL5fP52EySLNI8rjE844wztLJrPrBFGIZWHp9xkWbZf//9rTbf+MY3rLpf//rXzrxKbd7/zEw+4yLN89RTT1ltrrzySq0ct0+USiUr02mnnaaV99tvP2u5GTNmWHW///3vrbqrrrpKK99yyy1Wm+22204r//u//3v5wEqp5uZmK5OZR5JFmuf666+32lx00UVa2Zyzb801xrfddptW/pd/+RdrubFjx1p1d911l1VnzrmGDBlitbn99tutusMPP9wdWCk1YcIEK5OZR5JFmueaa66x2riO1VE2btyoXnzxxcg80rGR5HHNV5Jes9bX11uZfMcmjTwSixYtUqeffrpW19XVlVkWV19mXdz5z2yfZZYJEyZYbZYuXeq1vkqzVJLHdUyLm5Oa609zbMw8rnlhUg0NDVamZcuWxS7nuvaUzNdd2920adO08vPPPx/bT1we6bWDJM9vfvOb2H6SXqv45JBmee+996y6PfbYI1Ee13XJPvvso5UfeeQRUV8ffPCBVbfbbrtp5T/+8Y9Wm3vuuUcrv/vuu5HrKRQKau3atRVnUUqpP/3pT1adOS/bd999rTYffvihVnbtJ1vr6uqyMpl5JFmkeR577DGrjbl+81xm2rRpk5XJZ2ykea677jqrjTkncG0/WyxcuNCaB/mMizSL67zhmt+65opbuOZyZp5rr73WWs5V59oGzTyu++L/9E//pJXb29vL5lVKqRUrVljrN8uSLNI8rmOm2X/cObKpqcm6R+szNtI8lc5H8/m8lcdnXKRZ/uu//suqc91jiBKGofV7jdmH6/cclzvuuMOqGzx4sFZ2HXvef/99Uf9JSLJUkifpttLS0qJ+9atfaXWDBg3Synvuuae1nHn/T5pH8vttnKamJjV58mSt7rPPPkucJa08PrIYF/S9LO4dVMLM09TUZLUZN26cVo66L5dmFmke1/mk0nH2GRdpFtdvBnFz+q1xJAAAAAAAAAAAAAAAAEDN4yEZAAAAAAAAAAAAAAAA1DwekgEAAAAAAAAAAAAAAEDNk7+Y6a983ovtetev651Qo0aN0squ9+yZfYVhGLluM5+5vCSHNIurL59325mZfcZFmqe1tdVqs2rVKlHOLTo6OtRbb72l1ZnvEXVlqa+vt+pc7yM236/tGtPnnntOK2/cuLF8YLX5/e9mJrPflpYWa7nly5dbda7P8dJLL2nlk046yWpz8skna+W77767fGC1+Ts2M5l5JFmkeVzv233wwQe1cm9vb/nAjjw+4yLN4npnd9J3vjc3N1vvEfcZF2ke17u0v//971t1Z511ljOvUkpNmjRJzZ49W6t7/PHHtfJXv/pVaznXMWPnnXe26i688EKt/Pbbb1ttZs6cqZV//vOfl82rlFIff/yxlcnMI8kizXPFFVdE5pEYNGiQ9e5Vn7FJK0+cbbfdVh1zzDGRWdLM4ToWXHzxxYn66OzstM4lvnp6eqw6c99ds2ZNKutKI4tS/nnM80DccTkMQ6vNrFmztLLrXCy1YsUKrbx06VKrzeGHH66Vhw8fHtlnoVDwymRmkeZ59NFHrTavvvqqVnadb5NkkeSQZvniF79otYkbU1Nvb6+VyWdcKsnz9NNPa+UNGzZEtp8yZYq68847tbrXX39dKx9wwAFeWZRS6tlnn9XK9957r9XGHLMgCCL7/Oijj6xMkjxmFmke13XaggULtHJ3d3fs+rf28ssva+UbbrjBauPa91zXbOY7zF3vij/hhBMS5Wtubra2QTOPJIs0j7nN+TIz+Y6Nb55dd91VKy9evDiy/fjx49W//uu/RuapZGzMPB999JHVxtzf4u4LtLW1qWeeeSZxlunTp1t1rvmK5D6JWReX2WRu2/PmzYvNIc2yevVqq81hhx1m1bnOBVF8xkWa58knn7TaJL3+c63fd2wkeVzfT9J7XPl83po3HnLIIVrZPKe61iPNY95rUco+B91///3l4pZlrkeSRZrnzTfftNrsvffeWvlnP/tZ2WxBEFh5fMZFmuX444+32hx88MFl87ls3LhRvfjii1rdXXfdpZVd16uu+xbHHXecVXfUUUdp5X322cdq841vfEMru76/rY0aNUodeeSRkXm+/vWvx2Ypl2fGjBla2XWv8/bbb9fKcddkkyZNUjfeeGNkHkkWaZ5FixZZbST34bfW2tqqHnvsscg8kizSPCtXrrTamPcyo7aNQqFg5fEdF0mWI444wmpj7jtxuru71SeffBKZx5XFdY1r7hOuPE888YTVJul2kc/nrUxmHkkWaR7X7wLmfYm4819zc7O1f/mOjSSPZEyjMi9dulRdcMEFkX1KckizuParCRMmWHVR19lBEKjGxkatrqurSysXi8XYLEq5P8fUqVO1suvY+P7774v63yIMw9hMkixp5ZEIw9CaI+6www5a+eijj7aWc/0mWC1Dhw5VBx54oFbnuoZOS9JjWpZqNUtaffW3ftKQ9Do9SkNDg1U3duxYrew6F2SVSZLn448/TmVdleaQZnGNX3NzszgLf0kGAAAAAAAAAAAAAAAANY+HZAAAAAAAAAAAAAAAAFDzeEgGAAAAAAAAAAAAAAAANY+HZAAAAAAAAAAAAAAAAFDzckkXKJVKWrmuri7y311tyrXbtGmTVv7d734nWi4JSV4zhzRLpdnK9eM7LpI8jzzyiFW3ZMmS2OW2tnbtWvXLX/4yct2uLIVCwaq78847rbr/+7//i82wYsUKrdzb2xvZPgxD5/q3du+991p1CxcujM2ilFKrVq3SysOGDbPaHH744Vr5k08+ieyzpaXFyiTJY2aR5jnxxBOtNh9++KFW7urqil1/VBZJjkqyJN0n6+rqrEw+4yLNs2zZMqvNSy+9JMq6RWtrq3VMWLBggVbu7u6OzaKUUsuXL7fq/ud//kcrt7e3W20+//xzrRy3/5VKJSuTmUeSRZrH1Sap3t5eK5PP2KSVJ04+n1dr1qyJzJJmDtfxdM6cOYn6KBaLzvNvWnl+//vfa2XXPtDU1JTK+pNmkeZxzaeSHpdd57/nn38+NktjY6NV58rz0UcfaeXW1larzRe/+EWtbB6zTMViUW3cuDEyjySLNI/rO1u6dKlWbmtrKx/YwWdcpFlcbVpaWhLlKxaLViafcZHmcX1f5rmjWCyWD6w2n0O/973vReaRjo0kz0033WS1mTFjhlaOm3cUi0Xr2OszNtI8rr7++Mc/auWo424Yhta6zfI999xjLXfYYYdZdUEQWHXz5s3Tyvfff7/V5uKLL7bqnnzySWdepTaPp5nJzCPJIs3T3NxstRk1apRV9/Of/9yZt1wm37GR5DnmmGOsNnfffbdWXrlyZdmsSim1bt06K5Pv2EjyfPzxx5F5JEaPHq3OPPNMrW7cuHFa+ZxzzrGWu+KKK6y6d955J3Z9rv0viS984Qvqtdde0+ouu+wyrfzmm2+K+pJkueCCC6y6d99916p79NFHK1qXdFwkef7+7//eauM6v86dO7eiTNKxkeTZf//9rTbPPPOMVjbvbZiWLVumvvOd72h15rbiumZ1zV2mTZtm1T333HNa2XU/IelcbtiwYerLX/5yZB5JFmke1/Hx008/1cpR84xtt93WGkOfcZFmcS2X9F6Lay731a9+VSvPmjXLWs51T8lcTin7noLrvPm1r31NK8fd4xo6dKg6+OCDtbpFixZp5UMOOSQ2i1JKvfHGG1bdEUccoZU7OzutNj/84Q+1ctz16Mcff2xlMvNIskjzuORyyW7ld3R0WJl8xkaaZ8SIEVab2bNna2XX/rNFqVSy8viOiyTLxIkTrTYnnHCCVeea826xbt06dd9990XmGT58eGyWJHlM5hzrgQceiGxfX19vZfIdG0ke13Fu5syZWtm1HW6tq6tL/fnPf06cxzX/lORxHWuSGDlypJXPZ1ykWVzXvkl/49lnn33Uyy+/rNVNnjxZK0vvJbquYc37V/Pnz7faxN0HMAVBoOrr6yvOUkkeye+NW3NdZ5vXO+ZcQZrFlUfSJi7z+vXr1UMPPZRoGWkWabu0fxf2bZNGFqlqjEulkm5LtbZ+k5nHNT+ZNGmSVnbdc88iizSP63rUPM667qslySLJIc1S6e/C/CUZAAAAAAAAAAAAAAAA1DwekgEAAAAAAAAAAAAAAEDN4yEZAAAAAAAAAAAAAAAA1DwekgEAAAAAAAAAAAAAAEDNC8IwFDeuq6sLc7mcVlcqlbSy+e+uNuXE9f3XDFq5u7tblUqloFzexsbGxFmkn8HMUiaDVdfR0fF2GIb7l2lvjbHPuEjzDBo0yGrj6r+1tTUys9mPJE99fb1VN3jw4Ng8xWLRavOlL31JK7/++uuqra3NuV38dd1WZjOPdGxcddOnT9fKhULBajN//nyt3NraqvL5fNnMDQ0N4ahRoyLXLckizeNiLrdx40ZVKBScmXO5XDh8+PDILL45XMu6+urp6bHqent7y27Lw4YNC/fbb7/EeVzrluaRiMqcy+XCkSNHRq7blaW3t9cry4EHHmjV3XDDDVr5nHPOUQsWLIjc/8x9Pcs8Rx99tNXGXF/UuUQp97HZJ4s0j2s8zP07DEMVhqEzs2tb9hkXaZa4sdkin89HbsvmMcNct2sfch33JHkk56menp7I7SIIAq95kW8el4aGBq3c0dGhisVi5LZsLpNWFqXsz+v6/Gb/bW1tZc8lf20fNjU1VZxFmsfVpru7Wyu3t7eXHWdXXp9xkWaRnoPiziXDhg2LzCPJkiSPyVxf3P7X0NAQbrPNNpF5fLO48rjmheYxa9WqVaqnpyfR/NNnbKR5NmzYENvXpk2byu5/rnNfEOhNXfvm0KFDrbpNmzZZdWZfruUOOeQQq+63v/1t2W25vr4+HDJkSGS/kizSPIceeqjVZtGiRVbdDTfcEJm5ubk5Mo90bCR5HnroIavNmjVrtHLc/tfY2BiOHj06Mo90bKR54kTNi5RSavfddw/vv/9+rc6c5998883WcsuWLbPqurq6YvO4uOYs5TIPHjw4nDx5cmQW3xyuLK5jvDmXVEqpefPmld2WgyAIfeYQ0rmcmee6666z2ixfvtyqO/vssxNl9h0bSZ4rr7zSarNu3TqtnM/nE88/p02bppWvvvpqa7kVK1ZYdT/60Y+suvXr15dbdVlxmXfeeefw1ltvjcyTVhalKp/nNzQ0WMe4LLO47o3l83mrLmoul+Yc39VOMl8xt8vu7u7Y65K4Ob5vFlceyf3guPOf71yuknvlccvF7X+ubcNnbKR5zPvwrvXFzT99rv1cn0GSxTWnnjlzplV34403Ru5/ZmafcVHKfV/czPOVr3zFavPyyy9r5QceeECtXLkycls2559mHkkWaZ577rnHatPZ2amVfeafPmMjzSO5T1kqlcrO5Xbeeefw9ttvj8wiySHN4lLmGFp2W25sbAzHjBmj1bW1tWnlSuaf5vnO9RuPS6lUSn3+6Tr3SvJIlovaLpRyZ/YdG9/PYfLJbHL9eyXnF0lf5TJL7temnUWoKtdSKavoWipNkvmUZFtOK49r/zPzjB8/3mpj3hc58MAD1dtvv11RZkkWaZ6DDjrIavOXv/xFK0f9lhYEQWjm8RkXaZbdd9/damPeL1ZKqTlz5ji3Zf6SDAAAAAAAAAAAAAAAAGoeD8kAAAAAAAAAAAAAAACg5vGQDAAAAAAAAAAAAAAAAGpeEIbyV3C53iVlvnPM9V4w13sTXe3MvlzvYDOXa29vj3yPqfluUTOLJIc0i4vr3bQdHR2R7zE131PqMy7SPK53r7r6Wr16ddnM9fX14dChQxPncY2N632n5rtz29vbY9u0traqfD5f9j1uuVwuHD58eGQe19i43vHc0dHh6r/cqsvasGFD2W35r+sOR40aFZknrSxKud9ZZ9ZF7X+5XM5ru/DN4npnt2u57u7usttyLpcLR44cWXEWaR7Xcq53L2/atCkysznOZr+ufU36Oczj0aWXXmq1Md9X+5Of/EQtXbo08p3Y5v4lGRvXO5wlea666iqrTXd3t1YuFAqx7/E2t1+fsZHmkYjKvN1224XnnntuZJa0cmzJYnLt7/l8PtG27PuuY2meOD7vd88qSznm+ru6ulSxWEz0HuE08/j01d3dHZlZMs5pZSm3nHlM6uzsLJvZlTfLLK7jpWvuHfXu8fr6euu47JNFmkcy1457J71r/uk7NpI80uuSqG3ZNc4+YyPN4zr+mKL2P9e5LwgCs01sDmkW1xzIpa2tLfJayuzHZ1ykeVzHpp6eHqsuai7nyuyTRZrH9Y56V5uo93i7ztk+WaR5XNuZ+T2GYRiZubGxMTTnQuY8w7VtuOYikjwu5nKFQqFs5vr6+nDIkCGRWXxzlFtWIupc4jr/+YyLlGsbc12DRWV2zYuyzOM6LyXZLpTanDluXiE9DkvySM7/kjmzeRwz80iySPO4lkuSWbJdSOdFkizS5Xp7exOdS3zGRZpH0sZnu/AdG2k7k3lNHzXHV8q9//mMjTSPpK80MkuySPNUmtl1LvEZlyTtTK7Pvn79+kTXUlluFy7mvZy4a2xJZt8srjySc1Dc+a++vj5sbm5OnMd1n0t6ToxrUyqVymbO5XLhiBEjIrP45kjSzrFconOJa65ukrSpRFxm8xiXdZ44UduFUu55RpbM33OVcs+Zq5VZcg0jPaaXyyzJK72WkmSR5lVKRV6XxN27qORc53vMUDGZ4853aWaR3BuTXEuZ97WSPCNh9BWbx3UvZ/vtt9fKixcvVt3d3ZGZ08gizbNy5UqrjeveSdS1VNy9Q0kOaRbXdZnruNfe3u7clvlLMgAAAAAAAAAAAAAAAKh5PCQDAAAAAAAAAAAAAACAmsdDMgAAAAAAAAAAAAAAAKh5PCQDAAAAAAAAAAAAAACAmpertINCoaCVS6WS1aauzn4WR9LO7NvVJgzDstnCMFS9vb2Ry0vzSrK42rj6j2Mu4zMu0jyuNt3d3aKcW4RhqPL5fGQe899dWZRSKpezN8murq7YDGb/UdvFln/v6emJzOPKIh0b17iazPVJMpv9SvJIsrjyuBSLRStTOWEYirY3nxyuLJX0tTXJMc0ni2+eOK7tQrIe3yx77723VXffffdp5ba2tsT9ZpmnWrLOYn7PUfvf4MGDrTxZZvFtkwbpesx2rvOmqy6LPNLzuGTOkvT8J8kjHRtXHklfjY2NifK5zieSsZGOs5nHnDu6+q/0/CcdF0kW12dPekx1ZfYZl7TySLjm+VmMTdRySY8ZpVIpNnM188SR7HuuOZBvDtf1gk9fZqYs87g+v2tfqZR0bCR5pPPWKJLrP+nYSPK4judJz3elUkl1dHRodeb1oO+cYkv/W5Nen5dTKpVir4Ol9x8kWdI4fkiuS3zvtbiY359rfT58x0aSRzqfihO3jPScLcnj6st1rySK5JwtHRtJHsm80OcYkjSHNEsaXGPsMy7V5nPPKK11KWXfU8vyXopPHsl1mc/1n08WaR5J/0mvpXzGRZrFtc11dnaK+trCdf7zGRdpHsmxJm47dY2z79hIj32mpPt7qVSytgWfsUkrT5xSqaTa29sTZ/HNkdYcw5ybu65D0srT0NAQu/44rusSnyyV5EnjvJDm2GRx/0Up2X24tLJk8RmyzJLl3MBnPb55fI4jaWWW5kmjf5+5UxAEonZmHtfxaeHChVrZ57gvySP5HcGVRzKmlV5LSXJIsyR9nsHEX5IBAAAAAAAAAAAAAABAzeMhGQAAAAAAAAAAAAAAANQ8HpIBAAAAAAAAAAAAAABAzeMhGQAAAAAAAAAAAAAAANS8XNIFSqVSFjmUUkrV1enP7BQKBatNb2+vVg7DMPMc0iyu5bLII8kizeO7nMncLnp6erz6NJdTSqn6+nqtnM/nrTZNTU2i/pOQZJHmcfWVdJzDMLT68R0bSZ6GhgarTbFYFGVVanNeM4/vuPhmSTrGpVIpdtutZFzMvtI6Zpj9SI7TrnVL8nzhC1+w6i6//HKt3N3dHdtP3LqlYyPJ4+orl9NPf67jatwyPlmkeVzfYZLMzc3NVh6fcZFmcbVx9e86RkUtY+5vrn1Nmlmyn5hjKpljmP36jo0kT1bHDJ+xqSRPGnO5NMfGNRfqC745pJ896Rw+yzySuXYac3zp2EjypJE5CILYTNLrEGk7yXLluPL6jIs0i2sbjTtvmMIwtPrxHRdJniAIrDZJ5svl1uU7NmnlieMa56zGphyz/7j9zzXPl26rEj7z8SxIt2WfNj58x6Wa4xe3rqyz+PSfZaa445GrjeScndbnlOSRXOsmzeMzLtIskvX58BkXaV+Sa0SfuZzv2Phes/qI25YkWaR5pH3FMc93vmOTVh6f9fRFDqkwDGPHOE3SfcCnHx9p5UlDmlkqPVa75stZZknjvkAYhrHzdel6XL+hmJJe65UTl0mSRSn/PEnnGEEQxGaqZGyyui7xuXbyvTZJg881dl/r60x9vX5Tf8rje49Scj/Pp+8s86Qx7nH5pPc5q7EN8JdkAAAAAAAAAAAAAAAAUPN4SAYAAAAAAAAAAAAAAAA1j4dkAAAAAAAAAAAAAAAAUPMSvyAz7v1U0nfcut4l1dvbq5UbGxtFy5UTBIG1bsn7tcwclWSRvqcxSprjksW7j4MgUE1NTVqd5H3z0rEx+3ItZ76nMat3H7s+lySPi/neNZ/MPmMjzSPZDqIyu/Y/n3GRZnFJ4511PuNSjrlsQ0OD1WbXXXe16t56662yfYZh6DxmJc2ilPudyVOnTtXKgwYNstq0t7drZcn+H/deXkkWaR7XNpX0Paqu7dlnbKR5pO+zLycIAiuP77hIsqTxvm3Xe7zNY4Z0XCR5JH3F7e+uzL5j4/udm8eRuG07CILYTJVsf75jn5QkTyVj77O+SrP09VzO5x20abz3PE1ZjE3afWX5HmxJX2nOa8qRzmnNY3xW7x+W5HHNUX3z+GQ2M6Y5NmY733dXm8z5nu/YpJUnjuucHTcf7WtJ56v9QV+PYV+9x7wcyXnJvL8jvX7PIotSdh7X+KWxbfqMjTSPa95V6XbgOy6SLJXcR41axndcJHlcfSXdLsIwjD0OS8cmjTy+shibKEnvJQZBkPiasZI8km03i3O/6zP15TnBZ1x8+y7XfxTJ/lfNPFJxGauZRUKyHWS1rr4mmc9Uax+Vzq2qdf0XhmFspjSv/9Licy+jr68NttafsuBvh2vOI/ntLCuSPEEQ9Isc0iyVziv5SzIAAAAAAAAAAAAAAACoeTwkAwAAAAAAAAAAAAAAgJrHQzIAAAAAAAAAAAAAAACoeTwkAwAAAAAAAAAAAAAAgJqXq7iDnN5FqVQSLedqZ9a52pjrixKGYWweSQ5pFulnj+LK7Dsukjx1dfZzUkk/R6lUUp2dnVpdQ0ODV5+SPJI2YRhGricMQ1UoFCrOIm3nauOqiyLZNirJnDRPHN+8aeZoamqy6np7exP1kWUeV9+bNm1KbX1RGhsbrTpXnvb2dq38rW99y2oTty/55JFkkeaR7gNxzGV8x0aSx5XPPM4Xi8WyWZcsWWLl8R0XSRbXfuVzTow7RriymOcbpZTq6elJ3LerTZwgCKyxMPNIskjzuJYz+5d8hrTOJa46c1tw9WXu73HH5SAIrH4kY+PqN408EmYen3GRZkljW3bNi3zHxTdP3HfsEtemkrHxyRM3/3Sp1nzdh2su5zMulXB99qjzn1LZZsri80uuWSVZpHnq6+uturgx9ckj/UxZ5fHhuqZOY57ry8zTl1mQHfN79TmXpEmynVVr25T2m8W9OZ8svvflXNK4lkpzXCRz7zT69V3Gd96Yxv0e37HxzZP0fCu5/1nJ2KR9L1Ep2fVpljnS2L6rnaca1wZJ1uPzHfrIct/P4pjhOy5ZHb98pJklq3NJX+dJQ7Wvs/v6OFPt+wzSHEr17XbRX8YFf1v6+np0a9XIwl+SAQAAAAAAAAAAAAAAQM3jIRkAAAAAAAAAAAAAAADUPB6SAQAAAAAAAAAAAAAAQM3jIRkAAAAAAAAAAAAAAADUvFySxkEQqLo6/bmaUqmklc1/V0qpQqFg1bnaSZjri+LKa2bxzZE0S9bSzFLJmGzR3/KkxTeLdL9Io1/f5SR50v4uKhkXSZZisZgoj+SY4ZtFmmf9+vWivrYmOQ7HLVPOhg0btPKbb74Z21cYhqK+k+Yxs0jzpMXs12dsKtHd3S3uu6Ojw8qTZRYX13aYdJ+U6OrqErWT7Be5nD4t8skrySM9Zph5JPOpIAgi+wzD0OuYYWaR5nFtd729vVampHzGRppH+lmjmOv2GRdpljT2bdd24TsukjxpZA6CIHY7kK7HN0/S/U+yrkrGRrKdJeXTh3TfS2NdEr7jUqvXeq5zm7ntxh2XXceMvsyTlkrOJVmIu3fRn7ZRyPS370ySp1r7gLTPap1j4tpXMi5mO9dy1bov52rjmyeLeUclY5PV3MPnvrLvtuGaf1drfudqk1WepFn6KkeUap6j09rm0spYreODRJrHhzSWi8tTybhkdQ73yVTJOTyN9fv0kea2ksb+n+bYSPKkkTHNbbAax4isxiVL/S1PX+vLa3MX1z3Aat27MKWZJY2+0rqnU40x7j9PHwAAAAAAAAAAAAAAAAAZ4SEZAAAAAAAAAAAAAAAA1DwekgEAAAAAAAAAAAAAAEDNs1/imZD5HjDXe7ylGhsbY/sy3zEW9f6pMAy98pg5pFlc70SrZDzK5ZFkkeZxfdY03uNmvlu+vr7eapPP5626pqYmrzxpvK85zbEx8/T29sYu4/MuNd+xkeRJ4x1/cfuIJIc0SxrvJC6VSrGZpOPim6ejoyO2TZw0x6azszPx+iTbss97TCVZpH0l3TbCMLSW8RmbStqZ32PUMqVSSXV3dydehzRvkixSYRha5w6fcZEu5zqXmtulZFuO29+kYyPJIz3XRwmCIHYZSZa08ki45nNZjE2UpMc5yRwxrSyuvqv1nl7puJt1kvFwzROT5pGOjSRPpdcl5fiMjTRPpftFEATOY2zSHNIs0uuFpHzHRZInrXchx2270rHxzePzfmkzU5pjk9V7vH2uzfvTe+CzzNLf3u/umyeLzJWMjeS85JPZ51pKeuyr1jinOTbV+N77avurRJZ50uq7r/vx+Z6zyqJUuvfYt5bGcUYp/zyVXgulOS6SLD7HI59M0nGRtPP5jn2uWaWZq3U8zGpspOur9D55mlmkbao1V3KJu35IiyRPVteeUnHXe2lmSWs7SGvb6S/XHf0lx9aymJfUkv42Hlndt/CRZpY0+korTzXGmL8kAwAAAAAAAAAAAAAAgJrHQzIAAAAAAAAAAAAAAACoeTwkAwAAAAAAAAAAAAAAgJrHQzIAAAAAAAAAAAAAAACoebkkjcMwbMnn80vSWHGxWBTVCewQ8W8tYRhG5k0xR5LlIjMXi0Utc5Z5enp6pN1FZi4UCpHjXCgURCvp6uqS5okTlVeFYdjS29sbmbm7uzutLFKxmbu7u1PZ/1JUNrNkjNPU29srbVrRtiyVII9E5DjHHZfz+XyaWSRit+Wurq4Bsy0rNfAy99O8SsVk7utjXKlUMqsqPpeknMfiONZkklmSpUweicjMSqmWUqmkZfYcGxHhcpHnEjNvxlmkKjr/pZxFInK7KJVKLZ2dnRwzovNI2lQ0x3CRXruY7dK6ljKv/yT9pnlNWEZFmVPOItGn27Kn2HOJUmogZa5qXul5XlUpc4I8EhVlTjOLsK/E8yJfnucNl4r3P+k4p/h9JJrLZZgjSf8VZa6E5+fN5Lic8T6ZOHMfHCNMtTbO/XGMlcrg/JfxcaRPt2VPVTv/+Uq6Lff1cdnnXKIy3Jartf+lOL8R8TmX+NzjqiCPRL8/lzj02bmkAn12zKhAv7lmFaq1+wJKDbzM/TGvUmUyB2EYVjsIAAAAAAAAAAAAAAAAUFW8bgkAAAAAAAAAAAAAAAA1j4dkAAAAAAAAAAAAAAAAUPN4SAYAAAAAAAAAAAAAAAA1j4dkAAAAAAAAAAAAAAAAUPN4SAYAAAAAAAAAAAAAAAA1j4dkAAAAAAAAAAAAAAAAUPN4SAYAAAAAAAAAAAAAAAA1j4dkAAAAAAAAAAAAAAAAUPN4SAYAAAAAAAAAAAAAAAA17/8DQriOStzYqU4AAAAASUVORK5CYII=\n", + "image/png": "iVBORw0KGgoAAAANSUhEUgAACMkAAADiCAYAAABebgK1AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAABF5klEQVR4nO3debxeVX0v/vWcKSchBCKEMJSiiDgAYgG5t1jrADi1ilLnal9MgqhU0F4Ex4BoRVsVJ4peEScs3rbg9EIcKCKliMggIijKGAOBJECGk5zp2b8/uN6++nvx/e5k7zyck8f3+08/Z629nr3XXnvttZehU1VVAQAAAAAAAACAfjYw0w0AAAAAAAAAAIBes0kGAAAAAAAAAIC+Z5MMAAAAAAAAAAB9zyYZAAAAAAAAAAD6nk0yAAAAAAAAAAD0vaFN+eNOp1P1qiFtVFXVeaT/fba2t5SyoqqqRY8UaPPmE/WLUrR5c3L/PSq2uDb3U18uZctr82xtb9GXHxXa/Ohw//VeP/WLUra8Ns/W9pYtsC+XLbDN/dSXS9ny2jxb21u2wL5ctsA291NfLmXLa/NsbW/Rlx8V2vzocP/1Xj/1i1K2vDbP1vaWLbAvly2wzf3Ul0uZuTZ3OmGTSlVV6f03MBD/+xBVFf+cxYsXp22anJwMs5UrV6ZlS01fzn7vTKmqyv33KOinNs/W9pbg/tukTTJsNnfOdAMa2BLbDI9kS+zLW2Kb4ZHoyzBz3H/0iy2xL2+JbYZHsiX25S2xzfBIZl1fzj5AdbvdR7El0HOz7v6jf9V9KM8+8G+EtC8PDcWfC6emphqVa2tqaqon91/2DGuj7fOvzbO16W+ayTZnfSerN9usMjAwUEZHR8N8eno6zI488sgwK6WUe++9N8y+9KUvpWWnp6fDvtzpdHp6HzU1OTmZ3n+Dg4M9OW7TPrkx42PWr3p1/2VlN+a39qpvZMfOfk/2PCgl7xdNj9m2bHT/+c8tAQAAAAAAAADQ92ySAQAAAAAAAACg79kkAwAAAAAAAABA37NJBgAAAAAAAACAvjc00w0AAAAAAGDL0e12Z7oJAH2nqqrGZTudTqu6p6amGh23abmZNJPPsIGB+N8uaNOu2fhczn5rnaxfZX252+2WsbGxMF+wYEGYvfCFL0zbdNRRR6X5H5qsz7Xp5726R9qo68u9Ohd12oy/s/E8N9W0vf4lGQAAAAAAAAAA+p5NMgAAAAAAAAAA9D2bZAAAAAAAAAAA6Hs2yQAAAAAAAAAA0PdskgEAAAAAAAAAoO/ZJAMAAAAAAAAAQN+zSQYAAAAAAAAAgL43NNMNAAAAAAAAAJqpqmqmm8BG6Ha7M92EzSr7PQMD+b/T0Itz0el0ysjISJgfdNBBYXbiiSemdd9+++1hNj09Xdu2SFVVrco3NTg42Kp83fWN1I1VTftU23OYnY/JycnG9U5NTTUuW0q7e6xpvUND8faRuvs2y7NrX3f9Op1OmjfhX5IBAAAAAAAAAKDv2SQDAAAAAAAAAEDfs0kGAAAAAAAAAIC+Z5MMAAAAAAAAAAB9zyYZAAAAAAAAAAD6nk0yAAAAAAAAAAD0vaGZbgAAAAAAAADwh2NoKP9EOTU19Si1hF7odrtpnl3/urKRqqrK+Ph4mH/3u98Ns8HBwbTu6enpRm2q0+l0yvDwcJhn56JNm9r+nqxdAwO9+Tc6mvaL38valZ2PuuNm9WbZxvyerHw2RtaNr5m25znS6XR6Um9df4uurX9JBgAAAAAAAACAvmeTDAAAAAAAAAAAfc8mGQAAAAAAAAAA+p5NMgAAAAAAAAAA9D2bZAAAAAAAAAAA6Hs2yQAAAAAAAAAA0PdskgEAAAAAAAAAoO8NzXQDAAAAAAAAgD8cU1NTM92EvjEw0OzfROh2u5u5JRuvV9d/cHAwzKanpx/1Y9Ydt6qqMjk5GeZtrtHw8HCjNm2MqqrCLGtzp9NJ6x0aircutO0zTc9l3f2V5W3vsax8dtw256rpeNJLWX9r2pdn368EAAAAAAAAAIDNzCYZAAAAAAAAAAD6nk0yAAAAAAAAAAD0PZtkAAAAAAAAAADoezbJAAAAAAAAAADQ92ySAQAAAAAAAACg7w3NdAMAAAAAgNlpYCD+/9h1u91HsSUAQKTT6aR5VVWPUkuYCf02J+vV/HNwcDDMpqene1a2jexc1JmcnOxJvW3UjUVZm4eHh8NsamqqcZvq1PW5LG97npv297rj9mrMyK5v3XMq06Zs1Cb/kgwAAAAAAAAAAH3PJhkAAAAAAAAAAPqeTTIAAAAAAAAAAPQ9m2QAAAAAAAAAAOh7NskAAAAAAAAAAND3bJIBAAAAAAAAAKDv2SQDAAAAAAAAAEDfG5rpBgAzb+eddw6zPfbYI8yuvfbaXjQHANhEZ555Zpq/4x3vCLN58+aVpzzlKWF+zTXXNG5Xr8yfP7/st99+YT42NhZms/H3AMBs1u12Z7oJAMAsNTSUf2acmpp6lFqy8QYG8n8/wNxn82hznttcg+np6TAbHBzsSb1tdDqdtF2Tk5ON6667BrNRp9MJszbnopS8X7U5V1nZtuNJ07q3tHGsqqo0z/pFU1ve3QEAAAAAAAAAAJvIJhkAAAAAAAAAAPqeTTIAAAAAAAAAAPQ9m2QAAAAAAAAAAOh7NskAAAAAAAAAAND3bJIBAAAAAAAAAKDvDc10A4CZNzIyEmbPfe5zw+zWW2/tRXMAgE20ww47NC67aNGi8qY3vSnMjzrqqMZ198rcuXPLU57ylDBfuHBhmK1atSqt+3GPe1yY/fCHP6xvHAAAADzKqqpqVX5oqNnnwqmpqVbHpX91u92ZbsIWoaqqMj093ZO6s2swMNDu39EYHBwMsza/p+1Ylmn7myP91td79Xt6dW2b1utfkgEAAAAAAAAAoO/ZJAMAAAAAAAAAQN+zSQYAAAAAAAAAgL5nkwwAAAAAAAAAAH3PJhkAAAAAAAAAAPqeTTIAAAAAAAAAAPQ9m2QAAAAAAAAAAOh7Q5vyxyMjI2WnnXYK8zvvvLN1g4BH3/333x9mj3/848NsdHS0F80BADbRypUrG5edN29e2WeffcJ8YKD5vvput9u4bKaqqjI5ORnmBxxwQJh961vfSuv+wQ9+EGZz584Ns/Hx8bTeLc3IyEjZeeedw3zdunVhls0tAQCALVPdWvCGDRsepZbQC1NTUzPdhEdNr9YqmP2mp6fDbHBwsHHZtnrVJ9us6dXpVZs7nU6YVVXVqu6szb08V200Pc91v2cmxsFeXdus3qzu2XnFAQAAAAAAAABgM7JJBgAAAAAAAACAvmeTDAAAAAAAAAAAfc8mGQAAAAAAAAAA+p5NMgAAAAAAAAAA9D2bZAAAAAAAAAAA6HtDm/LHixcvLn/3d38X5rfcckuYfec730nrvuOOOzalKcBmNDk5GWaPfexjw2xkZKQHrQEANtV9993XuOzcuXPLvvvuuxlb818GBprtye92u2m+du3actVVV4X5scceG2a/+93v0rovv/zyMNtxxx3DbNmyZWm9O+20UznuuOPCfMmSJWn5prJ6zznnnLRsdh1e/epXh9knP/nJ2nYBAACzz/bbbx9mL3zhC9OyX/7ylzd3c2bUggULwmzt2rVhNjg4WLbZZpswX7VqVat29ZOhofwT5dTUVJpnaw516wozUS+z2/T09Iwdu2mfq1t3GxwcDLNe/t5OpxNmVVX17Lh1mq5TtpGNc3VjXF35Nn0jk5Wta3PTa5+V6xX/kgwAAAAAAAAAAH3PJhkAAAAAAAAAAPqeTTIAAAAAAAAAAPQ9m2QAAAAAAAAAAOh7NskAAAAAAAAAAND3bJIBAAAAAAAAAKDv2SQDAAAAAAAAAEDfG9qUPx4eHi477LBDmD/taU8Ls0MOOSSt+6/+6q/CrNvt1rYNNqdOpxNmVVU9ii15dGT32C677BJmw8PDYTYwMFDmz58f5qtXr964xlFrYCDe75hd26xcnV6Oy39o9x/AxhocHAyzlStXNq630+mkz/TZaHJysixdujTM2/yeM844I8weeOCBMJuenk7rveeee8qSJUvCvGlWp2nZiYmJcvfdd4f5nnvu2bBFAADAbLV+/fow+9M//dO07Je//OXN3Zye23HHHcPsvPPOC7MTTjghzHbbbbfy93//92H+6le/Osz6ce1zaGiTPkNuNk3XzDcmZ8uVra3VrevMlDbfcWbqN7UZy7Kybb8dZfd2m7Eqq7fteNLLurckvXg++pdkAAAAAAAAAADoezbJAAAAAAAAAADQ92ySAQAAAAAAAACg79kkAwAAAAAAAABA37NJBgAAAAAAAACAvmeTDAAAAAAAAAAAfW9oU/54ZGSkPPaxjw3zE044Icw+9KEPpXWPjo6G2djYWG3bHkmn0ykjIyNhPjExEWZVVTU6Ziml7Lfffml+7bXXhtnIyEjZeeedw/yuu+4Ks263W9+4GTAwEO/Fmq1tbnP9M51Op2fHbHOes7JNDQ4Olvnz54f52rVrG9c9W/tNps05nqnfO1PHze6F7B7q1TE3Rq/uvy2xrwMzY9ttt21cdnJysixfvjzM586dG2bT09Np3Vk+ODgYZuPj42m93W43/Zu3v/3tadnMf/zHf4TZ1NRU43p32mmnctxxx4X5kiVL0vIzIXs+Zn2u7nndq7k20J/mzJkTZnXPi36z6667ps+4U089NczWr1/fiyYB0GcmJyfDbJ999knLZu94de+OvVL3bvKyl70szL7//e+H2erVq8Osqqr029PQUPxZru69spfnMWtX9i7cRtt6s3XVXrWZLVube+g1r3lNmB1//PFp2T//8z9vfNxsXKj77tSm7Ezp5XfUzJb4LaZX13B4eDjMsnuozb6MXvblqM2z8w4AAAAAAAAAAIDNyCYZAAAAAAAAAAD6nk0yAAAAAAAAAAD0PZtkAAAAAAAAAADoezbJAAAAAAAAAADQ92ySAQAAAAAAAACg79kkAwAAAAAAAABA3xvalD9etmxZWbJkSZj/5je/CbNjjz02rfvII48Ms0996lNhdsABB4TZfvvtV6655pow/+EPfxhmN998c5iVUsoNN9wQZnvttVda9tprrw2zxYsXl5NOOinMs2y26na7M92EzarT6TQuW1VVq2MPDMT72ppmdfm6devCLLu2IyMjZddddw3zj33sY2F2+OGHh1kppTzzmc8Ms6uuuiotOxu1vUd6dY/V9ZtI2/Zk91ibe6jNvdtG0/PYy+P2qs8MDeXTiqmpqcbl68pCPxgcHGxcdqeddmpcdsOGDbVz36ay35Rlbcfsn/zkJ2HWZjzJyrad52XvWVnWK51OJx2Xt9pqqzCre/ZNT083bhfwh+eFL3xhmN13331p2SuvvHJzN2dGzZ8/vzzrWc8K8+222y7Mli5d2osmAdBnsrl63bvUnDlzwmxsbKxxm9qoe1ceHh4Os8MOOyzMvv3tb4fZ8uXLy1lnnRXm2bpc3Zpdr9ZNS+nd2lsv1/SyumdibZQtW9263De+8Y0wO//88xsfd3R0tOyxxx5h/stf/rJx3Zm631t3n2RjTi/Hql7Jngfj4+Np2Zn6BtRGdn2z35tdv6qqtqjvOFveVQMAAAAAAAAAgE1kkwwAAAAAAAAAAH3PJhkAAAAAAAAAAPqeTTIAAAAAAAAAAPQ9m2QAAAAAAAAAAOh7NskAAAAAAAAAAND3hjblj9euXVt+9KMfhfnk5GSYbdiwIa37iiuuCLMnPelJYXbHHXeE2X333Vc+8YlPhPnKlSvD7Pbbbw+zUkpZuHBhmF111VVp2cxWW21VDjzwwDAfGIj3NXW73cbHna1m6vd2Op0wq6qqcb2Dg4NhNj09XVu+6W8eGtqkW/2/Wb58eZhNTU2F2XbbbVeOPPLIMM+u7Qte8IK0TV/84hfD7MlPfnJatu4cZu3qVX+suz7Zea4rn7Wrzbno5f2X3WOjo6Np2fHx8Ub19tJMjc29Gi+OPvroMDv22GPTsvvvv3+Y7bvvvuV73/temH/84x8PszPPPDM9btaX99hjjzDL5hillDI8PFy23377ML/nnnvS8vD/VzcXmDt3bqNsY467atWqRu3K5jZ1ZbOsbsyuqip9Pmb3fZ2JiYnGZdtYsmTJjBw3UneOs2dy3bMky+vmPRszZwb6y0UXXRRmT3ziEx+9hswCVVWlz6k27/4AUEq7tcQsb7venpXPvPzlL0/zpz/96WF26623hln2vWv9+vXlxhtvrG/cI5ip9cteajM/qXs/hM1pzpw5aZ59e9h7770bH7fT6TS+T9p848m+6be1JY5l2XrTU5/61LTsr371qzBre56bfqdrszbaRnYes2d5XV9u8yyJ2uRfkgEAAAAAAAAAoO/ZJAMAAAAAAAAAQN+zSQYAAAAAAAAAgL5nkwwAAAAAAAAAAH3PJhkAAAAAAAAAAPqeTTIAAAAAAAAAAPQ9m2QAAAAAAAAAAOh7Q5vyx1VVlcnJyUYHqit3xx13hNng4GCYTU1Nhdm6devK1VdfHeYLFiwIs4suuijMSillZGQkzAYGmu89GhwcTNs1Z86cxnV3u91GWVvZudpzzz3TsrfddluYjY2NNW7TTJmenm5VPutbWVZ3fbOy2T1WVVWYLVq0qBx33HFhvuuuu6ZtyixcuDDM6u6R9evXh1mn0ylDQ/Gw+O53vzvMnve856XHPfHEE8PsmmuuSctmRkZGyo477hjm9913X5ht2LCh1XEjG/Oc6HQ6YfbqV786zI4//vi03p/85Cdhdsopp4TZxtybTe+xumdCL8ffSNamD37wg2nZpUuXhtnBBx/cuE2/+MUvypOe9KQwP//888PsIx/5SFr36aefHmZ77713mL3tbW9L63384x9fvvjFL4b5G97whjDLzmMppWy99dZhduedd6Zl+S9/9md/luZZ3/nNb34TZu9973vDbGBgoMybNy/Ms+dQnYmJiTCrm1Nl1q1bV37605+GeTZGtp3b9Eo2f6kbl9vMqfpNdi6yd7S99torrffss88Os5tvvjkte8QRR6T5TNh5553TfNmyZWG2yy67lLe85S1hfu+994bZueeemx53zZo1ac7Gyea9peRjM733q1/9aqabAP/P8PBwmr/uda8Ls+9///tp2br3B2Zer+aQixcvDrOVK1c2rretbP0sewf/9Kc/nda57bbbhvmKFSs2qm30Tt16fHYfZOvIGyMrn72bHH744Wm9W221VZi9+MUvDrOsP1ZVlc4Rs9+SrZnWlZ2tsvfzbCzppZlcr81+8+677x5m2TpVKX946xWZbEzI1I1xWb233HJLo2OW8vB3mptuuqlx+cyW2C9mapzLztV73vOetOw555wTZpdeemnjNtVpsx+hV7LnWJvnXy+eJbPv7AEAAAAAAAAAwGZmkwwAAAAAAAAAAH3PJhkAAAAAAAAAAPqeTTIAAAAAAAAAAPQ9m2QAAAAAAAAAAOh7NskAAAAAAAAAAND3hjblj6uqKtPT040ONDg4mOZjY2NhNjw8nLYpMj4+Xm6//fYwf/7zn5+2KZO1d2hok07rfzNnzpzy+Mc/PszPPvvsMHvlK1+Z1v3GN74xzL72ta+lZcfHx8Os0+mUOXPmhHnW5qxcKSXtb0ceeWSYTU1NpfWW8nC7I1m/aqOXxxwYaL7nLbsOO+64Y5hl9+bKlSvLeeedF+bZNaobZ+bPn9+oTaWUsn79+jDbdtttG48LxxxzTJqfdtppYfbXf/3XadnsXG233XblqKOOCvMFCxaE2cknn5we92lPe1qYffWrXw2zww8/PK13eHi47LDDDmF+8MEHh9nLX/7ytO5PfOITYZYdc8WKFWm9bXS73cZls/u6Tb2veMUrwmzbbbdNy7773e9ufNxMVVVpX8+ecXvvvXda95Oe9KQwe/3rXx9ma9euTeu96667ypvf/OYw//KXvxxmL3nJS9K6s/N8wgknhFn2vC6llN1337186EMfCvMPf/jDYVY3vznrrLPC7MYbbwyz97///Wm9dcfOxqrs+paS96u77rorzD7+8Y+H2cjISPmjP/qjML/tttvCrO75l82nd9ttt7RsZvXq1eUHP/hBo+PWmZiYaFRuY+ZFTcfBunJtxtfMPffcU5YsWdKobF25LM+yc845J8w6nU76HMrGyOOOOy7MSinl6KOPDrO///u/T8tmhoeHy+LFi8N82bJlYVZ33bNnSfYsKCUft1evXl2+973vhfm5554bZnfeeWd63G9+85th1qt+PluNjo6Wxz72sWH++c9/Psz++I//OK37ox/9aJh97GMfq23b5lb3vM7u66Zj9h+iuXPnpnl2nrP5Wt1axsDAQJk3b16Y183l6a23vOUtaf6YxzwmzD74wQ+mZf/mb/4mzBYsWFCe8YxnhPnFF18cZnV9+U1velOYnXjiiWH2ohe9KK33j//4j8s73vGOMD/ppJPCbKbGqqGhobJw4cIwz57Zhx12WFr3GWecEWavetWrGte7cOHC8rznPS/ML7jggrR8JltXyNbtsvFx7ty5ZZ999gnzH//4x2H22c9+Nsx+X3fk1FNPTcvecccdad5vsvWKv/zLv0zLHn/88WH2k5/8JMyuv/762nZl6+o777xzmD3rWc9K691///3D7MEHHwyzpt/JSundN4JsXC2llDPPPDPM6t6lli5d2rhdmY35jtNUr9552qzX7rbbbuU973lPmG+zzTZhlq3zl9Lue+dM2XXXXcMs+/7whje8IcwWLFhQDjrooDC//PLLw6xuXTV712rTl+u+vWf9qu67YC/f/ZuOZVm5UvLf1GbsLaX59/t//Md/TPNsL0Lbbzy9uoZt+lWmzbXP9OL+8y/JAAAAAAAAAADQ92ySAQAAAAAAAACg79kkAwAAAAAAAABA37NJBgAAAAAAAACAvmeTDAAAAAAAAAAAfc8mGQAAAAAAAAAA+p5NMgAAAAAAAAAA9L2hTfnjTqdTBgcHe9WW0PT0dJhVVRVm3W63rF27Nsx33nnnxm0aGIj3Fw0NbdJp/W86nU6ZM2dOmO+9995hduCBB6Z1X3311WF22WWXpWXvuOOOMNt6663LM57xjDDPfs9xxx2XHveXv/xlmO24445htnz58rTeUvK+08YLX/jCMLvtttvCLDvHv5f1uzaGh4fDbN999w2zuXPnhtmqVavK17/+9TAfGxsLs6zPlJKfh7r7evXq1WG23Xbblde//vVhfvzxx4fZhg0b0uNm48LIyEhaNqv7/vvvL//0T/8U5j/72c/C7Pzzz0+P+zd/8zdh9tGPfjTM6u6/+fPnlz/7sz8L89tvvz3Mtt9++7TuXXfdNczWrFkTZtlzZmNkfbLb7Taut03ZrM+99a1vDbO3vOUtjY/Z9reuX78+zLNzvP/++6d1X3zxxWG2zTbbhFnWnlJKmZycLPfcc0+YZ/fY4Ycfntb9nOc8J8xe97rXhdlFF12U1js6Olqe9KQnhfl3vvOdMHvooYfSuo844ogw+8hHPpK2KbNo0aLyyle+Msyzc3XGGWekdWfX6Ctf+UqYrVq1KswmJibK0qVLwzwbb+rm2Vl+wQUXpGUz4+Pj5be//W2Ytx0jZ0I2HtXNp3o1ptdZsmRJo6zO+973vjD71re+lZbNniXZPLCuL2f97Xvf+15aNjM6Olqe8IQnhPmyZcvCrG6Oceqpp4bZb37zm/rGBfbcc89y6aWXhvmnPvWpMPv2t7+d1t3L/joTtt566zBbt25dWna33XYrn/vc58L8qKOOCrO6uW12jzU1PDyc9skPf/jDYVb3PrRo0aIwq/stF154YZo31el00rxX7+6l5GN+1mcOO+ywtN7tttsuzE4//fQwO+ecc9J6JyYmyl133RXmT33qU8Ps5z//eVr3bBwz6q59Xd9pKqs3ezY+97nPTet9+ctfHmZTU1P1DQusXbu2/PjHPw7z7Jn9zW9+M607my8/7WlPC7MHH3wwrXfu3Llpfz322GMbtamUkq4FT0xMpGUzW2+9dXn2s58d5pdffnmYPe95z2t83O9+97thlq1xlfLwec7WkrNxvW5MyMbBN73pTWGWvVeuWbMm7cvZ83p8fDzMSinl3HPPDbNs/bmUUubNmxdmnU4nfafN+lzdOX7iE5+Y5pG6NeZOp5OOZdk6SbZmV0opP/nJT8Ls1ltvDbO6ddXBwcGyYMGCMM/eay655JK07l//+tdhNjk5GWZ1z6gsz54zbZ5tr3jFK9L8zDPPDLO6d6lsnaNO1t/aPP/aqFsXyO7Po48+Oszq5st1a4knnHBCmNWt+X3hC18IsyOPPDItO1OyPpl9s8zW+Xfbbbdy9tlnh3n2vavuWZKtm990001p2ex5Xid7N8zOYSml7LfffmF21llnpWXPO++8MKt7lrz0pS8Ns1NOOSU97vz588PsiiuuCLPsPauUh8e53XffPczvvPPOMLvhhhvSunupV9+Fe7H+2el00u+d2byobm9F1qa6stHz3L8kAwAAAAAAAABA37NJBgAAAAAAAACAvmeTDAAAAAAAAAAAfc8mGQAAAAAAAAAA+p5NMgAAAAAAAAAA9D2bZAAAAAAAAAAA6HtDm/LHVVWV6enpMB8cHGzckOHh4Ub1Tk5Ohlmn0ykjIyNhPjU1FWZDQ/mpmZiYCLNut5uWzVRVVcbHx8P83HPPDbPly5endX/9618PsyOPPDIt+773vS/MFi5cWA4//PAw/853vhNm559/fnrcbbbZJsweeuihMMv66e9l/Sq7hgMD+d6yV77ylWH21a9+NcyWLVuW1tvpdNJjt+l32X2U3SdVVYXZxMREufvuuxu1J7sHSinl5z//eZg99alPTcvecsstYVY3ZmzYsCHMXv/616fHPeCAA8LspS99aVr2S1/6Uph1u90yNjYW5u9+97vD7B//8R/T4+68885hdsYZZ4TZgw8+mNY7NDRUFi1aFOZbbbVVmH3iE59I6z7ttNPCLOtXWV/+vV7df704Zt14sdNOO4XZXXfdtXGNewTZs7yUfDxpo+78P+1pTwuzL3/5y2GWjY2lPPx7Vq5cGebZWPWkJz0prfvf//3fw+xzn/tcmF133XVpvbfddlt57WtfG+Y33nhjmP3Lv/xLWnf2e9///veH2T333JPWu/3225cjjjgizLM52zve8Y607mzu9OY3vznM1q5dG2Z18+XsObMx85fIP/zDPzQuW/f823fffcPshhtuaHzcXo2d/HfZszGbfw4MDJQ5c+aE+eLFi8Psm9/8Ztqm7NrXjQmZTqeTPouyfn7WWWeldWdj59lnn13fuMAvfvGLsscee4R5dj569Vydrbbddtswy+bpv5fNjW6//fYwqzvP/+t//a8wy9q8Zs2aMNtzzz3Lv/3bv4X5G9/4xjCrmwvsuOOOYXbzzTenZTudTpjtsssu6bMzW1OoO8fPec5zwiybM9XZYYcdyqtf/eowf/rTnx5mT37yk9O6s/72zne+M8zq1tXmzp1bnvKUp4T5c5/73DDL1iNmq6zP9cqiRYvKy1/+8jC//PLLw+xnP/tZWnfW19vMi7rdbjoOZs+p7373u2nd2fvS3/3d34XZeeedl9Y7Z86c8oQnPCHM//mf/znMDj300LTugw46KMze/va3h1ndWLT11lun99g555wTZtl6bimlXHDBBWH2qle9Kszq5iDLly8vH/vYx8I8+82jo6Np3dk4l/XHbP2lbi3jxBNPDLNsbbqUvE/WPTszCxYsKM961rPCfP369WG23377pXU/7nGPC7Osn2fPtlIefofO5vIXXnhhmNWtYf7yl78Ms2zuU/cuPDo6Wp74xCeGefaMq1srzjRdlxseHi7bb799mGdrlNk3j7o2LVy4MC2bmZycLPfff3+Y130va6qu3l6989R948kcf/zxYXbFFVekZVesWFE+//nPh3k2F8zugVJKOeSQQ8LspJNOCrPs3vy9pudr7733TvP9998/zLJ3i9WrV4fZzTffXA488MAwz/pUtgZSSknrzdYMSsnnXHVrGVmWfY8spZSTTz45zL7whS+kZbNn55577ln+9//+32Ge9au6b2nZOPgXf/EXYbZu3bq03r322qtcc801Yf4//sf/CLM3vOENad3ZGnSb70p1a3PPfvazwyz7BldKSdccPvzhD4fZBz/4wTDbZ5990r7+kpe8JMzq1piz89h0jPIvyQAAAAAAAAAA0PdskgEAAAAAAAAAoO/ZJAMAAAAAAAAAQN+zSQYAAAAAAAAAgL5nkwwAAAAAAAAAAH3PJhkAAAAAAAAAAPqeTTIAAAAAAAAAAPS9oU35406nUwYHB8N8enq6dYM2p8HBwbJw4cIwv+mmm8JsamoqrXtoKD51AwPN9x5NTk6W+++/P8zvvvvuxse9+OKLw+y1r31tfeMC22+/fTnmmGMa1f2Vr3wlrfspT3lKmE1OToZZVVVpvaOjo+Wxj31smB9//PFh9rOf/Syt+4gjjgizU045JczWr1+f1ttL4+PjYZZdv9tvvz3MpqamyvLly1u1K5LdB7/5zW8a17tgwYJyyCGHhPkFF1wQZldccUVa99Of/vT0uL3yne98J8zOPffctOxdd90VZt1ut3GbxsfHy29/+9sw/+QnPxlmhx9+eFr3lVde2bhdbRx44IFhdtVVV6Vls7G76Xmuqiotm92bBx98cFr3hRdeGGbZuFyn0+mk5yJ7LmfPt1Lyvv7Tn/40zLL7p5RSdtppp/R58ZKXvCTMXvCCF6R1P/WpTw2zl770pWFWN3+Zmpoq9957b5h/4hOfCLNszC8l76/33HNPmNX1m7q50fOf//wwO/vss9O6/+3f/i3Mmt5/AwMDZWRkJMyz532bufTExETjsiMjI2WXXXYJ88MOOyzMbrjhhsbHnSltnmG9tGTJkpluwn/T6XTK8PBwmO+www5h9qtf/arxcbP3rDoDAwNl3rx5YZ61eccdd0zr3nXXXcNsxYoV9Y0LTExMpHPbrL/Wvf/N1r7e1LJly8Ksbvy8//77yz/90z+FeTY3rXtm33zzzWH29re/PcyyZ1Rde3/0ox+lbcpkz93seV1nxx13LKeeemqYZ7936dKlad1HHXVUmF1zzTVp2TVr1oTZDjvsUE488cQw/9u//dswW716dXrczHve854wW7t2bVp2+fLl5aMf/WiYZ21m42Vrn9m6zqWXXtqL5tQaGRkpO++8c5gfeuihYXbSSSeldWfv4K95zWvC7Jvf/GZa780331z+5E/+JMyze6xuXS4bP9/5zneGWd1zc968eWXfffcN82233TbMNmzYkNa93XbbhVl2/erGz263W8bGxsI8m0uMjo6mdb/iFa8Is2ydOFM3l3viE58YZqeddlpad3auTjjhhPrGBaqqSp+tn/3sZ8PsoosuSut+73vfG2Ynn3xymNU9o7beeuvynOc8p3G7Mtl9lGV1a/ljY2PpvZ+tcxx33HFp3YsXLw6zbJ6fPbO33nrrdH3t05/+dJjVjZ/nnHNOmO2+++5p2czQ0FA6jmXvaXVjXFa2bh2rV+rG/Gx8zL5ZzZ07N613eno6nZ+++MUvDrOsXCn594nsm8cPf/jDtN462bnca6+90rLZGnX2zSobM6ampsqqVavS4zb13e9+N8zq1iDbyH7PHXfckZa9/PLLw2zRokVNm1TGxsbKtddeG+bZe0n2/aeUUq6++uowy77R1b1L3XnnneWNb3xjmH/uc58Ls7pz9a//+q9hln0fqhs/d9ppp3Se8hd/8Rdhtvfee6d1Z98lV65cGWbZuP273/2uvOtd7wrz008/PcyOPvroMCullAcffLBRmzL+JRkAAAAAAAAAAPqeTTIAAAAAAAAAAPQ9m2QAAAAAAAAAAOh7NskAAAAAAAAAAND3bJIBAAAAAAAAAKDv2SQDAAAAAAAAAEDfG9qUP66qqkxPT4f54OBgmGXlSillYCDer5OVraoqzLrdblmzZk2Y/5//83/Sspm6vKn777+/fOYznwnzW2+9tXHdy5YtC7Ntttmmcb2//vWvyyGHHBLmf/VXfxVmN954Y1r3+vXrw6zNNZiYmCi/+93vwvzHP/5xmK1duzate+nSpY3K9qpPbYypqakw+8Y3vhFmk5OTaZ0PPPBAmI+OjoZZNh6UUsr1118fZitXrkzLZm6++eby9Kc/Pcyz83TXXXeldWdlV6xYUd+4QFVVZWJiIsyz7Mwzz0zrPuyww8Ks7hpl1q1bV37yk5+E+XnnnRdmJ5xwQlr3vHnzwmzhwoVhdsEFF6T17r///uWaa64J87e+9a1hdvXVV6d1Z7Lz3GbMeOMb3xhm1113XVr2RS96UZhl93wppVx11VVhtt1226XPi4svvjjM7r333vS42X2d9anLL788rbfb7abj+rOe9aww27BhQ1p3do8ceuihYfbrX/86rbfb7ZaxsbEwf9/73peWzwwNxdPKbJ6RzR9LKWX58uXlIx/5SJjvsssuYXb44YendWeyMeOLX/ximD3taU9Lx4vtttsuzOrmGJlsvK+zfv36dE72y1/+Mszq5viZXo1xdXXXmck5WS9k9/W3vvWtMOt0Oul9/Ud/9Edhlr13lJLf99n8v05VVen1e8xjHhNmq1evTutuM1/L1LW5adaP2ow3q1atKl/96lfDPHvPvvTSS9O6TznllDDL3pOz9jz00EPp/ZmpG/+OP/74MPve977X6JillLJmzZpy2WWXhfnixYvD7L777kvr3nHHHcOszXhfVVX6TptlbWT1ZmtcpTz8XpmNR3vssUeYbb311mnd2Tj4hzTe1I0Xn/zkJ8Ps29/+dlp3r87j6OhoefKTnxzmd999d5jV3UM33HBDmH3ta18Ls1WrVqX1djqdMmfOnPRvInXP5G233TbMTj755DA755xz0nrHxsbS9+W99947zOrWqs4444wwy+b5deNU3ZpRpu7d5HWve12YZWtgmbpxOeuvH/jAB9K6ly9fHmbj4+P1jUvKZvOIxz72sWF29tlnp3Vn7+4333xzmNWtN4yMjKRz+ew8Z2vmpeTPseHh4TDbmPExe4fI1oWOOeaYtN63vOUtYfapT30qzLLz/OCDD5YLL7wwzLM51/777x9mpeTzpjZrCnVzjDaajgm9VNfnsvsge351Op203uHh4bLTTjuF+Tvf+c4w+/jHP57WnfXJU089NcxuuummtN66tYFM3XfH7BtR02MODAyUrbbaKszXrVvXqN5S8rleNpZsjKxPZn2ubg1lhx12CLP58+enZbMxodPppNfvkksuCbPjjjsuPe61114bZk996lPDLFuPLeXhd9KTTjopzLP13qc85Slp3c9//vPDbMGCBWH2/e9/P6132223LS9+8YvD/CUveUmYXXTRRWnd2TeGz372s2GW3UNVVaVj0Qte8IIwq5uLv+Y1rwmzpusC/iUZAAAAAAAAAAD6nk0yAAAAAAAAAAD0PZtkAAAAAAAAAADoezbJAAAAAAAAAADQ92ySAQAAAAAAAACg79kkAwAAAAAAAABA37NJBgAAAAAAAACAvje0KX/c6XTK8PBwmHe73TAbHR1N656ent6UpmyUsbGxcsMNN4R51t6Bgeb7h9qUXbFiRTn33HPDPGtz3TmcO3dumJ1++un1jQusX7++/PznPw/zLJuamkrrrsubqqqqbNiwIcy/8Y1vhNlBBx2U1n333XeH2dBQfMt1Op203rr7b3x8PMyyflPXrqbq2pv117q+fNZZZ4VZdsw64+Pj5bbbbgvz7DzNVF8upflY9qEPfSit9zOf+UyYZfdPXX+rqqpMTk6G+Tvf+c4we/azn53W/ZznPCfMbr/99jAbHBxM673pppvK3nvvHeZ33HFHWn4mZH3uF7/4RZj9z//5P9N63//+94fZ7373u7TsVVddFWYPPvhgOvauWrUqrTuTPYeOO+64MMvG1VJKWb58eToeZfdf3X0yNjYWZr/+9a/DrK7NVVWlx87u7TpZvddee22YZb/19/l1110X5i94wQvC7IgjjkjrfulLXxpmP/rRj9KykZ///Odl1113DfOJiYkw68V8eHPI2lwnG19n6+/dElVVFWannXZamC1btqzxMbPxZuutt07LZu3N5tJt3XXXXWF2wAEHpGVPOOGEMFuxYkVa9gtf+EKYjY6Olj322CPMs2c2myZ7Tl199dVh9rznPS+t9/rrrw+zr33ta2GW3UOLFy8ub3vb28L8S1/6UpidcsopYVZKft9nx6yzcuXKtF0jIyON67755pvDrM36y/33318+/elPh3n2XnLZZZc1Pu6hhx4aZnXzj6mpqfLggw+GeTbvPfDAA9O6f/CDH6T5H4rp6en0HF9xxRVhtttuu6V1t3k/yKxZsybtk6eeemqYPfTQQ2ndF198cZh94AMfSNuUmZycTMej7H127dq1ad3HHHNMmJ144olhdsEFF6T13nfffemYkb1LffKTn0zr/upXvxpm2TGvvPLKtN5Smo+TdX0yO1/ZMbN5YN160Wtf+9owu/DCC8OslFK+973vhdlHPvKRtGw2JkxNTZWVK1eGedYf26xzzJ8/P8zqrvmaNWvK5ZdfHubZNci+W9SVbfv+l5XPfnM2zyullP/8z/9s1J7sPbmqqjTPxrhLL700Pe4uu+wSZvfdd19aNlM3x8jUfVvo5bp4r2RjYDbfrlsb3Wuvvco111wT5u9+97vD7F//9V/TurP1l89//vNhln03LOXh+2urrbYK8/Xr14dZ9ltLKeXss88Os+w7T/YtbWBgoMyZMyfM161bl7Yps3r16jB717ve1bjeqqrSMS57d6xbV83G7bo1oWc84xlh9sADD5Svf/3rYX7nnXeGWTa/LCX/Xp29V2bff0p5eL3pb//2b8M8W3PPvjuVUsrrX//6NI/UzeUmJibK0qVLwzy7/tl3i1JKuhaVfRt80YteFGY77LBDeo4PPvjgMKt7/i1atCjM7r///rRsdA/5l2QAAAAAAAAAAOh7NskAAAAAAAAAAND3bJIBAAAAAAAAAKDv2SQDAAAAAAAAAEDfs0kGAAAAAAAAAIC+Z5MMAAAAAAAAAAB9r1NV1Ub/8cDAQDU0NJTlYTYyMpLWvWHDhjAbHh5Oy01PT3eC9lSjo6Nh2ay92e+sMzU1lebr1q37WVVVBzxSNjw8XG233XZh2excrF27Nj1uVrbO/fffH7Z5aGio2nbbbcOy3W63UVZKKZOTk43Kjo+Pl263+4j9opT6vpz115NOOinMSill0aJFYfbud787zNatWxf25VIebnOba5jJ+mx2niYnJ8PzPDAwUNXd9021OQ9r164N+/Lg4GA6ZmTnoq4vZ/nExERadmpqKmxzXb/IxrksKyX/vVmfycbl/3vcas6cOWH5LKuTnYvx8fEwq7v/BgcHq/nz54flx8bGwmxwcDDMSsnHuUy32y1VVT1imzudTjrGtZGd47rfkvXlTqdTNe2vdb81u/+yejfmWZKdj6bHbVM2G5f/b9m0zdm4XTfOZe3KytaNGYODg9W8efPSY0faPIeyuWmb+ef09HTjNrUs2/j+mwnZGFdK/TiXPad6df9tTJvTA7ewZMmSMHvf+94XZgcccEC55pprHrHNdfOiXXfdNczuueeeMCullPXr14dZ3Zi+fv36dC7X9Hm9/fbbp8c98sgjw+zWW29Ny/7Lv/xLev817cu9NFN9OZs3Ze/Jq1atKpOTk2Gbh4eHq8c85jFh+WOPPTbMXvWqV4VZKaW8973vDbP//M//DLMVK1aEbd59992rD3zgA2HZbL78sY99LMxKKeWGG24Is2y+XEopExMT6XtJ9txtMy63eUaNj4+n6y8LFy4My15//fVhdtNNN6XHvfHGG8PswAMPDLNjjjmm3HLLLWFfrlt/Ofjgg8PszW9+c5iVUsohhxwSZnXz/JkaM9rI3qU6nfDnlP333z/M6t5lr7zyyqw9adlSSuP7bybmeRs2bOjZu1Sd7Dpk84SVK1emz5K6eUZm7ty5ab711luH2dKlS8Os7p21bp6R9Y26OUjTd/SszXXtzc5/m7WkbI5YSilr1qxJ77/s2L1aG83m6XXzoqGhocZ9ue7ezN5Zsz61MWsZTa9x3RiYzX/q7pGsL9etB0bq3oey59A//MM/pGUPOuigsC+PjIxUixcvDsvee++9YVY3XrzsZS8Ls7vvvjste80116TvUmnhFrJrnz2/JiYmavtydv82XecvJX+XOv/888PsLW95S/n1r38dtnnRokXV4YcfHpb/53/+50ZtKqWUPffcM8yy+XTdulzT95I6bdblSjKXq1uXy/rM2972tvSgf/mXfxlmRxxxRFr2lltuSceMHXbYISybjWVf+cpX0uN+6EMfCrPsPWzZsmVlfHw87MsjIyNVtvaTPWf23XffMCullD/90z8Ns0suuSTMrrnmmrJ69eqwzQsWLKgOOOARL0EpJX+/r1tPz/rzs5/97DC75JJLysqVKx+xzfPnz6/22WefsOydd94ZZmeddVaYlVLKddddF2Yf//jH07LRWuLsWg0HAAAAAAAAAIAesEkGAAAAAAAAAIC+Z5MMAAAAAAAAAAB9zyYZAAAAAAAAAAD6nk0yAAAAAAAAAAD0PZtkAAAAAAAAAADoezbJAAAAAAAAAADQ94Y2Z2WTk5Nh1u1207IDA/F+nenp6TCrqqq+YYGpqalG7akrW/dbM1VVpedx/fr1jevO6m2jqqoyMTHRqGx2beu0Oc9t7LPPPmn+hS984VFqycbL+mubstn9V1VVen0HBwfDrE2/qLt36wwNxcNi1s/r+mOWtx0zmpZvMy73UjZW1bUp6zvZ9at7lnS73TI2Npb+TaRu7M2uQ5tr0OYZlxkfHw+zXo7LWd11z6Dsvm57/7UZX2dC3ZixYcOGMKvrN037Vd39V3ees6zuvm3aN+r6TdOxqI3suboxst80PDwcZjM5l+vV869XZdtYsmRJ47KnnXZamC1btizM6t5LbrvttjBrc56ycahOt9sta9euDfPsnl+1alVa90c+8pEwa/ss2NKeJb106aWXhtmf//mfh9kBBxyQ1rto0aJy9NFHh/mvfvWrMHvGM56R1t302ZmNn3fddVd505veFOZt1l6yuVwbbdYFZmpsnZ6eTseM/fbbL8yOOOKItO7sXHz4wx8Os5UrV6b11o1zF198cZg985nPTOvmv2Tz05/97Gc9qbeNumf2TGg7x8/Gz7r3jmxMeeCBB8JsY+a1TdeD6+Y3dfd+ZGP6VK/WozJN1xJLyduUPcPqnm9Zv2n7bGy6XtHpdNJ6s3OV1VvXl7vdbvqbe7UW1Uan00nn8r0aA7Nr22ZMb3Nf7rzzzmF2zz33NG5TL9e4nvOc54TZK17xirTsTjvttLmb01rb75XZNW66/lxKvi70/ve/P8zq+s0OO+xQTjjhhDA///zzw6zu/SCbVzW9T9r05TZrXL2Uteuyyy5Ly/71X/91mK1Zs6Zpk1q9/11yySVpvv3224dZNt63nWtnz8a6d4Crr746zLL+WPfNf2xsrFx33XVh3nTeVEp+LrN+lfWb8fHxcvvtt4d5di4+85nPhFkp+bf3urLRefYvyQAAAAAAAAAA0PdskgEAAAAAAAAAoO/ZJAMAAAAAAAAAQN+zSQYAAAAAAAAAgL5nkwwAAAAAAAAAAH3PJhkAAAAAAAAAAPre0Kb8cVVVZXp6uldt2ew6nU4ZGIj3AU1NTYXZxMREWndWbxtVVZXx8fEwHxwcDLPJycm07jlz5oRZdsyNkfWL4eHhRuXqtL0GQ0Ob1P3/nz/5kz9J85NPPjnMsjZ3Op203k6n0/hc1v3WbrcbZlmb6/pcJivbq/urTlVVtfd+U21+U3Z92oxzdW3KymaqqmpU7vey31vX57L7KLtH2ra5jew6ZOeijV7V20tNx6m2Zes0LV93DbI8G9M3pi83vf515Zo+VzfHsXtRb9NjtpkvZ/O8tmXr2tSr533TY27M+d8Sx7KmlixZ8qgfs6qqxvOI2Xptms5tSpm9v6nfHHrooWGWjXMbNmxI67333nvLhz/84TBvOp+uk5XNntndbreMjY2leZNjllLKyMhI47J1st+UzdPbzOXaqKoqfcatXLkyzM4+++zGx83WX+qe13Vjc3au3vrWt9Y3jloz+e6Y6eU7z0xoM0/v1bnodrvpmtGWeA1m63t2JBsj6/pFL9fymz6nZur516bumTzP2fMzO26vvj+0uT5t1k/23XffMPv3f//3xvVWVZXOqdvME7N5+AknnNC43jba9NU256JNX66T9cnrr78+zLL3jlIebtO8efPCfPvttw+ze++9N617JszGb9ydTid9382u7dKlS9O6H3rooUb11ul2u+mYkdX929/+Nq17dHQ0zJoes5SHr/26devCfO7cuY3rrjtuZGPeLZqOC3X3dtP3yrrfk12j7LfcdNNNYVZK/uys+25/2WWXPXJ70lIAAAAAAAAAANAHbJIBAAAAAAAAAKDv2SQDAAAAAAAAAEDfs0kGAAAAAAAAAIC+Z5MMAAAAAAAAAAB9zyYZAAAAAAAAAAD6nk0yAAAAAAAAAAD0vaFN+eNOp1MGBwfDfHp6OswGBvL9OE3rzVRVVSYmJsJ8ZGQkzLrdbqNjllL/W9toc44nJyc3d3NKKQ+f56mpqTBvcy4zbertdDrp+dpjjz3CbHR0NK177dq1YZadp6qq0nqrqkqvYXY+6vrG0NAmDQWbRZv7ZHh4eDO2ZPNo83t6dY+01at21fXl7Fy2aVPb35Pdvzys7j5ocw3aPLOza5eNf3Xjct2xs/PRZlyerX0xa1f2ezfm+der39y0zbNRNufdGE378kyOy/Redn9m7yWdTqcnx9wY+tXMa/Nu33Ysi/TyWdILdWsZbZ5RM/HuXkp+b9cdNyvbZrypO3Z2nsfHx9N6s76eHXNjxsCmY3Md4+fMquvLbZ6Pvbq2MzVfbvPe2fb+a3ou3V+bx2w9j9nY2+bebfNNpM3f9OpZ0qvzVHfcNv2mzVp+0+PWfXvYa6+9wuyiiy5qdMyN0Wad6t577w2zZz7zmY3b1MZMjSeDg4Nl2223DfOxsbEwq5t/ZrK1+Lq+vGrVqnL++eeH+cte9rIwO/vss9O6Z+u4/mhr872kzn/8x3+EWZvxvu4bazYurFq1Kq371ltvDbMNGzaEWV1f7nQ66ViW3WNtvkm26ed1axltruGcOXMal43UtTfrM3XrQV/60pfC7MUvfnFa9rLLLnvk9qSlAAAAAAAAAACgD9gkAwAAAAAAAABA37NJBgAAAAAAAACAvmeTDAAAAAAAAAAAfc8mGQAAAAAAAAAA+p5NMgAAAAAAAAAA9L2hzVnZ8PBwmHW73cb1Ni1bVVWZmppqVO/QUH5q2vyeTFVVad1ZNjCQ73lqU7ZO2/JNzJkzJ8yy6/57WZvXrl0bZsccc0xab9M+V1VVWm8peZuzLLs3SyllfHw8zJq2udPplMHBwTCfnp5udMxSStmwYUOYZces0+l00nt/YmKicd3Z9WkzntSNGb06bmZj+nJTbca5trK+sTFjDu20OcfZtetln8nUHbeX7erVfGAm5gJ1ZmL+2Ub2bCyl3TOujZm6T9qYiecf/6WXcwFmv7qxbKb0as5ct3bQtN6mz9WZnC/3SqfTmekmbLJenufsfLQZfz07Z1bbZ2fT6zcb5/CzWdPz3Ksxv61eHbdX72GzcS2/jTZtavNsrPtm0mYNJVuDnpycrG9coM36Z52Z6BtZexcsWJCWvemmm8LsV7/6VeM2dbvddM29jay/XXHFFT055mw1MDCQftc6+uijw+yhhx6qrTvyzGc+M8ze9a53pfWOjo6WJzzhCWG+dOnSRm3aEjX93tVL9913X5q/733v68lxu91u+r0se5ZceeWVad297DfZmN+rZ1jb78K96ltZvU3XoOue11mfGR0dTev+xS9+EWann356Wvbtb3/7I/7v/TVCAQAAAAAAAADAI7BJBgAAAAAAAACAvmeTDAAAAAAAAAAAfc8mGQAAAAAAAAAA+p5NMgAAAAAAAAAA9D2bZAAAAAAAAAAA6Hs2yQAAAAAAAAAA0PeGNrXAwEC8r6bb7TYqV0opk5OTYTY4OFjfsM0s+y0zqe48zlS9Tc9Xm+NOT083LltK3uYHH3wwzH760582rnemrF+/vnHZ7Bp1Op3G9Q4PD4dZNh6Uko8JbfpFVVVlYmIizIeG4iFzamoqrbuX/aJXdTcd79uajfdQKfk1nqlz1W969SyZqf7aZl7U9JhtZXVnY2Bd2dmq6XWo+61t5yiPdr2l9O5cND1m2/60JfbHNpYsWdIo65W6OWJVVY9SS+C/9OpduulcoE29/Wa2jgnZ/L/u+szU9Wsz//xD6nN/aHo15+rVuLql6tU73mxds+1lm5uo+52zsb/WPf96tf7ZSzPVrqZrdnV5ltWtBWeafpcopZQPfvCDjY/LzJucnCzLli0L849+9KNhts0226R1P+5xjwuz1atXh9lDDz2U1rtixYryxS9+McyvvvrqMMu+tdRpM3/Z0tblOp1Ouu6aHbeuTVney2/vbcbPpu/Ydc/VqqrSb4+92uMwZ86cMKtbP+t0Oum31F7Nx571rGeF2WWXXRZm8+bNK/vss0+Y77bbbmF2yCGHpG16/OMfH2ZnnHFGWjYy+2aHAAAAAAAAAACwmdkkAwAAAAAAAABA37NJBgAAAAAAAACAvmeTDAAAAAAAAAAAfc8mGQAAAAAAAAAA+p5NMgAAAAAAAAAA9L2hzVnZwEC852ZycnJzHmqjdDqdMjIyEuZTU1Nh1u1207qz35rV21ZduzJZm8fHxxvXW1VV2q7suHW/Z2go7qJZ2aqq0nrr2jw2NpaWz7S5Rr2qOzuPberNVFWV3vdt+sXg4GDjdvVKmzEjy+rq7nQ6tdc3UjdWZcftZT/Prm/ds6Tu3u+VXp6PXmhz/2XanofZ2q6ZMFNtnqn5Ta/Uja+Z2dhv2jz/pqenw6zT6bQae2dKr8aMLdGSJUtmugn/zUw9jyHT9J0VHkmb97829ZbSfIz9Q3s20nsztf7Vq7XRjdG0XXXHnY3z2tn4TtrGlngeO51Oo2xjbInv/jNxDbNz0at3nomJiZ7UW6eqqll77f+QZNfggQceSMuuXbs2zG655ZYwq/sWtnbt2vKjH/0ozNs8s2fj828m9PJb2vDwcON21cmOvWHDhkblSpm59/PsGsyZM6dxvdn39435lp21KzuXddc+Wyu+9dZbwyz7PYODg2XhwoVhPjo6Gmaf+9znwqyUUpYvXx5mWX/LWAkCAAAAAAAAAKDv2SQDAAAAAAAAAEDfs0kGAAAAAAAAAIC+Z5MMAAAAAAAAAAB9zyYZAAAAAAAAAAD6nk0yAAAAAAAAAAD0PZtkAAAAAAAAAADoe0Ob8sdVVa0YHx+/s1eNiXS73SzeLQp62d7p6ek2xdM2r1+//lE/xxshbHMpZcXU1FRP2jwxMdG0aNbe2Xqea9s8MTGxJbV5RbfbDdtbc1+nxsfHG5ctNfdfdo5b9Me2Gre5jRbXKO3LpZQV09PTYZtbjq9N1ba5lLJF3X8laW+b+6+lxmNGpoe/p2f9Yra2eYb6Rm2bm/aNHmp8/7XRy/nn5OTkFjPH/7+2tH5RSp89S2aQNvdev/XlUnr4/OvRnHk2nuNS+rDNvXqW9PJdyvNvs9CXe29W9uWZmn/O1ve/XmlxnmdjXy6lps1VVfWkzVNTU02LtlqXa6PFO2ttm5ue5x6uMzbuFzO09lnKDH3jaWlLe2bP2Fpi9s2k5ntK2uZut7tibGxsS5ozz8Z+UUqP1vLrzuHk5GRduzIz8v29hdq+vG7dukZtXr9+fbMW1ZuVbf7lL3+ZxWGbV69eveLiiy+ebf2ilKDNnaqqHu2GAAAAAAAAAADAo8p/bgkAAAAAAAAAgL5nkwwAAAAAAAAAAH3PJhkAAAAAAAAAAPqeTTIAAAAAAAAAAPQ9m2QAAAAAAAAAAOh7NskAAAAAAAAAAND3bJIBAAAAAAAAAKDv2SQDAAAAAAAAAEDfs0kGAAAAAAAAAIC+9/8B/DZx2GnogwAAAAAASUVORK5CYII=\n", "text/plain": [ "<Figure size 2880x2880 with 50 Axes>" ] @@ -348,16 +370,16 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 89, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "torch.Size([473, 1, 28, 8])" + "torch.Size([190, 1, 28, 5])" ] }, - "execution_count": 14, + "execution_count": 89, "metadata": {}, "output_type": "execute_result" } @@ -368,22 +390,22 @@ }, { "cell_type": "code", - "execution_count": 101, + "execution_count": 16, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "<matplotlib.image.AxesImage at 0x7f95e295e5d0>" + "<matplotlib.image.AxesImage at 0x7fddb02b4f90>" ] }, - "execution_count": 101, + "execution_count": 16, "metadata": {}, "output_type": "execute_result" }, { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAABH4AAABDCAYAAADqHsJ5AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAABsUElEQVR4nO29aXBcV3Ym+L3cM7EllkRiI3YSxEaAAAESXERqLS0luRZ32eWa6uqyXZ7oHvfY1T0x3dO/ZjzR7ZiOjvZ4Ytx2ONoeu1wVtktlqaySKEsUSYngThAECGLfQRBrJpDITCBX4M0P8FydvHhJ1SqJqvtFMIh8y13Pve+c7557rqbrOhQUFBQUFBQUFBQUFBQUFBQUPnswfdIFUFBQUFBQUFBQUFBQUFBQUFD4xUARPwoKCgoKCgoKCgoKCgoKCgqfUSjiR0FBQUFBQUFBQUFBQUFBQeEzCkX8KCgoKCgoKCgoKCgoKCgoKHxGoYgfBQUFBQUFBQUFBQUFBQUFhc8oFPGjoKCgoKCgoKCgoKCgoKCg8BnFz0T8aJr2vKZpo5qmTWia9u9/XoVSUFBQUFBQUFBQUFBQUFBQUPjZoem6/tO9qGlmAGMAngUwD+AWgK/quj708yuegoKCgoKCgoKCgoKCgoKCgsJPi5/F46cTwISu61O6rscB/B2AX/n5FEtBQUFBQUFBQUFBQUFBQUFB4WeF5Wd4txTAffZ7HsDRR72gaZquadrPkKWCws8Hshz+tJ5vPL2fJA3K/2fN9ycpT7prVA5+n7ePfO+nKQul89OWn37L5fpp0vp5lvPThp9WDo3AZcHob6P308mcUT5G19OlLedrVMdHyYimaeL+o8ojp/eT4lFt87Omka5f0/WJ0ViW85HH/0fl86j7Py/I/SjXQ87fZDLBbDZD0zQkEokfqy5G+f2kzxjl8ai2Tyd36fJ51PPp+taoTOnK96iypBtvmqZhZ2fnkXVQUFBQUFBQ+ETh03XdY3TjZyF+fixomvY7AH6Hflut1rRK2Y9jlHIFnu7z57hib5SexWLB9vZ2ivJC75tMJkOlhtIymUxpFcR01ynNRxnhjzKmjBQ8+b6RsZPOaJKvU9nSKb/U3nJ+RmXh9ZXLn65eRmnwa48yxuT2M5lM4rrZbMb29vYj66RpmuhTem5nZ0fco3oYySEZGrxvt7e397SX3DY8L26IpoNRevx5s9mc8mwymRTXeN/KdeLPUNl43alsdI2nZwTKh/eBxWJJkcWdnZ2UPuFlk9Pi6fD2MpvNe65tb2/vaSuehzxujQxJk8mU0kbyuKLfVC65vOkMKZlw4Nfl8hhdp/LIdfsoY1/ud6O5kv9P9adrvK+p3bksyPXn89yjxhyNG5IDeYwBqbIpyy5dozxlOaH7VB9Kn8sjz3NnZ0ekIbcNry/v90fNLXw88XrzNI3kno8R/hzPV/7mUV24fMvyStd4//L5mbcfLwuNKX7vo0gLo/pSHRKJRNpvGPBhn/N+5GVL931zOp1obGxEW1sbEokE/vEf/xFra2uG7UBpGs05NHfKbcjnNHkelOdzs9ksxoHJZEIymUz77bVYPlS9uNzKz5OMks4ij2VZhvmYMKqPyWSCxWJBPB4X12hcyOXkY4fPE1tbW1BQUFBQUFD41GI23Y2fhfh5AGAf+1328FoKdF3/cwB/DgAmk0lPp5iSQccNDlkBMkh7z2+urBkZQUaGyUcRJ49S1LmCbASj/GQjTK4jr4eRgswNXZ6PrKB+VP7pyiKX/1HP/bgEDj1rpHATuCJtJAPp8jXqX1J6jerBjXiu3PK00pXVyHgiQ0k2GOT8ZVKRG5ayIc/rLBuk8vNcFrkhILetUf3kMSEbF5QupWFk/Mjl5W3J/+ZGu2y8ULnlcsnpyv3O+1IuD+9bI9LEaIylkz9eDhny2DW6TnMcEQbpnjOqowyTyQSr1YpkMpl29T1dmkTg0N98zjUigQgymcvT52nyNHh/EkiW5Hlabj9ucMt1eNTYl9uNp/uTkhnp+tWoDwky0S/LHSfQ0s1xRmWTZV0un9we/L4s6/K3Uc6b52Ekj0bzCh+/Ru0iE7ZyekZkFN3npBCvj81mQ2FhIfbv34+DBw9icXFxzzxM6VE6Rt8xWRch+X/UWOTPGLWdnKaRHMlEqtF8zPstHUHD+5ePZ/kbJc9tdC2dDmM0NoyIbwUFBQUFBYXHAz8L8XMLwH5N06qwS/j8OoDf+KiX0hnFRjAyhvm9R103MgZkyKudctnkNGQFi+eXzmiX65HO0H8UASCXJ901I0XUKE2jsnKjwG63IycnB9vb2/D5fI80Lo3aSm7HH8d4/ig8ql/S1ctI8abf6ZR62VCT85GNNjldOQ+j/jKqW7p0jGDkEWHU5rxf6ZlHKe5GZTUy+uR2lfOiZ40MDf6+bMByr6OfpI34Kv5HyZpc3h+3LahO/J5c13RtJl83Gp/pxr2RoWqUj1G5jQxD+k1/ywa+TA4YGaBGbWLkDSZ7/xkZtnKa6TxQ0rUVlYvLtUxyymnydpCN/HRyJPcpvWvkbSaXR86Xk118jMh9TekQmSE/Y+S1alRm8gJxOBzIzs7Gzs4O/H5/CqErtzldIxLZyOuM55Xu+yXPsx81H/J0NM3Ymw/YnQOrq6vR1NSE6upqJJNJzM7OIh6Pp8xz/D2LxYJkMrlnvjLypklXPlmmjL659LfsxcRlSyYHuacOb38ZRvOBURl52jJpa7QwYPTt4GnJc7+CgoKCgoLC44efmvjRdT2padrvAngHgBnAX+q6PvhjvgsgVQE3UmJlZURWfGXlmV83MoSNFEKuiPH8fxLSRL6ezlhLB7mccrpGSpmRkWV0nxsPHLLySy7gmZmZKCoqQlVVFWKxGK5cuYJIJJKyPSId6fFRiqGs1HPjjf5/FIEmp5GuTSj9j9rGls5AN+qPdOWTnzEqIzeA5foYba8yWpHn7xqtnqdrA6OVe26QGJVbbgeeRrqVeTkdI+NZvic/Y5Reuvfk8tJ13ufpttbR39ygNboul4Eb0kbGEX833Zz0qDks3XwhG9y6rqd4+xgZcOm8CYzajjyRjOY9o3Lxe48iQo3a0cgji8sjvSOTmrKcyHXn/S1v7dO0XU8b2nrDDX65bY3qns74N7ou1zPdtyhdexp9n+RxyMmqdOlwmdI0TXjH1NbWYmtrCxsbG0gmk4ZbwNJ904zkM51ccXmXxwJvQ56PTEAZEXbUtvn5+ejs7ERdXR12dnZw79493L59G4lEQrxDdTObzcjKykJpaSn8fj/8fr94jiMd2cn/luVFHrvyXEBjhOYEee7mz8rfcEqbzyfca4n6TP5e8D41moflMSN/f+StkPx9Fd9HQUFBQUHh8cXPFONH1/WzAM7+uM+TEsaVlHRu7/S3bDxQOqSUGCmS9FtWeGSlRlbQDOqXNn5IOsNYfj+dMZWuzLKxI+ebziOC11Nuv3RGMzdQbDYbcnJycOjQIRw5cgRVVVXY3NzEwMAAYrHYHkOPG2fp2lv+WzYAjJRm/r+R0srbQa43f54UaVrhlfsnneG2s7OTEjfHyHBJZ8TR3zKZw8tuZFTRby7X/D7Fg5CJLN4f9I8MGpvNlnI/HSFB+aUrk9H4o3eN4u0AEAYlz4OX2agMXB55GxgZNLRdJh2ZxtOTtxvx/PmckE4Gjbay0d9yWgS+HZP3rVFf87QsFktKvbic83bZ2dlJket0BrlcD7pmtHWG9zPfjiXPN7wfeDwaozlULp+8PVBuF8pbngf5s7x9aIzTs9QmsjzL8svLmY68lK/zMm9vb+8ZF7LBL8eToXobzV3U77zucl68nXk/pKsjH08mkwl5eXloaGjA8ePHMT4+jrt37yIWixlur6Q2NiJeqDxmsxlmsxkWiwVWqxWxWCwlpow8r8h1p3IZzfWyzPJ5gBYnjh49iqamJmxsbOD27du4ceMGNjc398yHZrMZ2dnZOHr0KH7jN34D3d3dOHv2LJaWllLIUyPSRS6XPPfLOghdo3TSbbVKN8fRc/Kcw+c7I92Df4stFksKASpvJ5bnRSKSaE5IJBIpfWM0/ygoKCgoKCg8nviFB3fmkA05WXnlJIwcD4T/baRUyat1lIds8FFsDHqOEzuy4ZLOSJfrQ/XgShgvK3+ep5uOfCAYrVxT3bnXAU/DSNGjtOTyUBtarVY4nU688MILOHXqFEpKShAKhTA0NIRwOLwnGDZvV9mok+sn11tW7mUj3MhA5u/KdebGp1E7yW0qG+j0HPWh1WoV9/kJMVwhJwPIqG9ko8HIgDCCXGb+HvdM4HnL7UAGGRmMPG0yCrhRL48PDmoPInH49jLK3yhmCoEIND4mqM3587LBSXnye1Rn4NFBc7mhy8vHjfF0bS2TUUTeGJHLRoSLPM450Zaujfg1o/Lx/jMimahPZaKI11cen3KMME7EbG9vpxDzsnFOafD3KX9OzMjvy8HGdf3DgLKyt4KmabBarULOuIzTezyAufztkAkLXj+jft7e3k4Jdit7wBgFcacDCmhbEaXFZZXepWe3t7f3kHXcy4MTFrIcUX2pzunIBbnPqO2zs7Nx+PBhdHR0wGq1CtKHCAUuWwSaB6m/dX3Xy8xisYjxUVxcjJaWFtTX1+Ps2bMYGRkRbZwu0D2vL9WZ2kgeK0Z1czgcOHnyJE6fPo25uTlcunQJExMTiEQi4jmSS03TYLfbUVFRgd/7vd9DZWUlJicnRX7y3MLHnxG5wp+h6zabLeWbL/edPAelI7e2t7cN41/J/Ql8GMuN58Pbln/TCDw+EJ8XrFYrqqqqUFFRgWAwiJ6eHnHfKJag3J8KCgoKCgoKjw8+VuKHICuZRtdJ8SJDhCsi9CxgvNIJIMWA5Pdk5UpWTo0CsBopO7JSJZMjfNXOiLAwIkDoPVLmjAwV+j8dASKXjStv3HDgimVZWRm+9a1vob6+HouLi7h+/TpKS0sxMjIiDCKjk6J43eTyy+VI11YyqWK0oiiTL7Jiy401mVDi7SYbzvzUFZPJBKfTCY/Hg7q6OlRWVuLixYuYm5sTJ9JommYYNJrSk4NvymSbTFbICrt8TTbMZTmie/LqrCzXsqFIxiyVw2azIR6Pp5BHNFZkY0OWTQLPixtCskFLZZXjFPFxZ0TqEPj4pfToGq8XH8O8bDKhRn/LcpXO048TvFz2qU7y+JD7O918ZTTXcHJXJmCM5gYjYzBd2vw3J4u4RwefZ+i6PG8aeUtRf9I9GjNcVhOJhCAGZSOf2p/SoL7kssMJHk5YGc1L5AXHPXBkeZFPx+MeEHL7cRKN9zfvX5oLiDilvDl5wsmtdGNYJhA4Wc3Lyd+h/iKi7Pjx42htbcX29jbOnz+PkZGRlDbWNA0OhwN5eXmIRqNYX19PaT8qO88rIyMDhw4dwuc//3m43W7cv38fU1NTKTF2jAgTaiu5D9ORsFzeXC4XamtrcerUKVy/fh23b9/G6uoqotHonv7hMpFMJhEIBBAOhzE4OIhIJJLSTlwe5HbUNE3MjTJBRWnzZ3nZ+ZzJ601jSfYy4u3LZVj2HuNyYTabEY/HU7wF5TlEXozipLDb7ca3vvUt7N+/H/39/VheXsb8/PweGXzUN11BQUFBQUHh8cDHTvzIxqDs4cCVbFJOjYxV/gz9Ld+XFSuu8JDR4XA4kJGRAYfDgeXlZaGoy4SGkYEF7CqZj9pOxJVbXkajU8xkY1c24Hm6vD250Sanw5+XvWJsNhuKi4vx7W9/G01NTbh48SICgQC8Xi+SySTGx8dTys3bKyMjA3NzcyJd2UiX+4bKyo17ebXbyFDn6coeULwN5PbVNG2PMiwrsSaTSRhoWVlZqK+vxzPPPAOfz4fW1lYsLi4iHA5jdXU1JR3ZwJWJAJlo4Z42MvnAn+X9ZGS4U5m5McS9lfhx1UZGEPcI4uDGp9wPfCykc/Pn44z/5mNWToO8OuR3ZaJEHrvyqUjcoKF6cpmjcjzKAJL7k8rD+4TXS05DbnO577isy0albNgblV3ub9mbh8rE0093jZdFLg+1r1xuI/KY189onpQNd7k9ZRKMl1H2SqPnuaFuZNzyccblSr7G25gTSrw8RuPZqK052SvLBG9rInzk8UfPcHJKLiNPx2hOlecR4EOPnQMHDqCxsRGapmF0dBQDAwN7tpQ5nU4cO3YMv//7v4/p6Wn8p//0nxAIBBCPx1PKQPJhNptx5MgRHD58GIlEAt3d3ejr60uJryP3H0+HIHsn8jHCSUhN05CVlYXq6mqcOHECGxsbuHPnDnw+HyKRSFoPGEpL13UEg0H09vbi1q1bCIVCe/pXljNefupPTthQe8jfJXlO4PWSiTReRlk34HMM14Vo/HHiiI8v+g5o2odbH+Wxxr+P5GkUCATg8/mwtbUFXf/Qs46/S+NPQUFBQUFB4fHEJ+LxIyup9Les4HLF8VHeDHJ6ZHByEoDfp3w8Hg/q6+tx4MAB2O12XL58GXfv3k3JXy6zEQkjK49G9wl85VPOwwjy/XT5yASaXE5+jVzUCwsL8Wu/9ms4ceIEJiYmMD09jZycHGRnZ2Nubg4rKysppInNZkNJSQm6urowOzuLxcVFQ68qo7+NCDQjI0auM3/OyJCQDR+en9z//DoAYcSQl8+hQ4cQDofhcrkQCASwvr4uVpNlw47KIJNJRvXghpaszH9UWpx4kesg/5PbjrehEaFC/2SXfiO5l6/xNGXZo7+5JwnFA8nIyIDL5YLb7UY0GkU4HEYkEknZqpFODozkieojyxNdczgcKd5MsjzJ7UfXuZzwfuHvyOkZeb/w8sjEWbo06R6N02QyucdLLh0JalQHo/lPNkDpWSJQeDpGRLg8/8pGrlE/8XJRe/FyGJWZ9ws3so36R35eHh82mw1ZWVnIzc2F2+2G2WzGnTt3RPsaze1Gc4tscBvVjxvocltyspHGH29nmXjgefHfvN2N6p+RkYHOzk7k5eVhfHwcIyMjCIVCe+oHAPF4HJFIBM8++yzef/99XLhwIcUjktK3WCyoqKhAQ0MDbDYb+vr6cPXqVSwsLKSQclxeZNKK8pbJHble9B13Op2orq5GQ0MDotEoBgcHsbS0JIgm+V0+NpLJJJaXl/H9738fmqZhdXVVeHGl+85wcBmV+5mX1Sgto/T4HCy3E7UfEUry4o9MGsoLGvI8L9+Tr29vbyMajWJpaQmrq6sYHBwU8sE9NvncYKRXKCgoKCgoKDwe+MSIHyNF1eg5wHhrWDoj2uFwwOv1YnNzE2tra3tWukjhysvLQ0tLC9rb21FbW4tEIoHh4WEAxvFqONIpdEZGAFeYZIPlo0gfOT8jA042AIzeldvKYrHA4/Ggq6sLzzzzDPLz8/Hee+8hHo8jKysLADA4OChW/0iZLigoQHNzM44cOYLx8fEUpVc2uH7Sej2qLX4SZZPKQsGq6fQaUmK5YUCkD5F/mZmZmJ2dRW5uLnp6enD//n1EIhFD0oVDNr7S1ZOe5Qa+bODKBq2Rwi4bxkbGPYEbk/IqMj3PDTTZwJHzMbouG+xyG9lsNuTn56OoqAhFRUVwuVxwOp2IRCLY2tpCMBiEz+fD+vo6gsGgKJdsIKfzgjIqk9VqRXZ2Nurq6jA/P4/FxcU9hqIR8WNkEBrlaeSBYeQRZkROGKVHMkHPm81mZGZmYv/+/QiFQpifn8fm5maK4cg9DWXI/WzUTrJHJB/rVqsVmZmZyM7ORjgchs/n23Ps9aNImnRj2mgu431sNL64jMmekkZyKL9rMplgt9tRXV2NmpoalJaWIjs7G8FgEOPj49jY2Eg7Do1kmpM03BiW6yz/NvpbngPlMW9EmjwKRBBYLBZR32AwiMnJSTx48CBl2x3VY3t7G/fv38e7776LpqYmlJaW7vGwpbZ3u91oa2uD1+vFysoKent7MTMzsycemlH/yWOJ0jYKpk9bq8xmM0pLS1FTU4OcnBzcu3cPQ0NDiEQie7wficzgedI2rytXrsBisSAWi+3pX94eRjKYzgNTXqTi92XZ4SBSi+dHZaY6GMmHkRzInoZGJLicPvciTCaTGB4eRiQSwfT0NGKxmCiDEaEvfx8UFBQUFBQUHh98Ilu9uKt0OmXQyOiVFTOuxNBpHyUlJWhvb8f4+Dju3LkjFD3+fGZmJlpbW3HmzBns27cPOzs7wjiUtx/wvKmM/H9edll5N1KceFl42umIE9mwNjJs+P/pDGL6Z7FYkJOTg+bmZnzxi1+E2+3GysoK5ufn4Xa74XK5sLS0hLt376b0gcPhQF1dHY4dOwYAGB0dFYYnf04mu+RyfBRBZeTJwNvnUQo65cGPLh4aGkIgEBABUnl5XS4XmpubcfDgQVitVoyPj8Nms2FlZQWXLl1CMBjcs0VLJiN428onoqSTXSBVYedbbDgBIHsA0DXe37LxbmSEp/udrp1lwzPdlgVOFvG6kjFJJ/54PB40Njbi0KFD8Hg8wrsgmUzC6XQiHo9jeXkZi4uLGB0dxerqKuLxOJLJZAq5YbRlgf9P5bNYLMjNzUV9fT2ef/55XL16FYFAAIlEIiU+C28XI8ON/280/uQ+5h4MVB55LuB58j7k8kDHT9fU1ODll1/G+Pg4wuGwiLfFx8qj+vhRnjRGQZx5/JnMzEw0NzejsbERk5OTePfdd/e0jVEfyDIkl4nKKRvG6QghXkciI2QjXCaE5HFitVpRWlqKZ599Fi0tLcjPz0c4HEZvb684/Y6Xh9KViTxKj8ePkWWPl01eQJCJCZIJqhfFPeLv8ODS8hZHo61yFFA6Ozsbx44dg8ViQV9fHyYmJhAOh/d4zlLMmfn5efzoRz9CQ0MDlpaWBPnAPV7sdjsOHDiA9vZ2hMNhjI+PY3p6eg8pxucuHmuLk2V071HfWLPZjLy8PDQ2NsLj8WB1dRXDw8PY2NgQ7cu3bnNZkr8TnPSl7VAAxMleVqtVEEVG20GNZNgIfP6W5wr+N+8/6gsjUoXPH5yAMorDJssLeWzx+YW8L7e3t0XMot7eXsTjcbHQYyTX6caygoKCgoKCwuODT8Tj51EuykbEihxHhV8HdmMUFBcX49ChQzh+/DhKSkoQi8Vw7949EaeAFCKr1Ypjx47h61//OjIyMrCzs4O5uTm89957GB8fT7utQY4bwBV3XiYyosgITEeAGNVfhvycDFIg+W/+LFfu6ZrFYsHhw4fx+c9/Hg0NDeju7sbMzAwGBgbQ2tqKeDyOqakprK2tifTNZjPq6urQ3t6O7OxsvPHGG0JJpHalfuWKuLytjdrHqB5076PIFa6Qy7FpNO1Db6bTp08jIyMDU1NTe4gLIodaWlpw+vRphEIhjI+PY3V1FQ6HA9euXcPW1taecsjxZWS5lY1Hoy0h8qosGSm8/9ORW3L6Rm3C21M2po28FDhJw/uP+pM/w/tSNopk2aO8vF4vTp8+jQMHDiCRSODmzZvw+/3C88Dj8aCoqAgVFRU4cuQIurq68M4772BsbAwbGxt7yDIuX0Bq7BTKNz8/H01NTXjuuedQWFiIwsJC2Gy2lPbl/WI0Bikf2eij45K57PH68/HA+4Dy3dn5MOYSxdGgdMggdbvdqK+vx1e/+lVkZWVheHg4hWTl6VIacp9wIzGdPPGyUxrUnm63Gy0tLfjCF76A6elpfPDBByltYrS1S56/KV8ysnlbkLHN5YzIAKvVuieoMrUtxR1J13/8HYvFAqfTiX379uGrX/0qjh8/ju3tbWxsbGBychJXrlyB3+83nKf53Ef5cfnjhjRdl+dCeX6idHjMLS4XnECVY9BwcoDe421E85PZbIbT6UR9fT2am5vR19eH4eFh+P1+w28Il5319XX8wR/8AeLxeAqhAAAOhwNFRUV44YUXAABXr14Vp4NR3pws4uOJ6mZEVBF5wetHYzk7OxvPPvssiouLMTk5iVu3bmFzczPlyHJqP0qfnzJnBE3bPekrPz8fNpsNa2triMVi8Hg82NnZwebmpjianuYoOrlL7guaN2kMUz9QPeh9WY6p/JxU48HE+fzOQeQbjQEqA28H3va8bLJs0jvJZBJra2siDx4ni8gweSuigoKCgoKCwuOJj534IcVCNkq5YiUrSvKKF1eYS0pK8NJLL6GpqQklJSXIyMjA0NAQPvjgA2xtbYl8dX3Xa+X48eP4l//yXwpFdW5uDleuXEFPT0+KEiwbKTwdrphxw4buk7Ikr+ilu8chtwtf/ZTzN3pHdpsnBXB7exs2mw2dnZ348pe/jMOHD6Ovrw//7b/9N6ysrOD48eMoKirC+Pg4bty4IRR6TdOQm5uLF154AUVFRbhx4wauXLmSslotExjceCFjhAwCfqoXN+h4PWQQ4SKf2sRlh46kr6mpweHDh1FUVIRz585hdXU1pd3JMKqursY3vvENxGIxJBIJWK1WTE1NYXV1NcUIkQlA3tdGirm8Ei97fcgGAj3DSTzuuUL15co6P41HXlnnciCXnVazZQ8dq9W6x0jl9eGxPmS543LN24SMwdLSUjgcDvT19eHGjRvY3NzE5z73Obz44ou4ePEirl69igsXLiAnJweHDh3CF7/4Rfz2b/827t27h4sXL2JwcFCcWkPGKJ8rrFariNHicDhQVVWFp556CidOnIDJZML09DTee+89BAKBlLaWSRS53GRU8fmGt50RgUfPyMYsN3xljxDehxaLBU1NTTh69Cja2tqQk5ODmzdv4sKFCwiHw3uMRd4GJHckQ1xOjSCPJxpDVOZwOIxAIICNjQ0sLS2ltJ1Rm3HPL05ayAQ4nws5iULPUhl4Wai8ZBTzcvCTwWgesVgssNlsqKiowDPPPIOnn34ai4uLWFlZAQD09vbi/fffx/DwMHZ2dmC32xGLxVKCDMtkLL9uBE4G6LouSC15vNCz8nzJ50GSE7rG5Yd/NzhxR2VzOp0oLy/HSy+9BJ/Ph/Pnz2N5eVnMvfz7IxPG8XgcsVhMlJ3PA263G1/+8pehaRp++MMfYmJiAsFgMGWuk+VJ13XY7faUuY3f4zLM62U2m+FyufD1r38dBQUFuHbtGsbHxxGPx5GbmwuXy4XMzExkZWUhJycHWVlZcDqdiEajeP/997GwsLCnn6juDocDR48eRUdHB7KysjA/Pw+bzYbq6mqYTCbMzc1hdHQUU1NTWFlZEbHHZE8gi8WCRCJhGIicy61MRHGZ4gQmpWG1WvfIGc0NJF8kI5yIkuclmv85AcW/L9QevP+JrKTr/IQ/I71IQUFBQUFB4fHCx078kMs1wWi1VgatQpPyQ1s5nn76aeHhk5mZCYfDgbW1Nbz66qvC0CPFxu12o7m5Gb/5m78Jl8uFnZ0dXLp0Cd3d3RgbG0vZRsHLxRVumcCh9An8nrw6+OMQPZQfJ4Zk4iAdOcbLLYOU/oaGBnzlK1+B1+vFuXPn8N3vfhdLS0soKyvD4cOHsbCwgOHhYfh8PpFWZmYmvvzlL6OkpASDg4N4//33EYlEhOFodJQ2rUiS0s+NNN4mvOx0j5RUrgzLx+KSgci9P3JyctDe3o7GxkbY7XbcvHkTExMTKcYLGZTZ2dn4+te/jtzcXOzs7IhtNNFoVNRHJlK4UUvGGDfQuAwY/f0oTxAqn0zIUDn4M5SW7N0jG4E8Ddno5oYJ9RdvT15nThjRNX7qDPfsko1Xs9mMkZERcczzzs4Oamtr0dnZKbZVUvqrq6u4fPkyotEonnvuOVRXVyMUCiEWi2F4eDiFPOCkFvfK8Hg8OHbsGOrr62G1WuH3+/Hmm29idXVVlI+PS7rG/+ZjRiYo6DqRJJyI4cYRN7JlgoK8BHj/Ut+3tbXhueeeQ11dHXRdx9DQEF577TWEQiFx+o5MPMvbLfl8Qf3L+47PJbxe/Ohri8UijOtkMomxsTFBrnEY9TuXD9mbibcxH8s0T1AdySOC8jCa69LNH5qmwePxoKOjA11dXWhoaEAsFsPExAQqKipw6dIl9Pf3Y2lpScz59E3inpF8Dufjx4gETEeyykQ4N6b5eOVjNl05OFHC8+IebxaLBVVVVXj22Wfh9Xrx13/911hdXRVzqkz4yN83ThpQeS0WC0pLS/HCCy9g//79ePPNNzE9PY2tra0UUt/tdovtm3xLJfWnUXwoqh+lQ/Nzfn4+vvzlL+P48eOYnp5GTU0NGhsbUVhYCJNp9wS0zMxMaJom4viRrNJWUS6jlFd2djZefvllNDQ0IBwOY2VlBW63G5WVlZiZmUF+fj5qa2tRV1eHWCwmYpBtbGxgY2MD0WhUlJHGZ19fX8pY4PO6UYBvHsxaJoH5d4XaXyZS+XeW0uFjmZPWBJpvrFYrbDYbNE1DMBgU/W2xWGC32+FwOIS3XSwWQzgcFv1G/3OiT0FBQUFBQeHxwsdO/MjGr0z4pFsZM5vN4vjxpqYmNDU1oa6uDjs7u1u18vLysLOzgzt37ojAw2RsOZ1O1NXV4ZVXXkFxcTEWFxdx+fJlXL16FdPT0ynHu3IYrU7yMtLfRitvnKiR68uNH6N0eP3lPLm3k0xSyW1J7Wa329Hc3Iyvfe1rqK2txejoKG7cuIGVlRVomobDhw/DYrFgdHQUMzMzwiCy2+0oLy9HU1MTAoEAJicnU7YMkDKZm5uLsrIyeL1eRKNR9PT0CI8h3gbUjtxYJ5CxZ7PZxFYDue5y25GSW1xcjCeffBIHDhxAKBRCT08PBgcHhaLOPQHy8vLw7LPPorKyEhaLBXNzc+jr68PAwIA4wcuo33j5ubHHjWdO3nBiggxBXn6utMtkIO9fbiDIRiL3/OHl4zIkG4rccCbIz8teLrw8ZMw9ioCk9JLJpPBUsVqt8Hq9aGtrQzgcxrlz57C4uIi1tTWRXiQSwZ07dxCNRvHMM8+grKxMyCs3OIw8H1wuF5555hm0tbXB7XZjfn4e58+fx9jYWMopPkbjkbcB73+ZpOAkCTe4ZJkx8gqRY0VxrxKTyYSjR4/i+eefR2VlJba3tzE2NoY33nhDBKWWCTvqDyLs5LmLb4nicwWNP15no3ltZ2cH0WgUgUBAeIzI8yCXNVmmeDsYeT7SbxojvC1JxuQy8jmVk1h8/mtoaMCTTz6JqqoqbG9v4/z58zCZTFhcXMTAwABGR0fh8/kEESmTo1ymZK8LWYZkEoW3tRGJy8lAmdA1IpEBiJg9+/btQ15eHubn57G6uopoNApd10WaOzs7yMvLQ0NDA1paWuD3+7GwsIDS0lK4XK6UrUTr6+tYWFhIKR8vL80rFCutqakJjY2N8Pv98Pv9KC0thcfjgdlsRjQaFYHbA4EAenp6sLCwkOJty73QuBzH4/E94yA/Px9dXV04ceIEcnJyUFpaCpvNJgjCtbU1jI6OYnNzE9vb21hfX0dBQQEaGxthtVpFoG6+gEBbF5966ins378fU1NTGBkZga7rOHToELa2tnDu3DnYbDbs27cPRUVFKC4uRnl5ufA0i0ajKVu6Njc3MTU1tYc45gSvLPf8vtG8z+d7PtfzY9dl0ofeo/7lnrBEBhYXF4uYZ4WFhfD5fHj99ddhs9lw+PBhFBQUIDMzUwTdTyaT2NrawtTUFAYGBuD3+1O+AwoKCgoKCgqPJz6R4M4y+cPBFSngw6PHKyoqUFlZiQMHDqC2thYulwvT09OYmZnBvn37YLfb4fP5cPv2bXEkqd1uR1FREfbv34+Ojg7U1tZiYmICH3zwAe7cuYO5uTmhQD6qvFQu2cjlyjKtlsViMcTj8UeSWfR+utVsOT+j9+VnZFB6FosFXq8Xv/Irv4L6+npsbm5icHAQIyMj2NnZQUFBgSBMlpaWEA6HRZp2ux3Hjh1DVlYWenp6MDk5KWImUf0rKyvR1NSE2tpaeDweALvHCPv9fmxubiIQCAiPGjoxhE5roWO9PR4PMjMzsbGxgaysLBHod21tLaVduOxYLBZkZWWhtrYWra2taGpqQjgcFkp9IBAQBBMp2/n5+WhubkZXVxe2t7dx9+5dDAwM4N69e1heXhZ1A/aeNEPgq68kn/QvPz8fZWVlAIDl5WVhXPH2kmWb9yfvu3TEDy8DXeOGhWyApkuf14m/K29d4ASPkUxyI1fuK27g5uTkoKqqCm63G729vRgdHUUkEhHeAdygmp6exubmJrxeL3JyclK2Lciyr2m78Zra29vR3t6OvLw8LC4uoqenB/39/XsC2speGHIbGJE39A6PdSGvfHPSIp1xJPc9J7NPnjyJ2tpamEwmTExM4Pbt25icnBQEqtyfXHa4ochlQs6b5EY2Onndqa7BYBD9/f0IBAIYGxtLkT2jehnFPOLv8DYhA1gmrOR50aiv5H6itqRtfqdPn0ZZWRlWVlYwPj6O8fFxWCwWxONxrK2tYWNjY09sHNmDkPqG36Pn+bxAW32ItKK25X1t9K2TCSsuT9RuZrMZ2dnZOHDgACoqKlBSUgK73Y7x8XEMDAxgcnIypQ1sNhvq6urQ0tICr9eLeDyOM2fOwG63p2wFNJlMCAaDePfdd7GysrLnGHvqmyNHjsDr9aKkpARVVVUoKCjAzs4Ojh8/Lrwm6Thwt9st5vyBgYGU+hJhY7PZkJeXJ7Zq0bHs0WhUzM9ZWVnYv38/Tp48CbfbjdnZWYyMjKR4Z4VCISwsLCAYDIotWvT9v3fvHgKBQAqZRIc5HD58GAcOHMDs7Czu3r0Lv9+P4uJiZGVlYW5uDnNzc9B1HSsrK8jOzhZkltVqFf1P8cisViuWlpawvLyc0mb0Nyf/eFtwmebegvz7wINh8zYkmeBjTJYvKgdte87Ly8O+ffvg9XpRWVmJxsZGuN1ujI+P48KFC2htbUVbW5voR5IR2kKXk5ODeDyO4eFhrKys7CEsFRQUFBQUFB4vfCLBnWUjUQYpN7Slq6ioCEeOHMHBgwfh8XgQi8UwPj6Oy5cvIxaLobKyUhiM09PTMJlMyMzMxL59+1BfX4+2tjZUVFRgfX0d58+fx/nz58Ux37IRlI7o4de4caBpGpxOJyorK+FyubC4uIilpSWxIisbiUZGumwAGBnsXBnk5eBpyOnb7XYUFBSgvb0dJ0+eRCQSwcDAAAYHB7G8vAyr1YqSkhLk5eVhdHRUbCmhsuTl5aGjowMbGxuYmJjA0tKSyNdsNsPr9aKjowOHDh1CYWEhHA6H+H95eVl4C/h8PoRCIQSDQWxtbSEajYo2279/P4qLixGPx3Hv3j1kZWUhHo+nbDej8lCbu1wu5OXloaamBl1dXWhsbEQgEMD4+DiGhobg9/sBQBhttJpZU1ODY8eOobi4GD09Pbh69SrGxsYQDAYFWcdXZXkaNpsNsVhMHKnNV2PJeG9ubsb+/fuhaRr6+/vh9/vFscOyNxClzdOXjXKr1Sq2OHK3/Gg0usfzihvu9D+PWUIGtdE45CQClyeZ7DEas9yI57LJDRwaixUVFVhdXUVvby/C4XCK3FI5c3Jy4PV6YbfbEQqFsLGxkWKIy4SG2WxGUVERnnzySXHEdH9/P3p6erC+vp5i2PJ6ciNdbhdqa76Ni55LR7TyNk1HrPDAqdS/eXl56OrqQlNTE6xWK+bm5gQZSUQp7xeeJk9b9pDhRNFHQU5f0zRsbm5iaGgIY2NjIs4J1c9o3uF1p78fRVgDqbFs5DbkBrRcVk4amUy7Hp1VVVXo6upCeXk5ZmZm0NfXh8nJSRGHhhNKnGDSNC3FyOXBraPRKFwuF1wul3iPb/GLxWLY2toSJyKRvPI25XWl/OVg10ZtyUkQj8cjZOfAgQMIBoOYm5sT2/wors+RI0dQX18Pm80Gt9uNI0eOIBQKiS1LZrNZeHcMDAxgfX3dcPHBZrPh6NGjqKurg8fjgcPhQDweR3Z2NpqamhCNRrG5uYlIJIKMjAwUFhZC13Xcv38fm5ubYqsjtaPH40FGRgZKSkpEHL7V1VWxkEAE6L59+9Da2oqamhrMzMygu7sbd+7cEQGY+XaxZDKJ/fv3o7OzE2VlZZienkZfXx82NzcBfOhdREGp29rakEwm0dPTg8XFReTl5aGiogJ5eXm4ePGi6PvNzU0RI4i+7fS9aW5uhsPhwPb2Nm7fvo3FxUUx1uT+ozhJnMCRt1bK8448v8rzlJEewslsInvy8vLgdrvh9XpRVlYGXddRX1+Puro6bG9vY3x8HA6HA01NTSgtLcXg4CDm5ubE9vjc3Fzk5OSguroalZWVWF1dTfH6UVBQUFBQUHg88Yls9eJGJf9NCpTFYkFGRgY8Hg+OHDmC1tZW4bo/Pj6OK1euYGhoCJFIBJ///Ofh9XoxPDyMgYEB7OzswOv1oqmpSbj8OxwO3L9/H++++y4uXLggYhFwsoUbZ+Q+n0gkUrxAjIywzMxM1NXV4eWXX4bdbsfly5dx8+ZNLC0tCXdzYO8qPFfwuGHOjW2+4s2flw1L7nEBfOjiXVhYiI6ODvyzf/bPYLVaMTk5iXPnzmFkZEQEwy0tLUU0GsX4+LgwkmjrDB2//dZbb2F5eTklPoHL5cKTTz6Jo0ePIhaLYWpqCmazGYWFhaiurkZ2djai0SiampqQSCQQCoWwubmJ2dlZTE1NoaCgACdOnEBdXR3W19fx7rvv4vr169jc3EyJEUEgciQnJwc1NTVoa2vDkSNHUFhYiFgshvPnz+PWrVtYWFgQ9bdarcjJyUFdXR0KCwtRV1eHxsZGLC0t4e/+7u+wurqashJObUzv67qOrKwsFBYWoqCgADMzM2JlmN5xOBzwer344he/iK6uLlHeWCwmjsnlxIWsxJtMJhQVFcHr9SKRSCCRSAhyp6CgAC6XCxkZGbDZbLDb7cjKysLS0hLee+89hMPhFHk02h7DySxOOsgEpFFwZ24w83u8HvLzJH+Un8vlQlVVFfbv3w+n04kLFy6IE+M46URxJg4ePIhTp04hNzcXV69eRU9PT0o8FQ4yLLu6utDS0oLl5WVcuXIFvb29IiAxGdrcC0kmE+VVbCOCgt/jBJ48N8jp8H6WyQyn04nGxka8+OKLcLvdmJmZwaVLl9DT0yOCjPMtTdQX/NQdXv50RB4nSdJ5PHEDld5NJpPC44h7olD68vY2TvZQ+fj2FN4O8jYfuV58zpNlmP7W9d1gveXl5fjc5z6H4uJidHd349atWyL2i9x/vA+JRG5tbcWBAwcEQeJyuXDw4EFEo1FkZWWJ0x9pC5TZbBYeKMFgEIFAAEtLSxgfH8ft27fTEoQ0jnjMGy5D3EOGSJ+amhq8/fbbIh7NgQMHkJ2dDafTiVAoBE3TUFZWhldeeQVHjx5FTk4O1tfXsb6+jhs3bmBmZgYrKytIJBKCvOjq6kqJq2M0FjMzM5GZmSnquLa2Bp/Ph4WFBczPz4sj38vLy/Hyyy9je3sbV65cQTgcFvGhPB4PWlpa0NDQgHg8DqfTiaysLCSTSRGwneqemZmJlpYWtLW1IRQK4dVXX8Xt27dF/CW+FdBqtcLj8eCLX/wiWltbMTg4iLNnz4q5meSMiOSamhoUFxfjnXfewdzcHMzm3VMq6+rqEIlEMDIyIhaBKA++1Xh7exudnZ3Yv38/dF3H6OgohoeHRT6P+n7Q/EMk/tbWltgGJ88zPB4SJ5NIbmTCnf7Z7Xa4XC6UlJSgpaUFlZWVyMrKQiKRwNDQENbX11FeXo5wOIwHDx6gu7sbkUgEDx48gKZp6O7uxvT0NCKRiCh/VlYWnn76aeTn5yMrK0ucEPZRQeMVFBQUFBQUPr34RLZ6EbhhQoqR3W4XwYbPnDmD8vJyBINBfPDBB+jp6cHMzIxY1Wtvb8cLL7yA4eFhDA0NQdM0vPTSSzh+/DjKysrg8/kQi8UwMzODGzdu4OLFi8Jjg5QrUnptNpvwCikrK4PNZsPU1BT6+/uFkcKNdrPZjLy8PDzxxBN4+eWXoWkaRkdHsbS0hFAoJBRPAuUlx8qgtIBUZZPHEaH3+btym3IjU9M0lJaW4vjx43juuefg9XoRi8XQ09OD+fl5sfrr9XrR1dWF5eVlzM3NiSPM7XY7SkpK8Morr+DBgwe4cuUKFhYWkEwmhVJ4+vRpnDlzBjMzM7hw4QJmZ2dRUVGBpqYmTExM4I033kAymURZWRlKS0tRWFiI5uZmPPfcc4IQm5qawltvvYUrV67g/v37KeSD7AlCHkjPP/882tvbUVRUJO5NTU3h2rVrWFxcFMagx+PByZMnUVdXJ7ZfmUwmbG1t4ebNmylxS3jMGlLa6+rqxDbBsrIyRCIR/M3f/E0K6WGxWEQ7Pfnkk7h8+TIyMjIQj8eFJxEZyvyYXu69lJubi29+85vwer2iLOT943Q6xTWK2WKz2bCwsIArV64IRZ0ThdRHVD5u9MoEK6UNpJKJsnxxgoSMHH6MOI+nwdvTarWirq4OnZ2dAID+/n7cv39fGHOUD23JPHLkCJ577jnEYjG8/fbb6OvrSzHgZZLY7Xbj5MmT+Na3voWxsTH8/d//PQYHB7GxsZFC5nAvFLl+nNyicccJBj52jU794zE7eEBf3sYkV3SfCIfa2lp86UtfQmFhIZaWlvDuu++it7dXEJL0/s7OjtgqYzKZ4Pf7UwxiKiMnfqm9OHnF+4+T3Zyw4f3HjVkiOSldqh/f1spJJtnDQSY9eZvzOZCnxT3WgA+DE9MWzqysLDQ0NOD06dMoLCzEW2+9hZs3b6YY8bI3C5WHYk6dOXMGJ0+exOLiIpLJJNxuNwoLC5GTkwOr1YpAIICtrS2Ew2ERH2d1dVXEC8vPz0d+fj7Ky8uxb98+zM/PC08QInl0XUcsFoPdbhdyJxN0wIdBpr1eLxobG1FbW4v5+Xm8//77gih58OCBiDkD7BIm3/zmN9Ha2gqTyYSenh688cYbmJmZER4cNCcEg0HY7XY0NTVhY2Mj5SQzal+r1YqioiJUV1cjEAjg4sWL6OvrQygUSomN5HA4cOjQIXR0dCA7OxuvvfYa/H6/8F4rLy+Hy+VCIpHAgwcPcO/ePRQVFaGlpQWxWAzXrl0TxLXdbkdXVxfa29thsVhw/fp13LlzR8S1kU9bM5vN+Hf/7t/h0KFDuHr1Kt577z1MTU0J4pz612q1orKyEk888QSsVisGBweh6zpKS0tRXV0Ns9mMq1evCu9S6gs+toHd+amjowMFBQXo7e0VOgEF1d7e3kZWVhays7PFKWNEElqtVpSXl6O2thbZ2dm4fPkyvv/97+8Zn0S4kFzwQxNo7uBEIc3BDocDlZWVIqi9yWSCz+fD0NAQJicnMT4+js997nM4cuSI+PZdunQJoVAIr732GqxWKyKRSErQ8Z2dHSHvLpcLdrs9xdtKQUFBQUFB4fHEJxrcmX7bbDZ4vV6cPHkSnZ2dKC8vh8PhQCgUwve//310d3eLGCyk9BcXF+Nf/+t/DbfbjampKTz55JMoLi6Gx+PBxsYGzp07h+XlZXR0dGB1dRULCwvCm4KUNqvVCrfbjfLycrS2tqKlpQVZWVkAdpXgmzdvYnV1FTMzM3sMw6KiInR0dODll1+Gw+HAnTt38E//9E8YHx8XK7GcqOFKG62akYHFPYPoWSMjyYg0I0ONp1teXo7Tp0+jtLQUt27dwvr6OlZWVvD666+L4JdOpxM5OTkoKCgQ8SKsViscDgcqKirwyiuvoKCgQCj0ZLTm5uaio6MDv/qrv4ru7m4RKwIA/H4/BgcHcfnyZdy7dw/b29sYGRkRpwRVV1fjm9/8Jqqrq7GxsYHu7m68/fbb2NjYEG3DDVlShktKStDY2IgzZ84gkUjgxo0bWFtbw7Fjx1BaWorJyUm0tLSIQJXl5eXIz8+HxWLB6uoqzp49C5fLhYqKCtFXRiSAy+VCc3Mzjh8/jqysLLHa7/P5cOHCBbGiTG78Xq8X7e3tOHLkCN5++21sbW1B13VBUBJIZoFdgyQrK0uszra3tyMzMxP9/f3C4KeYHHTMNAVQraioQEFBAS5duoRgMJgiA7LHiRyrh4wFOT4QyRI/Op4b/eQZJ3sncGJE9uigcdLc3IyOjg4EAgEMDw9jdHRUGKHkcUZeCKdOnYLL5cKFCxdw+fJl+P1+4VVBRBjlQ1tbjh07hueffx5jY2P4z//5P4vTfPipUgDEuJfnHk400mk3nBQuLCzE008/DZPJhLGxMfT29iIWi6XMBbz9+THYdrsdOTk5yMzMRCwWS/FEy87ORnNzs/BIXFlZwV/8xV9gcHBQENO8XYuKinD69Gk0NDQgmUzi5s2beP/991PiIxEBRcamw+FAXV0dkskk1tbWsLy8LGSGH9NMfSunwz2NOBFktVpht9sFqRgKhVKOteZzE8mYTHJw4omTrpxQ5PMh9R2PzUPxxzo7O5GZmYkf/vCHuHXrliBY5bkykUiII+Lz8vLQ2tqKZ599Fpqm4c0338T169eRkZGB48eP4/jx43jw4AGys7PxJ3/yJ+LbQyebkdxQrJvOzk5UV1djbW1NkL1EotAcS/OSx+OBruv40Y9+hNnZ2ZSjtvliAHk48gWHRCKB0dFR8YzT6UR7eztqa2uhaRp6enrw3nvvoa+vT7QDtXNxcTE6Oztx8uRJLC8vC+Jb9g7LycnBN77xDTidTly6dEmQ8rwcRK5T/B8i+L/0pS8hJycHGxsbmJubw40bN3D37l1MTEygpKQEp06dQlZWFoaGhnDv3j0xp1RXV+PIkSNwuVy4c+cOfvjDH2Jzc1PIB/U7jdOsrCwcOHAAH3zwAV5//XWMjIykBHCneSsjIwP5+fkoKSnB+vo6IpEIvF4vXnrpJVGOmzdvppzoyRdnSCZdLheys7ORm5uLkpISNDU1ob6+XszH+fn5yMjIELHrNE3D3NwcIpEI7HY7cnNzUVFRgbKyMlRWVuL1118X86A83xJBy+dXPmfzLa8FBQU4c+YMTpw4gfv376O3txdDQ0NYXFwUJ29WV1fjd3/3d2EymfC9730Pb7/9tojjR/lwkpTmAPLAWllZEdsX5QUBBQUFBQUFhccLn4jHDxkd5JnR1dWFzs5O4cUxNzeHoaEh3L59G7Ozs9ja2hLGqd1uR35+Pl588UWUl5cjkUjg2LFjiMfjuH//Pn74wx/izp07WF9fR1tbGwAI13JSHIuKilBXV4fm5mYUFRUhKysLLpcLwWAQg4ODqKurw8LCAmZmZrCxsSGMHFrhKy4uxokTJ/Dss8+KFcp/+qd/wsLCQsrJUECqsiSvdPP73CiV/5a9eSh9UtL4SmFeXh5eeukl5OXlIRqNChLh1VdfFZ5Suq7DZrPB4/HA6XQKQy8zMxNNTU3o6urC/v378aMf/Qi3bt0S75FhcOLECWForK+vw2TaDfLc0dEBh8OB/v5+oVRS21NcpqGhIVRWVsLn88Hv96fEMaGVTiIbnE4niouL8fLLL8Pj8aCvrw9TU1PQdR0HDx5EXV0dVldX0djYKFzkk8kkNjY2cO3aNSwtLWFtbQ3r6+tob2+H1+tFJBKBz+cT5AAZb1VVVejs7ERNTQ1u3bqF8fFxHD16FKFQCH19fcJQof6y2WyoqalBe3s7srKyUFFRgaGhIdy4cUNsg+Ar1OXl5SgrK0NZWZnYOrZv3z44HA7cvn0b3d3dKdsDyXi0WCzipJvs7Gz4fD6MjY0Jo5DaKTMzEw6HA06nU8RY8vv9CAaDoo1l7weSBU4E0d+ypxm9x40rbrRw8slut6OmpgYnTpyAz+cTMSTI2DGZTKiursYTTzwhgsZOTEygr68Ps7OzCIVCKacuEWFis9lQWlqKjo4OHDx4EEVFRfD5fPibv/kbBAKBlMConGygrSsej0cEIO/t7U2pE+8vh8OBtrY2/NZv/RZaW1thNpvxwQcfYH19HRMTE0IOuBcfzU9msxmNjY1obW1FWVkZ7HY7VldX8Zd/+ZcirlFjYyNOnTqF+vp6+P1+/NVf/RUGBwcFccg9jXJzc/GlL30J9fX12N7eFiv53KuK+ozIzaeffhq5ubmi7pFIBKOjo3jrrbfEPEhELhnH+fn5GBkZEQY371uHw4Hc3FxUVVWhoqICXq8XTqcT6+vrOHfunBiTJENUdjmGGp/XZJmhf/QO92Di7U39WllZidraWtjtdgwNDWFoaGiP1xp/n58A2NHRgfr6euE5NzExIbx6rly5gpmZGRQUFOCVV15BZmYmpqenxQmBnEyJxWKoqanBwYMHYTKZMDk5CYvFgv379+PAgQOoqqqC1+tFYWEhvF6viDdmMpmER04gENizSEDbYqkesscZjYecnBy0tbVhZ2cHg4ODuHTpEgYGBoQsx2IxAEBFRQVefPFFHDt2DJFIRJDUfHuepmnIyMjAsWPHxLZqOkGMe/QR8QDskm9utxtOpxOtra24desWZmdnMTc3h5WVFYRCIbFV8NSpU6isrMSDBw8wOTmZIhtFRUWwWCyYnJzEnTt3hCcQgW+bIrLxz/7szzA5OYm5uTlRT5prqazb29vihEin04m2tjY88cQTyMzMxKVLl3Dt2jVBtMrfZE6qB4NBrK+vo7S0FG1tbTh48KDw6JmZmUE4HBbb31ZWVpBMJnH//n1sbW3B4XAgIyMDTzzxBH7t135NfHO4Bw/Pk3s3Ubnob07mUz/4/X50d3cL70hqb2qDzs5O5OTk4Dvf+Q6uX78uvtckSyRzeXl5cDgcSCQSyMzMxHPPPQer1Ypbt26JWEz8pEAFBQUFBQWFxw8fO/FDxgwp7vv27UN1dTUyMjIwOjqKiYkJzM7OYmFhAT6fTwQWtVgsOHjwIBoaGlBbW4uOjg64XC48ePAAg4ODmJqawtTUFCYnJ7G8vIxkMilOdioqKkJra6sILun1epGRkQGHw4FwOIzR0VGsrKxgfn4e+fn5OHDgAObm5nD//n2h8DudTpSWlqK+vl4cJ5+Xl4dLly7h/PnzgvSRPXVkoofuceVLNoy4ss8Vc2BvsGP+bkZGBj73uc+hpaUFFotFuOdfuHABy8vLKR4C8XhcGMs1NTU4evQo8vPzUVtbi9zcXAwMDODKlSsIhUIpCqLL5UJRURGSySSCwaA4TaelpQX79u1DX1+fMHCpXPS+xWJBVVUVIpGIIPV4G/BtI3RMfFVVFXw+H0ZGRjA3N4eNjQ3U1taivLwckUgE/f39KC8vx9TUFAKBADY3N7GxsSFOKKPtUBSfgLxMAMDtdqO+vh6VlZXweDwwmUy4fPkyBgYG0NzcDLvdjoWFBYyPjyMSiQhvBGqH4uJi7Nu3DzabTZyWtrGxAafTCY/Hg/z8fOTl5cFut6O4uBihUAjhcBgWiwVFRUVwu91YXV3F1atXMT8/n3KiGvWrxWJBc3MzKioq4Ha7EQgE0NzcjIyMDLjdbrGlwWazibhOtD2yv78f3d3d6Onp2UPWcMOKDAk5/hAngui3kWzTVgRd12G327Fv3z4cOXIEmvbh9kcK4kpeVS0tLbDb7ZiamsKDBw+wurqKpaUlbG1tpcTfIo8P8jQrKyvDwYMHsW/fPsRiMXR3d2NqakqQW1Q+8ugrKCiAx+PB/v37UVNTI44znp6eFh5CNI5o+057ezu+9rWvibFgs9lw8OBBtLe3Y25uLsU44/GPsrKycPLkSRw4cABOpxMbGxsIBoPCcKb+aWlpQU1NDcLhMC5evIjx8fE9BhsRsS+//DIOHTqEcDiM4eFh9Pf3Cw87vkqfmZmJ2tpaHD9+HCUlJZibm8PCwoLw7CPZj0ajok0tFgvKysrEUdh0mhoR5OThsG/fPhQXFyMjI0Nsp6FYJfn5+ZiZmdkTrFaWYz6vGckS1Yee4wGcZZm12WzYv38/SkpKBKFM7ce3sHGYTLvxXtrb21FXV4dYLIarV6+KsU0nG5Gs5uTkiDmLl42McxpzBw8eRGlpKba2tlBZWYmamhoxVgGI70tvby8ikQja29tRX1+/53hwblATOe3z+cQWZFr84ORkRUWF8J68fPmyOOacPDjouRMnTqC2thbLy8t4//33MT4+nuJpRW1aVFSEkydPwmw2Y2xsDMvLy2L+lPthc3MTd+/eFbI3Pz+PkZERQVpFo1HxvaH4ZBRnqKamBkVFReI49KWlJfT29sLv92N8fFwsnshzELURbRUjGaQy8a3S9DsYDGJ1dRWlpaV45plnEI1GceXKFRHcmh+7Tt84/l0lAu2dd97B2toacnJyEIlEEAqFEAgEMDc3h0Qigc3NTVHvnZ0dbG1tCU9Dq9Uq+snpdKaMXSov96ojbx95bBD5Q+RWKBTCwMAALBaL8MqV57NkMolbt27hxo0bIvi20RilkzHNZrM4gfG1117D1NSUIIM56aegoKCgoKDw+OET2epVWFiIQ4cOoaWlBQ6HA+vr67h79y6Gh4cxNzcnVq6AVKKDVtyam5vh9XoxMDCA27dv4+7du5idnRVEESlOi4uL8Pv9ImZQNBpFdnY27HY7xsbGMDY2hsXFRSwuLmJ1dRXr6+s4c+YMgF2FnYJ7ut1uVFZWoq6uDuXl5eIkqkAggKGhIUxPTwtXc66wydsNZG8J2Tjhz3LjW36f/6a/zWYzGhoacOzYMZSUlGBzcxPLy8si4CjfoqJpGqLRKJaXlzE/P4+ysjKcOnVKGPBjY2O4evUq7t+/v0fpTyaTCIVC8Hq9IqZAZWUl3G43lpeX0d/fv4cw0DRNEDlutxtjY2Po7+8XHi7cvZ6DvHeIFIjH48JTqbS0FOFwGKFQCIODg+jr60MgEBAGB3djp2DJiUQCRUVF6OzsxObmJgoKCsQxyaFQCBMTE7h27Rqi0agIGr24uCiOsiXioqioCGVlZairqxMGYjgcRnZ2togPQieJEXkUCASwsLAglGibzYZ4PI7e3l6MjY0hHA6nrKxzj5XS0lKUlJTA7XajuLhYBE+12+3Y2dlBJBIRxIfdbkdFRQVyc3PhdDoxMzMjAs7y/iAYefvwZ4zkjssneVJQnTweD+rr65GVlYWRkRFhbGiahuzsbNTV1aWQQuPj4+IUPIrPQfnYbDZkZ2ejuLgYpaWlqK2tRSKRENuZaAtZJBIRMkbbN71er/DwIeKIgrw6nc6UrV1kJBUWFqKhoQEdHR2w2+24evUqzGYzamtrkZGRgVOnTuHKlSvidDsAgiTxeDw4cOAAmpubEQqFcP/+fczPzyMQCIggzBUVFWhra0NjYyNsNhuGhoZw/fr1lK2O1KYUlPzpp5/GxsYG+vv7cffuXeFJIMPj8aCurg4VFRWYmZkR21Rp3O3s7KQQqxRXqbq6Gvn5+ZienhZERF5eHsrLy1FaWipOgbJardja2hIxlzIyMlBUVISCgoIUbyXZ84e3sdEcyIkgfl/eksiNcordUlRUhNnZWVEmLteUH98yQ0da2+12Ef8kFAqlxBciYtfj8SAcDgsylnvlUL+TrNE4r6urE0b3gwcP4PP5hBfI2toaotEoHjx4gLGxMczOziIWiwmPw2QyCb/fL7ye1tfX4fP54PF4hKxSvchTq7KyEhaLBXfu3BFHlFMZabxTP62trWF4eBg3b94UdeKeVC6XS/T52toabt++DZ/Pt8eLjt7Z2trCxMQE/H4/HA6H2ObGAxHzslB8naysLOEVQ+nOz8+LU8LW19f3vCt/DxOJBJaXl1MWVXidSaaSyaQglSKRCMLhMAYGBsTx5PJ8I5Nb/Js3MDCAzc1NOJ1OxONxbG5uIhgMYmNjI2Wro0ya01a1yclJvP/++/B6vXtIH15+8m4y0gE4Oajru3Gj6Eh5Pj6ozZLJJO7duwefz4fh4WFxkqlMjNL2MofDIQje+fl5DAwMpMRLk3UaBQUFBQUFhccLn8gRDZmZmSIA4vj4OHp6ejA8PIxQKCQUQiDVaCAliuLF+P1+/OAHP8Dly5cRDofF1hCCyWTCgwcPMDw8DKfTKQz8paUlBAIBnDt3DvPz8yJPAGJvu8m0Gzi2qqoKmqahsrISbW1tyMnJwcrKioi7QivfXJHiBruRpw+QGneFykrXydOHfstEkqwM0vtWqxVtbW1iO9Pc3Bx6enrQ3d1tGCA3FovB5/Ph1q1bwlCemprCvXv3MDg4KNzxuSK5s7ODQCCAgYEBeDwePPHEE9jZ2UEikcDU1BRu3ryJubm5PXUkhdRut2N6eho3b97E1NQUQqGQKJccdJZikwSDQXGdvMXy8/PhdrsRDAbh8Xhw7tw5zM3N7Qk0y08U8vv9WFpaQmVlJbq6uoSivbS0hMHBQYyMjGBmZgaxWAxerxcFBQVi+4fJZEJ2djaysrJQXFyMxsZG4QVBsTRcLheeeeYZFBQUwGKxYHNzEz6fD6urq+Lo31gsJrYZOp1OzM3N4fz581hbW0tZreZ9L69m5+XlwWazCeIuGAzC7/cL7zaSjyeffFLEnKCxQ+0iE5PU3rJBwI1n+Rq9S4YKxemqq6tDbW0tJicncevWLUEIZGVloaqqCidPnoTT6cT58+cxODiIcDicEuuKvFGcTify8/NRWVmJ+vp6ZGdnw+/34969e4IYiUQi2NjYgMvlgsPhQE5OjiAtiOQhInN6ehpFRUXIzs7G+vp6iocQsHt8dl1dHY4fP47MzEy8+uqr6OnpEW35wgsvoLOzE/X19eLEMACi3q2trWhubkYgEMClS5dSCBo6cefIkSN44oknUFRUhPv372NsbAwPHjxIMVjtdjvy8vJQW1uLU6dOoaamBn/7t3+LK1euYHFxUcg4n2c0TYPH40FJSQkSiQRu376Ne/fuCRmfnZ2Frutim5fL5RLEZW5uLpaWltDT04OtrS1YLBYcOHAAHR0dyM3NRSAQgM/nQzAYTAk+vH//fjQ1NSErKws/+MEPhAcLB98myE+1I6KatiRxuSVvFXnu5Glqmga32w232421tTVkZGSIe3ybDp+TAeDgwYMoLi4WJwvyLUXU9jk5OaioqEB5eTlmZ2exuLi459Q3HtDb7/djbW0NTqcTmqZhdnYWY2NjGB8fF0Q098S4ePEirl69Kk6WzM7ORkVFBex2O65fvy6+AXREPB8X1C5EtgLA5OQkLly4IIJT87pT+87Pz8Pv92NkZAR+v9/QeHc4HCgsLEQ0GhXxrCg4NLUh9/ggb5pwOJxCnPFvF3mn6LqOy5cvo7e3F06nE8lkEj6fT3i9rq2tibzI24u2QtE1SpPyftTCCT2bTCaxsLCAS5cuYWZmBqurq5ifnxdEBweNJy57/PtLpBH9Jo80iiUmk+bU/iTno6Oj8Pl8KCsrS/H04vLJ8+S/5fEul5EfVMHbf2dnR5yKKI8nrg8kEglBbB04cABWqxV9fX1CpvjzivhRUFBQUFB4fPGJePxQDADaLsRJG77qRr8pIDNt5aqursbw8DA++OCDlLg1PE4Cue5/8MEH4vjwaDQqjoGneDDcxRqAWHF84oknxGkgABCNRnH27Fn09PTgyJEjeOqpp1BZWYnnn38eU1NTWF9fTwkAyhVXboBwg4QUUO7hQWXnSi8pd+R1wk9r4mnS0cJ0pPHs7KzYqsaVcMo7GAzinXfeEbEYJicnRVBdvmLJyab19XVcvHgRAFBYWCiIopWVFeHmzskKKv/W1hbGxsbw3e9+V5AgfFWXB7rk5EI8HhfeH8DuEdh2u11srbFarSJQKf0jxZwbDHNzc+I4YY/Hg/n5eVy/fl2cwsYDfEajUZjNZhQUFKChoQEOhwMOhwMNDQ0iDtX09DT6+/vhdrtRW1srvNcCgQCuXbuGW7duYWpqCvF4HIlEQhjd2dnZMJvNGB4exq1btzA9Pb2H9OFxHXRdFwGyCwoKsLa2hsHBQWGQkwHJZd/hcKCzsxORSETIOk+T8pANcC6r/Dngw4DJfHySoURxt9rb29Hc3Iy1tTXcuHFDkHYWiwX79u3DsWPHUF5eju985zu4e/euMICoDNx7qKqqCs3NzSgpKUE0GsU777wjYskUFhaiuroaeXl5OHLkCPLz83Hw4EHU1tbC7XZje3tbtFNvby/u3buHrKwsHD58GFtbW+JkQJI1i8WCmpoaQZyePXtWBE/WdR3vvvsubDYb/sW/+Bc4dOgQuru7xbgoLi5Ge3s7Dh8+jFu3bqG7uxuhUEjUJzc3F42NjXjqqaeQk5MDk8mESCQi4oZQP9hsNmRkZKC6uhptbW3o7OxEaWkpotGoiFnGxyL1FY83s7W1BbfbLbbYyJ4zlE9nZyc+//nPY3l5Gb29vRgYGEAoFBJxlE6cOIHy8nKxfcflcqGhoQE1NTUiWC0RHW63G9XV1VhZWdlDOnBShcsbgeY6Pu9zo5rHXeJGudVqxdzcHGpqalBeXo6TJ09idXVVkPHceN7Z2REkX3Z2ttj+kkgkhCcNEaQlJSU4dOgQGhsbAQBnz55NIWVpTiK5jcfjYusUsBubzu/3Y2trK4V05kb39vY2IpHInlg5Z86cwezsLMLhMKxWKwoLC5GZmQmfz7dn62wymcTy8jLeeustuFwurK+vCwKak8WatruN6vXXXxftyb91VB5d1xGJRLCwsIB79+7h0qVLwus2HfnC5wmK3UUEFf9+83hDtG2b14We5YQ/nxdIxvmWSiKMuBzxbYRc9sjLigeo5oHeqfwUn4v3GdclKIYc1YtIUE7Ccc8cHmuItn5tbW1hfn4eAMQWQplI4R5OXN5oTBjpEDJxyseZ/DzVjQhS+t/v94uFIL5tk+sZMrGroKCgoKCg8HjhYyV+SPnw+Xx7vBu4QUDPcmWFYhUcOnQIo6Oj+NM//VMEAoGUd7nyY7VaEY/Hsba2Jk6y4qt88rYaUoiuXLkCh8OBw4cPIzMzE4uLi7hx4wamp6eFEbC+vo5kMolf//Vfx7Fjx1BbW4t79+4J9/eBgQGMjIykEC6yMkcGCI/HwA1zKptR3BVebroei8Xw+uuvGyrt3GDn8QuSySTC4TDee++9lPx4YGYyjCnvaDSKqakp3L9/f0+ZKC9Kh5RW3uaTk5N7yC0iafiRsfJqLhkzwWAQDx48wPT0NGKxGP7hH/5BnKIGpMZH4kbv1tYW+vv7MTg4CLPZjFgsJuSG+oDkIxQKYWlpCYWFhejq6kJHRwfW1tawuLiI73//+5icnBReVA6HQxzjG4vFsL6+LgIqc0OVgvBOTk4KsocblNwIonqQN9WdO3fQ39+fQoZxGeFkjKZpIo7E4uKi2ApAAUVlOeSkD5czWR6on3j56L2cnBx85StfQVtbG3w+H/7+7/9erOiTrFPw7Hg8jtnZWWGg81OkqDwUcDgYDGJychIjIyMi1ojJZMLU1BRWV1dRX1+PV155RRjSCwsL6O7uxsDAAKanp0WcjXg8LuJXjI6O4tq1aynjAwD279+PgwcPIpFIiK2bwK63jsfjQXZ2NoLBICYmJoSBS9t7GhoasLa2hosXL4r4JHl5eSJQel1dHQYGBvC9730PBw8exNNPP42ioiI0NjZiaGhIxBU7evQoCgoKoOu6iFlDwdHJMKV+oDFP/b6ysoKFhQURgJkIbi5fLpcLx44dwx/+4R8iFovhj//4j1OC6VIw+EQiIeJrdXV1YWdnR2xRGhgYEFuSampq8G/+zb/BgQMHxHZCfiKRkWcFyRXJndEY4PIlky3A7hHyly9fRllZGU6fPo2vfOUreO655+D3+xGNRrG6ugqz2QybzYa1tTX82Z/9GTY3NzEyMoKamhocPnwYlZWVGBgYQDKZREFBAXJzc1FYWIhEIoHx8XG88847Qk7lsvOxNjMzg+np6ZRrNF8RSLa5Jwu1y8bGBoaGhnD48GH8wR/8gehri8UCn8+Hs2fPpmyL4oR2OBzG1taWuMbnPB53i4h13p7yaWqhUAhXrlzBzZs3xdjgXiacZOSkL58b5Pg0VCbqO3k7FAef9/k8yPUC7rlCJ/3xBSLuOehwOFL6gZOHsswRqSEvPPC+JDng8ih79vL24t9BHpQb+FD3oPJwApp7vvKxw/PncgVAEFk0priOQSQwlz2ZzKK0OMlGZLz8beHfCgUFBQUFBYXHCx9J/Giatg/AdwB4AegA/lzX9T/WNO1/B/AtAKsPH/0Puq6ffVRafHVPJjj4/0DqSpbZbIbX60V5eTnW1tbw+uuvY2lpSSgqRtsf+HG2PH/uucANDco7HA7j3Xffxfnz50U5KKApsKuIBgIB4V7/G7/xGzCZdo97Hh0dxfr6espKvtVqTSE5ZMOaTt0iBZHKzLcsELhHkNHK3/b2Nu7fv78nL76aSumQok5KqazQUbtzQ44rlWSgyGXk+cjBIGUiir9LSqvNZkshwujZRCIBk8kkglVfvnwZuq6nBJ/mXip02hl5YPF85S1hvL5U/h/84Afo7u5GdnY2tre3sbi4iPX19T2xaGKxGEKhEBYWFkQfGq3MkqFAxADlJ2/d4H3FjTmjMUP9w/tU13XcvXsXb775piAsjZ6nd2RClBtasixyI4ruZWZm4ktf+hIOHz6M6elpvP3222L7Ejd0NjY2sLa2JgJih8NhYZRwo568TgYGBsRpcJykNZlMmJubw9mzZzE5OYnMzExsbGxgZGRExIwisoT6iU5GSyaTmJ+fx9TUVIoRSR4kMzMzqKurwxe+8AWMjY2huLgYRUVFyM/Px/b2Nt544w10d3cL0pbi5xQWFiIYDKK0tFRsmzx16hRycnIwPz+P733ve7h9+zZCoRCCwaCIOXbixAnU19cjJycHADA1NYWzZ89iamoKLpcLv/3bv42ysjJ8/etfx/j4OObn50W+7733nggyDOzOWxsbG8jJycHx48fx3nvvpch+fn4+Tp48iW9/+9tYWFjAn//5n+POnTsIBoOibWkLIQWrz8zMRDAYxMrKCoaHh0WbJhIJOBwO2O12xONxFBQUwGq1irhsXL6A9EFz+bzCnyM5JZmjZzgJ8eDBA/zDP/wD5ubmcPz4cXG6FHn2JRIJLC4uYnJyUvT13bt3UVdXJ+Ix1dbWIhgMYnt7W8T3Ghoawvj4uPDeJNJb9uDk3zI+NvmcJ89x5PXH66NpGjY2NvC9730PkUgEFRUV2NzcxP379zE6OoqRkRHhpUKGOMWUkRc9+DzIvxH85EciXEmG+feY5kZ5vHPwuZ3SpzFM/cq9fvgcYrQdieY58gzkXjuccOFzErWlTCBxr7BYLLZHrnjdOKmj67r4DvN3qH1pPibyld/jBBvw4beHfws4Mafr+h7SiurPPd3kBR65PCRD8lZeAi3cUP/KZBxdp626/Jssg8+VCgoKCgoKCo8vNCPlLuUBTSsGUKzreq+maVkAbgP4AoCvAAjruv5fftzMTCaTToa9QT4p/9PfpIzl5OSgpqYGOzs7mJqaSol/wJ83Uuq40kmGrpFiy1dU5S0SXCmk7VYUu2NnZzf2DRmztJWAk0vyqiBfUZPbgK/c8esyccVd4UmZ58o2/5sr3zxdah9+khKVl28N4IaXXBdZWSSShiuSVAYypDj4dU5E8XJyg5EbHEYKK683rxOXD0648D4gDy06jYXKRf0q9xmVjxRwvs2GrlP7ygSPbJgYKeDcK0k2FPh1LiMUHycUCiEUCgkSlNLn73PDgLcJ/UvXH1S2F198EV/+8pcxNDSEa9eu4d69eyLgKcmjybQbmP3EiRP42te+hsnJSfT29oo4EqFQKOUkH6MxzMtntVrhcDhgs9mE4RWNRgXBKhu/ZrMZeXl58Hg8CIVCmJ+fF/1ExlxFRQWOHj0qCJutrS1xWtvo6Cju3buHiYkJ3L9/P6Wdu7q6cPLkSZSVlSEYDCIvLw+bm5uYmZnByMgIJiYmsLKyImIZ2e12VFVViSPFacve/Py8CFBPx0C3trais7MTmqZha2sLd+/eFUQeBfcNhUKw2+3weDxoaGhAc3Mzzp07h7/6q7/C1taWqGNLSwu+/e1vIycnB3/0R38kPH34qj7Vi048JHKGTi3iMmu321FdXY1/9a/+Fe7cuYPvfve7KSdA0VhK16fcC0LelklzNJ+/+TigMU1xnXJzc+FwOIQ3Jn1jEomE2FYXj8dhtVpRXFyM2tpaVFRUiADkW1tboo/C4XCKtxUnxLlxT/IoG9xG8yH/LX9XOPFQUFCAjIwMsU2ZgrbzBQHKm5dDJt+5/Mtl4N8K+fQ8eT7g/cUJCKM0AaQQZbzPeZ/y93gby2SY7P3J5YqeNSLE+fxtRNQZtRUnn+TvndVqTfE25O1N13g7yzqDrM+QjPAxwOvOdQaeJs3hVqs1bR3494b3P7UD/37wcsjl5u/I9zRNE2SxgoKCgoKCwqcSt3VdP2J04yM9fnRdXwSw+PDvkKZpwwBKf5bSyMTDw7TF//w+KSmbm5sYGxuDru/GIzBSenkasuJkRHBxxYeTO7JCaWSE0japiYkJ8bz8Hhk28nYa3g5GpMWj3Kllo4GXj+dDZTVS1OW2obaQFWy5D/hKITfEeNr0t/weJxSM8qd0qQ+4AcIJLW588TLxMshyISvKVF9+TzYMiOiRjR4jOZLbWs6Ll102HIzSlI2GdGlzkoOTRhR0lbdXOrng942MPrmOvPw2mw3t7e2YmprC9evXxXHOnKihdNbW1tDf3499+/bh4MGDaG1tRUZGBh48eIDJyUmMjo4Kooa8WmRwg2lzczPldCKZUORtrus61tbWROwd+dhjAPD7/cLLqKysDHa7Hevr61haWsLMzAwWFxexsbGR0j7JZBIzMzPIzc1FZmYmMjMzEQqFcPfuXQwMDGB2djYl/grFd6ETnSYmJoTHGJWPZC4ej6Ovrw9ra2ui75aWlrC+vo7MzEw89dRTOHPmjNie5XA4xOlDPT09KVtDNE1DKBRCb28vNjY2cOfOHbFN1mgshsNhcRIbN9j5WKcTk1599VX4fD5RRw7uucNlm+7xMWw0j8jzOZ8Xd3Z2hPfd+vq6IDLi8XhKsGF6loighYUFhMNhzMzMwOFwYGVlRQRSNiIaZOOdykhjjXtQyO2ZjpyVCXQiL1dWVlIIAN42j2oXeVwaGe9c1jkpIKdhRM7Qb05yyHMIkQ7yN53PBem+wbIXCi+70TyVbl6XvUT5czxN+Rskywt/huRanj95H8pztZyvfI+el8cAbb9Kp4PI40T+htB3ltfBqN353EhEmQz5eyX3g4KCgoKCgsLjh58oxo+maZUADgO4AeAEgN/VNO2fA+gB8G91XV83eOd3APwO/eaKDFeMZMKEr6gBu6QGBXI2MuJlL6J0igsvB1foKB3+dzpSQVZQ0xECch15/Y2epXSI6DBSRHn5ZIVSJkd4/bl3kJEyx13Y+ao65f0oV3C5nJz4kRVX+fdHpW9UF6P25vcpTV5+ekYmOmQiRTYu5TLw9PjfRqfEyH1u1J/yPaMxYFRX2dA02iJgVJ5HGUZGxqWR9xtdN5lMCAQC6O3txeDgINbX11PanhtE0WgUc3NzIpg4eTbQliEuK/LWiHQkFIE8qvjWByNCgwcU5+UEIII++3w+5Ofnw+FwwOfzwe/3IxaLGcaf2tnZPRXu7t270DQNGRkZgkCi0/9oPPE5KhwOY3p6GrOzs6IcRgGE/X6/OIKZyyXFL8nOzobH40E0GkUoFBJHRo+MjKRskaFyvvnmm4hGo2J7kywfcnvxuU6eb2j73pUrVwBgz0lFvH3luRiA8MiQ5V0GN7DlMUWGK8k+pSePW26cE1EUCAT2EDw8P7kt5LErexPK359045bXi/LisVhkmZfllfKWvzvyuPyofqDrvEy8neT52ahvePq8LYy+cenaQk5b/pbJ+ciBv+V71MdG5ZXrTfoFH5ty21ObymWnd+Vx+1Hztiw7Rt9tnofR91KWETkfLgtyOxqVy4jsN9Kp0ukuCgoKCgoKCo8HPnKrl3hQ0zIBfADgP+q6/pqmaV4APuzG/fk/sbsd7DcflYbJZNL5iRoP0xWGj5Sf+FteSeT3+GlVXFEzcmPmaXNDRFaMyHuAG9P8VA1ZaeVkjRGBIRsipKRSrBm5fpz4kYkKep/aUd5uQO/KwTtJkePu/ZQuh9EKIeUjt50cz4L6ilYu6To3wOQ+MOpTOX9eVspTjk1kZJxS21LdeV/KJ0nxFXzZCEinyMur1fIzRu/y7V68veWtXkaxO+Q0OVHFDV++NUbuKyMPDKP+pb/JKJVlido6Pz9fHI/O24TLH+8Pm80Gu92eYsDxQNfcCOFtQeOF4mOQzNMKeTgchs1mE/fTxVqSCRzuWWREKnAZ46dM8W1klBadPmhEBsuxQLjRJ5MgfAzQ3CjLTV5eHp588kkUFBRgaWkJCwsLePDgAVZXd0Ou0XZLXgcu63IsFy5X8rYf8vCTjVO+9SVdwHGZAKR05fhfMqjc8nwmn1bEn6c0ZXmmPqe+5mXjMOoz+s3lmW/TTfdtkuc6eoZkgLYp0vMUm8rIo4V/2yhtOS6XEZHCvTlk2SayVCbTuBcS72M+N8mEC/cEInnh92SZobLJafDfctkoTYp1RF6B/HvL+0Kur0zq8/6T7/Ny07gx8oSiNjTyVqM85Guy3PN5VS5vuvrTczL4XMnbmPcdlxPqN5IT7vlDdeN5UdvyU+YUFBQUFBQUPnVIu9XrxyJ+NE2zAngTwDu6rv9Xg/uVAN7Udb3pUemYTCbdbrcD2GvwcoXKiAQwckfmSqpsqJHBwNMgJZYrcZwgMJvNYqWcK2L0f7qyktIpkz5cWZdPEqH7shHOFXNZ6SZljba2yIYK/eaGj0wA8Lxk41omH7hRypVJftoXV5xlwoQHf5YNDiPPEyqXTBow+UkhNbgRL4O3ldF13mc83ovs4ULPEmnGDR+qG9WJ2oq78lM9ZKOO9zcPxMqNEGp/eSVdNsjld2S55EG65XpxYoHXTSagjIgnSocTGtyIoe03Mni/8PyIWDEybmjM2mw2ABDxlqh+MkEjb9OT5xReTjpxjf/j/SXHdSIZ5LLCZVI+XYeIUJIhPrb5WObzCs9LPnmH7lssFjgcjpRg5fzkHpI7PgZoTMrBb/l4oHqlI094+ek+ERdGhiz1Ey+/ru9uo6H06FnZyORjPJ1HF801fBzzuY3PhUZeRpwQ43LJAyDLhBD3OJEXHOga/01jnOSc6k2/5XFF9aK2lb8rfJ7n8xmXV/6MPD55P8mkmhxjjvch1VOWUxny3M7nKFlu5DnOSOa5fPB68/fMZrOQazlf3jZyO8t6BO9L/p00+q7z7yX1AW8bui6TObzdeZvy9qYykWwDH55mRu0HQJxQyb/jAFKIG3ku5Hlz+aD5lGSffzsoT0X8KCgoKCgofKrx08f40Xa1i78AMMxJH03TivXd+D8A8EUA9z4qLV3XfdFodBO7nkK/UMgnzCgoKPxEKMDHME4VFBR+JqhxqqDweECNVQWFTz/UOFX4LKAi3Y0f51SvkwC6AQwAoKWu/wDgqwBasbvVawbA/8iIoEel15OOhVJQUPh0QI1TBYVPP9Q4VVB4PKDGqoLCpx9qnCp81vHjnOp1GYBRZMezP//iKCgoKCgoKCgoKCgoKCgoKCj8vJD+3HAFBQUFBQUFBQUFBQUFBQUFhccanwTx8+efQJ4KCgo/GdQ4VVD49EONUwWFxwNqrCoofPqhxqnCZxo/9nHuCgoKCgoKCgoKCgoKCgoKCgqPF9RWLwUFBQUFBQUFBQUFBQUFBYXPKD424kfTtOc1TRvVNG1C07R//3Hlq6CgkApN0/ZpmnZR07QhTdMGNU37vYfX8zRNO6dp2vjD/3MfXtc0Tft/Ho7du5qmtX2yNVBQ+OWCpmlmTdPuaJr25sPfVZqm3Xg4Jv9e0zTbw+v2h78nHt6v/EQLrqDwSwJN09yapv1A07QRTdOGNU3rUt9UBYVPHzRN+/ZD3feepml/q2maQ31TFX5Z8LEQP5qmmQH8CYAXADQA+KqmaQ0fR94KCgp7kATwb3VdbwBwDMD/9HA8/nsA53Vd3w/g/MPfwO643f/w3+8A+NOPv8gKCr/U+D0Aw+z3/wXgj3RdrwWwDuC3Hl7/LQDrD6//0cPnFBQUfvH4YwD/pOv6QQAt2B2v6puqoPApgqZppQD+ZwBHdF1vAmAG8OtQ31SFXxJ8XB4/nQAmdF2f0nU9DuDvAPzKx5S3goICg67ri7qu9z78O4RdBbUUu2Pyrx8+9tcAvvDw718B8B19F9cBuDVNK/54S62g8MsJTdPKALwE4L8//K0BeArADx4+Io9VGsM/APD0w+cVFBR+QdA0LQfAEwD+AgB0XY/ruh6A+qYqKHwaYQHg1DTNAsAFYBHqm6rwS4KPi/gpBXCf/Z5/eE1BQeETxEO31cMAbgDw6rq++PDWEgDvw7/V+FVQ+OTwfwP4XwHsPPydDyCg63ry4W8+HsVYfXh/4+HzCgoKvzhUAVgF8P893JL53zVNy4D6pioofKqg6/oDAP8FwBx2CZ8NALehvqkKvyRQwZ0VFH5JoWlaJoB/APD7uq4H+T1997g/deSfgsInCE3TPg9gRdf12590WRQUFNLCAqANwJ/qun4YwCY+3NYFQH1TFRQ+DXgYZ+tXsEvWlgDIAPD8J1ooBYWPER8X8fMAwD72u+zhNQUFhU8AmqZZsUv6fE/X9dceXl4md/OH/688vK7Gr4LCJ4MTAF7RNG0Gu1ukn8JuLBH3Qzd1IHU8irH68H4OAP/HWWAFhV9CzAOY13X9xsPfP8AuEaS+qQoKny48A2Ba1/VVXdcTAF7D7ndWfVMVfinwcRE/twDsfxg13YbdQFpvfEx5KygoMDzcn/wXAIZ1Xf+v7NYbAL7x8O9vAPhHdv2fPzyJ5BiADea+rqCg8AuCruv/m67rZbquV2L3u3lB1/WvAbgI4FcfPiaPVRrDv/rweeVloKDwC4Su60sA7muaVvfw0tMAhqC+qQoKnzbMATimaZrroS5MY1V9UxV+KaB9XPKradqL2I1VYAbwl7qu/8ePJWMFBYUUaJp2EkA3gAF8GDfkP2A3zs/3AZQDmAXwFV3X1x5+HP9f7LrDbgH4pq7rPR97wRUUfomhadoZAP+Lruuf1zStGrseQHkA7gD4H3Rdj2ma5gDwN9iN27UG4Nd1XZ/6hIqsoPBLA03TWrEbgN0GYArAN7G7uKq+qQoKnyJomvZ/APg17J5wewfAb2M3lo/6pip85vGxET8KCgoKCgoKCgoKCgoKCgoKCh8vVHBnBQUFBQUFBQUFBQUFBQUFhc8oFPGjoKCgoKCgoKCgoKCgoKCg8BmFIn4UFBQUFBQUFBQUFBQUFBQUPqNQxI+CgoKCgoKCgoKCgoKCgoLCZxSK+FFQUFBQUFBQUFBQUFBQUFD4jEIRPwoKCgoKCgoKCgoKCgoKCgqfUSjiR0FBQUFBQUFBQUFBQUFBQeEzCkX8KCgoKCgoKCgoKCgoKCgoKHxG8f8DOrxqAzw+EtMAAAAASUVORK5CYII=\n", + "image/png": "iVBORw0KGgoAAAANSUhEUgAABH4AAABDCAYAAADqHsJ5AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAydUlEQVR4nO3deXAc133o+++ZFYMdg30nCBIkAVLcSVFkRFGS5dBPMaPIsqw4smI7dlKJk2uXUym/2FV5SepW5SbOje6LX9lRbMeyymXFUiRREh1KLkmkNlISCRJcwQUkVmLfBsDMYLZ+fwCn3dMYgBQpATD5+1SpiOnu6T7d043R+eF3fkcZhoEQQgghhBBCCCGEuPk4FroBQgghhBBCCCGEEOLjIYEfIYQQQgghhBBCiJuUBH6EEEIIIYQQQgghblIS+BFCCCGEEEIIIYS4SUngRwghhBBCCCGEEOImJYEfIYQQQgghhBBCiJvUDQV+lFK/rZQ6p5S6qJT61kfVKCGEEEIIIYQQQghx45RhGNf3RqWcwHngE0An8AHwiGEYZz665gkhhBBCCCGEEEKI63UjGT9bgIuGYVwyDCMCPA3s+WiaJYQQQgghhBBCCCFulOsG3lsOdFhedwJb53qDUspQSt3AIYUQv8n0828YBj6fD4BYLEY0Gl3IZt0wpRSGYZjnl0gkFrhFQgghhBBCiFvMgGEYhalW3Ejg55oopb4KfFW/drvdH/chhRDXSQcwPqr36ECIUsrcLpFI4Ha7ufPOO+nu7qa9vZ1gMEiqoLDerzWwYhiG+bM1kGT9WQde9DbWNiqlSCQSM4I19nOZ7ViJRAKHwzFjncPhIJFIYBgGkUhEgj9CCCGEEEKI+dQ224obCfx0AZWW1xXTy5IYhvEE8ASAw+G4voJCQoh5cT01v+Z6jz0gYxgGDocDv99PR0cHg4ODhMPhqwaO4NcBF+tyHcSxBmHsx7QGcOzr9H6twSHr9tZj6dfWfV3LNRBCCCGEEEKIhXQjNX4+AJYrpWqUUh7gc8CLH02zhBA3i1RBksnJSbq7uwkEAsRisZQBotneq9kDNHb2jCC9zLre+m+qfdvbZG2Xdf/29wshhBBCCCHEYnHdGT+GYcSUUl8DXgGcwI8Nwzj9kbVMCLFoXM8QsNkYhsHIyMiMYVX213pbezvs+9JZRLO178O027q/VMfSGUe6LfZhYanaKIQQQgghhBAL6bqnc78eDofDkBo/QtzcZgvaWNdbh1Sleo8OsNiHZzmdTnNol5W15o42VyDmWoI0qYJJqWoKWfepz0lq/AghhBBCCCHm2VHDMDalWnEjQ72EuOnc6tkaH8X5X234lf7XOlzKnvmTKrhjGAaxWMwspGxdHo/Hk4Zj2QMzqQI4er01yGTdt67943A4ZmQA2Y9vzQZKlS0khBBCCCGEEAvlY5/VazFxOBzk5uZSUlLC+fPnicViC92kBbFs2TL8fj9tbW309vYmrSsqKiI/P59QKERra+vCNHABSZ2WZLMN8fowQ79S1e5JlRXkcDiIx+NJ61PtS29j3W62mjyp2INC1hnArO+zBnesdJHpVPWC9L7kPhJCCCGEEEIsFrdU4Ecphdvtpq6ujsuXL9+ygR+Px0NxcTHp6ekzAj+ZmZlUV1cDMD4+jt/v5/Lly0Sj0YVoqphnV8uUud59ftjtrNlAOgPIvs21ttNakNm+3F6gea7t9TL7sDLrMDRr5pEQQgghhBBCLAbzOtRroYdAJBIJwuEwxcXFs/41/1aQSCRwu90UFRXN6Pg6nU4yMjKoqqoiPT2dz3/+8+Tk5Mz6uWVnZ5OVlYXX652v5otr4HK5rvtZSzVdeaptbkSqGjnWmjhzZfxcbb/XWmTZPquX/Wf70LFUU71bt9PP0kL/nhNCCCGEEEIIq3mNfni9XkpLS8nMzMTlWphko8nJSfx+/y0d+NGZTrm5uTPWhcNhotEoBQUFxGIxli9fTlZWFk6nM+W+fD4f5eXlFBYWfpxNFh+C1+uloqKCzMzMWbdxOp243W4Wa7F1a+2fDzOkzP6fXm5fn4rX6yUnJwen02kGb+bafq7AlRBCCCGEEEIsFvMa/fD5fKxfv56amhqys7Pn89DA1F/mE4kEeXl5swYybgWRSASAnJwcc5nu4AYCAYaHh8nIyGBsbIzTp0+jlEqZQaKUIhaLUVNTYw4PEwsvPT2djRs3UlBQkHK90+mksLCQ6upqSktLr2mfqYIgN5rVMtuMWan2m+reS7WNPWBjLc48WxBJZ+tkZWVRUlKC0+mckb1jn80r1fF17SEZ5iWEEEIIIYRYTOZ9qNeKFSu48847WbZs2Zwdx+vtVF7LUJDc3Fx8Pt+HCv7MlTXwmzSsQ9dLiUajKbOeQqEQY2NjZud13759DA0NEY1GU9Z/GRoaIj09PWX20GKnO/g3m8nJSerq6mb9THJzc3nooYf41re+xSOPPHLNQ6Lsr28kwGF/fvS9OFtxZmvdHfvsWtb36NnAEolEygycVPs3DAOPx0NaWhpOpzNpub3Ys35+Uv0+0MtvxntKCCGEEEII8ZtrXgM/LpeL/fv3MzIyYnayYKrYsLWz5Pf7qa2tpaCg4EPX+khV/DXV1M1erzfpL/uzma3461yFaFMNMVlMXC4XDoeDYDCI2+02h7boTm9bWxtPP/00iUSC5uZmJiYmkuqbWBmGQVpaGunp6YvuPD0ez5zrv/KVr1BZWXnTZX/FYjGWLl1Kbm7ujOCeUoo77riDYDBozmzndDqTrsFs9/xHEfCx7g+gpKSErVu3cs899yQtn2s6eGtAZrbhVfa6VS6Xy3zedQBH3/f63i8oKKCurm7G+Vr3U1BQgNvtTtk+p9OZFBQSQgghhBBCiMVgXgvtBAIBEokEw8PD5Ofnc//997Nr1y5Wr17NsWPH6OrqoqurC5fLRU5ODgMDAzz77LMp9zVX5zPVsAxrEGZycpLs7GzWr19PQ0MD0WiUn//857S3t1/TsexZL9aAyFxDUlJ1nK1ZBB/3EBF9fB1oGxoaSlqur1MwGOTSpUvmTF56piK9jbWdTqfTLOycKjC0UJxOJ5s3b2Z0dJSLFy8SDodnbNPQ0MCRI0fo7+8nFAotQCs/HvF4nFgsZgY0rMGR/Px8PB4P9fX1DA8Ps3fvXjOAMltNHf3amtHyUXzOSikGBwcpKiqirq6OkpISenp6ktZbj5vqOXQ6ncRisTkDwXr9XLN0xeNxXC4Xubm5KQO2OTk5rFixgnvuuYd///d/Z2BgQGbvEkIIIYQQQvxGmNfAz+TkJEopwuGwORRjdHSUWCxGd3c3fX19KKWIRqMEAgGqqqrMTtls7B0662t7XQ+d/TA2NkZ9fT05OTm0t7cTCoXYvXs3TzzxxIx92Ot7pMp8MAwDt9udcirn2X52Op2sXbuWdevWkZ+fT29vL8ePH+fkyZMopcjPzycSiTAxMfGRTTtvGIaZ/aA/B3tmgw4S6ECUPqfZsj2s+9QZFLMNs5lPiUSCM2fOkJeXN2f2iM74uJnY70Hr57t161ZWrlyJw+Ggu7ubtrY28z2z+Sgzfaz0vdLZ2Uk0GjWzjlLNnjXbsR0OB8XFxQwODs56z9ln+tL3rKYDr4lEAofDQVpaWlIg0DAMQqEQHR0dXLhwgYKCAoaGhojH41cN7AohhBBCCCHEQpvXwE8ikSAej5NIJBgfHycajTI0NERVVRWGYXD27Fmzk5abm0tdXV1S8MYeOMnKyjIDI7P9NV+zdtCi0Shr1qyhsbGRlpYWsrOzWbdunTnUqb6+npKSEkZGRmhpaSEQCODz+QiFQmYQxppJVFdXx4YNGxgYGOD06dN0dXUlHdfeNq/XS319PcXFxfT09DA6OkpOTg5Lly7l3LlzNDQ0UFFRgdfrpampie7ubrNjPDExcdXr7Pf7WbZsGbm5uQwNDXHs2DGzU2wYhjnUa2Rk5JprqczF4XCQnp5OTU0NmzdvZnBwkF/96ldJATulFBkZGSxfvpxwOExnZycTExPXHCDSQYJrZRgGw8PDJBKJWQNn19tBdzgclJeXE4/HGR4evuFsobS0NCKRSNL5OZ1O1q9fz9mzZwmFQh/63KPR6Iz3KKXYunUrlZWVnDhxgmPHjpmfwY1k8SilSEtLI5FIMDk5eU3b63bCVCB2cnIyaaa/2dpiv1ddLhcbNmygqamJvr6+pHOZbWiidb0OVsJU0fNwOIzf76e7uzvp/tUzoJ06dYqRkZGU11b/K4EfIYQQQgghxGIyr4Efa5AhEokQj8eJx+N0dHRQWlpKZ2cnwWAQh8NBdXU1mZmZVFZWkpmZSV9fHyMjI0xOTprTVRcWFjI4OEhHR0fKzneqGXh0Zy87O5uOjg7a29tZtmyZWfMnOzubhoYGSktLGRkZAeDEiRNUV1cTDAbp6upCKYXf7ycnJ4eOjg62bdvG0qVLzbo5fX19KYshw1QHMj8/n7Vr19Lc3MyZM2eIx+M0NDRQUFBAdnY2mzdvJhwOU1RUBEwFivr7+yksLOT06dMsWbKEwcFBYrEYWVlZ5ObmEgwG6ejowOfzUVtby4oVK6iuriYWi3HmzBkzuwemAg0Ag4ODc2YnparzYqe3ycvLY/Xq1axduxaPx8M777zD+Pi4mV2RmZlJdXU1S5cuxe/38/bbb9PW1kYwGJz1ftEBpcrKSnw+Hz09PQwODiYFF66WZTE6Ojpjn3p7h8ORcrayqykpKWHVqlU4HA7a29tpb29nfHwcpRS5ubnmcKHx8XH6+/tnDO3LyckhKyvLvO8rKiro7OxkaGiIcDhs1pLZvHkzfX199PT04PP5yMnJIZFI0NfXZ87MlkpBQQG5ubkUFxdTWFhIIBBgYmICr9fLhg0b6Ozs5MSJE5w/f97M7NLXoKioiHA4zPj4eFKwMNXwSX1epaWlZiHp4eFhuru7Z7TJ4XCQkZFBSUkJXV1dTE5OmvuPx+PXHDyzZ+M5nU78fj+ZmZkMDAwkDZmcLehjPw/9XzgcZmxsjIqKCgKBAPn5+QwODhIMBsnKyqKiooLTp08zNjZmBpjcbjcul4t4PE4kEpH6PkIIIYQQQohFZ8HGuOhMjGg0ysDAADk5Oeawrmg0ilKKoqIifud3foc//dM/5a677qKkpASv10tRURF79uxh06ZNPPjggxQUFJiFW3VnT3fI7J07XT/o9OnTjIyMmNlCHo8Hl8vFpk2bUErx1ltvMTAwwIYNG3C73axdu5YHHniA/Px8SkpK2LVrF1/4whcoKiqivr6ef/u3fyMSiZCXl4fP5zPP0z5kLDc3lzVr1qCU4ujRowSDQWKxGCdPnuTQoUOUlpaSk5PD888/z3/9139RWVnJxo0bKSsr49577yUjI4NHH32UDRs2UF9fz65du/jSl77EQw89hM/nY9myZdTX19PR0cHFixdZvnw5aWlpSdfA7XaTSCTo7u5O6sTbM6tmG2pjrwkUjUYpKipiyZIl/OQnP8Hr9ZKXl4fb7UYpRWZmJsuXL2fnzp309PSwdu1ali5dSnZ29pwd5fT0dFauXMkXv/hF/uzP/oxPfvKTFBYWmp9jRkYG2dnZ5OXl4fV65xyy5fF4yM7OJj8/n4KCAtLT0819XK0ItN1v/dZvUVlZyfLly9mxYwcVFRXAVB2YtWvXsmfPHh5++GF27tyJz+dLOsf09HRWr17N/fffj8/nY/fu3fzRH/0Rd911lxnoS0tLY9OmTSQSCXJzc8nIyDDfc99991FRUZG0Tx3A0s/Apk2bWL9+PTt27GDr1q1UVVXhcrmoqamhtraWl156iRMnTpgBNOtzs3PnTlauXDnjmtiDgvq11+tl586dbN++nU984hPce++9Sfe/lpGRwapVq3jssccoLy832+pyucwMNL3P9PR0MjMzSUtLM9fpe8rv95OdnW2ui0QiHDp0iN7e3hlZOHMNedMZZLFYzBzOGAqFGB0dZfny5dTW1vLwww+zevVqcnNzycrKoqyszCzsnJaWZi6rqamhtLSUjIwMM6AoASAhhBBCCCHEYjGvGT8wszByIpEgHA6Tl5dndpj0X/I9Hg9lZWX84z/+I3/+53/OxMQEDoeDJUuWYBgGP/rRj3j88ccpLCykv7/fLODrcDjYsmUL3d3d9PT0mNkRer9VVVU8++yzZp0OmBpak5OTw5133sl//Md/MDg4iN/vN4ef7N+/nx/84Ac0NjZSWFhIWloa7733HuvWraOrq4ucnByKi4txOp1mO6wdap1hUFxczMaNG3n66afN5br4bFZWFqWlpZw4cQKfz0cgEKCpqcnM/li9ejV33nknFRUVfO1rXyMWi/H+++/zy1/+kgceeID77ruPNWvW8Morr7BmzRrq6up4//33zewNfX37+/uZmJhgdHR0ziE1qQr52mso6c9J1zhqb2/n8OHDFBUVMTg4iMvlYsWKFdx1110MDg7y7W9/m1dffZVjx47R29s76/GdTidLly7lwQcf5F//9V8ZGRnhwQcfxOPx4PV6qampYffu3eTk5JCdnc3+/ftpbGykv79/RjuVUmzbto0dO3ZQU1NDWloae/fuJRAIUFhYSG5uLsPDwymLB+trYT3fhoYGXnjhBU6ePGkON/J6vXz5y1/m1KlTPPfccyxdutQ85qFDhwiHw8RiMcrKysjIyODKlSvcdtttAJw8edLM3Lpy5Qrp6els2rSJxx9/nImJCdatW0d5eTm9vb10dHTwd3/3d/zJn/yJWSvL7/dTUFBAPB6ntbWVzMxMent7OXv2LM3NzWa22OOPP86xY8e4cOECo6OjSfWb9Llu3LgRl8vF+fPnU97H9muzbds2IpEIr7zyCtXV1dx+++189rOf5cknn0zafuXKlezcuZMLFy6wY8cOurq6iMVilJaWkpmZycjICD09PaSnp7Nnzx6qq6u5fPkyx48f58qVK2zbto1HH32UtLQ0Ll26xIEDBzhy5Aijo6N0dXXNqL9l//xcLldSdpO1NpVeFgwGGRkZobKyku3bt/P666+zfft2lFIEAgHy8vKIx+M4nU7uuusuduzYQUZGhpn909LSwhNPPLHg9a2EEEIIIYQQwmreAz+aLqyrO1+6Q6U7a3r4x8GDBxkZGaGxsZG8vDx27NhBWloap06d4pvf/CZ5eXlmfRSd8eFyubj77rt5/fXXGR4eJhKJmEEmPePR8PCwWQdlYmKCnp4eqqqq8Hg8xGIxVq9ezdatWyktLWXLli289957/OQnP+ETn/gEZWVlHD16lDfeeINdu3bh8/n49re/zd69ezl69CjxeHxGTRp9vh6PxxyWAsyYdUkpRXV1NQcPHiQej9PV1WVmdLS0tPCNb3yDv/iLv+A73/kOvb29nD59mpMnT5JIJPjc5z6Hy+Xi/vvvJx6P09jYyL59+8xOru68t7W1mQEbv99Pb28v8OHq+sBU1pbb7SYnJ4dIJMLFixdJJBL09/ebbd6xYwe7d+9myZIl/OpXv+KP//iPGR4eJhgMzlkLxeVy4fF4MAyDnp4e4vE4P/vZz3C73ezatYv169fz4osv0t7eTm5uLt/61re44447eOONN3j77bcBWLduHY2NjWzdupW1a9fS3t7O97//fVwuF7t376aqqop4PM7ly5dpbW1NOTRI8/l8fOpTn6KpqYnLly8zMjKSFDR86KGHaG5u5vDhw9TW1nLbbbfh9Xo5cuQIa9asYfny5ezbt4/KykrS09Pp7u7m85//PO+88w533HEH3/ve9zh//nxSDSY9dGjLli309vbS09PDnj17aGtrY+fOnYTDYZYsWUJVVRXV1dWUl5fz2GOP8eqrr7J161bOnz9PR0cHDoeDmpoa9u7dS11dnRmo8Pl8+Hw+otGoOVQtIyMDh8MxY6YseyBNP2s7d+7kZz/7GUNDQzQ0NFBTU0NNTQ3PPPOMObywsLCQ2tpaMjIyeP75582sPofDQW5uLjU1NUQiEfr7+3E6nWzYsIHDhw9TU1PD3XffbdYE+9u//VvGx8dpaGhg+/btbNq0iWeeeYZt27bxi1/8wpyFzu/3U1ZWhsfj4fjx4zz88MPs3r2b559/ngMHDpj3p1IqqUCz/h20Zs0aXnzxRT744AOWLFmC1+s1g3tKKX7v936PdevWsXfvXlpaWszMKb1esn2EEEIIIYQQi8m8B35SDR3KzMyko6PD7OjqTub4+Lg55XhLSwsrVqxg1apVLFmyhLGxMU6dOsXf//3f097eTlZWFkuXLmXVqlV0dXVxzz338J//+Z9J03jrorc//vGP6e3tNYv+6iLTg4OD1NfX85WvfIXOzk6OHj1Ka2urmTHU2NjIww8/jN/vx+FwEA6HGRgY4K/+6q/4h3/4B95//30zoONyuSguLqa9vT1pKFUsFiMSiVBXV0dTU1NSHZVAIMClS5f40pe+xKlTp8xhZK2trea+b7/9doaGhnjrrbeIxWKMjIyY7Z+YmKChoYHGxkYaGxtpb283g1DWAth66mpdUFoHfmb7rOZapuv3JBIJcz9KKbxer5nFcvbsWX7wgx/Q19dHIBC4plnKotEoXV1dnDx5kn/+53/mu9/9Lt3d3axfv57s7Gw++OADLl68SDQaJRaLMTY2xsaNGwmFQjQ1NREKhbj33ns5f/48W7ZsYWBggGPHjpkBjv/+7//GMAxuv/12/H7/VdsDU8PFHnzwQdra2sx7QmeKrVy5kn379lFeXs66devo6+vjxRdfZGJigs7OTh555BHeeustCgsLycvLM2ehysjI4Hvf+x6XL182Aw9ZWVls3boVh8Nh1oQqLy+nq6uL3t5eXn75Zerq6vjGN77BlStXOHjwIG+++Saf+tSn2LhxIwcPHjQDnbrgcnNzM+Pj49x///0UFRUxPDxMTk4OW7du5bOf/SyHDx/m1VdfpbKykjNnzpjDJFMVftb3stPppKSkhEgkQllZGQUFBYyNjTE2NkZdXZ0ZkPT7/fh8PsbGxohEImb9K4CRkRGCwaD5GYRCIQKBAG1tbRw9epTq6mruu+8+tm/fzjPPPMPQ0BDHjx9ndHSU+vp6HnroIfx+P++99x7t7e2sXr2a1atXU1NTQ2ZmJh6Ph0gkwr59+6iurmbbtm289tprZraUPh891EwPTXz33XcZHx8nEAiYgR2Hw0FmZiZ5eXk4nU7S09Nxu93mcE1dt0wCP0IIIYQQQojFZEEzfmAqayQajXLs2DGzM6aUYmxsjBMnTjA4OAhgFlru7e2lpaWF8vJyrly5wpUrV5icnMQwDCYmJhgbGwOmghvj4+MzpoKPxWIcPXqUQCBgdvQmJyfp7+9nfHycn//854yOjprDxPRx4/E4o6OjNDU1UV9fTyKRIBKJcOHCBZqbm1m9ejW9vb1UVFSQn59PYWEhp06doqOjI+l8+/r6aGxs5Hd/93fNYr66OPPg4CD9/f08//zzrFixgmg0SmNjI5cvX6a3t5fJyUmeeuopJiYmOHDgADBVvFhnL7zzzjuUlpZSXFxMWVkZTqeTtLQ0/H4/L7/8shlwMQyDSCTCyMgIFy5cuOowmbk+Qx2o0J+RzqKAqVotTqeTkZERLl++TDQaTTnlfSqJRILh4WEOHToEwBe+8AVef/111q5dy+joqJlpoZSioaGBvr4+8vPziUajZGZmEo1GKSkpMTM1YrEYwWDQvL9GR0fp7e0lJyeHnJwcXC5XUkDCLhqN0tTUxB/8wR/w1FNPmddSB2ai0SihUIjx8XF6e3sJBoOMjo6SSCQIBALk5uZy22234fP5iMfj5OXlMT4+zqFDh+jq6iIYDJpButHRUV555RUzMOFyuRgcHOT48eNmwejs7Gxqa2s5fPgwJ0+eJBKJ0NbWRllZGQ6Hg9bWVgKBgBlIjUQiXLlyhRdeeIG6ujoikQg9PT00NTURi8Vwu91MTk6aBdetz81cAUCPx0N6ejoNDQ14PB6OHTtGPB5n27ZtnDlzBpjKlkpLS0sayqnPbXh4mMHBQdLT080AzOTkpBmsjEQiZtu3bdtGe3s7Xq+XsrIy8vLyzJm8amtrUUqxatUqIpEITU1NbN++nZUrV3L+/Hm8Xi+bN2+moqKCRCLB/v37Z9zvOvijAz6GYTAwMGAWINefz6FDh3A4HOzatYu7776bnp4ejh8/zrFjx+asMyWEEEIIIYQQC2HBavxYAz+Dg4Nm5gpMdSrHxsZobGw0a5Ho2Yy6u7tpb2+nvLycwcFBcxhXJBKhr68Ph8NBdna2OZW3vaaQUsrMytEd0WAwyIULFwiHw7zxxhsATExMJHWEdVvfffdd+vr6zJm7ent7ef7556mrq2PFihWEw2Gz4HEgEJhx/iMjI5w8eZKVK1eybt06xsbGGB0dJRQKmcGrN954gzVr1jA+Pk5ra6t5LgMDA7z11ltEo1FaW1vNTAyXy2UGynQB3PLycvLy8giFQoRCoRmBnHg8TjAYNIdcXUugZ7bPUdf30ddJB990QCEnJ4fy8nIzM2lsbOyajhMOh81MsIqKCu666y6ysrKIxWJkZmbi9/uprq6mtraWM2fOMDExQSAQMIfc6MBQT08PHo+HgoICOjs7ganhQKFQiEgkgs/nIzs72wwyphKPx2lrazNrDFlnB5ucnCQSiXDnnXcSDAYJBAIMDAyYQwnD4TDnz5+ntraWnp4eurq6cLlc+P1+8vLyCAQChMNhMygWCATMwI9hGFy6dMnMoBofH8fpdDIwMEBTUxMnTpygp6eHzMxMM/CUSCQ4fvx4Ug0lwzAIh8O8/vrrLF26lGAwyPj4OAMDA3R1dVFcXEwkEmFiYsJsy1yZKzp419rayoYNG8z6RDrIsn37dtxuN4D5OeTk5JhDG/W5hcNh+vv7zRo8DoeDaDTK0qVLyc/Px+VycfnyZbq6uti6dSunT58mLS2NnJwcs+ZRVVWVGdDx+/1cuHCBixcvmjW3ysvLycrKYmRkhOLiYm6//XYOHDhgPnP2YuVDQ0PmUDd9nqFQiJ6eHmKxGJcuXcLn87Fu3ToKCgrw+/1s27aNiooK9u/fn/S8CSGEEEIIIcRCm9fAj+4MWTvNOqBjLzIbDoe5cOECgBlUaGlpMTMpurq6ZgRlRkdHiUQi5lCb2QoU29swNjZGc3MzDoeDkZERc3Yx3Tm11jZpamqiubkZwMxg2bdvH+3t7axatcps58mTJxkdHZ1R4HlycpKuri5+8YtfcN999+H1emlpaaGnp8eczWxwcJCDBw8mtVdnGOkixNb9xuNxent7cTqdvPDCC6xdu5aamhqUUnR3d5t1h+znkirzJtW1SkVnR+g6RNZAXl9fH5mZmYyOjhIMBikpKWH9+vUMDw9z6tQpMzA0F/256Vo37777Lo899hjt7e04nU7Wrl1LVlYWt912GwMDAzQ2NtLZ2UlaWhqxWIxEIsHZs2eJx+OcOnWKzZs3s2rVKq5cuUI8Hqe+vp7h4WFaWlrwer1UVVUxNDQ0Z7HrcDjM0aNHzWFOMBXU0Nlh27Zto7W1lRMnTpjFxvXwwv3797Nlyxa6uro4d+4cBQUFOBwONm3axMTEhDnDHEAkEuHcuXPmNT506BAbNmxg1apVeL1erly5wtjYGE8++SQtLS1m8DMQCJjnfOLECbOmlfWcOjs7aWtrM89Jf4atra2UlZUxMTFBKBRKKvysZ4Gz7yuRSPD6669z9913c+7cOZqbm+no6CA7Oxufz0deXh5DQ0MMDw8zNDREYWGh+dlq8XjczHzTvw8mJiZYuXIlY2NjXLp0iaNHjzI+Ps4nP/lJ1q5dSzgc5uLFi5w7d45wOIzT6aSiosIsEj0yMsLQ0BBHjhzB7/dz++2343a7efvtt6mpqWH9+vUUFRXR0dFhnpMOdDmdTvr6+sxhdxkZGaSlpdHb28ulS5fMQuanTp3i3LlzZGVlsWrVKj796U/zmc98hjNnznDp0iUJ/AghhBBCCCEWDXWtGR4fBYfDYfh8PvOv/npoh8680Z1p3SH1eDxJQz+sWSnWTqt1tp68vDw+/elPk5mZyU9/+lPGxsaSiifrDl3SRZgOMFgDU3r/usNr3UYXSrYOd4FfF2rWU0TrQI61fdYOodPpTOpM62thDexYs5as2Ql6tjHdfus6XSTbPl219drNxhrAse43VYBI16DJzc0FIBAImEV7nU4nkUiEtLQ0lixZwooVK+jp6eHw4cMzht/Z6eFZHo+HjIwMdu3axfbt23nuuedobm6msrKSTZs2EYvFeO2112hvbzezxaxt18E9j8fDnj17WL16NQMDA/T19VFVVcUPf/hDqqqq2L59O6FQiB/96Edz1h9yOBysWLECt9tNW1ubmdHldDpnDPGx3qPWc7LeC263m7S0tKTZ5azD7vT94fF4kqY810PS3G530j2rpzfX18D6n26fzkaztlOfQ319PY8++ihvvvkmr7zyilnbprq6mnA4zNDQkLl/vS997tbnOD09nT/8wz/kwoULvPPOO4RCIXN69u7ubjMwZz2+9Rx27dpFQ0MDTU1NnD59mkQigcfj4W/+5m945plnuHjxImNjY8TjcbxeL+Xl5dTW1nL27FlisRiBQIBAIGDO2LVixQqCwSBDQ0PU19ezbds22traePrpp4lGo+Y55OXlsXz5choaGnjqqadwOp088MADXL58mZaWFvLy8li1ahXNzc1m1lp6ejrl5eVs2bKF3//93+ef/umf2Ldvn5kRKIQQQgghhBDz5KhhGJtSrZj3oV72jqg1a8Ua7ADMoIWVtTNtD2bozvSSJUvYt2/fjE683peu1WENcsCvhy3pjqk1+GOfpcva0dfBDmt9FL2tNXPCHsixBgH0tvpYettUASPd+dfL7RkUqTKdrNfRGkiyzipmD+7Yr7P9/TqwNDg4mPS56Ouup7Zvbm7mzJkz1zzN9c6dO1m5ciVer9cczvOXf/mX5pChvr4+jh07Zt5L1sLVVtZgyt69e/nggw9YuXIl0WiUl156iVAoxNjYGLm5uWRkZFy1XYlEwsz2sh7PXhPHep2sn4MeAqRfh8NhM3hkz8bS+7UG7/TyWCxmBtb0vWI/rm6vvhd0zRxrcNV6Xkop8vPzk4b/aSUlJVRVVXHw4MEZhcCtwVHr9XjppZf4+te/TmNjI4FAwCxmbb0HUt1viUSCCxcusGrVKuDXw8QcDgeHDx/G5/OZz6K+hjowo6+XPdB77tw585o3NzeTnp7OI488wrPPPpuUMTg5OUlPTw/9/f3m8NHXXnuNSCRiDvscGRnhscceo7i42Azs6OGq3/nOd3jnnXeSgmNCCCGEEEIIsdCuGvhRSlUCPwWKAQN4wjCM/6OU+n+ArwD905v+tWEYv7za/qyBDssxMAwjaQpp62vd6bV2ZHU2hH6/DuR4PB6WLl1Ke3s7iUQiKftFZ+rYA0zW9tj/Sm/dt+4g63bAr7OD9Ixk1nX287Neg7mujTVwY22rDtRYA13WzBb7vuzZPfb6SjB7QGgu9qDHbO+zBpQ+TGbZW2+9ZRbQ1fvQw5nsQS09Q5m1w2/9vHQ7QqEQbW1t9PT0mAEDHbB48803zX1dTarzSHVPW8/bnjllXT7b/nU2nHU/MPMzmi1zxpr1ZA0UWp8Z/T4dDFq2bBmtra309PQkZYmdOXOG2trapPvMGpyyvoapz0oXYc7MzGRoaMjMHprrOuq2BgIBgsGgOZRS3/vd3d1mYDYSiZBIJMz6UtZ96uusZ9nS19LhcBAIBOjp6cHr9bJ8+XKam5vN3zPj4+NMTEwk1SHSU787HA4mJiZ4//33OXXqFF6v1xwCpwOd4XCYycnJOYuECyGEEEIIIcR8u5aMnxjwTcMwGpVSWcBRpdSvptf9i2EY3/2wB02VpWIPytiHN1kzcOxBDb2Nz+cjPz8fj8djzuBkzb6xBlOsAQtr8MTaebTXwUk17El3Gu1BoWvt+NnbYM0wsgeAdBvsmTjWbCnr8tnYAwnX+r659mXPSLqe/Vk77DoAZw+OpAqC2LM87NfPep10QV9rwOTDfmap2pwqwDbXNqmyXeYKCNpZa9LYr4k1gGUNlqZ6djS3201lZSXvvvuuWeRav0cHPEZHR1O2KS0tbcazFovFaGxspKamxgy2XO36WocX9vb2JhUBj8fjjI2NMTAwQCAQMLeNxWLms2v/HQGY10evi8fjDAwMcOTIETZs2MDFixfN9dZAof1+tgZ0JyYmzGwr+3Ht2YRCCCGEEEIIsdCuGvgxDKMb6J7+eUwpdRYov94DWgMsqTJv7MOBrIEf6zL7PnVnKxQKceDAgaQ6ItY6KLqDZ++MWzMk7EOw7FkmqdphX2d//WHY22bPGJlNqrbZO7CpAilzteFafRQZDlcLjKSq85TqntHr7Z+f/jnV5wvX/5nZg5VznVuqAJl1P6neo+/NVJ+lNdBhDUBa92cfWmkPZCqlSEtLIxgMmoWjrW2Lx+N0dnbOyNrR3G43xcXFeDweczhZYWEhDoeDoqIifD7fNV1bvX5ycpIzZ86YdXy0jo4OxsfHzewv6/lZzy3Vcay/C4LBIJcvX+aOO+7gueeeM/cHv8460vuzXjtrJpm1WLo1KKTvUyGEEEIIIYRYLD5UjR+l1BJgPfAesB34mlLqC8ARprKChlO856vAV6d/Tpm5Yh2io9fZO+XWZakyKnSmw9DQEK+99lpSZ8zeYbZ2ePUye0Fg+7FTBaTmuE5X3cZutiwc67pU1yJVe+w/X2uwZzH6KNtvz0SaLWh0o+27GmvA4mrBoqsFE+33uHVIo/2ema0d+lloamqit7eXycnJGduHQqFZ2xmPx3G73aSnpwPg8/morq4mFAqZw7yuJbim2zM5OUlbW9uM9vX19SV9Zvbg12z7y87ONocEpqenU1paSnp6elK2kPU49vfbg0w6s8d+jSXLRwghhBBCCLEYXfOsXkqpTOAg8D8Nw3hOKVUMDDBV9+fvgVLDML401z6cTqfh9XqTOmvWbAl7p2u2bBV7BoT+2el0mp1ee6FXe2aR/jdVO6yzjunjWduh//Kf6trZMzCudn1TZQpZ92Vfbl1mH56W6lqlWj/bPj9MwCrVud1IltNcx9FtutGMHPvPuk7U9QTqZmuj9bVmrVVkzRyb7T5JleGl35cqi0cvt86UZw+K6PXW91jXWWeJs2e0XAs9w5uexr2goIBYLEZbWxvj4+NmHRwdoJ1tH4ZhmHW59DLrM2m9znp/urC6fVZAfa51dXUYhkFOTg4VFRVUVlaSlZXFgQMH+OCDD5icnJyx70QiMWPmPGub9DIrp9NpnufY2JjM6iWEEEIIIYSYT7PO6nVNgR+llBt4GXjFMIz/nWL9EuBlwzBWz7Ufp9Np6Hog1mCPvVCz7rjpor1A0hAsvd30PlMGjuzDP6ydY+t6zdpZtrZH78M+rMRy7maH71qzR2YL9swV3JgtOGDfxm62baxZUqnev5iyglIFQq7n/dbX2kdxnrMFjq523KsFnK41g8Q+BM4e4JwtWJjqubGum6tts7XXGoCF5CCSNTg11z6sQzft7bLev5rL5ZrxvOpzNQyDgoICysrKyMvLIxQK0dfXx+joKJFIxKwnZf2dZJ+9zxocs86OZs2s0r+fdOAnGAzOmFVQCCGEEEIIIT5G1x/4UVM9rCeBIcMwvm5ZXmpM1f9BKfUNYKthGJ+7yr76gQmmMoWEEItXAfKcCrHYyXMqxG8GeVaFWPzkORU3g2rDMApTrbiWwM8O4C3gJKD/FP7XwCPAOqaGerUCf6wDQVfZ35HZolBCiMVBnlMhFj95ToX4zSDPqhCLnzyn4mZ3LbN6vQ2kGnPyy4++OUIIIYQQQgghhBDioyLzDgshhBBCCCGEEELcpBYi8PPEAhxTCPHhyHMqxOInz6kQvxnkWRVi8ZPnVNzUrnk6dyGEEEIIIYQQQgjxm0WGegkhhBBCCCGEEELcpOYt8KOU+m2l1Dml1EWl1Lfm67hCiGRKqUql1BtKqTNKqdNKqf8xvdyvlPqVUurC9L9508uVUur/nX52TyilNizsGQhxa1FKOZVSx5RSL0+/rlFKvTf9TP6nUsozvdw7/fri9PolC9pwIW4RSqlcpdSzSqlmpdRZpdQ2+U4VYvFRSn1j+v99Tymlfq6USpPvVHGrmJfAj1LKCfx/wG6gHnhEKVU/H8cWQswQA75pGEY9cDvwZ9PP47eA1wzDWA68Nv0app7b5dP/fRX4/vw3WYhb2v8Azlpe/y/gXwzDWAYMA1+eXv5lYHh6+b9MbyeE+Pj9H2C/YRgrgbVMPa/ynSrEIqKUKgf+AthkGMZqwAl8DvlOFbeI+cr42QJcNAzjkmEYEeBpYM88HVsIYWEYRrdhGI3TP48x9T+o5Uw9k09Ob/Yk8LvTP+8BfmpMOQzkKqVK57fVQtyalFIVwP8F/HD6tQLuBp6d3sT+rOpn+FngnunthRAfE6VUDnAn8CMAwzAihmGMIN+pQixGLsCnlHIB6UA38p0qbhHzFfgpBzosrzunlwkhFtB02up64D2g2DCM7ulVPUDx9M/y/AqxcB4H/gpITL/OB0YMw4hNv7Y+j+azOr1+dHp7IcTHpwboB/5jekjmD5VSGch3qhCLimEYXcB3gXamAj6jwFHkO1XcIqS4sxC3KKVUJvBfwNcNwwhY1xlT0/3JlH9CLCCl1P1An2EYRxe6LUKIWbmADcD3DcNYD0zw62FdgHynCrEYTNfZ2sNUsLYMyAB+e0EbJcQ8mq/ATxdQaXldMb1MCLEAlFJupoI+PzMM47npxb063Xz6377p5fL8CrEwtgOfVkq1MjVE+m6maonkTqepQ/LzaD6r0+tzgMH5bLAQt6BOoNMwjPemXz/LVCBIvlOFWFzuBS4bhtFvGEYUeI6p71n5ThW3hPkK/HwALJ+umu5hqpDWi/N0bCGExfT45B8BZw3D+N+WVS8Cj03//Biw17L8C9MzkdwOjFrS14UQHxPDMP5vwzAqDMNYwtT35uuGYXweeAP4zPRm9mdVP8Ofmd5esgyE+BgZhtEDdCilVkwvugc4g3ynCrHYtAO3K6XSp/9fWD+r8p0qbglqvu5fpdSnmKpV4AR+bBjG/5yXAwshkiildgBvASf5dd2Qv2aqzs8vgCqgDfisYRhD01+O32MqHTYIfNEwjCPz3nAhbmFKqbuAvzQM436l1FKmMoD8wDHgDwzDmFRKpQFPMVW3awj4nGEYlxaoyULcMpRS65gqwO4BLgFfZOqPq/KdKsQiopT6W+Bhpma4PQb8EVO1fOQ7Vdz05i3wI4QQQgghhBBCCCHmlxR3FkIIIYQQQgghhLhJSeBHCCGEEEIIIYQQ4iYlgR8hhBBCCCGEEEKIm5QEfoQQQgghhBBCCCFuUhL4EUIIIYQQQgghhLhJSeBHCCGEEEIIIYQQ4iYlgR8hhBBCCCGEEEKIm5QEfoQQQgghhBBCCCFuUv8/S8iNDFw5qb8AAAAASUVORK5CYII=\n", "text/plain": [ "<Figure size 1440x1440 with 1 Axes>" ] @@ -401,7 +423,7 @@ }, { "cell_type": "code", - "execution_count": 46, + "execution_count": 17, "metadata": {}, "outputs": [], "source": [ @@ -410,31 +432,30 @@ }, { "cell_type": "code", - "execution_count": 59, + "execution_count": 18, "metadata": {}, - "outputs": [], + "outputs": [ + { + "ename": "TypeError", + "evalue": "__init__() missing 1 required positional argument: 'pad_token'", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mTypeError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m<ipython-input-18-388038927ee3>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mtarget_transform\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mCompose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtensor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mAddTokens\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minit_token\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"<sos>\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0meos_token\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"<eos>\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", + "\u001b[0;31mTypeError\u001b[0m: __init__() missing 1 required positional argument: 'pad_token'" + ] + } + ], "source": [ "target_transform = Compose([torch.tensor, AddTokens(init_token=\"<sos>\", eos_token=\"<eos>\")])" ] }, { "cell_type": "code", - "execution_count": 60, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "IAM Lines Dataset\n", - "Number classes: 82\n", - "Mapping: {0: '0', 1: '1', 2: '2', 3: '3', 4: '4', 5: '5', 6: '6', 7: '7', 8: '8', 9: '9', 10: 'A', 11: 'B', 12: 'C', 13: 'D', 14: 'E', 15: 'F', 16: 'G', 17: 'H', 18: 'I', 19: 'J', 20: 'K', 21: 'L', 22: 'M', 23: 'N', 24: 'O', 25: 'P', 26: 'Q', 27: 'R', 28: 'S', 29: 'T', 30: 'U', 31: 'V', 32: 'W', 33: 'X', 34: 'Y', 35: 'Z', 36: 'a', 37: 'b', 38: 'c', 39: 'd', 40: 'e', 41: 'f', 42: 'g', 43: 'h', 44: 'i', 45: 'j', 46: 'k', 47: 'l', 48: 'm', 49: 'n', 50: 'o', 51: 'p', 52: 'q', 53: 'r', 54: 's', 55: 't', 56: 'u', 57: 'v', 58: 'w', 59: 'x', 60: 'y', 61: 'z', 62: ' ', 63: '!', 64: '\"', 65: '#', 66: '&', 67: \"'\", 68: '(', 69: ')', 70: '*', 71: '+', 72: ',', 73: '-', 74: '.', 75: '/', 76: ':', 77: ';', 78: '?', 79: '_', 80: '<sos>', 81: '<eos>'}\n", - "Data: (7101, 28, 952)\n", - "Targets: (7101, 97)\n", - "\n" - ] - } - ], + "outputs": [], "source": [ "dataset = IamLinesDataset(train=True, init_token=\"<sos>\", pad_token=\"_\", eos_token=\"<eos>\", target_transform=target_transform)\n", "dataset.load_or_generate_data()\n", @@ -443,7 +464,7 @@ }, { "cell_type": "code", - "execution_count": 61, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ diff --git a/src/tasks/train.sh b/src/tasks/train.sh index 71a68be..9be5be6 100755 --- a/src/tasks/train.sh +++ b/src/tasks/train.sh @@ -62,6 +62,6 @@ then train_command="${train_command} -$verbose" fi -train_command="${train_command} $test $notrain" +train_command="${train_command} $test $notrain ." echo $train_command eval $train_command diff --git a/src/text_recognizer/datasets/transforms.py b/src/text_recognizer/datasets/transforms.py index 8deac7f..1105f23 100644 --- a/src/text_recognizer/datasets/transforms.py +++ b/src/text_recognizer/datasets/transforms.py @@ -3,7 +3,8 @@ import numpy as np from PIL import Image import torch from torch import Tensor -from torchvision.transforms import Compose, Resize, ToPILImage, ToTensor +import torch.nn.functional as F +from torchvision.transforms import Compose, ToPILImage, ToTensor from text_recognizer.datasets.util import EmnistMapper @@ -16,6 +17,18 @@ class Transpose: return np.array(image).swapaxes(0, 1) +class Resize: + """Resizes a tensor to a specified width.""" + + def __init__(self, width: int = 952) -> None: + # The default is 952 because of the IAM dataset. + self.width = width + + def __call__(self, image: Tensor) -> Tensor: + """Resize tensor in the last dimension.""" + return F.interpolate(image, size=self.width, mode="nearest") + + class AddTokens: """Adds start of sequence and end of sequence tokens to target tensor.""" diff --git a/src/text_recognizer/models/__init__.py b/src/text_recognizer/models/__init__.py index 28aa52e..53340f1 100644 --- a/src/text_recognizer/models/__init__.py +++ b/src/text_recognizer/models/__init__.py @@ -2,18 +2,16 @@ from .base import Model from .character_model import CharacterModel from .crnn_model import CRNNModel -from .metrics import accuracy, cer, wer -from .transformer_encoder_model import TransformerEncoderModel -from .vision_transformer_model import VisionTransformerModel +from .metrics import accuracy, accuracy_ignore_pad, cer, wer +from .transformer_model import TransformerModel __all__ = [ - "Model", + "accuracy", + "accuracy_ignore_pad", "cer", "CharacterModel", "CRNNModel", - "CNNTransfromerModel", - "accuracy", - "TransformerEncoderModel", - "VisionTransformerModel", + "Model", + "TransformerModel", "wer", ] diff --git a/src/text_recognizer/models/base.py b/src/text_recognizer/models/base.py index cc44c92..a945b41 100644 --- a/src/text_recognizer/models/base.py +++ b/src/text_recognizer/models/base.py @@ -49,7 +49,7 @@ class Model(ABC): network_args (Optional[Dict]): Arguments for the network. Defaults to None. dataset_args (Optional[Dict]): Arguments for the dataset. metrics (Optional[Dict]): Metrics to evaluate the performance with. Defaults to None. - criterion (Optional[Callable]): The criterion to evaulate the preformance of the network. + criterion (Optional[Callable]): The criterion to evaluate the performance of the network. Defaults to None. criterion_args (Optional[Dict]): Dict of arguments for criterion. Defaults to None. optimizer (Optional[Callable]): The optimizer for updating the weights. Defaults to None. @@ -221,7 +221,7 @@ class Model(ABC): def _configure_network(self, network_fn: Type[nn.Module]) -> None: """Loads the network.""" - # If no network arguemnts are given, load pretrained weights if they exist. + # If no network arguments are given, load pretrained weights if they exist. if self._network_args is None: self.load_weights(network_fn) else: @@ -245,7 +245,7 @@ class Model(ABC): self._optimizer = None if self._optimizer and self._lr_scheduler is not None: - if "OneCycleLR" in str(self._lr_scheduler): + if "steps_per_epoch" in self.lr_scheduler_args: self.lr_scheduler_args["steps_per_epoch"] = len(self.train_dataloader()) # Assume lr scheduler should update at each epoch if not specified. @@ -412,7 +412,7 @@ class Model(ABC): self._optimizer.load_state_dict(checkpoint["optimizer_state"]) if self._lr_scheduler is not None: - # Does not work when loadning from previous checkpoint and trying to train beyond the last max epochs + # Does not work when loading from previous checkpoint and trying to train beyond the last max epochs # with OneCycleLR. if self._lr_scheduler["lr_scheduler"].__class__.__name__ != "OneCycleLR": self._lr_scheduler["lr_scheduler"].load_state_dict( diff --git a/src/text_recognizer/models/metrics.py b/src/text_recognizer/models/metrics.py index 42c3c6e..af9adb5 100644 --- a/src/text_recognizer/models/metrics.py +++ b/src/text_recognizer/models/metrics.py @@ -6,7 +6,23 @@ from torch import Tensor from text_recognizer.networks import greedy_decoder -def accuracy(outputs: Tensor, labels: Tensor) -> float: +def accuracy_ignore_pad( + output: Tensor, + target: Tensor, + pad_index: int = 79, + eos_index: int = 81, + seq_len: int = 97, +) -> float: + """Sets all predictions after eos to pad.""" + start_indices = torch.nonzero(target == eos_index, as_tuple=False).squeeze(1) + end_indices = torch.arange(seq_len, target.shape[0] + 1, seq_len) + for start, stop in zip(start_indices, end_indices): + output[start + 1 : stop] = pad_index + + return accuracy(output, target) + + +def accuracy(outputs: Tensor, labels: Tensor,) -> float: """Computes the accuracy. Args: @@ -17,10 +33,9 @@ def accuracy(outputs: Tensor, labels: Tensor) -> float: float: The accuracy for the batch. """ - # eos_index = torch.nonzero(labels == eos, as_tuple=False) - # eos_index = eos_index[0].item() if eos_index.nelement() else -1 _, predicted = torch.max(outputs, dim=-1) + acc = (predicted == labels).sum().float() / labels.shape[0] acc = acc.item() return acc diff --git a/src/text_recognizer/models/transformer_encoder_model.py b/src/text_recognizer/models/transformer_encoder_model.py deleted file mode 100644 index e35e298..0000000 --- a/src/text_recognizer/models/transformer_encoder_model.py +++ /dev/null @@ -1,111 +0,0 @@ -"""Defines the CNN-Transformer class.""" -from typing import Callable, Dict, List, Optional, Tuple, Type, Union - -import numpy as np -import torch -from torch import nn -from torch import Tensor -from torch.utils.data import Dataset -from torchvision.transforms import ToTensor - -from text_recognizer.datasets import EmnistMapper -from text_recognizer.models.base import Model - - -class TransformerEncoderModel(Model): - """A class for only using the encoder part in the sequence modelling.""" - - def __init__( - self, - network_fn: Type[nn.Module], - dataset: Type[Dataset], - network_args: Optional[Dict] = None, - dataset_args: Optional[Dict] = None, - metrics: Optional[Dict] = None, - criterion: Optional[Callable] = None, - criterion_args: Optional[Dict] = None, - optimizer: Optional[Callable] = None, - optimizer_args: Optional[Dict] = None, - lr_scheduler: Optional[Callable] = None, - lr_scheduler_args: Optional[Dict] = None, - swa_args: Optional[Dict] = None, - device: Optional[str] = None, - ) -> None: - super().__init__( - network_fn, - dataset, - network_args, - dataset_args, - metrics, - criterion, - criterion_args, - optimizer, - optimizer_args, - lr_scheduler, - lr_scheduler_args, - swa_args, - device, - ) - # self.init_token = dataset_args["args"]["init_token"] - self.pad_token = dataset_args["args"]["pad_token"] - self.eos_token = dataset_args["args"]["eos_token"] - if network_args is not None: - self.max_len = network_args["max_len"] - else: - self.max_len = 128 - - if self._mapper is None: - self._mapper = EmnistMapper( - # init_token=self.init_token, - pad_token=self.pad_token, - eos_token=self.eos_token, - ) - self.tensor_transform = ToTensor() - - self.softmax = nn.Softmax(dim=2) - - @torch.no_grad() - def _generate_sentence(self, image: Tensor) -> Tuple[List, float]: - logits = self.network(image) - # Convert logits to probabilities. - probs = self.softmax(logits).squeeze(0) - - confidence, pred_tokens = probs.max(1) - pred_tokens = pred_tokens - - eos_index = torch.nonzero( - pred_tokens == self._mapper(self.eos_token), as_tuple=False, - ) - - eos_index = eos_index[0].item() if eos_index.nelement() else -1 - - predicted_characters = "".join( - [self.mapper(x) for x in pred_tokens[:eos_index].tolist()] - ) - - confidence = np.min(confidence.tolist()) - - return predicted_characters, confidence - - @torch.no_grad() - def predict_on_image(self, image: Union[np.ndarray, Tensor]) -> Tuple[str, float]: - """Predict on a single input.""" - self.eval() - - if image.dtype == np.uint8: - # Converts an image with range [0, 255] with to Pytorch Tensor with range [0, 1]. - image = self.tensor_transform(image) - - # Rescale image between 0 and 1. - if image.dtype == torch.uint8: - # If the image is an unscaled tensor. - image = image.type("torch.FloatTensor") / 255 - - # Put the image tensor on the device the model weights are on. - image = image.to(self.device) - - (predicted_characters, confidence_of_prediction,) = self._generate_sentence( - image - ) - - return predicted_characters, confidence_of_prediction diff --git a/src/text_recognizer/models/vision_transformer_model.py b/src/text_recognizer/models/transformer_model.py index 3d36437..968a047 100644 --- a/src/text_recognizer/models/vision_transformer_model.py +++ b/src/text_recognizer/models/transformer_model.py @@ -13,7 +13,7 @@ from text_recognizer.models.base import Model from text_recognizer.networks import greedy_decoder -class VisionTransformerModel(Model): +class TransformerModel(Model): """Model for predicting a sequence of characters from an image of a text line with a cnn-transformer.""" def __init__( @@ -50,10 +50,7 @@ class VisionTransformerModel(Model): self.init_token = dataset_args["args"]["init_token"] self.pad_token = dataset_args["args"]["pad_token"] self.eos_token = dataset_args["args"]["eos_token"] - if network_args is not None: - self.max_len = network_args["max_len"] - else: - self.max_len = 120 + self.max_len = 120 if self._mapper is None: self._mapper = EmnistMapper( @@ -67,7 +64,7 @@ class VisionTransformerModel(Model): @torch.no_grad() def _generate_sentence(self, image: Tensor) -> Tuple[List, float]: - src = self.network.preprocess_input(image) + src = self.network.extract_image_features(image) memory = self.network.encoder(src) confidence_of_predictions = [] @@ -75,7 +72,7 @@ class VisionTransformerModel(Model): for _ in range(self.max_len - 1): trg = torch.tensor(trg_indices, device=self.device)[None, :].long() - trg = self.network.preprocess_target(trg) + trg = self.network.target_embedding(trg) logits = self.network.decoder(trg=trg, memory=memory, trg_mask=None) # Convert logits to probabilities. @@ -101,7 +98,7 @@ class VisionTransformerModel(Model): self.eval() if image.dtype == np.uint8: - # Converts an image with range [0, 255] with to Pytorch Tensor with range [0, 1]. + # Converts an image with range [0, 255] with to PyTorch Tensor with range [0, 1]. image = self.tensor_transform(image) # Rescale image between 0 and 1. diff --git a/src/text_recognizer/networks/__init__.py b/src/text_recognizer/networks/__init__.py index 6d88768..2cc1137 100644 --- a/src/text_recognizer/networks/__init__.py +++ b/src/text_recognizer/networks/__init__.py @@ -1,25 +1,20 @@ """Network modules.""" from .cnn_transformer import CNNTransformer -from .cnn_transformer_encoder import CNNTransformerEncoder from .crnn import ConvolutionalRecurrentNetwork from .ctc import greedy_decoder from .densenet import DenseNet from .lenet import LeNet -from .loss import EmbeddingLoss from .mlp import MLP from .residual_network import ResidualNetwork, ResidualNetworkEncoder from .sparse_mlp import SparseMLP from .transformer import Transformer from .util import sliding_window -from .vision_transformer import VisionTransformer from .wide_resnet import WideResidualNetwork __all__ = [ "CNNTransformer", - "CNNTransformerEncoder", "ConvolutionalRecurrentNetwork", "DenseNet", - "EmbeddingLoss", "greedy_decoder", "MLP", "LeNet", @@ -28,6 +23,5 @@ __all__ = [ "sliding_window", "Transformer", "SparseMLP", - "VisionTransformer", "WideResidualNetwork", ] diff --git a/src/text_recognizer/networks/cnn_transformer.py b/src/text_recognizer/networks/cnn_transformer.py index 3da2c9f..16c7a41 100644 --- a/src/text_recognizer/networks/cnn_transformer.py +++ b/src/text_recognizer/networks/cnn_transformer.py @@ -1,4 +1,4 @@ -"""A DETR style transfomers but for text recognition.""" +"""A CNN-Transformer for image to text recognition.""" from typing import Dict, Optional, Tuple from einops import rearrange @@ -11,7 +11,7 @@ from text_recognizer.networks.util import configure_backbone class CNNTransformer(nn.Module): - """CNN+Transfomer for image to sequence prediction, sort of based on the ideas from DETR.""" + """CNN+Transfomer for image to sequence prediction.""" def __init__( self, @@ -25,22 +25,14 @@ class CNNTransformer(nn.Module): dropout_rate: float, trg_pad_index: int, backbone: str, - out_channels: int, - max_len: int, backbone_args: Optional[Dict] = None, activation: str = "gelu", ) -> None: super().__init__() self.trg_pad_index = trg_pad_index - self.backbone = configure_backbone(backbone, backbone_args) self.character_embedding = nn.Embedding(vocab_size, hidden_dim) - - # self.conv = nn.Conv2d(out_channels, max_len, kernel_size=1) - self.position_encoding = PositionalEncoding(hidden_dim, dropout_rate) - self.row_embed = nn.Parameter(torch.rand(max_len, max_len // 2)) - self.col_embed = nn.Parameter(torch.rand(max_len, max_len // 2)) self.adaptive_pool = ( nn.AdaptiveAvgPool2d((adaptive_pool_dim)) if adaptive_pool_dim else None @@ -78,8 +70,12 @@ class CNNTransformer(nn.Module): self.transformer.decoder(trg=trg, memory=memory, trg_mask=trg_mask) ) - def preprocess_input(self, src: Tensor) -> Tensor: - """Encodes src with a backbone network and a positional encoding. + def extract_image_features(self, src: Tensor) -> Tensor: + """Extracts image features with a backbone neural network. + + It seem like the winning idea was to swap channels and width dimension and collapse + the height dimension. The transformer is learning like a baby with this implementation!!! :D + Ohhhh, the joy I am experiencing right now!! Bring in the beers! :D :D :D Args: src (Tensor): Input tensor. @@ -88,29 +84,19 @@ class CNNTransformer(nn.Module): Tensor: A input src to the transformer. """ - # If batch dimenstion is missing, it needs to be added. + # If batch dimension is missing, it needs to be added. if len(src.shape) < 4: src = src[(None,) * (4 - len(src.shape))] src = self.backbone(src) - # src = self.conv(src) + src = rearrange(src, "b c h w -> b w c h") if self.adaptive_pool is not None: src = self.adaptive_pool(src) - H, W = src.shape[-2:] - src = rearrange(src, "b t h w -> b t (h w)") - - # construct positional encodings - pos = torch.cat( - [ - self.col_embed[:W].unsqueeze(0).repeat(H, 1, 1), - self.row_embed[:H].unsqueeze(1).repeat(1, W, 1), - ], - dim=-1, - ).unsqueeze(0) - pos = rearrange(pos, "b h w l -> b l (h w)") - src = pos + 0.1 * src + src = src.squeeze(3) + src = self.position_encoding(src) + return src - def preprocess_target(self, trg: Tensor) -> Tuple[Tensor, Tensor]: + def target_embedding(self, trg: Tensor) -> Tuple[Tensor, Tensor]: """Encodes target tensor with embedding and postion. Args: @@ -126,9 +112,9 @@ class CNNTransformer(nn.Module): def forward(self, x: Tensor, trg: Optional[Tensor] = None) -> Tensor: """Forward pass with CNN transfomer.""" - h = self.preprocess_input(x) + h = self.extract_image_features(x) trg_mask = self._create_trg_mask(trg) - trg = self.preprocess_target(trg) + trg = self.target_embedding(trg) out = self.transformer(h, trg, trg_mask=trg_mask) logits = self.head(out) diff --git a/src/text_recognizer/networks/cnn_transformer_encoder.py b/src/text_recognizer/networks/cnn_transformer_encoder.py deleted file mode 100644 index 93626bf..0000000 --- a/src/text_recognizer/networks/cnn_transformer_encoder.py +++ /dev/null @@ -1,73 +0,0 @@ -"""Network with a CNN backend and a transformer encoder head.""" -from typing import Dict - -from einops import rearrange -import torch -from torch import nn -from torch import Tensor - -from text_recognizer.networks.transformer import PositionalEncoding -from text_recognizer.networks.util import configure_backbone - - -class CNNTransformerEncoder(nn.Module): - """A CNN backbone with Transformer Encoder frontend for sequence prediction.""" - - def __init__( - self, - backbone: str, - backbone_args: Dict, - mlp_dim: int, - d_model: int, - nhead: int = 8, - dropout_rate: float = 0.1, - activation: str = "relu", - num_layers: int = 6, - num_classes: int = 80, - num_channels: int = 256, - max_len: int = 97, - ) -> None: - super().__init__() - self.d_model = d_model - self.nhead = nhead - self.dropout_rate = dropout_rate - self.activation = activation - self.num_layers = num_layers - - self.backbone = configure_backbone(backbone, backbone_args) - self.position_encoding = PositionalEncoding(d_model, dropout_rate) - self.encoder = self._configure_encoder() - - self.conv = nn.Conv2d(num_channels, max_len, kernel_size=1) - - self.mlp = nn.Linear(mlp_dim, d_model) - - self.head = nn.Linear(d_model, num_classes) - - def _configure_encoder(self) -> nn.TransformerEncoder: - encoder_layer = nn.TransformerEncoderLayer( - d_model=self.d_model, - nhead=self.nhead, - dropout=self.dropout_rate, - activation=self.activation, - ) - norm = nn.LayerNorm(self.d_model) - return nn.TransformerEncoder( - encoder_layer=encoder_layer, num_layers=self.num_layers, norm=norm - ) - - def forward(self, x: Tensor, targets: Tensor = None) -> Tensor: - """Forward pass through the network.""" - if len(x.shape) < 4: - x = x[(None,) * (4 - len(x.shape))] - - x = self.conv(self.backbone(x)) - x = rearrange(x, "b c h w -> b c (h w)") - x = self.mlp(x) - x = self.position_encoding(x) - x = rearrange(x, "b c h-> c b h") - x = self.encoder(x) - x = rearrange(x, "c b h-> b c h") - logits = self.head(x) - - return logits diff --git a/src/text_recognizer/networks/loss/__init__.py b/src/text_recognizer/networks/loss/__init__.py new file mode 100644 index 0000000..b489264 --- /dev/null +++ b/src/text_recognizer/networks/loss/__init__.py @@ -0,0 +1,2 @@ +"""Loss module.""" +from .loss import EmbeddingLoss, LabelSmoothingCrossEntropy diff --git a/src/text_recognizer/networks/loss.py b/src/text_recognizer/networks/loss/loss.py index cf9fa0d..cf9fa0d 100644 --- a/src/text_recognizer/networks/loss.py +++ b/src/text_recognizer/networks/loss/loss.py diff --git a/src/text_recognizer/networks/neural_machine_reader.py b/src/text_recognizer/networks/neural_machine_reader.py index 540a7d2..7f8c49b 100644 --- a/src/text_recognizer/networks/neural_machine_reader.py +++ b/src/text_recognizer/networks/neural_machine_reader.py @@ -1,180 +1,201 @@ -from typing import Dict, Optional, Tuple +"""Sequence to sequence network with RNN cells.""" +# from typing import Dict, Optional, Tuple -from einops import rearrange -from einops.layers.torch import Rearrange -import torch -from torch import nn -from torch import Tensor +# from einops import rearrange +# from einops.layers.torch import Rearrange +# import torch +# from torch import nn +# from torch import Tensor -from text_recognizer.networks.util import configure_backbone +# from text_recognizer.networks.util import configure_backbone -class Encoder(nn.Module): +# class Encoder(nn.Module): +# def __init__( +# self, +# embedding_dim: int, +# encoder_dim: int, +# decoder_dim: int, +# dropout_rate: float = 0.1, +# ) -> None: +# super().__init__() +# self.rnn = nn.GRU( +# input_size=embedding_dim, hidden_size=encoder_dim, bidirectional=True +# ) +# self.fc = nn.Sequential( +# nn.Linear(in_features=2 * encoder_dim, out_features=decoder_dim), nn.Tanh() +# ) +# self.dropout = nn.Dropout(p=dropout_rate) + +# def forward(self, x: Tensor) -> Tuple[Tensor, Tensor]: +# """Encodes a sequence of tensors with a bidirectional GRU. + +# Args: +# x (Tensor): A input sequence. + +# Shape: +# - x: :math:`(T, N, E)`. +# - output[0]: :math:`(T, N, 2 * E)`. +# - output[1]: :math:`(T, N, D)`. + +# where T is the sequence length, N is the batch size, E is the +# embedding/encoder dimension, and D is the decoder dimension. + +# Returns: +# Tuple[Tensor, Tensor]: The encoder output and the hidden state of the +# encoder. + +# """ + +# output, hidden = self.rnn(x) - def __init__(self, embedding_dim: int, encoder_dim: int, decoder_dim: int, dropout_rate: float = 0.1) -> None: - super().__init__() - self.rnn = nn.GRU(input_size=embedding_dim, hidden_size=encoder_dim, bidirectional=True) - self.fc = nn.Sequential(nn.Linear(in_features=2*encoder_dim, out_features=decoder_dim), nn.Tanh()) - self.dropout = nn.Dropout(p=dropout_rate) +# # Get the hidden state from the forward and backward rnn. +# hidden_state = torch.cat((hidden[-2, :, :], hidden[-1, :, :]), dim=1) + +# # Apply fully connected layer and tanh activation. +# hidden_state = self.fc(hidden_state) + +# return output, hidden_state + + +# class Attention(nn.Module): +# def __init__(self, encoder_dim: int, decoder_dim: int) -> None: +# super().__init__() +# self.atten = nn.Linear( +# in_features=2 * encoder_dim + decoder_dim, out_features=decoder_dim +# ) +# self.value = nn.Linear(in_features=decoder_dim, out_features=1, bias=False) - def forward(self, x: Tensor) -> Tuple[Tensor, Tensor]: - """Encodes a sequence of tensors with a bidirectional GRU. +# def forward(self, hidden_state: Tensor, encoder_outputs: Tensor) -> Tensor: +# """Short summary. - Args: - x (Tensor): A input sequence. +# Args: +# hidden_state (Tensor): Description of parameter `h`. +# encoder_outputs (Tensor): Description of parameter `enc_out`. - Shape: - - x: :math:`(T, N, E)`. - - output[0]: :math:`(T, N, 2 * E)`. - - output[1]: :math:`(T, N, D)`. +# Shape: +# - x: :math:`(T, N, E)`. +# - output[0]: :math:`(T, N, 2 * E)`. +# - output[1]: :math:`(T, N, D)`. + +# where T is the sequence length, N is the batch size, E is the +# embedding/encoder dimension, and D is the decoder dimension. + +# Returns: +# Tensor: Description of returned object. + +# """ +# t, b = enc_out.shape[:2] +# # repeat decoder hidden state src_len times +# hidden_state = hidden_state.unsqueeze(1).repeat(1, t, 1) + +# encoder_outputs = rearrange(encoder_outputs, "t b e2 -> b t e2") + +# # Calculate the energy between the decoders previous hidden state and the +# # encoders hidden states. +# energy = torch.tanh( +# self.attn(torch.cat((hidden_state, encoder_outputs), dim=2)) +# ) + +# attention = self.value(energy).squeeze(2) + +# # Apply softmax on the attention to squeeze it between 0 and 1. +# attention = F.softmax(attention, dim=1) + +# return attention + + +# class Decoder(nn.Module): +# def __init__( +# self, +# embedding_dim: int, +# encoder_dim: int, +# decoder_dim: int, +# output_dim: int, +# dropout_rate: float = 0.1, +# ) -> None: +# super().__init__() +# self.output_dim = output_dim +# self.embedding = nn.Embedding(output_dim, embedding_dim) +# self.attention = Attention(encoder_dim, decoder_dim) +# self.rnn = nn.GRU( +# input_size=2 * encoder_dim + embedding_dim, hidden_size=decoder_dim +# ) + +# self.head = nn.Linear( +# in_features=2 * encoder_dim + embedding_dim + decoder_dim, +# out_features=output_dim, +# ) +# self.dropout = nn.Dropout(p=dropout_rate) + +# def forward( +# self, trg: Tensor, hidden_state: Tensor, encoder_outputs: Tensor +# ) -> Tensor: +# # input = [batch size] +# # hidden = [batch size, dec hid dim] +# # encoder_outputs = [src len, batch size, enc hid dim * 2] +# trg = trg.unsqueeze(0) +# trg_embedded = self.dropout(self.embedding(trg)) - where T is the sequence length, N is the batch size, E is the - embedding/encoder dimension, and D is the decoder dimension. +# a = self.attention(hidden_state, encoder_outputs) + +# weighted = torch.bmm(a, encoder_outputs) - Returns: - Tuple[Tensor, Tensor]: The encoder output and the hidden state of the - encoder. - - """ - - output, hidden = self.rnn(x) - - # Get the hidden state from the forward and backward rnn. - hidden_state = torch.cat((hidden[-2,:,:], hidden[-1,:,:]), dim = 1)) - - # Apply fully connected layer and tanh activation. - hidden_state = self.fc(hidden_state) - - return output, hidden_state - - -class Attention(nn.Module): - - def __init__(self, encoder_dim: int, decoder_dim: int) -> None: - super().__init__() - self.atten = nn.Linear(in_features=2*encoder_dim + decoder_dim, out_features=decoder_dim) - self.value = nn.Linear(in_features=decoder_dim, out_features=1, bias=False) - - def forward(self, hidden_state: Tensor, encoder_outputs: Tensor) -> Tensor: - """Short summary. - - Args: - hidden_state (Tensor): Description of parameter `h`. - encoder_outputs (Tensor): Description of parameter `enc_out`. - - Shape: - - x: :math:`(T, N, E)`. - - output[0]: :math:`(T, N, 2 * E)`. - - output[1]: :math:`(T, N, D)`. - - where T is the sequence length, N is the batch size, E is the - embedding/encoder dimension, and D is the decoder dimension. - - Returns: - Tensor: Description of returned object. - - """ - t, b = enc_out.shape[:2] - #repeat decoder hidden state src_len times - hidden_state = hidden_state.unsqueeze(1).repeat(1, t, 1) - - encoder_outputs = rearrange(encoder_outputs, "t b e2 -> b t e2") - - # Calculate the energy between the decoders previous hidden state and the - # encoders hidden states. - energy = torch.tanh(self.attn(torch.cat((hidden_state, encoder_outputs), dim = 2))) - - attention = self.value(energy).squeeze(2) - - # Apply softmax on the attention to squeeze it between 0 and 1. - attention = F.softmax(attention, dim=1) - - return attention - - -class Decoder(nn.Module): - - def __init__(self, embedding_dim: int, encoder_dim: int, decoder_dim: int, output_dim: int, dropout_rate: float = 0.1) -> None: - super().__init__() - self.output_dim = output_dim - self.embedding = nn.Embedding(output_dim, embedding_dim) - self.attention = Attention(encoder_dim, decoder_dim) - self.rnn = nn.GRU(input_size=2*encoder_dim + embedding_dim, hidden_size=decoder_dim) - - self.head = nn.Linear(in_features=2*encoder_dim+embedding_dim+decoder_dim, out_features=output_dim) - self.dropout = nn.Dropout(p=dropout_rate) - - def forward(self, trg: Tensor, hidden_state: Tensor, encoder_outputs: Tensor) -> Tensor: - #input = [batch size] - #hidden = [batch size, dec hid dim] - #encoder_outputs = [src len, batch size, enc hid dim * 2] - trg = trg.unsqueeze(0) - trg_embedded = self.dropout(self.embedding(trg)) - - a = self.attention(hidden_state, encoder_outputs) - - weighted = torch.bmm(a, encoder_outputs) - - # Permutate the tensor. - weighted = rearrange(weighted, "b a e2 -> a b e2") - - rnn_input = torch.cat((trg_embedded, weighted), dim = 2) - - output, hidden = self.rnn(rnn_input, hidden.unsqueeze(0)) - - #seq len, n layers and n directions will always be 1 in this decoder, therefore: - #output = [1, batch size, dec hid dim] - #hidden = [1, batch size, dec hid dim] - #this also means that output == hidden - assert (output == hidden).all() - - trg_embedded = trg_embedded.squeeze(0) - output = output.squeeze(0) - weighted = weighted.squeeze(0) - - logits = self.fc_out(torch.cat((output, weighted, trg_embedded), dim = 1)) - - #prediction = [batch size, output dim] - - return logits, hidden.squeeze(0) - - -class NeuralMachineReader(nn.Module): - - def __init__(self, embedding_dim: int, encoder_dim: int, decoder_dim: int, output_dim: int, backbone: Optional[str] = None, - backbone_args: Optional[Dict] = None, patch_size: Tuple[int, int] = (28, 28), - stride: Tuple[int, int] = (1, 14), dropout_rate: float = 0.1, teacher_forcing_ratio: float = 0.5) -> None: - super().__init__() - self.patch_size = patch_size - self.stride = stride - self.sliding_window = self._configure_sliding_window() - - self.backbone = - self.encoder = Encoder(embedding_dim, encoder_dim, decoder_dim, dropout_rate) - self.decoder = Decoder(embedding_dim, encoder_dim, decoder_dim, output_dim, dropout_rate) - self.teacher_forcing_ratio = teacher_forcing_ratio - - def _configure_sliding_window(self) -> nn.Sequential: - return nn.Sequential( - nn.Unfold(kernel_size=self.patch_size, stride=self.stride), - Rearrange( - "b (c h w) t -> b t c h w", - h=self.patch_size[0], - w=self.patch_size[1], - c=1, - ), - ) - - def forward(self, x: Tensor, trg: Tensor) -> Tensor: - #x = [batch size, height, width] - #trg = [trg len, batch size] - - # Create image patches with a sliding window kernel. - x = self.sliding_window(x) - - # Rearrange from a sequence of patches for feedforward network. - b, t = x.shape[:2] - x = rearrange(x, "b t c h w -> (b t) c h w", b=b, t=t) - - x = self.backbone(x) - x = rearrange(x, "(b t) h -> t b h", b=b, t=t) +# # Permutate the tensor. +# weighted = rearrange(weighted, "b a e2 -> a b e2") + +# rnn_input = torch.cat((trg_embedded, weighted), dim=2) + +# output, hidden = self.rnn(rnn_input, hidden.unsqueeze(0)) + +# # seq len, n layers and n directions will always be 1 in this decoder, therefore: +# # output = [1, batch size, dec hid dim] +# # hidden = [1, batch size, dec hid dim] +# # this also means that output == hidden +# assert (output == hidden).all() + +# trg_embedded = trg_embedded.squeeze(0) +# output = output.squeeze(0) +# weighted = weighted.squeeze(0) + +# logits = self.fc_out(torch.cat((output, weighted, trg_embedded), dim=1)) + +# # prediction = [batch size, output dim] + +# return logits, hidden.squeeze(0) + + +# class NeuralMachineReader(nn.Module): +# def __init__( +# self, +# embedding_dim: int, +# encoder_dim: int, +# decoder_dim: int, +# output_dim: int, +# backbone: Optional[str] = None, +# backbone_args: Optional[Dict] = None, +# adaptive_pool_dim: Tuple = (None, 1), +# dropout_rate: float = 0.1, +# teacher_forcing_ratio: float = 0.5, +# ) -> None: +# super().__init__() + +# self.backbone = configure_backbone(backbone, backbone_args) +# self.adaptive_pool = nn.AdaptiveAvgPool2d((adaptive_pool_dim)) + +# self.encoder = Encoder(embedding_dim, encoder_dim, decoder_dim, dropout_rate) +# self.decoder = Decoder( +# embedding_dim, encoder_dim, decoder_dim, output_dim, dropout_rate +# ) +# self.teacher_forcing_ratio = teacher_forcing_ratio + +# def extract_image_features(self, x: Tensor) -> Tensor: +# x = self.backbone(x) +# x = rearrange(x, "b c h w -> b w c h") +# x = self.adaptive_pool(x) +# x = x.squeeze(3) + +# def forward(self, x: Tensor, trg: Tensor) -> Tensor: +# # x = [batch size, height, width] +# # trg = [trg len, batch size] +# z = self.extract_image_features(x) diff --git a/src/text_recognizer/networks/stn.py b/src/text_recognizer/networks/stn.py index b031128..e9d216f 100644 --- a/src/text_recognizer/networks/stn.py +++ b/src/text_recognizer/networks/stn.py @@ -13,7 +13,7 @@ class SpatialTransformerNetwork(nn.Module): Network that learns how to perform spatial transformations on the input image in order to enhance the geometric invariance of the model. - # TODO: add arguements to make it more general. + # TODO: add arguments to make it more general. """ diff --git a/src/text_recognizer/networks/util.py b/src/text_recognizer/networks/util.py index b31e640..e2d7955 100644 --- a/src/text_recognizer/networks/util.py +++ b/src/text_recognizer/networks/util.py @@ -24,7 +24,7 @@ def sliding_window( """ unfold = nn.Unfold(kernel_size=patch_size, stride=stride) - # Preform the slidning window, unsqueeze as the channel dimesion is lost. + # Preform the sliding window, unsqueeze as the channel dimesion is lost. c = images.shape[1] patches = unfold(images) patches = rearrange( diff --git a/src/text_recognizer/networks/vision_transformer.py b/src/text_recognizer/networks/vision_transformer.py deleted file mode 100644 index f227954..0000000 --- a/src/text_recognizer/networks/vision_transformer.py +++ /dev/null @@ -1,159 +0,0 @@ -"""VisionTransformer module. - -Splits each image into patches and feeds them to a transformer. - -""" - -from typing import Dict, Optional, Tuple, Type - -from einops import rearrange, reduce -from einops.layers.torch import Rearrange -from loguru import logger -import torch -from torch import nn -from torch import Tensor - -from text_recognizer.networks.transformer import PositionalEncoding, Transformer -from text_recognizer.networks.util import configure_backbone - - -class VisionTransformer(nn.Module): - """Linear projection+Transfomer for image to sequence prediction, sort of based on the ideas from ViT.""" - - def __init__( - self, - num_encoder_layers: int, - num_decoder_layers: int, - hidden_dim: int, - vocab_size: int, - num_heads: int, - max_len: int, - expansion_dim: int, - dropout_rate: float, - trg_pad_index: int, - mlp_dim: Optional[int] = None, - patch_size: Tuple[int, int] = (28, 28), - stride: Tuple[int, int] = (1, 14), - activation: str = "gelu", - backbone: Optional[str] = None, - backbone_args: Optional[Dict] = None, - ) -> None: - super().__init__() - - self.patch_size = patch_size - self.stride = stride - self.trg_pad_index = trg_pad_index - self.slidning_window = self._configure_sliding_window() - self.character_embedding = nn.Embedding(vocab_size, hidden_dim) - self.position_encoding = PositionalEncoding(hidden_dim, dropout_rate, max_len) - self.mlp_dim = mlp_dim - - self.use_backbone = False - if backbone is None: - self.linear_projection = nn.Linear( - self.patch_size[0] * self.patch_size[1], hidden_dim - ) - else: - self.backbone = configure_backbone(backbone, backbone_args) - if mlp_dim: - self.mlp = nn.Linear(mlp_dim, hidden_dim) - self.use_backbone = True - - self.transformer = Transformer( - num_encoder_layers, - num_decoder_layers, - hidden_dim, - num_heads, - expansion_dim, - dropout_rate, - activation, - ) - - self.head = nn.Sequential(nn.Linear(hidden_dim, vocab_size),) - - def _configure_sliding_window(self) -> nn.Sequential: - return nn.Sequential( - nn.Unfold(kernel_size=self.patch_size, stride=self.stride), - Rearrange( - "b (c h w) t -> b t c h w", - h=self.patch_size[0], - w=self.patch_size[1], - c=1, - ), - ) - - def _create_trg_mask(self, trg: Tensor) -> Tensor: - # Move this outside the transformer. - trg_pad_mask = (trg != self.trg_pad_index)[:, None, None] - trg_len = trg.shape[1] - trg_sub_mask = torch.tril( - torch.ones((trg_len, trg_len), device=trg.device) - ).bool() - trg_mask = trg_pad_mask & trg_sub_mask - return trg_mask - - def encoder(self, src: Tensor) -> Tensor: - """Forward pass with the encoder of the transformer.""" - return self.transformer.encoder(src) - - def decoder(self, trg: Tensor, memory: Tensor, trg_mask: Tensor) -> Tensor: - """Forward pass with the decoder of the transformer + classification head.""" - return self.head( - self.transformer.decoder(trg=trg, memory=memory, trg_mask=trg_mask) - ) - - def _backbone(self, x: Tensor) -> Tensor: - b, t = x.shape[:2] - if self.use_backbone: - x = rearrange(x, "b t c h w -> (b t) c h w", b=b, t=t) - x = self.backbone(x) - if self.mlp_dim: - x = rearrange(x, "(b t) c h w -> b t (c h w)", b=b, t=t) - x = self.mlp(x) - else: - x = rearrange(x, "(b t) h -> b t h", b=b, t=t) - else: - x = rearrange(x, "b t c h w -> b t (c h w)", b=b, t=t) - x = self.linear_projection(x) - return x - - def preprocess_input(self, src: Tensor) -> Tensor: - """Encodes src with a backbone network and a positional encoding. - - Args: - src (Tensor): Input tensor. - - Returns: - Tensor: A input src to the transformer. - - """ - # If batch dimenstion is missing, it needs to be added. - if len(src.shape) < 4: - src = src[(None,) * (4 - len(src.shape))] - src = self.slidning_window(src) # .squeeze(-2) - src = self._backbone(src) - src = self.position_encoding(src) - return src - - def preprocess_target(self, trg: Tensor) -> Tuple[Tensor, Tensor]: - """Encodes target tensor with embedding and postion. - - Args: - trg (Tensor): Target tensor. - - Returns: - Tuple[Tensor, Tensor]: Encoded target tensor and target mask. - - """ - trg_mask = self._create_trg_mask(trg) - trg = self.character_embedding(trg.long()) - trg = self.position_encoding(trg) - return trg, trg_mask - - def forward(self, x: Tensor, trg: Tensor) -> Tensor: - """Forward pass with vision transfomer.""" - src = self.preprocess_input(x) - trg, trg_mask = self.preprocess_target(trg) - out = self.transformer(src, trg, trg_mask=trg_mask) - logits = self.head(out) - return logits diff --git a/src/text_recognizer/networks/wide_resnet.py b/src/text_recognizer/networks/wide_resnet.py index aa79c12..28f3380 100644 --- a/src/text_recognizer/networks/wide_resnet.py +++ b/src/text_recognizer/networks/wide_resnet.py @@ -2,7 +2,7 @@ from functools import partial from typing import Callable, Dict, List, Optional, Type, Union -from einops.layers.torch import Rearrange, Reduce +from einops.layers.torch import Reduce import numpy as np import torch from torch import nn diff --git a/src/text_recognizer/weights/CharacterModel_EmnistDataset_DenseNet_weights.pt b/src/text_recognizer/weights/CharacterModel_EmnistDataset_DenseNet_weights.pt index 4c28b51..f2dfd84 100644 --- a/src/text_recognizer/weights/CharacterModel_EmnistDataset_DenseNet_weights.pt +++ b/src/text_recognizer/weights/CharacterModel_EmnistDataset_DenseNet_weights.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:2a7fe47aec144ea79e50a145e2fbc29e0174092c9e702c0cd0c6ef105ca99760 -size 1273881 +oid sha256:8a69e5efedea70c4c5cb8ccdcc8cd480400f6c73e3313423f4dbbfe615644f0a +size 4500617 diff --git a/src/text_recognizer/weights/CharacterModel_EmnistDataset_WideResidualNetwork_weights.pt b/src/text_recognizer/weights/CharacterModel_EmnistDataset_WideResidualNetwork_weights.pt index cd3e37a..e1add8d 100644 --- a/src/text_recognizer/weights/CharacterModel_EmnistDataset_WideResidualNetwork_weights.pt +++ b/src/text_recognizer/weights/CharacterModel_EmnistDataset_WideResidualNetwork_weights.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:4d54658ffc40698149d0db1f9c0b4e446809f6dc237d9a384c54cb2e69219af2 -size 14953410 +oid sha256:68dd5c98eedc8753546f88b4e6fd5fc38725dc0079b837c30fb3d48069ec412b +size 15002754 diff --git a/src/training/run_experiment.py b/src/training/run_experiment.py index 0510d5c..e6ae84c 100644 --- a/src/training/run_experiment.py +++ b/src/training/run_experiment.py @@ -9,7 +9,6 @@ import re from typing import Callable, Dict, List, Optional, Tuple, Type import warnings -import adabelief_pytorch import click from loguru import logger import numpy as np @@ -17,19 +16,17 @@ import torch from torchsummary import summary from tqdm import tqdm from training.gpu_manager import GPUManager -from training.trainer.callbacks import Callback, CallbackList +from training.trainer.callbacks import CallbackList from training.trainer.train import Trainer import wandb import yaml from text_recognizer.models import Model -from text_recognizer.networks import loss as custom_loss_module +from text_recognizer.networks.loss import loss as custom_loss_module EXPERIMENTS_DIRNAME = Path(__file__).parents[0].resolve() / "experiments" -DEFAULT_TRAIN_ARGS = {"batch_size": 64, "epochs": 16} - def _get_level(verbose: int) -> int: """Sets the logger level.""" @@ -107,11 +104,7 @@ def _load_modules_and_arguments(experiment_config: Dict,) -> Tuple[Callable, Dic criterion_args = experiment_config["criterion"].get("args", {}) or {} # Optimizers - if experiment_config["optimizer"]["type"] == "AdaBelief": - warnings.filterwarnings("ignore", category=UserWarning) - optimizer_ = getattr(adabelief_pytorch, experiment_config["optimizer"]["type"]) - else: - optimizer_ = getattr(torch.optim, experiment_config["optimizer"]["type"]) + optimizer_ = getattr(torch.optim, experiment_config["optimizer"]["type"]) optimizer_args = experiment_config["optimizer"].get("args", {}) # Learning rate scheduler @@ -277,11 +270,6 @@ def run_experiment( # Lets W&B save the model and track the gradients and optional parameters. wandb.watch(model.network) - experiment_config["train_args"] = { - **DEFAULT_TRAIN_ARGS, - **experiment_config.get("train_args", {}), - } - experiment_config["experiment_group"] = experiment_config.get( "experiment_group", None ) @@ -351,7 +339,7 @@ def run_experiment( "--pretrained_weights", type=str, help="Path to pretrained model weights." ) @click.option( - "--notrain", is_flag=False, is_eager=True, help="Do not train the model.", + "--notrain", is_flag=False, help="Do not train the model.", ) def run_cli( experiment_config: str, |