diff options
| author | Gustaf Rydholm <gustaf.rydholm@gmail.com> | 2021-09-30 23:10:59 +0200 |
|---|---|---|
| committer | Gustaf Rydholm <gustaf.rydholm@gmail.com> | 2021-09-30 23:10:59 +0200 |
| commit | 7b8705f382b1642cf171cf7fcd01295104b9deef (patch) | |
| tree | 20be695e258e53c21dbbfaa677e69afbcddee019 /notebooks | |
| parent | 9ee84b0557d1348211a2267e649db392e640dad0 (diff) | |
Delete legacy notebooks
Diffstat (limited to 'notebooks')
| -rw-r--r-- | notebooks/00-scratch-pad.ipynb | 1669 | ||||
| -rw-r--r-- | notebooks/05a-test-end-to-end-model.ipynb | 80 | ||||
| -rw-r--r-- | notebooks/05c-test-model-end-to-end.ipynb | 898 | ||||
| -rw-r--r-- | notebooks/06-try-transformer-model-predictions.ipynb | 358 |
4 files changed, 0 insertions, 3005 deletions
diff --git a/notebooks/00-scratch-pad.ipynb b/notebooks/00-scratch-pad.ipynb deleted file mode 100644 index d0f4215..0000000 --- a/notebooks/00-scratch-pad.ipynb +++ /dev/null @@ -1,1669 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "%load_ext autoreload\n", - "%autoreload 2\n", - "\n", - "%matplotlib inline\n", - "import matplotlib.pyplot as plt\n", - "import numpy as np\n", - "from PIL import Image\n", - "import torch.nn.functional as F\n", - "import torch\n", - "from torch import nn\n", - "from torchsummary import summary\n", - "from importlib.util import find_spec\n", - "if find_spec(\"text_recognizer\") is None:\n", - " import sys\n", - " sys.path.append('..')\n", - "\n", - "from text_recognizer.networks.transformer.vit import ViT\n", - "from text_recognizer.networks.transformer.transformer import Transformer\n", - "from text_recognizer.networks.transformer.layers import Decoder" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": {}, - "outputs": [], - "source": [ - "t = torch.randint(0, 5, (4, 4))" - ] - }, - { - "cell_type": "code", - "execution_count": 19, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "36" - ] - }, - "execution_count": 19, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "576 // 16" - ] - }, - { - "cell_type": "code", - "execution_count": 22, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "40" - ] - }, - "execution_count": 22, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "640 // 16" - ] - }, - { - "cell_type": "code", - "execution_count": 24, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "1440" - ] - }, - "execution_count": 24, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "36 * 40" - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "tensor([[0, 1, 2, 1],\n", - " [1, 2, 3, 3],\n", - " [2, 2, 3, 3],\n", - " [4, 0, 2, 4]])" - ] - }, - "execution_count": 16, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "t" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [], - "source": [ - "t = torch.randint(0, 5, (1, 4, 4, 4))" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "tensor([[[[2, 3, 3, 3],\n", - " [3, 4, 4, 2],\n", - " [2, 3, 0, 0],\n", - " [4, 3, 4, 0]],\n", - "\n", - " [[3, 0, 3, 0],\n", - " [1, 4, 1, 3],\n", - " [2, 3, 3, 3],\n", - " [2, 3, 3, 1]],\n", - "\n", - " [[1, 1, 0, 3],\n", - " [1, 3, 0, 4],\n", - " [3, 1, 4, 2],\n", - " [3, 1, 4, 3]],\n", - "\n", - " [[3, 2, 3, 4],\n", - " [3, 2, 3, 3],\n", - " [0, 2, 2, 3],\n", - " [4, 0, 3, 4]]]])" - ] - }, - "execution_count": 12, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "t" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "torch.Size([1, 4, 16])" - ] - }, - "execution_count": 13, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "t.flatten(start_dim=2).shape" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "tensor([[[2, 3, 3, 3, 3, 4, 4, 2, 2, 3, 0, 0, 4, 3, 4, 0],\n", - " [3, 0, 3, 0, 1, 4, 1, 3, 2, 3, 3, 3, 2, 3, 3, 1],\n", - " [1, 1, 0, 3, 1, 3, 0, 4, 3, 1, 4, 2, 3, 1, 4, 3],\n", - " [3, 2, 3, 4, 3, 2, 3, 3, 0, 2, 2, 3, 4, 0, 3, 4]]])" - ] - }, - "execution_count": 14, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "t.flatten(start_dim=2)" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "ename": "TypeError", - "evalue": "__init__() got an unexpected keyword argument 'dim'", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mTypeError\u001b[0m Traceback (most recent call last)", - "\u001b[0;32m/tmp/ipykernel_6532/3641656095.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mflatten\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mFlatten\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdim\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", - "\u001b[0;31mTypeError\u001b[0m: __init__() got an unexpected keyword argument 'dim'" - ] - } - ], - "source": [ - "flatten = nn.Flatten(stdim=2)" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "True" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "torch.cuda.is_available()" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "loss = nn.CrossEntropyLoss()" - ] - }, - { - "cell_type": "code", - "execution_count": 51, - "metadata": {}, - "outputs": [], - "source": [ - "o = torch.randn((4, 5, 4))" - ] - }, - { - "cell_type": "code", - "execution_count": 52, - "metadata": {}, - "outputs": [], - "source": [ - "t = torch.randint(0, 5, (4, 4))" - ] - }, - { - "cell_type": "code", - "execution_count": 53, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "torch.Size([4, 5, 4])" - ] - }, - "execution_count": 53, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "o.shape" - ] - }, - { - "cell_type": "code", - "execution_count": 54, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "torch.Size([4, 4])" - ] - }, - "execution_count": 54, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "t.shape" - ] - }, - { - "cell_type": "code", - "execution_count": 55, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "tensor([[0, 1, 3, 2],\n", - " [1, 4, 4, 4],\n", - " [1, 4, 2, 1],\n", - " [2, 0, 4, 4]])" - ] - }, - "execution_count": 55, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "t" - ] - }, - { - "cell_type": "code", - "execution_count": 56, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "tensor([[[ 0.0647, -1.3831, 0.0266, 0.8528],\n", - " [ 1.4976, 0.4153, 1.0353, 0.0154],\n", - " [ 1.4562, -0.3568, 0.3599, -0.6222],\n", - " [ 0.2773, 0.4563, 0.9282, -2.1445],\n", - " [ 0.5191, 0.3683, -0.3469, 0.1355]],\n", - "\n", - " [[ 0.0424, -0.3215, 0.5662, -0.4217],\n", - " [ 2.0793, 1.2817, 0.1559, -0.6900],\n", - " [-1.1751, -0.3359, 1.7875, -0.3671],\n", - " [-0.4553, -0.3952, -0.8633, 0.1538],\n", - " [-1.3862, 0.4255, -2.2948, 0.0312]],\n", - "\n", - " [[-1.4257, 2.2662, 0.2670, -0.4330],\n", - " [-0.3244, -0.8669, -0.2571, 0.8028],\n", - " [ 0.9109, -0.2289, -1.2095, -0.9761],\n", - " [-0.0156, 1.2403, -1.1967, 0.6841],\n", - " [-0.8185, 0.2967, -2.1639, -0.7903]],\n", - "\n", - " [[-1.0425, 0.1426, 0.1383, 0.9784],\n", - " [-1.2853, 1.4123, -0.2272, -0.3335],\n", - " [ 1.5751, -0.7663, 0.9610, 0.5686],\n", - " [ 0.9697, -1.5515, -0.8658, -0.5882],\n", - " [-1.2467, 0.0539, 0.1208, -1.0297]]])" - ] - }, - "execution_count": 56, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "o" - ] - }, - { - "cell_type": "code", - "execution_count": 57, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "tensor(1.8355)" - ] - }, - "execution_count": 57, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "loss(o, t)" - ] - }, - { - "cell_type": "code", - "execution_count": 63, - "metadata": {}, - "outputs": [ - { - "ename": "TypeError", - "evalue": "unsupported operand type(s) for |: 'int' and 'Tensor'", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mTypeError\u001b[0m Traceback (most recent call last)", - "\u001b[0;32m/tmp/ipykernel_9275/1867668791.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mt\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m2\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m2\u001b[0m \u001b[0;34m|\u001b[0m \u001b[0mt\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m2\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", - "\u001b[0;31mTypeError\u001b[0m: unsupported operand type(s) for |: 'int' and 'Tensor'" - ] - } - ], - "source": [ - "t[:, 2] == 2 | t[:, 2] == 1" - ] - }, - { - "cell_type": "code", - "execution_count": 60, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "torch.Size([4, 1])" - ] - }, - "execution_count": 60, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "torch.argmax(o, dim=-1)[:, -1:].shape" - ] - }, - { - "cell_type": "code", - "execution_count": 43, - "metadata": {}, - "outputs": [], - "source": [ - "class LabelSmoothingLossCanonical(nn.Module):\n", - " def __init__(self, smoothing=0.0, dim=-1):\n", - " super(LabelSmoothingLossCanonical, self).__init__()\n", - " self.confidence = 1.0 - smoothing\n", - " self.smoothing = smoothing\n", - " self.dim = dim\n", - "\n", - " def forward(self, pred, target):\n", - " pred = pred.log_softmax(dim=self.dim)\n", - " with torch.no_grad():\n", - " # true_dist = pred.data.clone()\n", - " true_dist = torch.zeros_like(pred)\n", - " print(true_dist.shape)\n", - " true_dist.scatter_(1, target.unsqueeze(1), self.confidence)\n", - " print(true_dist.shape)\n", - " print(true_dist)\n", - " true_dist.masked_fill_((target == 4).unsqueeze(1), 0)\n", - " print(true_dist)\n", - " true_dist += self.smoothing / pred.size(self.dim)\n", - " return torch.mean(torch.sum(-true_dist * pred, dim=self.dim))" - ] - }, - { - "cell_type": "code", - "execution_count": 44, - "metadata": {}, - "outputs": [], - "source": [ - "l = LabelSmoothingLossCanonical(0.1)" - ] - }, - { - "cell_type": "code", - "execution_count": 45, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "torch.Size([1, 5, 4])\n", - "torch.Size([1, 5, 4])\n", - "tensor([[[0.0000, 0.0000, 0.0000, 0.0000],\n", - " [0.0000, 0.0000, 0.0000, 0.0000],\n", - " [0.9000, 0.9000, 0.0000, 0.9000],\n", - " [0.0000, 0.0000, 0.0000, 0.0000],\n", - " [0.0000, 0.0000, 0.9000, 0.0000]]])\n", - "tensor([[[0.0000, 0.0000, 0.0000, 0.0000],\n", - " [0.0000, 0.0000, 0.0000, 0.0000],\n", - " [0.9000, 0.9000, 0.0000, 0.9000],\n", - " [0.0000, 0.0000, 0.0000, 0.0000],\n", - " [0.0000, 0.0000, 0.0000, 0.0000]]])\n" - ] - }, - { - "data": { - "text/plain": [ - "tensor(0.9438)" - ] - }, - "execution_count": 45, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "l(o, t)" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [], - "source": [ - "from text_recognizer.networks.encoders.efficientnet.efficientnet import EfficientNet" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [], - "source": [ - "en = EfficientNet(\"b0\")" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [], - "source": [ - "def generate_square_subsequent_mask(size: int) -> torch.Tensor:\n", - " \"\"\"Generate a triangular (size, size) mask.\"\"\"\n", - " mask = (torch.triu(torch.ones(size, size)) == 1).transpose(0, 1)\n", - " mask = mask.float().masked_fill(mask == 0, float(\"-inf\")).masked_fill(mask == 1, float(0.0))\n", - " return mask" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "tensor([[0., -inf, -inf, -inf],\n", - " [0., 0., -inf, -inf],\n", - " [0., 0., 0., -inf],\n", - " [0., 0., 0., 0.]])" - ] - }, - "execution_count": 10, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "generate_square_subsequent_mask(4)" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [], - "source": [ - "from torch import Tensor" - ] - }, - { - "cell_type": "code", - "execution_count": 52, - "metadata": {}, - "outputs": [], - "source": [ - "tgt = torch.randint(0, 4, (1, 4))\n", - "tgt_mask = torch.ones_like(tgt).bool()" - ] - }, - { - "cell_type": "code", - "execution_count": 53, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "tensor([[True, True, True, True]])" - ] - }, - "execution_count": 53, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "tgt_mask" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [], - "source": [ - "def target_padding_mask(trg: Tensor, pad_index: int) -> Tensor:\n", - " \"\"\"Returns causal target mask.\"\"\"\n", - " trg_pad_mask = (trg != pad_index)[:, None, None]\n", - " trg_len = trg.shape[1]\n", - " trg_sub_mask = torch.tril(torch.ones((trg_len, trg_len), device=trg.device)).bool()\n", - " trg_mask = trg_pad_mask & trg_sub_mask\n", - " return trg_mask" - ] - }, - { - "cell_type": "code", - "execution_count": 54, - "metadata": {}, - "outputs": [], - "source": [ - "t = torch.randint(0, 6, (0, 4))" - ] - }, - { - "cell_type": "code", - "execution_count": 55, - "metadata": {}, - "outputs": [], - "source": [ - "t = torch.Tensor([[0, 0, 0, 3, 3, 3]])" - ] - }, - { - "cell_type": "code", - "execution_count": 58, - "metadata": {}, - "outputs": [], - "source": [ - "tt = t != 3" - ] - }, - { - "cell_type": "code", - "execution_count": 59, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "tensor([[ True, True, True, False, False, False]])" - ] - }, - "execution_count": 59, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "tt" - ] - }, - { - "cell_type": "code", - "execution_count": 43, - "metadata": {}, - "outputs": [], - "source": [ - "t = torch.cat((t, t))" - ] - }, - { - "cell_type": "code", - "execution_count": 44, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "torch.Size([2, 6])" - ] - }, - "execution_count": 44, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "t.shape" - ] - }, - { - "cell_type": "code", - "execution_count": 45, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "tensor([[[[ True, False, False, False, False, False],\n", - " [ True, True, False, False, False, False],\n", - " [ True, True, True, False, False, False],\n", - " [ True, True, True, False, False, False],\n", - " [ True, True, True, False, False, False],\n", - " [ True, True, True, False, False, False]]],\n", - "\n", - "\n", - " [[[ True, False, False, False, False, False],\n", - " [ True, True, False, False, False, False],\n", - " [ True, True, True, False, False, False],\n", - " [ True, True, True, False, False, False],\n", - " [ True, True, True, False, False, False],\n", - " [ True, True, True, False, False, False]]]])" - ] - }, - "execution_count": 45, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "target_padding_mask(t, 3)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "target_padding_mask()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "summary(en, (1, 224, 224));" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "torch.cuda.is_available()" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [ - { - "ename": "TypeError", - "evalue": "__init__() missing 4 required positional arguments: 'attn_fn', 'norm_fn', 'ff_fn', and 'rotary_emb'", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mTypeError\u001b[0m Traceback (most recent call last)", - "\u001b[0;32m/tmp/ipykernel_13932/689714588.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mdecoder\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mDecoder\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdim\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m128\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdepth\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnum_heads\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m8\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mff_kwargs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m{\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mattn_kwargs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m{\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcross_attend\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", - "\u001b[0;32m~/projects/text-recognizer/text_recognizer/networks/transformer/layers.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, **kwargs)\u001b[0m\n\u001b[1;32m 104\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m__init__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mAny\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 105\u001b[0m \u001b[0;32massert\u001b[0m \u001b[0;34m\"causal\"\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"Cannot set causality on decoder\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 106\u001b[0;31m \u001b[0msuper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__init__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcausal\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", - "\u001b[0;31mTypeError\u001b[0m: __init__() missing 4 required positional arguments: 'attn_fn', 'norm_fn', 'ff_fn', and 'rotary_emb'" - ] - } - ], - "source": [ - "decoder = Decoder(dim=128, depth=2, num_heads=8, ff_kwargs={}, attn_kwargs={}, cross_attend=True)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "decoder.cuda()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "transformer_decoder = Transformer(num_tokens=1003, max_seq_len=451, attn_layers=decoder, emb_dim=128, emb_dropout=0.1)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "transformer_decoder.cuda()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "efficient_transformer = Nystromer(\n", - " dim = 64,\n", - " depth = 4,\n", - " num_heads = 8,\n", - " num_landmarks = 64\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "v = ViT(\n", - " dim = 64,\n", - " image_size = (576, 640),\n", - " patch_size = (32, 32),\n", - " transformer = efficient_transformer\n", - ").cuda()" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [], - "source": [ - "t = torch.randn(4, 1, 576, 640).cuda()" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "EfficientNet(\n", - " (_conv_stem): Sequential(\n", - " (0): ZeroPad2d(padding=(0, 1, 0, 1), value=0.0)\n", - " (1): Conv2d(1, 32, kernel_size=(3, 3), stride=(2, 2), bias=False)\n", - " (2): BatchNorm2d(32, eps=0.001, momentum=0.99, affine=True, track_running_stats=True)\n", - " (3): Mish(inplace=True)\n", - " )\n", - " (_blocks): ModuleList(\n", - " (0): MBConvBlock(\n", - " (_depthwise): Sequential(\n", - " (0): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), groups=32, bias=False)\n", - " (1): BatchNorm2d(32, eps=0.001, momentum=0.99, affine=True, track_running_stats=True)\n", - " (2): Mish(inplace=True)\n", - " )\n", - " (_squeeze_excite): Sequential(\n", - " (0): Conv2d(32, 8, kernel_size=(1, 1), stride=(1, 1))\n", - " (1): Mish(inplace=True)\n", - " (2): Conv2d(8, 32, kernel_size=(1, 1), stride=(1, 1))\n", - " )\n", - " (_pointwise): Sequential(\n", - " (0): Conv2d(32, 16, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", - " (1): BatchNorm2d(16, eps=0.001, momentum=0.99, affine=True, track_running_stats=True)\n", - " )\n", - " )\n", - " (1): MBConvBlock(\n", - " (_inverted_bottleneck): Sequential(\n", - " (0): Conv2d(16, 96, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", - " (1): BatchNorm2d(96, eps=0.001, momentum=0.99, affine=True, track_running_stats=True)\n", - " (2): Mish(inplace=True)\n", - " )\n", - " (_depthwise): Sequential(\n", - " (0): Conv2d(96, 96, kernel_size=(3, 3), stride=(2, 2), groups=96, bias=False)\n", - " (1): BatchNorm2d(96, eps=0.001, momentum=0.99, affine=True, track_running_stats=True)\n", - " (2): Mish(inplace=True)\n", - " )\n", - " (_squeeze_excite): Sequential(\n", - " (0): Conv2d(96, 24, kernel_size=(1, 1), stride=(1, 1))\n", - " (1): Mish(inplace=True)\n", - " (2): Conv2d(24, 96, kernel_size=(1, 1), stride=(1, 1))\n", - " )\n", - " (_pointwise): Sequential(\n", - " (0): Conv2d(96, 24, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", - " (1): BatchNorm2d(24, eps=0.001, momentum=0.99, affine=True, track_running_stats=True)\n", - " )\n", - " )\n", - " (2): MBConvBlock(\n", - " (_inverted_bottleneck): Sequential(\n", - " (0): Conv2d(24, 144, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", - " (1): BatchNorm2d(144, eps=0.001, momentum=0.99, affine=True, track_running_stats=True)\n", - " (2): Mish(inplace=True)\n", - " )\n", - " (_depthwise): Sequential(\n", - " (0): Conv2d(144, 144, kernel_size=(3, 3), stride=(1, 1), groups=144, bias=False)\n", - " (1): BatchNorm2d(144, eps=0.001, momentum=0.99, affine=True, track_running_stats=True)\n", - " (2): Mish(inplace=True)\n", - " )\n", - " (_squeeze_excite): Sequential(\n", - " (0): Conv2d(144, 36, kernel_size=(1, 1), stride=(1, 1))\n", - " (1): Mish(inplace=True)\n", - " (2): Conv2d(36, 144, kernel_size=(1, 1), stride=(1, 1))\n", - " )\n", - " (_pointwise): Sequential(\n", - " (0): Conv2d(144, 24, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", - " (1): BatchNorm2d(24, eps=0.001, momentum=0.99, affine=True, track_running_stats=True)\n", - " )\n", - " )\n", - " (3): MBConvBlock(\n", - " (_inverted_bottleneck): Sequential(\n", - " (0): Conv2d(24, 144, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", - " (1): BatchNorm2d(144, eps=0.001, momentum=0.99, affine=True, track_running_stats=True)\n", - " (2): Mish(inplace=True)\n", - " )\n", - " (_depthwise): Sequential(\n", - " (0): Conv2d(144, 144, kernel_size=(5, 5), stride=(2, 2), groups=144, bias=False)\n", - " (1): BatchNorm2d(144, eps=0.001, momentum=0.99, affine=True, track_running_stats=True)\n", - " (2): Mish(inplace=True)\n", - " )\n", - " (_squeeze_excite): Sequential(\n", - " (0): Conv2d(144, 36, kernel_size=(1, 1), stride=(1, 1))\n", - " (1): Mish(inplace=True)\n", - " (2): Conv2d(36, 144, kernel_size=(1, 1), stride=(1, 1))\n", - " )\n", - " (_pointwise): Sequential(\n", - " (0): Conv2d(144, 40, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", - " (1): BatchNorm2d(40, eps=0.001, momentum=0.99, affine=True, track_running_stats=True)\n", - " )\n", - " )\n", - " (4): MBConvBlock(\n", - " (_inverted_bottleneck): Sequential(\n", - " (0): Conv2d(40, 240, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", - " (1): BatchNorm2d(240, eps=0.001, momentum=0.99, affine=True, track_running_stats=True)\n", - " (2): Mish(inplace=True)\n", - " )\n", - " (_depthwise): Sequential(\n", - " (0): Conv2d(240, 240, kernel_size=(5, 5), stride=(1, 1), groups=240, bias=False)\n", - " (1): BatchNorm2d(240, eps=0.001, momentum=0.99, affine=True, track_running_stats=True)\n", - " (2): Mish(inplace=True)\n", - " )\n", - " (_squeeze_excite): Sequential(\n", - " (0): Conv2d(240, 60, kernel_size=(1, 1), stride=(1, 1))\n", - " (1): Mish(inplace=True)\n", - " (2): Conv2d(60, 240, kernel_size=(1, 1), stride=(1, 1))\n", - " )\n", - " (_pointwise): Sequential(\n", - " (0): Conv2d(240, 40, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", - " (1): BatchNorm2d(40, eps=0.001, momentum=0.99, affine=True, track_running_stats=True)\n", - " )\n", - " )\n", - " (5): MBConvBlock(\n", - " (_inverted_bottleneck): Sequential(\n", - " (0): Conv2d(40, 240, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", - " (1): BatchNorm2d(240, eps=0.001, momentum=0.99, affine=True, track_running_stats=True)\n", - " (2): Mish(inplace=True)\n", - " )\n", - " (_depthwise): Sequential(\n", - " (0): Conv2d(240, 240, kernel_size=(3, 3), stride=(2, 2), groups=240, bias=False)\n", - " (1): BatchNorm2d(240, eps=0.001, momentum=0.99, affine=True, track_running_stats=True)\n", - " (2): Mish(inplace=True)\n", - " )\n", - " (_squeeze_excite): Sequential(\n", - " (0): Conv2d(240, 60, kernel_size=(1, 1), stride=(1, 1))\n", - " (1): Mish(inplace=True)\n", - " (2): Conv2d(60, 240, kernel_size=(1, 1), stride=(1, 1))\n", - " )\n", - " (_pointwise): Sequential(\n", - " (0): Conv2d(240, 80, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", - " (1): BatchNorm2d(80, eps=0.001, momentum=0.99, affine=True, track_running_stats=True)\n", - " )\n", - " )\n", - " (6): MBConvBlock(\n", - " (_inverted_bottleneck): Sequential(\n", - " (0): Conv2d(80, 480, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", - " (1): BatchNorm2d(480, eps=0.001, momentum=0.99, affine=True, track_running_stats=True)\n", - " (2): Mish(inplace=True)\n", - " )\n", - " (_depthwise): Sequential(\n", - " (0): Conv2d(480, 480, kernel_size=(3, 3), stride=(1, 1), groups=480, bias=False)\n", - " (1): BatchNorm2d(480, eps=0.001, momentum=0.99, affine=True, track_running_stats=True)\n", - " (2): Mish(inplace=True)\n", - " )\n", - " (_squeeze_excite): Sequential(\n", - " (0): Conv2d(480, 120, kernel_size=(1, 1), stride=(1, 1))\n", - " (1): Mish(inplace=True)\n", - " (2): Conv2d(120, 480, kernel_size=(1, 1), stride=(1, 1))\n", - " )\n", - " (_pointwise): Sequential(\n", - " (0): Conv2d(480, 80, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", - " (1): BatchNorm2d(80, eps=0.001, momentum=0.99, affine=True, track_running_stats=True)\n", - " )\n", - " )\n", - " (7): MBConvBlock(\n", - " (_inverted_bottleneck): Sequential(\n", - " (0): Conv2d(80, 480, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", - " (1): BatchNorm2d(480, eps=0.001, momentum=0.99, affine=True, track_running_stats=True)\n", - " (2): Mish(inplace=True)\n", - " )\n", - " (_depthwise): Sequential(\n", - " (0): Conv2d(480, 480, kernel_size=(3, 3), stride=(1, 1), groups=480, bias=False)\n", - " (1): BatchNorm2d(480, eps=0.001, momentum=0.99, affine=True, track_running_stats=True)\n", - " (2): Mish(inplace=True)\n", - " )\n", - " (_squeeze_excite): Sequential(\n", - " (0): Conv2d(480, 120, kernel_size=(1, 1), stride=(1, 1))\n", - " (1): Mish(inplace=True)\n", - " (2): Conv2d(120, 480, kernel_size=(1, 1), stride=(1, 1))\n", - " )\n", - " (_pointwise): Sequential(\n", - " (0): Conv2d(480, 80, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", - " (1): BatchNorm2d(80, eps=0.001, momentum=0.99, affine=True, track_running_stats=True)\n", - " )\n", - " )\n", - " (8): MBConvBlock(\n", - " (_inverted_bottleneck): Sequential(\n", - " (0): Conv2d(80, 480, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", - " (1): BatchNorm2d(480, eps=0.001, momentum=0.99, affine=True, track_running_stats=True)\n", - " (2): Mish(inplace=True)\n", - " )\n", - " (_depthwise): Sequential(\n", - " (0): Conv2d(480, 480, kernel_size=(5, 5), stride=(1, 1), groups=480, bias=False)\n", - " (1): BatchNorm2d(480, eps=0.001, momentum=0.99, affine=True, track_running_stats=True)\n", - " (2): Mish(inplace=True)\n", - " )\n", - " (_squeeze_excite): Sequential(\n", - " (0): Conv2d(480, 120, kernel_size=(1, 1), stride=(1, 1))\n", - " (1): Mish(inplace=True)\n", - " (2): Conv2d(120, 480, kernel_size=(1, 1), stride=(1, 1))\n", - " )\n", - " (_pointwise): Sequential(\n", - " (0): Conv2d(480, 112, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", - " (1): BatchNorm2d(112, eps=0.001, momentum=0.99, affine=True, track_running_stats=True)\n", - " )\n", - " )\n", - " (9): MBConvBlock(\n", - " (_inverted_bottleneck): Sequential(\n", - " (0): Conv2d(112, 672, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", - " (1): BatchNorm2d(672, eps=0.001, momentum=0.99, affine=True, track_running_stats=True)\n", - " (2): Mish(inplace=True)\n", - " )\n", - " (_depthwise): Sequential(\n", - " (0): Conv2d(672, 672, kernel_size=(5, 5), stride=(1, 1), groups=672, bias=False)\n", - " (1): BatchNorm2d(672, eps=0.001, momentum=0.99, affine=True, track_running_stats=True)\n", - " (2): Mish(inplace=True)\n", - " )\n", - " (_squeeze_excite): Sequential(\n", - " (0): Conv2d(672, 168, kernel_size=(1, 1), stride=(1, 1))\n", - " (1): Mish(inplace=True)\n", - " (2): Conv2d(168, 672, kernel_size=(1, 1), stride=(1, 1))\n", - " )\n", - " (_pointwise): Sequential(\n", - " (0): Conv2d(672, 112, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", - " (1): BatchNorm2d(112, eps=0.001, momentum=0.99, affine=True, track_running_stats=True)\n", - " )\n", - " )\n", - " (10): MBConvBlock(\n", - " (_inverted_bottleneck): Sequential(\n", - " (0): Conv2d(112, 672, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", - " (1): BatchNorm2d(672, eps=0.001, momentum=0.99, affine=True, track_running_stats=True)\n", - " (2): Mish(inplace=True)\n", - " )\n", - " (_depthwise): Sequential(\n", - " (0): Conv2d(672, 672, kernel_size=(5, 5), stride=(1, 1), groups=672, bias=False)\n", - " (1): BatchNorm2d(672, eps=0.001, momentum=0.99, affine=True, track_running_stats=True)\n", - " (2): Mish(inplace=True)\n", - " )\n", - " (_squeeze_excite): Sequential(\n", - " (0): Conv2d(672, 168, kernel_size=(1, 1), stride=(1, 1))\n", - " (1): Mish(inplace=True)\n", - " (2): Conv2d(168, 672, kernel_size=(1, 1), stride=(1, 1))\n", - " )\n", - " (_pointwise): Sequential(\n", - " (0): Conv2d(672, 112, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", - " (1): BatchNorm2d(112, eps=0.001, momentum=0.99, affine=True, track_running_stats=True)\n", - " )\n", - " )\n", - " (11): MBConvBlock(\n", - " (_inverted_bottleneck): Sequential(\n", - " (0): Conv2d(112, 672, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", - " (1): BatchNorm2d(672, eps=0.001, momentum=0.99, affine=True, track_running_stats=True)\n", - " (2): Mish(inplace=True)\n", - " )\n", - " (_depthwise): Sequential(\n", - " (0): Conv2d(672, 672, kernel_size=(5, 5), stride=(2, 2), groups=672, bias=False)\n", - " (1): BatchNorm2d(672, eps=0.001, momentum=0.99, affine=True, track_running_stats=True)\n", - " (2): Mish(inplace=True)\n", - " )\n", - " (_squeeze_excite): Sequential(\n", - " (0): Conv2d(672, 168, kernel_size=(1, 1), stride=(1, 1))\n", - " (1): Mish(inplace=True)\n", - " (2): Conv2d(168, 672, kernel_size=(1, 1), stride=(1, 1))\n", - " )\n", - " (_pointwise): Sequential(\n", - " (0): Conv2d(672, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", - " (1): BatchNorm2d(192, eps=0.001, momentum=0.99, affine=True, track_running_stats=True)\n", - " )\n", - " )\n", - " (12): MBConvBlock(\n", - " (_inverted_bottleneck): Sequential(\n", - " (0): Conv2d(192, 1152, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", - " (1): BatchNorm2d(1152, eps=0.001, momentum=0.99, affine=True, track_running_stats=True)\n", - " (2): Mish(inplace=True)\n", - " )\n", - " (_depthwise): Sequential(\n", - " (0): Conv2d(1152, 1152, kernel_size=(5, 5), stride=(1, 1), groups=1152, bias=False)\n", - " (1): BatchNorm2d(1152, eps=0.001, momentum=0.99, affine=True, track_running_stats=True)\n", - " (2): Mish(inplace=True)\n", - " )\n", - " (_squeeze_excite): Sequential(\n", - " (0): Conv2d(1152, 288, kernel_size=(1, 1), stride=(1, 1))\n", - " (1): Mish(inplace=True)\n", - " (2): Conv2d(288, 1152, kernel_size=(1, 1), stride=(1, 1))\n", - " )\n", - " (_pointwise): Sequential(\n", - " (0): Conv2d(1152, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", - " (1): BatchNorm2d(192, eps=0.001, momentum=0.99, affine=True, track_running_stats=True)\n", - " )\n", - " )\n", - " (13): MBConvBlock(\n", - " (_inverted_bottleneck): Sequential(\n", - " (0): Conv2d(192, 1152, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", - " (1): BatchNorm2d(1152, eps=0.001, momentum=0.99, affine=True, track_running_stats=True)\n", - " (2): Mish(inplace=True)\n", - " )\n", - " (_depthwise): Sequential(\n", - " (0): Conv2d(1152, 1152, kernel_size=(5, 5), stride=(1, 1), groups=1152, bias=False)\n", - " (1): BatchNorm2d(1152, eps=0.001, momentum=0.99, affine=True, track_running_stats=True)\n", - " (2): Mish(inplace=True)\n", - " )\n", - " (_squeeze_excite): Sequential(\n", - " (0): Conv2d(1152, 288, kernel_size=(1, 1), stride=(1, 1))\n", - " (1): Mish(inplace=True)\n", - " (2): Conv2d(288, 1152, kernel_size=(1, 1), stride=(1, 1))\n", - " )\n", - " (_pointwise): Sequential(\n", - " (0): Conv2d(1152, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", - " (1): BatchNorm2d(192, eps=0.001, momentum=0.99, affine=True, track_running_stats=True)\n", - " )\n", - " )\n", - " (14): MBConvBlock(\n", - " (_inverted_bottleneck): Sequential(\n", - " (0): Conv2d(192, 1152, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", - " (1): BatchNorm2d(1152, eps=0.001, momentum=0.99, affine=True, track_running_stats=True)\n", - " (2): Mish(inplace=True)\n", - " )\n", - " (_depthwise): Sequential(\n", - " (0): Conv2d(1152, 1152, kernel_size=(5, 5), stride=(1, 1), groups=1152, bias=False)\n", - " (1): BatchNorm2d(1152, eps=0.001, momentum=0.99, affine=True, track_running_stats=True)\n", - " (2): Mish(inplace=True)\n", - " )\n", - " (_squeeze_excite): Sequential(\n", - " (0): Conv2d(1152, 288, kernel_size=(1, 1), stride=(1, 1))\n", - " (1): Mish(inplace=True)\n", - " (2): Conv2d(288, 1152, kernel_size=(1, 1), stride=(1, 1))\n", - " )\n", - " (_pointwise): Sequential(\n", - " (0): Conv2d(1152, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", - " (1): BatchNorm2d(192, eps=0.001, momentum=0.99, affine=True, track_running_stats=True)\n", - " )\n", - " )\n", - " (15): MBConvBlock(\n", - " (_inverted_bottleneck): Sequential(\n", - " (0): Conv2d(192, 1152, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", - " (1): BatchNorm2d(1152, eps=0.001, momentum=0.99, affine=True, track_running_stats=True)\n", - " (2): Mish(inplace=True)\n", - " )\n", - " (_depthwise): Sequential(\n", - " (0): Conv2d(1152, 1152, kernel_size=(3, 3), stride=(1, 1), groups=1152, bias=False)\n", - " (1): BatchNorm2d(1152, eps=0.001, momentum=0.99, affine=True, track_running_stats=True)\n", - " (2): Mish(inplace=True)\n", - " )\n", - " (_squeeze_excite): Sequential(\n", - " (0): Conv2d(1152, 288, kernel_size=(1, 1), stride=(1, 1))\n", - " (1): Mish(inplace=True)\n", - " (2): Conv2d(288, 1152, kernel_size=(1, 1), stride=(1, 1))\n", - " )\n", - " (_pointwise): Sequential(\n", - " (0): Conv2d(1152, 320, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", - " (1): BatchNorm2d(320, eps=0.001, momentum=0.99, affine=True, track_running_stats=True)\n", - " )\n", - " )\n", - " )\n", - " (_conv_head): Sequential(\n", - " (0): Conv2d(320, 1280, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", - " (1): BatchNorm2d(1280, eps=0.001, momentum=0.99, affine=True, track_running_stats=True)\n", - " )\n", - ")" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "en.cuda()" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "torch.Size([4, 1280, 18, 20])" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "en(t).shape" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "o = v(t)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "caption = torch.randint(0, 90, (16, 690)).cuda()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "o.shape" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "caption.shape" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "o = torch.randn(16, 20 * 18, 128).cuda()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "caption = torch.randint(0, 1000, (16, 200)).cuda()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "transformer_decoder(caption, context = o).shape # (1, 1024, 20000)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "efficient_transformer = efficient_transformer(num_landmarks=256)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "efficient_transformer()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from omegaconf import OmegaConf" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "path = \"../training/configs/vqvae.yaml\"" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "conf = OmegaConf.load(path)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "print(OmegaConf.to_yaml(conf))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from text_recognizer.networks import VQVAE" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "vae = VQVAE(**conf.network.args)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "vae" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "datum = torch.randn([2, 1, 576, 640])" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "vae.encoder(datum)[0].shape" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "vae(datum)[0].shape" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "datum = torch.randn([2, 1, 576, 640])" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "trg = torch.randint(0, 1000, [2, 682])" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "trg.shape" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "datum = torch.randn([2, 1, 224, 224])" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "en(t).shape" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "path = \"../training/configs/cnn_transformer.yaml\"" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "conf = OmegaConf.load(path)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "print(OmegaConf.to_yaml(conf))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from text_recognizer.networks.cnn_transformer import CNNTransformer" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "t = CNNTransformer(input_shape=(1, 576, 640), output_shape=(682, 1), **conf.network.args)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "t.encode(datum).shape" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "trg.shape" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "t(datum, trg).shape" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "b, n = 16, 128\n", - "device = \"cpu\"" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "x = lambda: torch.ones((b, n), device=device).bool()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "x().shape" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "torch.ones((b, n), device=device).bool().shape" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "x = torch.randn(1, 1, 576, 640)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "576 // 32" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "640 // 32" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "18 * 20" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "x = torch.randn(1, 1, 144, 160)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from einops import rearrange" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "patch_size=16\n", - "p = rearrange(x, 'b c (h p1) (w p2) -> b (h w) (p1 p2 c)', p1 = patch_size, p2 = patch_size)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "p.shape" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.7" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/notebooks/05a-test-end-to-end-model.ipynb b/notebooks/05a-test-end-to-end-model.ipynb deleted file mode 100644 index 7723b12..0000000 --- a/notebooks/05a-test-end-to-end-model.ipynb +++ /dev/null @@ -1,80 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "The autoreload extension is already loaded. To reload it, use:\n", - " %reload_ext autoreload\n" - ] - }, - { - "ename": "ImportError", - "evalue": "cannot import name 'ParagraphTextRecognizor' from 'text_recognizer' (/home/akternurra/Documents/projects/quest-for-general-artifical-intelligence/projects/text-recognizer/src/text_recognizer/__init__.py)", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mImportError\u001b[0m Traceback (most recent call last)", - "\u001b[0;32m<ipython-input-4-f0e40de01802>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m 16\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mtext_recognizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdatasets\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mIamDataset\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 17\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mtext_recognizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdatasets\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mIamParagraphsDataset\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 18\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0mtext_recognizer\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mParagraphTextRecognizor\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", - "\u001b[0;31mImportError\u001b[0m: cannot import name 'ParagraphTextRecognizor' from 'text_recognizer' (/home/akternurra/Documents/projects/quest-for-general-artifical-intelligence/projects/text-recognizer/src/text_recognizer/__init__.py)" - ] - } - ], - "source": [ - "%load_ext autoreload\n", - "%autoreload 2\n", - "\n", - "import cv2\n", - "%matplotlib inline\n", - "import matplotlib.pyplot as plt\n", - "import numpy as np\n", - "\n", - "from omegaconf import OmegaConf\n", - "\n", - "from importlib.util import find_spec\n", - "if find_spec(\"text_recognizer\") is None:\n", - " import sys\n", - " sys.path.append('..')\n", - "\n", - "from text_recognizer.datasets import IamDataset\n", - "from text_recognizer.datasets import IamParagraphsDataset\n", - "from text_recognizer import ParagraphTextRecognizor" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "ParagraphTextRecognizor" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.2" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/notebooks/05c-test-model-end-to-end.ipynb b/notebooks/05c-test-model-end-to-end.ipynb deleted file mode 100644 index 23361b6..0000000 --- a/notebooks/05c-test-model-end-to-end.ipynb +++ /dev/null @@ -1,898 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 32, - "id": "1e40a88b", - "metadata": {}, - "outputs": [], - "source": [ - "%load_ext autoreload\n", - "%autoreload 2\n", - "\n", - "%matplotlib inline\n", - "import matplotlib.pyplot as plt\n", - "import numpy as np\n", - "from PIL import Image\n", - "import torch\n", - "from torch import nn\n", - "from importlib.util import find_spec\n", - "if find_spec(\"text_recognizer\") is None:\n", - " import sys\n", - " sys.path.append('..')\n", - " " - ] - }, - { - "cell_type": "code", - "execution_count": 33, - "id": "38fb3d9d-a163-4b72-981f-f31b51be39f2", - "metadata": {}, - "outputs": [], - "source": [ - "from hydra import compose, initialize\n", - "from omegaconf import OmegaConf\n", - "from hydra.utils import instantiate" - ] - }, - { - "cell_type": "code", - "execution_count": 47, - "id": "74780b21-3313-452b-b580-703cac878416", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "callbacks:\n", - " model_checkpoint:\n", - " _target_: pytorch_lightning.callbacks.ModelCheckpoint\n", - " monitor: val/loss\n", - " save_top_k: 1\n", - " save_last: true\n", - " mode: min\n", - " verbose: false\n", - " dirpath: checkpoints/\n", - " filename: '{epoch:02d}'\n", - " learning_rate_monitor:\n", - " _target_: pytorch_lightning.callbacks.LearningRateMonitor\n", - " logging_interval: step\n", - " log_momentum: false\n", - " watch_model:\n", - " _target_: callbacks.wandb_callbacks.WatchModel\n", - " log: all\n", - " log_freq: 100\n", - " upload_ckpts_as_artifact:\n", - " _target_: callbacks.wandb_callbacks.UploadCheckpointsAsArtifact\n", - " ckpt_dir: checkpoints/\n", - " upload_best_only: true\n", - " log_image_reconstruction:\n", - " _target_: callbacks.wandb_callbacks.LogReconstuctedImages\n", - " num_samples: 8\n", - "criterion:\n", - " _target_: text_recognizer.criterions.vqgan_loss.VQGANLoss\n", - " reconstruction_loss:\n", - " _target_: torch.nn.L1Loss\n", - " reduction: mean\n", - " discriminator:\n", - " _target_: text_recognizer.criterions.n_layer_discriminator.NLayerDiscriminator\n", - " in_channels: 1\n", - " num_channels: 32\n", - " num_layers: 3\n", - " vq_loss_weight: 1.0\n", - " discriminator_weight: 1.0\n", - "datamodule:\n", - " _target_: text_recognizer.data.iam_extended_paragraphs.IAMExtendedParagraphs\n", - " batch_size: 8\n", - " num_workers: 12\n", - " train_fraction: 0.8\n", - " augment: true\n", - " pin_memory: false\n", - " word_pieces: true\n", - "logger:\n", - " wandb:\n", - " _target_: pytorch_lightning.loggers.wandb.WandbLogger\n", - " project: text-recognizer\n", - " name: null\n", - " save_dir: .\n", - " offline: false\n", - " id: null\n", - " log_model: false\n", - " prefix: ''\n", - " job_type: train\n", - " group: ''\n", - " tags: []\n", - "mapping:\n", - " _target_: text_recognizer.data.word_piece_mapping.WordPieceMapping\n", - " num_features: 1000\n", - " tokens: iamdb_1kwp_tokens_1000.txt\n", - " lexicon: iamdb_1kwp_lex_1000.txt\n", - " data_dir: null\n", - " use_words: false\n", - " prepend_wordsep: false\n", - " special_tokens:\n", - " - <s>\n", - " - <e>\n", - " - <p>\n", - " extra_symbols:\n", - " - '\n", - "\n", - " '\n", - "model:\n", - " _target_: text_recognizer.models.vqgan.VQGANLitModel\n", - "network:\n", - " encoder:\n", - " _target_: text_recognizer.networks.vqvae.encoder.Encoder\n", - " in_channels: 1\n", - " hidden_dim: 32\n", - " channels_multipliers:\n", - " - 1\n", - " - 2\n", - " - 4\n", - " - 8\n", - " - 8\n", - " dropout_rate: 0.25\n", - " decoder:\n", - " _target_: text_recognizer.networks.vqvae.decoder.Decoder\n", - " out_channels: 1\n", - " hidden_dim: 32\n", - " channels_multipliers:\n", - " - 8\n", - " - 8\n", - " - 4\n", - " - 1\n", - " dropout_rate: 0.25\n", - " _target_: text_recognizer.networks.vqvae.vqvae.VQVAE\n", - " hidden_dim: 256\n", - " embedding_dim: 32\n", - " num_embeddings: 1024\n", - " decay: 0.99\n", - "trainer:\n", - " _target_: pytorch_lightning.Trainer\n", - " stochastic_weight_avg: false\n", - " auto_scale_batch_size: binsearch\n", - " auto_lr_find: false\n", - " gradient_clip_val: 0\n", - " fast_dev_run: false\n", - " gpus: 1\n", - " precision: 16\n", - " max_epochs: 256\n", - " terminate_on_nan: true\n", - " weights_summary: top\n", - " limit_train_batches: 1.0\n", - " limit_val_batches: 1.0\n", - " limit_test_batches: 1.0\n", - " resume_from_checkpoint: null\n", - "seed: 4711\n", - "tune: false\n", - "train: true\n", - "test: true\n", - "logging: INFO\n", - "work_dir: ${hydra:runtime.cwd}\n", - "debug: false\n", - "print_config: false\n", - "ignore_warnings: true\n", - "summary: null\n", - "lr_schedulers:\n", - " generator:\n", - " _target_: torch.optim.lr_scheduler.CosineAnnealingLR\n", - " T_max: 256\n", - " eta_min: 0.0\n", - " last_epoch: -1\n", - " interval: epoch\n", - " monitor: val/loss\n", - " discriminator:\n", - " _target_: torch.optim.lr_scheduler.CosineAnnealingLR\n", - " T_max: 256\n", - " eta_min: 0.0\n", - " last_epoch: -1\n", - " interval: epoch\n", - " monitor: val/loss\n", - "optimizers:\n", - " generator:\n", - " _target_: madgrad.MADGRAD\n", - " lr: 0.001\n", - " momentum: 0.5\n", - " weight_decay: 0\n", - " eps: 1.0e-06\n", - " parameters: network\n", - " discriminator:\n", - " _target_: madgrad.MADGRAD\n", - " lr: 0.001\n", - " momentum: 0.5\n", - " weight_decay: 0\n", - " eps: 1.0e-06\n", - " parameters: loss_fn.discriminator\n", - "\n", - "{'callbacks': {'model_checkpoint': {'_target_': 'pytorch_lightning.callbacks.ModelCheckpoint', 'monitor': 'val/loss', 'save_top_k': 1, 'save_last': True, 'mode': 'min', 'verbose': False, 'dirpath': 'checkpoints/', 'filename': '{epoch:02d}'}, 'learning_rate_monitor': {'_target_': 'pytorch_lightning.callbacks.LearningRateMonitor', 'logging_interval': 'step', 'log_momentum': False}, 'watch_model': {'_target_': 'callbacks.wandb_callbacks.WatchModel', 'log': 'all', 'log_freq': 100}, 'upload_ckpts_as_artifact': {'_target_': 'callbacks.wandb_callbacks.UploadCheckpointsAsArtifact', 'ckpt_dir': 'checkpoints/', 'upload_best_only': True}, 'log_image_reconstruction': {'_target_': 'callbacks.wandb_callbacks.LogReconstuctedImages', 'num_samples': 8}}, 'criterion': {'_target_': 'text_recognizer.criterions.vqgan_loss.VQGANLoss', 'reconstruction_loss': {'_target_': 'torch.nn.L1Loss', 'reduction': 'mean'}, 'discriminator': {'_target_': 'text_recognizer.criterions.n_layer_discriminator.NLayerDiscriminator', 'in_channels': 1, 'num_channels': 32, 'num_layers': 3}, 'vq_loss_weight': 1.0, 'discriminator_weight': 1.0}, 'datamodule': {'_target_': 'text_recognizer.data.iam_extended_paragraphs.IAMExtendedParagraphs', 'batch_size': 8, 'num_workers': 12, 'train_fraction': 0.8, 'augment': True, 'pin_memory': False, 'word_pieces': True}, 'logger': {'wandb': {'_target_': 'pytorch_lightning.loggers.wandb.WandbLogger', 'project': 'text-recognizer', 'name': None, 'save_dir': '.', 'offline': False, 'id': None, 'log_model': False, 'prefix': '', 'job_type': 'train', 'group': '', 'tags': []}}, 'mapping': {'_target_': 'text_recognizer.data.word_piece_mapping.WordPieceMapping', 'num_features': 1000, 'tokens': 'iamdb_1kwp_tokens_1000.txt', 'lexicon': 'iamdb_1kwp_lex_1000.txt', 'data_dir': None, 'use_words': False, 'prepend_wordsep': False, 'special_tokens': ['<s>', '<e>', '<p>'], 'extra_symbols': ['\\n']}, 'model': {'_target_': 'text_recognizer.models.vqgan.VQGANLitModel'}, 'network': {'encoder': {'_target_': 'text_recognizer.networks.vqvae.encoder.Encoder', 'in_channels': 1, 'hidden_dim': 32, 'channels_multipliers': [1, 2, 4, 8, 8], 'dropout_rate': 0.25}, 'decoder': {'_target_': 'text_recognizer.networks.vqvae.decoder.Decoder', 'out_channels': 1, 'hidden_dim': 32, 'channels_multipliers': [8, 8, 4, 1], 'dropout_rate': 0.25}, '_target_': 'text_recognizer.networks.vqvae.vqvae.VQVAE', 'hidden_dim': 256, 'embedding_dim': 32, 'num_embeddings': 1024, 'decay': 0.99}, 'trainer': {'_target_': 'pytorch_lightning.Trainer', 'stochastic_weight_avg': False, 'auto_scale_batch_size': 'binsearch', 'auto_lr_find': False, 'gradient_clip_val': 0, 'fast_dev_run': False, 'gpus': 1, 'precision': 16, 'max_epochs': 256, 'terminate_on_nan': True, 'weights_summary': 'top', 'limit_train_batches': 1.0, 'limit_val_batches': 1.0, 'limit_test_batches': 1.0, 'resume_from_checkpoint': None}, 'seed': 4711, 'tune': False, 'train': True, 'test': True, 'logging': 'INFO', 'work_dir': '${hydra:runtime.cwd}', 'debug': False, 'print_config': False, 'ignore_warnings': True, 'summary': None, 'lr_schedulers': {'generator': {'_target_': 'torch.optim.lr_scheduler.CosineAnnealingLR', 'T_max': 256, 'eta_min': 0.0, 'last_epoch': -1, 'interval': 'epoch', 'monitor': 'val/loss'}, 'discriminator': {'_target_': 'torch.optim.lr_scheduler.CosineAnnealingLR', 'T_max': 256, 'eta_min': 0.0, 'last_epoch': -1, 'interval': 'epoch', 'monitor': 'val/loss'}}, 'optimizers': {'generator': {'_target_': 'madgrad.MADGRAD', 'lr': 0.001, 'momentum': 0.5, 'weight_decay': 0, 'eps': 1e-06, 'parameters': 'network'}, 'discriminator': {'_target_': 'madgrad.MADGRAD', 'lr': 0.001, 'momentum': 0.5, 'weight_decay': 0, 'eps': 1e-06, 'parameters': 'loss_fn.discriminator'}}}\n" - ] - } - ], - "source": [ - "# context initialization\n", - "with initialize(config_path=\"../training/conf/\", job_name=\"test_app\"):\n", - " cfg = compose(config_name=\"config\", overrides=[\"+experiment=vqgan\"])\n", - " print(OmegaConf.to_yaml(cfg))\n", - " print(cfg)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "205a03e8-7aa1-407f-afa5-92693715b677", - "metadata": {}, - "outputs": [], - "source": [ - "net = instantiate(cfg)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c74384f0-754e-4c29-8f06-339372d6e4c1", - "metadata": {}, - "outputs": [], - "source": [ - "from torchsummary import summary" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "5ebab599-2497-42f8-b54b-1663ee66fde9", - "metadata": {}, - "outputs": [], - "source": [ - "summary(net, (1, 576, 640), device=\"cpu\");" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "6ba3f405-5948-465d-a7b8-459c84345034", - "metadata": {}, - "outputs": [], - "source": [ - "net = net.cuda()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "5c998137-0967-488f-a572-a5f5a6b86353", - "metadata": {}, - "outputs": [], - "source": [ - "x = torch.randn(16, 1, 576, 640)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "920aeeb2-088c-4ea0-84a2-a2532d4f697a", - "metadata": {}, - "outputs": [], - "source": [ - "x = x.cuda()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "119ab631-fb3a-47a3-afc2-0e66260ebe7f", - "metadata": {}, - "outputs": [], - "source": [ - "xx, l = net(x)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "7ccdec29-3952-460d-95b4-820b03aa4997", - "metadata": {}, - "outputs": [], - "source": [ - "xx.shape" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "a847084a-a65d-4072-ae1e-ae5d85a1664a", - "metadata": {}, - "outputs": [], - "source": [ - "l" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "9b21480a-707b-41de-b75d-30fb467973a4", - "metadata": {}, - "outputs": [], - "source": [ - "vq(x)[0].shape" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "cba1096d-8832-4955-88c9-a8650cf968cf", - "metadata": {}, - "outputs": [], - "source": [ - "import os" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "443a52d9-09f3-4e24-8a23-e0397a65f747", - "metadata": {}, - "outputs": [], - "source": [ - "import glob" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "78541477-6f02-42da-ad75-4a47bb043e79", - "metadata": {}, - "outputs": [], - "source": [ - "from pathlib import Path" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "bdedced3-e08b-4bec-822c-e5dcd521c6b8", - "metadata": {}, - "outputs": [], - "source": [ - "list(Path(code_dir).glob(\"**/*.py\"))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "79771541-c474-46a9-afdf-f74e736d6c16", - "metadata": {}, - "outputs": [], - "source": [ - "for path in glob.glob(os.path.join(code_dir, \"**/*.py\"), recursive=True):\n", - " print(path)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "a79a2a20-56df-48b3-b964-22a0def52117", - "metadata": {}, - "outputs": [], - "source": [ - "e = Encoder(1, 64, 32, 0.2)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "5a6fd004-6d7c-4a20-9ed4-508a73b329b2", - "metadata": {}, - "outputs": [], - "source": [ - "d = Decoder(64, 1, 32, 0.2)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "82c18401-ea33-4ab6-ace4-03cb6e2e4435", - "metadata": {}, - "outputs": [], - "source": [ - "z = e(x)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "64f99b20-fa37-4614-b258-5870b7668959", - "metadata": {}, - "outputs": [], - "source": [ - "xh = d(z)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "4a81e7de-1203-4ab6-9562-37341e135daf", - "metadata": {}, - "outputs": [], - "source": [ - "xh.shape" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "204d167b-dce0-4dd7-b0e1-88a53859fd28", - "metadata": {}, - "outputs": [], - "source": [ - "a = [2, 2]" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "b77a6e8a-070d-46d3-9470-a5729eace57f", - "metadata": {}, - "outputs": [], - "source": [ - "a += [1, 1]" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "741adac8-acc4-4715-afe9-07d3522cab62", - "metadata": {}, - "outputs": [], - "source": [ - "a" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "49b894be-5947-4e06-b698-bb990bf2c64c", - "metadata": {}, - "outputs": [], - "source": [ - "x" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "4371af97-1f3b-4c5e-9812-3fb97d07c1cb", - "metadata": {}, - "outputs": [], - "source": [ - "576 // (2 * 4)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "28224cc8-79e0-481f-b24c-85bd0ef69f0a", - "metadata": {}, - "outputs": [], - "source": [ - "16 // 2" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "d3a6146b-94b1-4618-a4e4-00f8e23ffdb0", - "metadata": {}, - "outputs": [], - "source": [ - "from hydra import compose, initialize\n", - "from omegaconf import OmegaConf\n", - "from hydra.utils import instantiate" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "764c8736-7d68-4261-a57d-face10ebbf42", - "metadata": {}, - "outputs": [], - "source": [ - "# context initialization\n", - "with initialize(config_path=\"../training/conf/\", job_name=\"test_app\"):\n", - " cfg = compose(config_name=\"config\", overrides=[\"+experiment=vqvae\"])\n", - " print(OmegaConf.to_yaml(cfg))\n", - " print(cfg)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c1a9aa6b-6405-4ffe-b065-02340762476a", - "metadata": {}, - "outputs": [], - "source": [ - "mapping = instantiate(cfg.mapping)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "969ba3be-d78f-4b1e-b522-ea8a42669e86", - "metadata": {}, - "outputs": [], - "source": [ - "network = instantiate(cfg.network)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "6147cd3e-0ad1-490f-917d-21be9bb8ce1c", - "metadata": {}, - "outputs": [], - "source": [ - "x = torch.rand(1, 1, 576, 640)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "a0ecea0c-abaf-4d5d-a13d-c085c1e4d282", - "metadata": {}, - "outputs": [], - "source": [ - "network.encode(x)[0].shape" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "a7b9f249-7e5e-4f31-bbe1-cfd6d3701cf0", - "metadata": {}, - "outputs": [], - "source": [ - "t, l = network(x)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "9a9450d2-f45d-4823-adac-68a8ea05ed1d", - "metadata": {}, - "outputs": [], - "source": [ - "l" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "93b8c90f-788a-4095-aa7a-55b34f0ddaaf", - "metadata": {}, - "outputs": [], - "source": [ - "from torch.nn import functional as F\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c9983788-2dae-4375-a821-a64cd1c68edf", - "metadata": {}, - "outputs": [], - "source": [ - "F.mse_loss(x, t) + l" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "29b128ca-80b7-481e-bb3c-44f109c7d292", - "metadata": {}, - "outputs": [], - "source": [ - "t.shape" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "23c9d90c-042b-423e-ab85-18449e29ded4", - "metadata": {}, - "outputs": [], - "source": [ - "576 / 4" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "047ebc09-1c74-44a7-a314-1099f09722fe", - "metadata": {}, - "outputs": [], - "source": [ - "t = torch.randint(0, 1006, (1, 451)).cuda()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "87372dde-2b1a-432b-ab79-0b116124c724", - "metadata": {}, - "outputs": [], - "source": [ - "z = torch.rand((1, 36 * 40, 128)).cuda()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "cf7ca9bf-cafa-4128-9db7-046c16933a52", - "metadata": {}, - "outputs": [], - "source": [ - "network = network.cuda()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "dfceaa5f-9ad8-4d33-addb-c56e8da48356", - "metadata": {}, - "outputs": [], - "source": [ - "network.decode(z, t).shape" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "9105fbbb-4363-4d3e-a01e-bc519c3b9c3a", - "metadata": {}, - "outputs": [], - "source": [ - "decoder = decoder.cuda()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c5797ec4-7a6a-46fd-8adc-265df44d0341", - "metadata": {}, - "outputs": [], - "source": [ - "decoder(z, t).shape" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "a23893a9-a0da-4327-a617-dc0c2011e5e8", - "metadata": {}, - "outputs": [], - "source": [ - "OmegaConf.set_struct(cfg, False)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "a6fae1fa-492d-4648-80fd-1c0dac659b02", - "metadata": {}, - "outputs": [], - "source": [ - "datamodule = instantiate(cfg.datamodule, mapping=mapping)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "514053ef-fcac-4f3c-a7c8-72c6927d6798", - "metadata": {}, - "outputs": [], - "source": [ - "datamodule.prepare_data()\n", - "datamodule.setup()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "4bad950b-a197-4c60-ad89-903124659a98", - "metadata": {}, - "outputs": [], - "source": [ - "len(datamodule.train_dataloader())" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "7db05cbd-48b3-43fa-a99a-353126311879", - "metadata": {}, - "outputs": [], - "source": [ - "mapping" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "f6e01c15-9a1b-4036-87ae-78716c592264", - "metadata": {}, - "outputs": [], - "source": [ - "config = cfg" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "4dc475fc-31f4-487e-88c8-b0f445131f5b", - "metadata": {}, - "outputs": [], - "source": [ - "loss_fn = instantiate(cfg.criterion)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c5c8ed64-d98c-47b5-baf2-1ba57a6c882f", - "metadata": {}, - "outputs": [], - "source": [ - "import hydra" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "b5ff5b24-f804-402b-a8ab-f366443025ca", - "metadata": {}, - "outputs": [], - "source": [ - " model = hydra.utils.instantiate(\n", - " config.model,\n", - " mapping=mapping,\n", - " network=network,\n", - " loss_fn=loss_fn,\n", - " optimizer_config=config.optimizer,\n", - " lr_scheduler_config=config.lr_scheduler,\n", - " _recursive_=False,\n", - " )\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "99f8a39f-8b10-4f7d-8bff-52794fd48717", - "metadata": {}, - "outputs": [], - "source": [ - "mapping.get_index" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "af2c8cfa-0b45-4681-b671-0f97ace62516", - "metadata": {}, - "outputs": [], - "source": [ - "net = instantiate(cfg)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "8f0742ad-5e2f-42d5-83e7-6e46398b4f0f", - "metadata": {}, - "outputs": [], - "source": [ - "net" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "40be59bc-db79-4af1-9df4-e280f7a56481", - "metadata": {}, - "outputs": [], - "source": [ - "img = torch.rand(4, 1, 576, 640)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "d5a8f10b-edf5-4a18-9747-f016db72c384", - "metadata": {}, - "outputs": [], - "source": [ - "y = torch.randint(0, 1006, (4, 451))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "19423ef1-3d98-4af3-8748-fdd3bb817300", - "metadata": {}, - "outputs": [], - "source": [ - "y.shape" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "0712ee7e-4f66-4fb1-bc91-d8a127eb7ac7", - "metadata": {}, - "outputs": [], - "source": [ - "net = net.cuda()\n", - "img = img.cuda()\n", - "y = y.cuda()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "719154b4-47db-4c91-bae4-8c572c4a4536", - "metadata": {}, - "outputs": [], - "source": [ - "net(img, y).shape" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "bcb7db0f-0afe-44eb-9bb7-b988fbead95a", - "metadata": {}, - "outputs": [], - "source": [ - "from torchsummary import summary" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "31af8ee1-28d3-46b8-a847-6506d29bc45c", - "metadata": {}, - "outputs": [], - "source": [ - "summary(net, [(1, 576, 640), (451,)], device=\"cpu\", depth=2)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "4d6d836f-d169-48b4-92e6-ca17179e6f85", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.6" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/notebooks/06-try-transformer-model-predictions.ipynb b/notebooks/06-try-transformer-model-predictions.ipynb deleted file mode 100644 index 8366088..0000000 --- a/notebooks/06-try-transformer-model-predictions.ipynb +++ /dev/null @@ -1,358 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "%load_ext autoreload\n", - "%autoreload 2\n", - "\n", - "import importlib\n", - "import cv2\n", - "import yaml\n", - "\n", - "%matplotlib inline\n", - "import matplotlib.pyplot as plt\n", - "import numpy as np\n", - "import torch\n", - "from torch import nn\n", - "from importlib.util import find_spec\n", - "if find_spec(\"text_recognizer\") is None:\n", - " import sys\n", - " sys.path.append('..')" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "def convert_y_label_to_string(y, dataset):\n", - " return ''.join([dataset.mapper(int(i)) for i in y])" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "from text_recognizer.models import TransformerModel\n", - "from text_recognizer.datasets import IamLinesDataset" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [], - "source": [ - "dataset = IamLinesDataset(train=False,\n", - " init_token=\"<sos>\",\n", - " pad_token=\"_\",\n", - " eos_token=\"<eos>\",\n", - " transform=[{\"type\": \"ToTensor\", \"args\": {}}],\n", - " target_transform=[\n", - " {\n", - " \"type\": \"AddTokens\",\n", - " \"args\": {\"init_token\": \"<sos>\", \"pad_token\": \"_\", \"eos_token\": \"<eos>\"},\n", - " }\n", - " ],\n", - " )\n", - "dataset.load_or_generate_data()" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [], - "source": [ - "config_path = \"../training/experiments/TransformerModel_IamLinesDataset_CNNTransformer/1213_175148/config.yml\"\n", - "with open(config_path, \"r\") as f:\n", - " experiment_config = yaml.safe_load(f)" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "'CNNTransformer'" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "experiment_config[\"network\"][\"type\"]" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2020-12-30 01:24:06.949 | DEBUG | text_recognizer.models.base:load_weights:432 - Loading network with pretrained weights.\n" - ] - } - ], - "source": [ - "model = TransformerModel(network_fn=experiment_config[\"network\"][\"type\"], dataset=experiment_config[\"dataset\"][\"type\"], dataset_args=experiment_config[\"dataset\"])" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2020-12-30 01:25:47.777 | DEBUG | text_recognizer.models.base:load_from_checkpoint:379 - Loading checkpoint...\n" - ] - } - ], - "source": [ - "ckpt_path = \"../training/experiments/TransformerModel_IamLinesDataset_CNNTransformer/1213_175148/model/best.pt\"\n", - "model.load_from_checkpoint(ckpt_path)" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [], - "source": [ - "model.eval()" - ] - }, - { - "cell_type": "code", - "execution_count": 23, - "metadata": {}, - "outputs": [], - "source": [ - "data, target = dataset[11]\n", - "sentence = convert_y_label_to_string(target, dataset) " - ] - }, - { - "cell_type": "code", - "execution_count": 24, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "torch.Size([98])" - ] - }, - "execution_count": 24, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "target.shape" - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "metadata": {}, - "outputs": [], - "source": [ - "data = data * (data > 0.1)" - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "metadata": {}, - "outputs": [], - "source": [ - "from torchvision import transforms" - ] - }, - { - "cell_type": "code", - "execution_count": 19, - "metadata": {}, - "outputs": [], - "source": [ - "ra = transforms.RandomAffine((-1.1, 1.1), scale=(0.8, 1))" - ] - }, - { - "cell_type": "code", - "execution_count": 20, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/home/akternurra/.cache/pypoetry/virtualenvs/text-recognizer-N1c_zsdp-py3.8/lib/python3.8/site-packages/torchvision/transforms/functional_tensor.py:876: UserWarning: Argument fill/fillcolor is not supported for Tensor input. Fill value is zero\n", - " warnings.warn(\"Argument fill/fillcolor is not supported for Tensor input. Fill value is zero\")\n" - ] - } - ], - "source": [ - "data = ra(data)" - ] - }, - { - "cell_type": "code", - "execution_count": 25, - "metadata": {}, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAADSUAAACECAYAAADcSq9PAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAABrl0lEQVR4nO3dd5wfVbn48edsSTa9ElJJoxcpEgIKghB6LyIi5Spi43KtoCIqigI/RZFyuagXAQUElAsIKL0TSkJCSSCNFFJJIcmmbbbN74/9cnjOw87ZmW++380GPu/Xy5fP7DPlnDNnzpwZGL4uSRIBAAAAAAAAAAAAAAAAAAAAAAAAgKwqNncBAAAAAAAAAAAAAAAAAAAAAAAAAGxZ+CgJAAAAAAAAAAAAAAAAAAAAAAAAQC58lAQAAAAAAAAAAAAAAAAAAAAAAAAgFz5KAgAAAAAAAAAAAAAAAAAAAAAAAJALHyUBAAAAAAAAAAAAAAAAAAAAAAAAyIWPkgAAAAAAAAAAAAAAAAAAAAAAAADkwkdJAAAAAAAAANDBOefmOufGbe5yfNw4525wzv0k47o3O+d+WebyOOfcTc65lc65l8t5LAAAAAAAAAAAAABoS9XmLgAAAAAAAAAAAOXinLtZRBYkSXJx3m2TJPl66Uu0SfYXkUNFZGiSJOvKfTDn3AgRmSMi1UmSNJb7eAAAAAAAAAAAAAC2LPxSEgAAAAAAAABEOOe23hL3/VHhnOM/rvWB4SIytz0+SOponHPVzrm+Zdx/Z+dcr3LtHwAAAAAAAAAAAPgo4qMkAAAAAAAAADCcc72dc99wzr0sIjerv//AObfQObfGOTfdOXdI4e+dnXO/d84tKvzv9865zoVcf+fcA865Vc6595xzzzrn3n83e7Nz7mXn3Nedc73bKNYY59ybzrmVzrmbnHM1qlzHOOdeLRxjvHPuEyo3zDn3f865Zc65Fc656wp/H+2ce6Lwt+XOudt0GZxzc51zFzjnXnfOrXPO3eic29o59+9C/R9zzvVR6+9bOPYq59xrzrmDIu27l3NucmE/f3fO3emc+2Uhd5BzbkGhrZeIyE3OuQrn3A+dc28XynuX/kClsI8lzrnVzrlnnHO7FP7+VRH5oohc6Jxb65y7v5WyOOfcVc65pc65WufcG865XQu5m98vV2H5Qufc4sI5/opzLnHObat218c592ChXi8550a3dYxWyjPYOffPQl+Z5Zw7t/D3c0Tkf0Vkv0Jdfp6y/Zedc28V+snDzrnhKreLc+7Rwr7fdc5dVPj7Ps65iYWyveuc+11hk2cK/7+qcMz9MvSbD/U351ynwjF3U+sNcM6td85t1Vo91Hq7Oud+KyILpOVXot7/e6zP7+Sce6qQm+qcO07ljnIt19Ea13Itf7+Q6i8i8wv1Gec+uEYBAAAAAAAAAAAApOAfqgEAAAAAAACAiBQ+fDnMOfc3EZknIoeJyK9E5LhCfgcR+U8RGZMkSQ8ROVxE5hY2/7GI7Csie4jI7iKyj4hcXMh9T1o+qNhKRLYWkYtEJCnkjhORywr7muecu905d2jKBxFfLKw3WkS2f3//zrk9ReTPIvI1EeknIn8QkX+6lg+lKkXkgUJ9RojIEBG54/0qi8jlIjJYRHYSkWEicok55snS8iHI9iJyrIj8u1D+raTl/fJ/FcowREQeFJFfikhfEfm+iNzd2gcnzrlOInKPtHzs1VdE/iYiJ5rVBhZyw0XkqyJyvoicICIHFsq7UkT+W63/bxHZTkQGiMgkEblNRCRJkj8W4l8nSdI9SZJjbXmk5Tx/plDHXiJyqoisaKXcR4jId0VknIhsKyIHtbKv00Tk5yLSR0RmSUv/yXyMgjukpb8MFpFTROQy59zBSZLcKCJfF5EXCnX5WStlPF5azs9J0nKOnpWW9hXnXA8ReUxEHirse1sRebyw6dUicnWSJD2lpX/dVfj7Zwr/37twzBck0m/S+luSJPWFep2hivsFEXk8SZJlrdSjj3Pum865CSLyiIg0i8jBSZLcWcjH+ny1iNxf2G6AtPSd2wrXr4jIjSLytcI1vKuIPCEikiTJQmk5P5NF5CoRmeOc+4VzbpQtHwAAAAAAAAAAAIAWfJQEAAAAAAAA4GPPOfef0vKB0RUi8oKIjE6S5MQkSe5LkqShsFqTiHQWkZ2dc9VJksxNkuTtQu6LIvKLJEmWFj6y+LmInFnINYjIIBEZniRJQ5IkzyZJkoiIFJbvTZLkRGn5GORFEfl/IjK3UCbtuiRJ5idJ8p60fOzyhcLfvyoif0iS5KUkSZqSJLlFRDZKy0dS+0jLxyMXJEmyLkmSuiRJnisce1aSJI8mSbKxUObfSctHP9q1SZK8W/hg41kReSlJkslJktRJy4dFexbWO0NE/pUkyb+SJGlOkuRREZkoIke10tz7ikiViFxTqP//icjLZp1mEflZoWwbpOVjnB8nSbIgSZKN0vIRzCnOuapCXf6cJMkaldvdOderlWO3pkFEeojIjiLikiR5K0mSxa2sd6qI3JQkydQkSdbLhz/gEhG5J0mSl5MkaZSWj6H2yHMM59wwEfm0iPygcK5elZZfRzorY12+LiKXF/bfKC0fvO1R+LWkY0RkSZIkvy3se02SJC+p8m3rnOufJMnaJEleTDtAG/0mtb+JyC0i8gXnnCssnykifzX17+mcu0NE5kjLR18/E5FhSZJckCTJVLVqrM/vKyLdReSKJEnqkyR5Qlo+lHr/emmQlmu4Z5IkK5MkmaTqtiRJkiuTJNlNWj7s6i0iLxZ+dWn3WMMDAAAAAAAAAAAAH0d8lAQAAAAAAAAAIiOl5ddtXhWR16SVX7FJkmSWiHxbWj5GWeqcu8M5N7iQHiwtvw7zvnmFv4mI/EZafjXnEefcbOfcD1PKsEJEXi+UoU+hTNr8lP0PF5HvOedWvf8/afn1msGF/59X+EAl4JzbulCHhc65WhG5VUT6m9XeVfGGVpa7qzJ8zpRhf2n5GMsaLCIL3/8wq5W6iYgsK3z49L7hInKP2vdb0vKR2NbOuUrn3BXOubcL9Zhb2MbWpVWFj1auk5ZfXlrqnPujc65nSrl1OW2ZRUSWqHi9FNon5zHeS5JkjfrbPGn5xaEshovI1aqd3pOWXzYaIi194e2U7c6Rll8Jmuacm+CcOybtAG30m9T+VvgAar2IHOSc21Fafqnpn2a1amn59aL3pOU6mJIkSVNKPdP6/GARmZ8kSbNaX7fhydLysdw859zTzrn9Uqo6U1rGglnS8jFZ75T1AAAAAAAAAAAAgI8tPkoCAAAAAAAA8LGXJMn3pOWXiqaIyLUiMsc5d6lzbjuz3u1JkuwvLR9FJNLyq0YiIosKf3vfNoW/SeEXab6XJMkoETlORL7rnDvk/RWdc9s55y6Vll+HuVpE3hCRUYUyacNa27+0fBzzqyRJeqv/dU2S5G+F3Dbv/6KQcVmhDrslSdJTWn7tyLWyXhbzReSvpgzdkiS5opV1F4vIEPWLObZuUiiX3f+RZv81hV9wOl1EjheRcSLSS0RGFLZxKfv6kCRJrkmS5JMisrO0fJxzQUq5h0bKXIpjLBKRvs65Hupv24jIwoyHmS8iXzPt1CVJkvGF3KiUss1MkuQLIjJAWvr0P5xz3aT1tov1m1h/E2n5taQzpOVXkv5hPjyTJElWJEmyq4h8XlraepJz7gnn3H8457qrVWN9fpGIDHPO6X/+4dswSZIJSZIcX6jrvSJy1/srFT5wO9I59zcReUdEjhaRy0VkaJIkT6fUCQAAAAAAAAAAAPjY4qMkAAAAAAAAABCRJEmWJknyuyRJPiEtv6bSW0RecM79WUTEObeDc+5g51xnEamTll8Kev/XWP4mIhc757ZyzvUXkZ9Kyy/IiHPuGOfctoWPcFZLyy/8NBdyfxaRFwrHOilJkt2TJLkqSZJlrRTxPOfcUOdcXxH5sYjcWfj7n0Tk6865sa5FN+fc0YUPW16Wlo9prij8vcY59+nCdj1EZK2IrHbODZHWP5LJ6lYROdY5d3jhw44a59xBzrmhraz7QqEN/tM5V+WcO15E9mlj/zeIyK+cc8NFRArtfLyqx0Zp+aWprtLy0Yz2rqR8jFPY15hC21WLyDppObfNrax6l4h8yTm3k3Ouq4j8pI0y5z5GkiTzRWS8iFxeaMNPSMuvGN2a8VA3iMiPnHO7FI7byzn3uULuAREZ5Jz7tnOus3Ouh3NubGG9M5xzWxV+XWhVYf1mEVlW+H/dfrF+E+tvUqjHidLyYdJf0ipR+HDom9Ly60Z/kJaPlBY5544orBLr8+//ItOFzrlq59xBInKsiNzhnOvknPuic65XkiQNIlIrH1yLA0RkgbT0nxdFZNskSU5KkuT+1n75CQAAAAAAAAAAAAAfJQEAAAAAAADAhyRJ8kqSJOeLyGBp+dBDRKSziFwhIstFZIm0/NLKjwq5X4rIRBF5XVp+6WhS4W8iItuJyGPS8iHHCyJyfZIkTxZyN4jI4CRJzk+SZFIbxbpdRB4Rkdki8vb7+0+SZKKInCsi14nIShGZJSL/Ucg1ScsHGdtKyy+/LJCWDzxERH4uIntJy4dSD4rI/2Vpm9YUPqY5XkQukpYPWeZLy8cqH3oHnSRJvYicJC0f26ySlg9UHpCWD4vSXC0i/xSRR5xza6Tlo5GxhdxfRGSetPwSzpuFnHajiOzsnFvlnLu3lX33lJaPXFYW9rNCRH7TSrn/LSLXiMiT0tLG7x8nVu5cxyj4grT82tMiEblHRH6WJMljGY4hSZLcIy2/dHSHc65WWn7568hCbo2IHCot/WGJiMwUkc8WNj1CRKY659ZKS1ufliTJhiRJ1ovIr0Tk+UL77SuRftNGf3u/n0ySll9aelZExDk31Tn3xZT6bEyS5M4kSY4UkR1FZHrh77E+X18ow5HScq1eLyJnJUkyrbDbM0VkbqF9vi4i7x97vYgckSTJnkmSXJ0kyfI2GxwAAAAAAAAAAAD4mHNJkmzuMgAAAAAAAAAAPsaccy+JyA1Jkty0ucuSlXNuJ2n56Kczv6STXeHXwRYlSXLx5i4LAAAAAAAAAAAAgE3DLyUBAAAAAAAAANqVc+5A59xA51yVc+5sEfmEiDy0ucvVFufcic65zs65PtLyi0T380FSds65EdLyK1k3buaiAAAAAAAAAAAAACiBqs1dAAAAAAAAAADAx84OInKXiHQTkdkickqSJIs3b5Ey+ZqI3CwiTSLytIh8c7OWZgvinLtURL4jIpcnSTJnc5enGM65i0TkolZS3URkHX/n7/ydv/N3/l7wbJIkR7bydwAAAAAAAAD4yHFJkmzuMgAAAAAAAAAAAAAAAAAAAAAAAADYglRs7gIAAAAAAAAAAAAAAAAAAAAAAAAA2LLwURIAAAAAAAAAAAAAAAAAAAAAAACAXKryrOycS5xz5SpLydgydu7c2cdJkgS5jRs3lrUs9njaltCW6Nh0//oo9Cd7veg6xXLoOMrRJ2P7zNMvPmrXC7Clae9rsBRzsI587ylH2dpjnzHlOC+M/R8tWc9nR7t2eSYEilPstdzRxoBS+6jXD0DHFntX1x42xzEBAAAAAAAAAAAAeMuTJNnK/jHvR0lSXV3tl8vxL1fp7Zqbm4vah/4ISURk9OjRPrYfIc2ePTvTPov9B56xOlRUlOaHqjb3PwzeEuX5lxjbW1sfXWi6f5WqP21O9nrRdYrl8rRZsfL0mcrKSh83NTUVvZ8tUTn6ZGyfsX7RHmXrSLJ+xBe7Xor9eKCt7bhPbR7tMTbm0d7XYCnmYHnGmPZWjrK1xz5jynFePupjf3tr6z5cblnPZ0e7dsvxTFiOe2spPvjY3PcalFd7P8sUey13tDFAK8X10pHrt6Uq9rzEnrPa+x5pdaT7RGwfWctm+3nWYxT7jq21bbPupyPJ2g9i7WTbXr/zamhoiO6nmOO1pb6+PvO6AAAAAAAAAAAAAEpuXmt/5N9cAAAAAAAAAAAAAAAAAAAAAAAAAJCLy/NfIqyoqEiqqj74caWsv8oQU+x/WdLuX/9XG88+++wgN2rUKB83NjYGufvvv9/HkydPDnL6v5BZql8b4L/W/tFVrl+koM+UT55fdMszxm3O/2JuR/tllI6ko54zq9y/NlDsr1zE7rv2F8Hy+DiNcR/3/6p9rG+3xy9LlkJH6q/l+AWiYhX7y0UxpdhHW/uJ6UjnulQ68q85lFuePlqO+3BH7U+xX52wudi9fnP/UlJHat/N3RblUI5f9tG/svFx/rWg2LNbR6uf/kVy+wvk7a3YPpn1F3g3d1u3h3I8Gxe7z1h7F/urvlpHu5aKpcdNWyfdNsW+E8rzS0mx85K1H9j18lzX/FISAAAAAAAAAAAAsFm9kiTJ3vaPW+Y/iQUAAAAAAAAAAAAAAAAAAAAAAACw2fBREgAAAAAAAAAAAAAAAAAAAAAAAIBcqvJuUFHxwXdMSZKkrhfLac65oraz9t13Xx8PGjQoyF1++eU+Hj58eJD7yle+4uOpU6cGubq6Oh/reouINDc3+9iW2dZJq6ysTM0VW/eOTLfFR7F+MbG6x/q9zdm+91ETayd9nRXbDrG23pQ+GTtnMaWoU8zH7TrLo9i20efM0uewVPezciu2XHa7pqamUhQnqtzXSzlk7S8fR7G+15HaJtbv2ruc5ehPdp9Z9xPbrj3apb3Hg47UJ0ulVHOgLVHsfLbH/bvc/anYOsSu6zztEFt3S7yXb4qOem1tSj8vRZ3suS/HPrVi73XtrT36S6nGuI0bN5aiOCVRbB1i/aCj9pFy0W0Yq3uxY0XsHUmsT1ZVha+pGxoaMm0Xy9n65bkmOtI7zazP36V63tfyPJPoc9jY2Jh6DHu8LeVZEQAAAAAAAAAAAEDr+Kd6AAAAAAAAAAAAAAAAAAAAAAAAAHLhoyQAAAAAAAAAAAAAAAAAAAAAAAAAufBREgAAAAAAAAAAAAAAAAAAAAAAAIBcqsq144qKD753am5uTl0vSZLUnHMuNVdVFRZ9jz328PFf/vKXINfQ0ODjd955J3WfXbt2DZbr6up8HKuDrqtd19ZB1zdWv3LRx7RtH8sVu888+ym1WNvH5DnXxey/tXXL3ReKbYvYfmL7sG2Ytd3sdpWVlZmOF9MefTDPtRRrC7SI9Z88fatU9DFi11Kp+lqxY1Xseil2fM9qU8aYjnpNxOoU64cdtT4fB7bti70+s85h23v8aYsua6m2i9U/ltPsPottt2LPS7Ht0h46ctmKUar5ppbnvl9se27O55VSKdV9P/YcW6xi+3ae63xLuX425zX/UejneZSjfbPe98p1/GJ93M49PhAbx2PPNqUY//PMCXSusbEx03pt5Ur1nnJLvH6ynvdSsftsamra5OOVY04JAAAAAAAAAAAAoLw6zr8pAQAAAAAAAAAAAAAAAAAAAAAAAGCLwEdJAAAAAAAAAAAAAAAAAAAAAAAAAHKpyrtBc3Ozj51zqeslSZJpf3Yfejub08fu1q1bkFu+fLmPly5dmrpPvQ8RkS5duvi4c+fO0bKl7TMmtl4sZ8tZUVHc92N2O7vfrOWJKXY73b7F7iPG7jPr8WybxfqBzsXaNrZdW+UphaampmC52P4Uuz71MUrVX8vdLlae8ajYa7sUih0fiu13pRqPssozbrVH2co9VhUrz/VS7nE6z/5j/TDPOFosfYxYf8lznygHfYxir4H2vnY3RdbzknUfpRKbE3Sk8UAk3m7Ftm/Wfca2y3NeSjV/KMU+SyVrO3UkeeYL5b4mStVfy6GysjJYLvf9LM+YXuwcdnOPcaW4F2Tdf7mOodm233vvvX08c+bMILdmzZqylgWbrth7YjmO1x7a+51Fsbak+a4We5azbV3s2FiOd3zFvifNOjcs9rnSbhe7D8aU6vjl2C6rTXk2L8Xx9LtBu/+sx8vzTggAAAAAAAAAAABAx7Bl/NN6AAAAAAAAAAAAAAAAAAAAAAAAAB0GHyUBAAAAAAAAAAAAAAAAAAAAAAAAyIWPkgAAAAAAAAAAAAAAAAAAAAAAAADkUpV3g4qKD75jSpLEx865YD2di4mt19zcHCzrY+y9995BbtmyZT5uaGhI3WfXrl2D5dra2tTt9PGy1sduZ2Xdj27nPMez+7dt2JHkadNiFNsnrdh2sVzsvBTbn4qtQ57+9FGXNobZnIhIU1NT6rqluM4tvU997FIpVf8pRZ8s9vh2TMvTt/W2ebZrjzqmKdU41t7Hi52n2D4391hVbB8px/FKMV8oVR3aox+WoqybMj605z43h1i5i+33xbZFbEwv97FjSnWuY/uxuVKMObHjVVZWpubsPEPvJ881nnXd9r6flUqs3HnmaqU417HtbDlj/U6vG7v3ZH3OaWvdPMoxVsW2K3ZOm7UsI0eODJaPP/54Hz/wwANBbsKECZmP35G093NBR7UlzQ+y9t9yXBPl0JHbPjZWxsZmK8/4n3a8PGLXtc7F5nF53oHpfcbarFTv9GLHyHN/i22XdR95ZB1v8/Q7O1dM2y4mdm+1c6WszwGx7QAAAAAAAAAAAAB0TPxTPQAAAAAAAAAAAAAAAAAAAAAAAAC58FESAAAAAAAAAAAAAAAAAAAAAAAAgFyqNmVj55yPm5ubM63X2rKWJImPKyrCb6aqq6t9fOihhwa53//+96n71/vs1KlTkGtqavJxXV1dajlj+9Rxa8tZ9xlrwxhdB9tmxYrVtxzblUOeY2ftk+U6frH70eW26+n+VKp+kbVcHVnsOtPXkhXrI7F187RTbPzTih0rSqXYc1/uPmnZc1ZZWZm6bin6sz0vpaijLVesbxXb72LHi4kdrz3Ob7FiZcta7va415Xj3m5l7TNbynhfjnZq777ckeZRm0Ps/lbuc2H3H7tnFVvOUtUhVrbhw4f7+IwzzghyQ4YM8fGMGTOC3E033eTj1atXl6Sc2setL1v6POXpP7FzHRvDY3PaYtaz8pzP9p7/WbG2z5qL7dPS52XcuHFBbu7cuT6eNWtW6nbWx/36QdtK1X+L1ZHn/sXI056xazfPvC72LqDY948xWctd7PGKfQ9SbJvlUY5n7Kzv7dpS7ntBnnciWdsldr3kee9djvflAAAAAAAAAAAAANrPR+vfHAAAAAAAAAAAAAAAAAAAAAAAAABQdnyUBAAAAAAAAAAAAAAAAAAAAAAAACAXPkoCAAAAAAAAAAAAAAAAAAAAAAAAkEtV3g2am5tb/btzLlhOkqTV2K5rt9PLdrshQ4b4uKamJsjV1tb6uKIi/NZKL2+zzTZBbtmyZT5uaGiQrGzZtMrKSh83NTWlbhdrl9j+LVvfUogdP8+57qhsHTb38XW7xXJWbLty9IsYfTw7ThRblti1pBXbnnna2spzjWi6bYptl9h2eeqbtl65lKJP5tlHe49H7XHNZR0PrFL0uzzH29zKUV+tI9+/88ha1k0ZK0sh6/lsj3KWu2+1h7S5fFs5K2v97Xr6PI0ZMybIvf766z7OMy8vhVK1S3vQbaqfj0REfv3rX/v41VdfDXK33Xabjz/72c8Gua997Ws+vvLKK4NcrP7Fjn8xseusHMeLKccx8owdsXVLUbZSzdljSvWMUO55ZKnqrvez6667BrlHHnnEx6tXr07dR7HPGcX2iXI9E3VUm3teUw5b6pwkq/YYq3S/KPaZs633TP379/dxnz59gty8efN8XF9fn1q2crz7yPN+rBzv0rLe2/X7IZGwX5TqOi5HG5bqHV9WpXpWjYntM+s74jzt8lEf4wAAAAAAAAAAAICPAv6pHgAAAAAAAAAAAAAAAAAAAAAAAIBc+CgJAAAAAAAAAAAAAAAAAAAAAAAAQC5VeTdwzvk4SZJW49h6bW0XM2jQIB/PnDkzyNXX1/u4qakpyDU3N7e6DxGR+fPnt7pea8tZ6ePrdhCJ1zdPW5RCRUX4TZqub6zcpSqnPoY9Z7Zsm7p/keLrEOvLxSrHuc5zDZZbKc6fyIf7hZa1fqW65kq1rm4bO8boXJ6xI+t6xfaDYssSG0NL1Ufaw+a8lizdpnnaMLZusfssVqzfl0p7969Yv4j1n1jbd6RrJE+5S9Gfiu0jpbo+y3FNZN3n5h5jYkrVFvqaOOmkk4JcZWWlj19++eWi9l/svSfrOJl33azb5aHb6Vvf+laQmzBhgo+vu+66INfQ0ODjqVOnBrnLLrvMx0OGDAlyCxYs8HHsua5U43ux10h7389iiu0z7X0fLtU4reXZ50dlrqjV1NT4uH///kFu9uzZPi52fm2V4r6R51m8PeZx5ba53wWUwqbclzrSeJRVe5Sl2POry1ZdXR3kxo0bFywfeOCBPrb32j/96U8+fvbZZ1PLFjufsTE19r4oDz0HaWxsjB5Di5VTb2fLFXtvWCrFPstl3Uc5xvdS3UOKPX6x78uz7r8c74gBAAAAAAAAAAAAlBf/VA8AAAAAAAAAAAAAAAAAAAAAAABALnyUBAAAAAAAAAAAAAAAAAAAAAAAACAXPkoCAAAAAAAAAAAAAAAAAAAAAAAAkEtV3g2SJPGxc67Vv9ucji2ba25uTs0NHDjQx6tWrUrdzqqurvbxzjvvHOSefvppHzc1NQU5WyctT502db22ypKHPmaszWLHq6gIv2XT6+ZpM72u3admyxlbN3a8tGPbdW0uVk5dtmLbpa2yZRVr380t1r4fZ7G+3JHaqdiyZL1WO7pynIvY2BFTbJvGjleOfcbExtG21t2csta32HtWsdr7eG0dQ+eKvS9t7vOetX6l2mcexV53xZZFHy/W1/L0Q73uVVddFeSOOOIIH0+cODF1u9g+y9F/yjVOF1vubbfd1seDBg0KcrpNbZvp5dra2iD32muv+Xj//fcPcn/7298yl21zKse5L3aMbY97u5annKU4XqnaOs+coKOydejbt6+PN2zYEOTmzZvnY/vuIeu7gXI8K7Z3f22tPGna45ko67uAPNuVW7nmSpt7DlaM2DjSHu9IOnXq5OOzzz47yNXU1ATL1157rY8vuOCCIDd69Ggfv/jii0Gurq4uU1lsfWPv/3RbVFZWBrkuXbr4uGfPnkFu2LBhPrb1W7JkiY8XLVoU5NavX59azrQy23LbnH7Xa8dUuxxT7LWc9Z14bLvYfc/uQ/f1PPUrxX0jT/3S/rmBXbZ11znbJwEAAAAAAAAAAAB0fFvev3EAAAAAAAAAAAAAAAAAAAAAAAAAYLPioyQAAAAAAAAAAAAAAAAAAAAAAAAAuVRtysZJkhSVi6moqGg1FhHZeuutfbx48eIgV1NT4+OmpqYg17NnTx8PHDgwyE2bNs3HzrkgZ5fTVFWFzaiPb9sh6z5jYvuwuUGDBgXLgwcP9nF9fX2QmzVrlo/r6uqCXHNzs49tnfSyPX6sH2Rti8rKytR9VldXB7nu3bv72J6XDRs2tBq3Rdddx5atq+6/drs8/ULn7H50rthrLna8Yvdpy2mv5azHK0VZ8vTJcrDH79Spk4/teNS1a1cfz549O8ht3LixJMfX2rstdD/Ic/8oth/Y8aGxsbGo/ejj2/bU45M+fyIi69ev93Gx127sWspzncVyxdpS9tnebZinDqWo76bso9z3kFNPPTXIvfnmmz6eMmVKkGvv8Whz2pT7kj7fdn6k2blwVnmul7T18li+fHmw/I9//KOo/ZRbueYSWdvNHn/ffff18cyZM4PcihUrfKzveyLxcr/99ts+/uxnPxvk9NyloaEhdZ+x+uQZb0uhVOes3OUslY5cztjYYWXtQ+09X4ixfUvP7+0Yp5+/Y+8eyvFMVKpxy7aTvhflOUYp3hPkofdpnxH0cr9+/YKcPp+jR48OcrW1tT5+5ZVXgtzSpUt9nOcaKIeOPDZrxfZR/S5QRKRXr14+XrNmTZDTz2dW1nmOfc90yimn+HjdunVB7rbbbguW9X3ZvnPT/dCWRedi9+E87+r0s+t+++0X5I444ggfDxgwIMjp9xT2vdrIkSMlzYQJE3x89913Bzl7ntLY+sWe72NjU6nGqmIVO/6V4p1YsfeXYudVdrvYvTw2pyz22QYAAAAAAAAAAABA++m4//YSAAAAAAAAAAAAAAAAAAAAAAAAgA6Jj5IAAAAAAAAAAAAAAAAAAAAAAAAA5MJHSQAAAAAAAAAAAAAAAAAAAAAAAAByqWqPgzjnostaRUX6d1KdO3f2cX19fZBraGhI3W6//fbz8dtvvx3k6urqUrfTYmVubm4OlquqPmjWxsbG1O2SJEk9hs3F9OnTx8cHH3xwkDvqqKOC5fXr1/t42LBhQW7WrFk+/uMf/xjkZs6c6ePKysogp+tv2yImax27desWLI8dO9bHBx54YJCrrq728dq1a4PcLrvs4uObbropyD355JM+bmpqSi1LrC/H6mO3s/1cb2vXjeWyls2el1i5sx7P0tvZ+mU9XmyfedolVve09doqS2zb2Hb2Ovvyl7/s4759+wY5PXYsWrQoyF133XU+XrVqVaZytVW2ctDn3ra9Xs5zPoutgx1/s16jPXv2DHLbb7+9j+2YM3DgQB/bfq/P4YYNG4Lc7Nmzffzqq68GuXfffTe1nFrsfmnbXq9bbPuWq2/F6lHu/Zf72KVSqvEvzzksdk6ix7H9998/yOl53JQpUzLv86OmVPM/Ox7V1NT4+NFHHw1ysXmytrmvl6zz8jzzzVIo1XhX7L3Ptr2eCz/++ONBTt/78rTT4sWLU8uSVi67bqyd8vSf2D6zHq+95z8dWWxO0B6yzg3turH9lEOx+7fb6Wfz1atXB7li58Ixpbhe8og9x+ah3ynY5+9SlDU2P9Lvh0TC+6meo4uIzJ0718fz588Pctttt52Pv//97we5e++918fPP/98kOtI41N7jwelet7WuXPPPTfI7bXXXj5es2ZNkNPn5dlnnw1y9h1jmkMPPTRY7t27t49vvvnmIBd7HrVzs6zHt3RbxM7nqFGjguUvfOELPu7fv3+Qe+KJJ3w8efLkIKfnC3reLyLSvXt3H/fo0SPInXbaaT7+zne+E+R+//vf+7i2tjbIxfpBbEzN8x4olss6l8qzz6zvzvLM4/LcN9LKkqcOWXOxazemvef6AAAAAAAAAAAAADbdlvFv4wIAAAAAAAAAAAAAAAAAAAAAAADoMPgoCQAAAAAAAAAAAAAAAAAAAAAAAEAuVXk3cM61GYuIVFV9sOv6+vpN3r+ISL9+/XxcV1eXul2nTp2C3D777OPjW2+9Ncg1Nzf7OEmSIKfrECtnQ0NDkKuoSP/WSx/DrqfLYuuude7cOVg+/fTTfbx8+fIg97Of/SxYXrZsmY979OgR5I488kgfn3/++UHuu9/9ro9LdT5jttlmGx9/61vfCnL6+A8//HCQmzhxoo/t+fvMZz7jY1u/RYsW+Xj69OlBTp8z20fS1rNs3fv06RMsjxkzxseDBg0Kcl27dvXxm2++GeRef/11H69evTrINTU1+biysjI1F+uHtk563Vh9Lb1unn6gxbazZcl6nW2KWJ26devm42OPPTbI6T767LPPBrmNGzf6+Etf+lKQO/HEE3188803ZypXucTORdb2zVNOfbxi+51VXV0dLO+1114+Pumkk4LcihUrfDx+/PggN2fOHB/X1tYGuZqaGh/b6/pTn/qUj4877rggd/fdd/v48ccfb70C8uH66XbKeh/Koz36VrFsn9RlteOfXrbb6ftLOeobK6c9Z7E5SKycxd6nil3X1knXo0uXLkFOz+Os2P2l2LKUe7tSKfZ4a9euDZb/4z/+w8d6vicS3ns68rVcrNiYV+w+dP/V93WRD1+f+j5hnwti8yqds2OVNnjw4GB5yJAhPn7rrbeCXLHPGno7ff9qbd207Ypln926d+/uY9su+hlwzZo1m3zsvIqdk3RUdt5Wimspplz71/UoxzFi14vtBwMHDvTx3Llzg5x+BmvrGGnHi9kc97Nin+10W5Sq3LGy7LDDDj4++eSTg9wdd9zh40mTJqWW05owYYKPhw4dGuT0O4ypU6cGuVWrVvl4c4wjsXc0sXuIvi/ad0n6XcSGDRtS95lH1u1s++r5kX1208/U9l7373//O/XY+r77yU9+Mshde+21PrZzAEvPH+z7m8bGxtTtsr57tbm9997bx+ecc06Q03396quvDnL2HKax72V1/e2z8TXXXOPjr3zlK0FOl+1//ud/Uo9R7HOPffbff//9g+UXXnjBx/Z9p942Nh7ExN41232W43qJvSeI3c9ic9PYfEhvF6tf7H3GR2GOBQAAAAAAAAAAAHzc8EtJAAAAAAAAAAAAAAAAAAAAAAAAAHLhoyQAAAAAAAAAAAAAAAAAAAAAAAAAufBREgAAAAAAAAAAAAAAAAAAAAAAAIBcqvJukCSJj51zqes1NjamrldZWZlp/xUV4TdTGzZs8HFDQ0OQa25u9vHQoUODXJcuXXz8zjvvpG4XK1dTU1Pqdracmq5PW2L70WXbfffdg1zfvn19fPPNNwe5+vr6YFnXY/Xq1UHu6aef9vFnPvOZ1LLEzrutg66/bYvq6mof77fffkHulFNO8fH//d//BbkJEyb4WPczkbBf2LI89NBDPu7Ro0eQGzFihI+nTZsmWcXO71ZbbeXjo48+Osjts88+wfLcuXN9vHDhwtTcyJEjg9xRRx3l42eeeSbIPfHEEz7euHFjarltHWLnN20fVqdOnYJlfS5sn9TXUkysXDZXivrlOb7ta4cffriPbVs88sgjPrb9V1/nzz33XJA78sgjfayvHbsfW07dvrF2snWIbaeXY/0g1mZVVeHtJ2sd8rD70cc87rjjgtwhhxzi41tuuSXIvfbaaz6211KsLdavX+/jNWvWBLnp06f72N6zLrzwQh+vXLkyyE2aNEm2RFn7TNZ9WLY/7bzzzj7ed999g9ycOXN8bK8lfX6nTp0a5JYuXepjOyfQYvdBe7zhw4f7WPdBkbAOts30+P74448HuXXr1qWWrVTXVkyfPn18PHDgwCD36quvZipLseN0sX2r2O2KFRuL82w7ZcqUIHfXXXf5+Nvf/naQO/fcc32sx6YtVWzOHGPbftiwYT621+Buu+3mY3uO+vXrFyzre9hTTz0V5O69914f2+tT1yPWD+y1tGzZMh8vWrQoyBU73vbu3dvH9jlL79O2vc7FxkY7HzrooINajUXC57W33noryOlj6HupSNgumyL2nBdr09h2WZ8dY9sVK8/xYvS51s/XIuH9M9YPYvsUKX48LnZMyFoWWyd9PPsOQT+b23E6ds1nrXueObu2KW2tj2n7ZOx5ohRzvmL3oZ/FRcL74D333BPkJk6c6GP7fBa7BnU59TO7iMjs2bN9vP322wc5PZ+3421WeeZKsXWHDBkSLI8dO9bHe++9d5DTzzP2nqnf0Vx00UVBLtaGWd8v2n6g+5qe24uIDB48ODV3zTXX+FjPAWw5u3XrFuROPvlkH995551Bbu3ata2WS+TD51fXY/ny5UGutra21fVs2ewx9HPQXnvtFeS+9rWv+fi6664LcpMnT/axbXs95uW5L+nt7D71/POOO+4IcpdeeqmP7Rxr/vz5qcfLOo+yZRk9enSwrOcdts/YMSFLWWx57DOgFnsvYsWul6zXkj1/sftn7HzG7gtZ70uxY7fHcysAAAAAAAAAAACA0uKXkgAAAAAAAAAAAAAAAAAAAAAAAADkwkdJAAAAAAAAAAAAAAAAAAAAAAAAAHKpyrtBRcUH3zE553ycJEnqNno9EZHm5uZW9yciUllZmZpbt26djzt37hzkOnXq5OOjjz46yD322GM+bmxsDHLV1dWp5W5oaPBxVVXYVE1NTT62ddf1i7Hb6brbNtP1PfLII4Pcgw8+6OP6+vrUcoqE9bDH32677Xy8ePHizNtlra89n7vssouPTzvttCB3zTXX+HjOnDmpx7PtpNvQllMvz58/P8hNnz49tdy67rY9dVmGDRsW5M4//3wfL1y4MMj9/ve/D5bnzZuXeoy0soiI9O/f38ff+MY3gtw222zj4//5n/9J3Wceug1t2+vlU089NcjtuOOOPv7FL34R5PR1Zs+ZPUZWervY+FPs/u22O++8c5A76KCDfHz55ZcHuVh99XK/fv2C3LJly3wc6yOx6zE2The7XUysfWN1yLPPWJ+xY87222/v409/+tNB7qqrrvLx7Nmzo8dIO16sbBs3bgxy+t5jx6O//vWvPj799NOD3NSpU31cV1eXWq4Y2y6x+hV77vMcP+s1abcbMWKEj08++eQgp/fz0EMPBbl33nnHx/p6FBEZNWqUj4844oggt2LFCh8/8sgjQU6fC9tm+lr+3Oc+F+QOOeQQH7/44otB7pZbbvFx9+7dg9zhhx/u46OOOirIXXnllT5+++23g1zW+3Ue9pwNGjTIxwMHDgxytq/H9lMKWeepWfeRZz+x7Up1XW3YsCFYfvLJJ3382c9+Nsjts88+Pn766adTy7a5FXvOYtvpueHuu+8e5PT885VXXgly//3f/+3jd999N8jp5w6RsN//13/9V5Dbd999fXzxxRcHuVWrVkkaXachQ4YEOX3u7b0gdp3H2ql3794+1nMOkXCstPu3Y7Om296O03p8+MMf/hDkFi1a5GP77Jb12Jui2P3Gtovl9HkpxzhtxZ7FY/Q98qyzzgpyuo/YsX7atGk+njJlSpDTz/d5FDs2x8T2kefc6vHg8ccfTz1GsXXI894la99q6x5c7LhSbK5Y+lnZ9tE1a9b4+IUXXghydj6YJta+lh6nbfvp56A8/SB2PJurqanx8Q477BDk9BzB9l99/epnEpFwLnzPPfcEOf3uw76z0O+o8vSDtHePImGb2ndXo0eP9rG9f+rngBkzZgQ5XW49bxIRWbt2rY/nzp0rWcXOmZ1LaPa86GV7X9Q5+x7m0Ucf9bEdf7VYv7PHi9UpNt/V5Vy6dGmQmzBhgo/1eyy7rn2mjl1L+nzaOuj7kkh4vu09LDYP0fMc+34j9p5gwIABPrbvkvT7a6vY+UIp3qtZsfc5Wd/HWbH+05GeVwAAAAAAAAAAAAC0jl9KAgAAAAAAAAAAAAAAAAAAAAAAAJALHyUBAAAAAAAAAAAAAAAAAAAAAAAAyIWPkgAAAAAAAAAAAAAAAAAAAAAAAADkUpV3A+ecj5MkSV1P56qrqzPvv7m52ceVlZVBrra21sc9e/YMcltvvbWPt9tuuyB38803ZzqeVVGR/s2Wztn66eWNGzcGuYaGhtTtdFls23br1s3H22+/fZD74x//mLpdVVX6Kdb7FBE54ogjfPzggw8GuaamplbLadnj62V7zs455xwf33bbbUFu9uzZmfZp6XLG2mLSpElBrq6uLnW7+vp6H9s+se222/r4/PPPD3KPP/64j5966qnU44mINDY2Shp9HWzYsCHILV682Me/+93vgtyPf/xjH++yyy5BbsqUKT7W17Q9nm5PkbBt7HbDhg3z8eGHHx7kOnXq5ONevXoFueXLl7d6bMuWRR/fnhe7rqbXtXWI9e1YOx177LFBbvr06T5evXp16vFtX9P73GmnnYLcSy+9VFQ5Y2O2XrZtGDtGVnYfWe8fMXY7W1+tb9++wfIXv/hFH+cZc2LHKLadYtf8tGnTfGz7wciRI1tdz7Jl1nUqxbnNSx8/69xBJLxPHnDAAUHu1FNP9fE//vGPIPfcc8/52N6HNdvv33rrLR/PmzcvyP3nf/6njxcuXBjk9D1FnyMRke9+97s+tmP/D37wAx8vWLAgyOm2sGPaiy++6GPbLt/73vd8/NOf/jTILV26NFiO9e2sY4edZ+j6r1ixIsjNmjUr9Xhp+xeJ35ey7sf2+9j9Jmt/jY2bxY5xbYndh/Uc4frrrw9yX/3qV308efLkILdmzZrU4+k62ePpc2/bU1+7eh4lEs6F7XmJ1S/r2GHLcuKJJ/r4wAMPDHJ6Dj116tQgF6v7+vXrg2V9r7/44ouD3C9+8Qsfn3feeUHu17/+davHEwnrVFNTE+T0Pcu2rxa7xu1ziG6bhx56KMjp6872+9h50efCPp9dd911Pl61alXqPqzY2LilKtd4kSb2jBszd+5cH+vzJxKOB4MHDw5y++yzj49PP/30IPfwww8Hy88884yP161bF+Syjs3tzV5L+npdtmxZkIvVoZj3LFa55nixsSRrWWNjep7963HF5vr37+/jcePGBbmLLrrIx7ad9DURe36JPQ/q510RkX79+vnYzr+y9t+s8zQRkR49egTLZ555po/Hjh0b5B599FEf2/FeP5vneVbW+7H3pWLvIVnf1dn59fjx432snxdERK655hof23cr+j3FmDFjgtyNN96YqSyx/mr16dMnWJ4xY0bqfjSb03Nv+47vySef9LFtp2Ln17GyaLF3SfZ60c82W221VXQ/aWzf0s/b9nllzpw5wfIZZ5zhY319iIisXLky0zFtOXXOHn/06NE+tvc6fU+x7avn0HkUO7+O3ZdiY7HeLnYdx8bbcryfAgAAAAAAAAAAAFBe/FISAAAAAAAAAAAAAAAAAAAAAAAAgFz4KAkAAAAAAAAAAAAAAAAAAAAAAABALnyUBAAAAAAAAAAAAAAAAAAAAAAAACCXqjwrO+ekouKD75iampp8XF1d/aF1W1uvLWn7FxFZtmyZj3fZZZcgt9122/n4+eefD3Jr165NPZ4+RmVlZZDTy7Ysupy27ieccIKPN27cGOTuvfdeHzc0NAS5qqoPToduPxGRzp07+3jDhg1Brra2VrLSxzj99NOD3MKFC3388ssvB7kkSXxs26mxsdHHtty6bcaNGxfkpk2b5uOJEyem7tOyx9di/Udbt25dsKzrFzveJz7xiSB3zjnn+Pjuu+8OcuPHj/dx7FyLhOW2dFvEyrlq1apgWfe1T33qU0FOt73dZ3Nzc2pOl9uW+bTTTvPx5MmTg9zYsWN9PHLkyCD33nvvpR5PL9vj6XLquC2x69rSx7TH6NSpk49Hjx4d5G644QYf23Mf2+dOO+3k46222irITZ8+3ce2nex1p+ljZF2vrXW1WJ/Ms64+Xp59WlnHuNdffz3Ixdop1r90WWNtlrXuIuGYvnjx4iC38847+3jJkiVBTo8B9nrRfd3WR68bq0P37t2DZX1vbatPZq2/LffRRx/t4/333z/IXX755T5etGhRkIvdl3RZbFvoZXtvfe6553w8aNCgIKfHtR/84AdB7pVXXvHxnXfeGeSy3r9tu9TX1/v4ySefDHLDhw/38fnnnx/kfvazn6Uew7ZF7L6v17Xj6KhRo3y8fPnyIDdv3rzU48eupdj1mXW8sG24KeNMln0UW848YtfrggULguX58+f7+KCDDgpy//znP31s26lbt24+3nXXXYPcscce6+OhQ4cGOd0vbB+97777fLxmzZpWyy+Sr810ue1YccQRR/j45z//eZCzY0faPmPzE5HwGlm9enWQu/LKK3187bXXBrmamhof22cG3Yb9+vULcuvXr08td+x+quukz62ISO/evX08derUIKfrF5svxPqkfpYRCetux5jY/CzreWlrbhibe5dbnvtwOcaOYuk5rX4ut+zc5dVXX/XxjjvuGOQOOOCAYPmSSy7x8f333x/k9H0469xMJPscM9b2seeQLl26BDn9nGmfz2LHyzr3jil23GxrjCuHWH2LbQv9fGafwfR4H6tvnucAPVbp+Y9IOObZ+ZB+XrHvPbK2i70vXHDBBcGyLvdFF10U5PQzUWyOZ3P6+HYMeOCBB3wcu/eUgz3eSy+95ONTTjklyPXv39/H9llq33339bF9Vly5cmXq8fI8A+o+ap/39bNVbDyyuR122MHHs2bNCnIrVqzwsX0HpfeZZ9wsdkxNO7ZIeL3Y96uazelrMNbvbFns2Kz7gn3XrPuTHVeyjg/6HImIHHbYYT6+7LLLgpw+RuwddbFjR56cFnuWyjNu6jrE+o+dC5biHgkAAAAAAAAAAACgvPilJAAAAAAAAAAAAAAAAAAAAAAAAAC58FESAAAAAAAAAAAAAAAAAAAAAAAAgFyqNmVj55yPkyQJcnq5srIyyDU3N6fuU+c6d+4c5CZPnuzjb3/720Fu+vTpPr7llltS99nU1JR67IqK9G+0dF3t8saNG4Pc4MGDfbxhw4Yg9/nPf97H//jHP1KPZ9uzpqbGx506dQpyVVUfnMaGhobUnIjIkUce6ePRo0cHuSuuuMLHsXayZdNtYc91//79fXzUUUcFuUsuucTHtk/oc2Fz+viNjY2pZbHnTNfJllMfz9ava9euPj7nnHOC3L333uvj8ePHBzldbnseYmx9dVlj1449ZzNmzPDx9ttvH+R0He12+nj2mtBtus022wS5/fff38cXX3xxkNt77719PHDgwCCn28bWT5fNnpfq6mophj6G7Qe2P8XO4dChQ33cs2fPIDd37lwf22tSt2GvXr2C3Oc+9zkfP/HEE0Guvr4+tdy2bdLYa0JvFxvjYtd8LJenLLqts+6jNbpfbLfddkHuN7/5TavHs8fM2p552GtJH8MeT/enPn36BLnzzjvPxwMGDAhyN9xwg4+LvdfFzufhhx8e5NavX+/jhx56KMjFxqqYcePGBcvHHXecj3/1q18Fufnz5xd1vNi51m1j22np0qU+rqurC3KnnXaajxcvXhzkbr/9dh/b8SCtXK2VLW1dW/c77rjDx2eddVaQs+OvLWua2H3YjsW77rqrj5988skgp+dExfaRYq/P2LiSta3zHL8c44gVa0M7N33++ed9fMIJJwS5xx57zMc777xzkPvCF77g41WrVgW5Z555xsd6Hi4SzlsPPvjgIKfvdX/961+DnL0Pa7F71qBBg3x89tlnB7nf//73Pl6yZEnqPvPcz+y6esy12y1cuLDVWCR81lm7dm2Q0/OOLl26BLk5c+b42F6DepyJjff77rtvsDx79mwfL1++PMgV2+91WV5++eUgp/vFXXfdFeRizxa6j8TKFat7HsWOzXY9fb3muQ9nLVs5xhxbd31d2zlsbW2tj+29Ttd92rRpQc4u6zHo1FNPDXJjxozx8d/+9rcgp+9nsfcSxY7psfFWPxNYsWepmFLNvXXOliXPfVhfh7E5Zuz4Vp57rxa7lvS4aedqsfrq+sXmavZ4PXr08LG99zzwwAOpZcnaJ+3x9PK5556bup2IyKWXXupj/Rxp143VN9ZmpRpzYu99tDzPsfoZ5aWXXgpy+t5jn1923313H19//fWxYgdiz7H2Ht23b18f23H03XffTT1GbO49YsQIH69YsSLIxZ4Z0tZrKxd711vs+wX9zu3VV18NcrFyF/ve0L7b0XOUT37yk0Fu4sSJPo6Nf3afur4HHHBAkNPzZjtnj81NY3Og2Hhf7HxXs/WLzdmznpfYO77YezQAAAAAAAAAAAAAHRO/lAQAAAAAAAAAAAAAAAAAAAAAAAAgFz5KAgAAAAAAAAAAAAAAAAAAAAAAAJALHyUBAAAAAAAAAAAAAAAAAAAAAAAAyKUqz8pJkkhTU5NfrqysTN9x1Qe7bmxsDHIVFenfQul9xraz+3jqqad8vGLFitTtnHNBrrm52cdJkgQ5vWxzDQ0NPu7cuXPq8ebOnRvkzjrrLB8/99xzQW7p0qWSZu3atT7u1q1bkOvSpUtqOQ899NBg+ZhjjvHx5ZdfHuRWrVrlY9tOMbH27du3b6vlFBGpq6tL3U73H32ORCTogzG2LfQxqqurg5w+hj3e2LFjfbxy5cogN378+NTj6XaxfTlWNtu3Y9dcbDvdn2699dbUfeq2bq1smm63U045Jcg98cQTPn7nnXeC3MyZM328xx57BLmHHnoo9Xia7SOxfhDrv7oNY+dBJKyvPYe9e/f2sR4PRETq6+tb3YdIeJ5OOOGEIKfHrhdeeCHI6fracuo62f5bLL0f2+9ifSSWi4ndF3R9bf1sW/Tr18/HNTU1Qc7eG7IeQ5ct1r6xunft2jVY7tmzp49t/znkkEN8bO8he+21l49nz54d5GL9vtjzore77777gtxJJ53kY9vW69evz3yM/v37+/j0008Pcr/73e98vGDBgsz7jNF1sm2mc7ZPdurUyce2vp/61Kd8/I1vfCPIbdiwwcd2vE07thXr97YOevx58803g5xuaxGRZcuWpR5DL9u20LkBAwYEuU984hM+/tWvfiVpYm0fa4ti+3meayDWvlnLmYc+xqbsU58nu5+33nrLx6eddlqQ+8pXvuLjQYMGBbn//d//9bEdc/TYFeujCxcuDHKXXHKJjw877LAgp+cEtt/Z+7Cmx01bzunTp2faR+xct7VujK7Hu+++G+Ts+K/pNrVzCf0sYNtel9uOOXq7448/Psj99re/TT2ebjc7/4qNmzqnn9VEwvFBz6dFRObMmZN6vNjctxzXZ579xNaNzXOKVYo62r680047+fjwww8Pcrq/Ll++PLUsdg6rnwvaeo6bMmWKj/Xzg4jI0Ucf7eMf/ehHQe7aa69N3S42d4uNm1n72pAhQ4LcvHnzfGyv8djYEStn7D6R9d63Kc8IWa/zWNnsdqW696Udzz6/9OrVy8f2mSC2nS63zen754wZM4Lcyy+/3Oo+ROLvHmLnbNttt/XxDjvsEOR++MMfBst6Pljsva9U98GYrMfIc4/W44y9v5xxxhk+tvMTPX/Q798se2/V17ktZ6zt9bOFiMi6det8nGdOq58z9Ts2W7bY/TTPs4Y+vt1n7P2Jbje73SuvvOJj+xwZG7tiOX08ew3asXny5Mk+1u9MRURGjBjh42nTpgU5PSbYY+jc6NGjg9wNN9yQup1m6xd7T5q1LWzd9fm087+tt97ax9///veDnH5Onz9/fuqxY2OFzemyxcZNAAAAAAAAAAAAAB0Tv5QEAAAAAAAAAAAAAAAAAAAAAAAAIBc+SgIAAAAAAAAAAAAAAAAAAAAAAACQS1XeDZxzPm5ubk5dr6Lig++d9DZWkiSpy126dAlyZ5xxho8nTZoU5HbbbTcfP/LII0Fuw4YNqcfT5bT10Tm7XVVVVavriYg8//zzPt5rr72C3Pz583286667BrnHHnvMx9XV1UFu/fr1Pl65cmWQO/LII308dOjQIDd48OBg+be//a2PFy5cGORi51bnbH0rKytbXU9EZNmyZT7u2rVrkNNlXbduXZBrbGyUNLpstiwxet2GhoYg17lzZx/r+oiI7L///j5+8cUXg5yub1NTU5Cz+0nbzpbN7kf3Nbudbgu7naavAZGwP9u2iPX7gQMH+vgTn/hEkPvhD3/o440bNwa5e++918dnnnlmkOvWrVtqOWPnV7ev7a+6zfr165d6vKVLlwa51atXB8u6bWpqaoLc2rVrfWyvV102XRYRkbFjx/p4xx13DHJXXnmlj2PXoD0v+tzHxttYv7P71GJjvd2n3k+xuZi2tuvUqZOPY/3H5mLtVsx69hj19fVBbtiwYT7eY489gpzuh7Nnzw5y+r5ox6O0Y4uE7ZSnb2l2jJk7d66PdbuLiNTV1QXL+pi2DQ899FAfv/3220Hurbfe8rEdq2L3pVg9sp5De11379691VhE5J133vGxHVf0eBC7t8XYcSQ23uu2njhxYpCzdco6dtj21Pv54he/GOT0HEXPAVrbjxabj2XdRx6x/qPbpVTHi4mNjVm3s8ux8W/FihXBsp5D//SnPw1yy5cv97Htd7HzpHN2TvDnP//Zx7/5zW+C3OTJk328ePHi1P3bZ4SDDjrIxzfeeGOQ09dd7B4SGxuLvWeJiPTu3dvHL7/8cpBbs2ZN6jE0/RxgyxobV+w+jz766NR96mcUO95mvX/HnqVqa2uD3P333+/j7373u0HuwgsvTC3nR92m9LWs9Hk56aSTgtx+++3nY3stzZw508e238Wulzx0fe3Y8c9//tPHdgzQY9f3v//9IKfvy7F546hRo4Kcvvfa7XR/HjFiRJDT7WTF5lx57sPFyHM8+xwbm8fF9qmPGdtnW2VNY/fx7rvv+tieh5EjR/pYj3ci4Zhn75/6GGeffXaQ69Gjh4+vv/76IKevkdg4Haurnf+NGTPGx1OnTg1y9h1RbK6Ydd4RO595+lNMrP7F9gPd1+bNmxfkevbs6eNx48YFueuuu67kZbHtot9FxN492OPFrsE333zTx/q5SuTDz2ha1nmjFXt+0LnYc4cdD3TOju/6ObrY9yD2vNhlff3YPqPfe82YMSPIxd6n6HeM+ryLfPj+libP9RGrv24ne671OGP7i37XbHOrVq3ysT3Xui/bcUzn8tz3NmVuAQAAAAAAAAAAAKB98EtJAAAAAAAAAAAAAAAAAAAAAAAAAHLhoyQAAAAAAAAAAAAAAAAAAAAAAAAAufBREgAAAAAAAAAAAAAAAAAAAAAAAIBcqvKs7JyT6upqv9zc3OzjJEmCdZuamnxcWVkZ5CoqPvgWyjkX5Hr27Onjc889N8itXr3ax3fccUeQ+8lPfuLjo48+Osjdd999rZZLRKShoSG1LLZOWlXVB03X2NgY5F5++WUfDx48OMgdd9xxPv7Xv/4V5Dp37uzjHj16BLnevXv7eMOGDUHuvPPO8/Hdd98d5C699NJgWbehLbeuvz7PIh9utzS2zXRZH3300SD3jW98w8d33nlnkJswYYKPN27cGORsf0orp+5nls3pfmD3r3O77LJLkHv66aczlStWFpHwWrL70XWy7avPmd1On197fL1drGz2mjjyyCN9/NRTTwW55cuXp243e/ZsH0+ZMiXIfepTn0rdp9atW7dgedddd/XxXnvtFeS222671P0sW7bMx/ZasuPK/PnzfazPkYjIypUrfWyvl6222iq1LIcddpiPr7766iBXW1vrY3vN6fMbux5tOe250GJjXGy7WE73J7v/WF/T5bbr6f3ErgERkfXr1/u4pqYmdV3bTrFyx66X2H1QL9tyTp482cdz584NcgcffLCPTzvttCB31VVX+biuri7I6T5ix/esYvfBWM5eS7Fzb8/LIYcc4uM//vGPQU6PvzF5+nLsfOrcqFGjgpzuWzvssEOQ0+fClkWfizz3CX2d23bQ5bT1031Sl7m1dfVyrA1tufW4ttNOOwW5GTNmpB5f78deg3Y5rZxWrNy6TWNjY9Y5TlvlipUltl0esWsytt/Ro0f72M5lFixY4ONVq1YFudgcJK1ctiy2fd9++20fjx8/PsgdddRRPr7pppuCnD6H/fr1C3L6+WHq1Kmp28W0NVfT9HOAiEjXrl19bMeO+vp6Hz/33HNBLnYN6vLYubBuU/38YHOf/OQng5x+Dvnxj38c5PQxsvZlKzYeWXrOp+dNIiKf//znfWyfEfT9JnbfLZVi74vlOF6e7WL70HNvPZ8WCZ8d7XgQO17WMbytuWHW9wv6+V5EZM899/TxCSecEORuueUWH/fq1SvInXPOOT4+/PDDg9ycOXN8bK95PZexz/v33HOPj1988cUgp5/FrWLvDVnvdXnmqbH9FHtPjr17yHO82DH0eGvvL8ccc4yPJ02aFOT0M1jfvn2D3Fe/+lUfd+rUKchdc801Po7N1WJzPJuzx9D0/UU/X7cmdu6z5mLzjDxjQDlkLYs9L/o9k73m9fO9PS+6/9p5jR4fbM6WrU+fPj62z4Cx+Wis3+v7qR7fRUSOPfZYH//73/8Ocvp6sddn1ueCWFvYcVP37e7duwe5QYMG+djOCQYMGODjRx55JMjpc2bbWtfJ5mJzPvtO6tBDD/Vxnvekel5lx35dX/3OyZa12OvM1k8v23PdpUsXH9s5gV73+eefD3K67rEx3F6DseslphxzPAAAAAAAAAAAAAClxS8lAQAAAAAAAAAAAAAAAAAAAAAAAMiFj5IAAAAAAAAAAAAAAAAAAAAAAAAA5FKVd4Pm5uYPNq76YPOePXsG661cudLHFRXp3z5169YtWP7a177m440bNwa5u+66y8d1dXVB7qqrrvLxRRddFOQaGhp8/NBDDwW5pqYmHzvngpyua2VlZep2NTU1QW7IkCE+3mWXXYLcokWLfDx27Nggp5cHDx4c5NatW+fjSZMmBblVq1b5eLfddgtyxx13XLB83333+Xj16tVBLkkSH9u20GxOn1/dJ6w777wzWJ42bZqPTz/99CCn+8Hjjz8e5F566SUf2zrotqivrw9yjY2NqWXTdbf9VZf7d7/7XZDbeuutfXz//fcHuaVLl/r4vffeC3L2GLHj6/7bpUuX1isgYd1FRDZs2JC6btqxRcK+bq/rMWPG+NheZ7q99fUhEl4jTz31VJC75JJLfLzffvsFudraWh/bvq3b9Nlnnw1y+pytWLEiyOn6ffaznw1y//Vf/xUs/+xnP/Ox7U/6mrT98MILL/SxHkdERK644gofL1iwQNLYMSerWN+yYv2u2PFAb2frrte12+nl2HZt0fceW/cdd9zRx2+88UaQ08eMXRN2HNHtZsut92OviaFDh/r4jDPOCHL6Ol+8eHGQmz17dmo5Y/ezWD/Q68budXY80PvU41RbunbtGiz36dPHx/oemYftv/ZcpLHr6fMyYsSIIPfMM8/4eOTIkZnLErsvxvqIPi+xeVTs+PbY22+/fbCsx9j58+cHOb1t7969g9yXv/xlH8+ZMyfI6bHRXi+xMSd2LcXodrL9PLaf2BgXu16KWa9U27VFX7/2OjvrrLN8fPvttwe5z3/+8z6urq4OcnouXuy4EhsbH3vssSB37rnn+vjWW28NcnqcsfXTcx5bB70cG6ti81t7DXzmM58Jlj/96U/72I6V2o9+9KPUY8TYeUa/fv18bM+DLuvZZ58d5K6//nof22s+do/W8vRf3Sftdnpe9Y9//CPI6XLr8UZE5Oabb/bx+vXriy5bVlnnUXnE2jfP+Je1LPZ5+6CDDvLxbbfdFuR0X4vN1fLUPbZunrmiZt8F6OewU089Ncjpa/Koo44Kco8++qiP9fUh8uF3EZq+n9rrWu/TjjlZ29Dmso4Vts30vTzPXM0eP1burM8MxW4Xq5OdZ+h1x48fH+ROO+00H59yyilBbsqUKa2uZ/fz73//O8jpOY+dx8We5ex9StP7seddj3k777xzkMszB9LynDO9rq2vlmdeXuw1kbaPtrZ7++23fdy3b98gp5/B7P0lNp/X11Zbz/B6/mKfe3R/tm2m92vbXs/n//SnPwW58847z8f6XY5I+G50+vTpQS52fep3O/b+ott01KhRQa5Tp04+7tWrV5DTzxP2+Vc/wy9ZsiTIPffcc5JGH8/2A3s+dV4/b4uIDB8+3Mf9+/cPcvqdX+w52p5rfS4mT54c5HR/suXUfSSWs9egPocDBgwIckcccYSP7dxQv++0fTJ2PM2WM/aePeuzGwAAAAAAAAAAAICOiV9KAgAAAAAAAAAAAAAAAAAAAAAAAJALHyUBAAAAAAAAAAAAAAAAAAAAAAAAyIWPkgAAAAAAAAAAAAAAAAAAAAAAAADkUpV3gyRJWv17Y2NjsOyc83F1dXWQ22GHHXx8wgknBLlVq1b5+C9/+UuQq6ur83FTU1OQe+edd3x85ZVXBrkLL7zQx/vss0+QmzBhgo9nz54d5JYsWeLjzp07B7mRI0f6eNy4cUFu11139fGDDz4Y5K677jof2zarqvrgdNg2W7NmTWqusrLSx7vsskuQs2X7+c9/7uMpU6YEuaVLl/p4/fr1QW7dunWpubVr17a6nkhYR5vTbf/WW28FucMPP9zHn/70p4PcMccc42PbH999910fv/fee0Fu3rx5Pm5ubg5yun11HxQJ+90vfvGL1HJ+/vOfD3L19fWp+9RtbTU0NKTmLH2d9e7dO8i9+OKLrcYiYbvpfVhjxowJlnX7rl69OnU73Zcte16uuOIKH++9995BTveZxx9/PMhNmzbNx7Yf2Gsrzfjx44PlQw45JFjeeuutfazHA5HwutPtIiJy9NFH+/iss84Kcnpd2/Z6XLN9VC9XVITfk+plu50upx039fFtWXSb2vbV69rj6XVtOfV2ulwi8XMW66+x/vv3v/89WP7Od77j4wsuuCDILVu2zMex/mvH39jx+/Xr52M9bomI7Ljjjj7+17/+FeReeeUVH59//vlBLjZ26LKk3avtepbtI5q9v8yZMyfTPm157Bin+1CfPn2C3MKFC1vdh122udg1oelrXCQc8+yYo9tG379ERI466igfb7/99kFuxowZqeWMtVus3Ho721/79+/v4xNPPDHI7bvvvsHyY4895uNbb701yOm+fuaZZwa5xYsX+7hr165BTs8t7HWu2zDW12Ji+8wjdo1osbExlivXPtu61t5nr1c9xr3++utB7nOf+5yP7XxXz/li43RsvLf0GGDn3t26dfOxPdd6uxUrVgQ5ve7YsWODnO7nnTp1CnK6n/fq1SvI7bnnnj4+7LDDgpwem0RE/vCHP/jYjtOXXXZZq/sUCceZWH3tHGSPPfbwcY8ePYLcl770JR/b9p04caKP9TzVytqXReJzAr0c68sbN24McrfffruPv/nNbwa5M844w8d//vOfg1ysTrE+mae+MVnvw6U6XtaybLPNNkFu1KhRPp4/f36QK3aMa4/21ey1rOtYW1sb5PR9wj7HvvHGGz62/dCOa5q+99pn6jfffNPHsT5pr3nN3tuyzsstPW5nvX+0JjYniYk9F+h9Flu2WN+y7x7uvPNOH//qV78KcnqsvP7664Pcs88+62PbDrpO9nzGxr/YPFUv236g7xn6GVpEZPTo0cHyrFmzUvcTK5uuR6xsMbH6xubzlu4Xtv/Erom0Y4uEz2f6HZBIOIe284zYHCjP+KfHp6FDhwY5fZ5i7zdsnfR2+vlMJHx/Zecg+j2bfUbQ80F7Lelxzb7bWbBggY8nT54c5HSb2u10/eyzxcknn+xjO8eKjdP6uXL58uWp64mE53DDhg1BTh9z4MCBQU7XI3Z9PPzww8Gyfi/76quvBjk9V7P7jM259Jy2pqYmyA0ePNjHBxxwQJDT9zA9boiE7+vtO4vYOKLZnO6v9trJ+vwLAAAAAAAAAAAAoGPil5IAAAAAAAAAAAAAAAAAAAAAAAAA5MJHSQAAAAAAAAAAAAAAAAAAAAAAAAByqcq7QUXFB98x1dTU+PhTn/pUsF5dXZ2PDz744CDXq1cvH//9738Pcm+88YaP6+vrg5xzzseVlZVBTi/PmTMnyP30pz/18Z577hnk9ttvPx8fe+yxqcezZdHt8NxzzwW5G2+80cfLli0Lck1NTZJGt1mSJKllaW5uDnJ6+dVXXw1yU6ZMCZa7du3q42HDhgW56urqVmOrsbExWF69enWrsciH2y2N3e6uu+7y8d133x3kdFt069YtyOn69e3bN8jpcq9cuTLIrVmzptX17LI9L9OnT0/N6T4ZO+8iYZ0aGhqCnN7WHkPbeuutg+X+/funrqv3WVUVDgO63AceeGCQmzRpUqvriYT9MJaz9Zs3b56P586dm1pme150uW37xnLaxo0bg2V7ve61114+njhxYpA78cQTfazHA5Gwnb75zW8GuT/84Q8+1v1HJGwb206aPZ5uX9tHdE73M7uuHVdix9Pb2VzaenbZ1k+XLbbPWN3t8lNPPRXktt12Wx9ffvnlQe6///u/fTxjxozUfdq+PWLECB8fc8wxQW7IkCE+tmPzdddd5+MlS5YEuc6dO/t4/fr1QW7t2rU+jrWT7fe67W0/iN1bhw4d6uN+/foFufHjx/vYnofYMWydHn30UR9//etfD3KXXHKJj+19QtfJjg+x+upcp06dgtwzzzzjY93Wdj92rLj33nt9fMEFF6Tmnn766SBXW1vr49h1pudbIiIDBw708ZgxY4LcPvvs4+PJkycHufvuuy9Y7tmzZ6v7FBE5+uijfTxy5Mggd8UVV/j4q1/9apDTbWP7ha5T7D4Ru9fFxqpSsX0mLdfWeFTqY1u2nXR5xo0bF+R0P9RzHhGRdevW+bhPnz5Bbvny5UWVJXYNanY80NeE7p8iIhs2bPCxHQ/+/Oc/+/i73/1ukNPPKHquLSIyYMAAH++0005B7rXXXvPxn/70pyBnnzVifVbfU/bYY48gZ8cETc9l7H1im2228fG3vvWtIKfb9IYbbghyel5u5/qxeUeMvk/Yef/SpUt9bK8XfZ+yOd0n9f1SROSMM87wsR1//vd//ze1LMXOc2Ln1sp674kpx7hirzN9T7Fjcex6jdVBbxd7jo2NW7F9ioTXhL2WTjrpJB9feeWVQU6PHa+//nrq8fLMP0eNGuVje31qtn1jzyVZx80828X6cuwYefph1nPf1lwxK102PWcWCd8F7L333kFu33339fHDDz8c5PTcyZZTn8PYc0ds7m2ft2PPYLp+NvfWW2/5+P777w9yP/nJT4Llyy67zMdvvvlm6jGsrNe5rVPWdxaxfcbKkrUPxvZhl2276HnAzJkzM+8zDz23sO8C9Hmx/Unfs2PvPmw76fnSs88+G+T0sp0T6Gcr+5yln59ic/1YHfLM9QcPHuzjRYsWBTndZnYeo5/TrVWrVgXLsfLod0Z2bhp7HtVlmz17dpC75ZZbfGznMgcddJCP9XslkbD+erwTEdlxxx19bN+F6vuUfS+hxxU7purtFi5cGORi44Fus9g1aO+RsftZezwDAgAAAAAAAAAAANg0/FISAAAAAAAAAAAAAAAAAAAAAAAAgFz4KAkAAAAAAAAAAAAAAAAAAAAAAABALnyUBAAAAAAAAAAAAAAAAAAAAAAAACCXqjwrO+eksrLSLzc0NPi4qakpWHfUqFE+fvrpp4PcxIkTfdzY2Bjk9HJVVVVqrqIi/J5KH1+XUURkxYoVPn7ssceC3COPPOLjLl26BLnq6mofr1+/PrUsViyny93c3BzkdH2TJAlytn01XV+7z3Xr1gXLGzZs8PHatWuD3MaNG31s6+CcSz2+ztlyF5vT9a2vr0/dzp4XnVu8eHGQ0+1r21O3W6yctix6Xdtm+rzY49k+qsWuCbud7k+6n4uILF261Me2X3Tu3NnHup/bY9j6jh8/PrWcertYf9Hjhki8b9sxQNN1su2ij9GpU6cgp8+ZPdf3339/sPzlL3/ZxyNGjAhyr732mo/1mCYi0qNHDx+fcsopQe473/mOj2fNmhXknnjiCR8vX748yNm+runzZPtBbJzW/ce2RYw+v/Yc6fNij6e3s31Eb2dzsXHT3gv0cl1dXZC7+eabfXzSSScFuS996Us+tm2hl21/0u39xhtvBLmbbrrJx++9916Q09ePPV7//v19XFNTE+R0m8ba19LHsG2m2fvgnnvu6eNnnnkmyGUdU+3xbe6ee+7x8V577RXkrrjiCh//9re/DXLz5s1rdf/2GHZ80G2/aNGi1O1i8wybe/LJJ328cOHCIPfFL37RxwceeGCQ0+O0vZa6du3q4+7duwe5NWvW+FiPyyIiv/zlL32s7/kiIttuu22wfMEFF/h41113DXKzZ8/28a9//esgt3r1ah9369YtyOm+HutrsT5i+3Js3I71+7R92O3yjEexvhyb16St11Y57bJuU9u++t6j7/MiInPmzPGx7Wv6PhG7t8fmt7acem5ht9PltnMCPaYOHjw4yOnrxXr22Wd9rOsqIrLPPvv42I4HkyZN8vEf//jHIKevAcvWKXaf0PPR7bffPsjpvmDLFptj6jmJne9ee+21Po49v8T6VuwZzPZf3dfOPffcIKfPxbRp04Lc66+/7mM739THs+PYXXfd5ePzzjsvyOl7+x133BHksj7LiMTHo9icpNh96uXYPmPHiB1vyZIlQW7BggU+PvLII4PcAw884GN7XtKObY8Xy9l+Hnu22WqrrYLcYYcd5uP99tsvyF133XU+njx5cpDT/entt98OcrF7u2bPi56fzZ8/P8jFxris13ysv+Z5js2qre1i7xs024ZZr5HYuKLnQyLhvGPHHXcMcgcccICPbf/9+9//7mN7znR/+sY3vhHkhg8f7mP7rKjnY7FzFmsH2w/0+GvbUx/j1ltvDXL2vc/FF1/s49tvvz3IzZgxw8d6Pi0S9tnYNRHrM7FneCv2LkCz7Zu1T9r50LBhw3z84osvBrnddtvNx/YZzF7LaWWJrWfF1o2No7Z9Y7ms8139LrCtssSe93VbxMpi+4++Xm2/+9e//uVje83reZy9lvT7lM985jNB7vnnnw+Wa2trfWzncdtss42P9XxTJP4OV7Nt+Pjjj/vYzo/08+IhhxwS5HTZ7BxPPxM+99xzQU4/n8b6nT1n+tli2bJlQU7P9W3d9XVnjxd7Roi9QwUAAAAAAAAAAADQ8fFLSQAAAAAAAAAAAAAAAAAAAAAAAABy4aMkAAAAAAAAAAAAAAAAAAAAAAAAALlU5Vk5SRJpamryyzp++umng3Wrq6t93NjY+KH9pBao6oMiVVZWpuZ0LCKyceNGH3fq1CnI6eM3NzcHOeecj3V9RETWrFnj44qK8PstXYfOnTsHOb2urbvdT1pZYtvF2kW3g4hITU1NsGz3m3aMLl26BDndNrYN9Xa6DiJhO9XX1wc53UdiudjxLL2u7Qdp5RIJ29Tuf/369a2WSyRsF5uLsedQ198eX9cj1n9tfRsaGlKPF6OP8eqrrwY5fU3YNoyVU7eN7aN6u1j/tNe8bid77er62j6ZVi4RkbfffjtY/vnPf95qOW1Z6+rqUnM33XRTkBs4cKCPx4wZE+SOOOKI1LKtWrXKxwsXLgxy06dP9/HatWuDXGyMs+dJ0+1rz7Veju3D9rvYPvW69lxrtg72/OoxL3Zebr/99iD3z3/+08fdunULcvrc19bWBrl169b52I5jum1sfWN17N69e6v7EIlf87FrIlYWvZ/9998/yL311ls+Xr16dZDTdbDjlm2LWH31fi+++OIgd+aZZ/r4iiuuCHKzZ8/28cyZM4PckiVLfDxnzpwgt3z58tSy6HLb/qPHrth9SF+PIiKXXHKJj3v16hXkdH/VY4NI2F91fURE3nvvPR/bPhKbY82aNStYvuCCC3xs5zIrVqxIPYY+n7addNvYdtLXue2jWuw6j93PYm1h+6Be1+Zi9yK9na2fzsXmBHbciu3T0terPdc77rijj/XYJBKf5+hl2xZ6O1snXW4957Db2XMWG6d1H+3Tp0+Q02OjPUe6Td99990g98ADD/g46xzS7tPWPXYO7XnRcydLX3e2bLr+3/ve94KcHseWLVsW5PQ4Zu9nehyz15k+vj2f+rnAtv0777zj46uvvjrI7bbbbj4+5phjgtxpp53m48WLFwe5N954w8d2DNft+8orrwS5k046ycf/+te/gpw9D7HnF31ebFtosXEl9uxo6Ta114tetvuIHS82Vv7tb3/z8eWXXx7k9ttvPx/fddddQU7Pk+18Pus8rl+/fsHyDjvsECzrPjN69Oggp/vab37zmyCnr3vbFro89lzrazs2pttn4+HDh/vYzkFiY2psXMl6f4n1u9h1bemyxPqWXdfuU+diz132GLG22H333X188sknBzl9Ll577bUg9/jjj/vYzsf08W0dJkyY4GM759Jj16WXXhrkdJ/U+xAJxyc7L46NB7H21NvZNnvkkUeC5SlTpvj4+OOPD3Jf+tKXfKzHWxGRRYsW+Xju3LlBbunSpT62z5x6jLXjpu6Xti2yvqewfTv2vkgvb7XVVkFu2LBhPrbP1Los9prXcxd7/vR4qOcqrZVbs+c3dv1osXctWZ/vRcKxJNa+9pzpde3zg2bvE3qf9rzr+6B9h/nwww/72NZB31NsHfQzpn6XIiLy/e9/P1jW17LtM2nriYj07NkzdV19HdhzG3uWevDBB32s6y6S/V2H1aNHDx/H3p0NGjQodZ92zOnbt2/qsYudD23YsMHHdt4W2w4AAAAAAAAAAABAx8AvJQEAAAAAAAAAAAAAAAAAAAAAAADIhY+SAAAAAAAAAAAAAAAAAAAAAAAAAOTCR0kAAAAAAAAAAAAAAAAAAAAAAAAAcnFJkmReubKyMunSpYtfrqio0LnU7fR6dt2qqqogN3r0aB/X1NSk7qehoSHI6XXt8fS6zrkgV11dnbpP3TZ6PRGR5uZmH9u66+Nv3LgxdTtbFr2fpqam1LLY+tk21BobG1NzsfNi2yJ2PF2n2DFsWXSb1tfXp5bFljN2PH2M2LmO7cOe67q6utR96rLZ7fQ5tNvZ606Xp1OnTqn7sedFX492n/qYNqf3ac+n3m6rrbYKcitWrJA0up3stRurQ2zs0O0Sa1+rc+fOPrZ118u27rav6XXtOUxbr611Y9e5rqPdpy6bLbe+fuyx9RgUa0O7z9gYEOtbOmevs1gd9LUbG9MsO67Y6yft+FZszNFtGNuHpesfGzdtmYcOHerjMWPGBLkHHnjAx+vWrQtyej+xcdO22TbbbNPqsUVEXnjhBR/bc637k+13sevc7kdfr7GxcuDAgUFu11139fHgwYMlje33seMtXLjQx9OnTw9y7777ro/tPUu3qW1ffe5tP9DXoB0Li82lrSfy4bbX5bHr6v3GjjFq1KhgecmSJT62fVS3t+0juo/a/quXbU7v08659Lr6fmnXtX1El822i76/2XaJjcW6LDanl2Ntbfdj+5O+fm19p06d6uPu3bsHuXHjxvn4iSeeCHJr165NLbcuq70m9HVmr4nYeDBkyBBJo/uWnWfocxa7Xuz5jF2fsTmAPYbO2/rqZxs7jj3//PM+tveCAw44oNWyiIjMnj3bxzvvvHOQe/LJJyWNrr+9lnQudv+2ZdFtr8+7SNimtg379evn4wEDBgS5rl27+rhXr16pObvPuXPn+vi1114LcrH7VOxZLjaHtvXV596OK2n7sGxZdDljc/bYfMieT73dyJEjg5yed9j66fvge++9F+T0GGDrp58n+vTpk1pOEZHa2lofT5s2LcitWrXKx7a+sbm3zsXudba+sfnfXnvtlVrONWvWtLp/kXi/i4nNjWPPtTGxeZwd02N0m9qxOfbMoNvCbhebJ2ux69Neg7ExXdfX3j+1Hj16BMsjRozwsT2fejzSfVck/n5Kn2t7HvS6bT0v6f5r7/vdunXzsX4OsMeMzY9s2y9dujS1LLr+dv6X9RnQXruxeWPs3j5s2DAfr169Osjp+80777wT5GJz2Fg57bLuz7Hn6NgzZ+weHSuP7aOxd5P6mtywYUPq8ew+Y+8f8zzjZ6X3aY8dm5/oa0BEpHfv3j62bbF8+XIf22s59o469t4n9h5P78fWSW8X64dW7D6hy6nbQURk66239rF9NtZls+O0fjcYG8Nt/V566SUf/7//9/+i6+pjAAAAAAAAAAAAAGh3ryRJsrf9I7+UBAAAAAAAAAAAAAAAAAAAAAAAACAXPkoCAAAAAAAAAAAAAAAAAAAAAAAAkEtV3g2SJPFxc3Nz6noVFR9876S3scvV1dVB7uyzz/Zx9+7dU7dzzqXm9LHbKmdNTY2PN2zYEOT0fhoaGlL3Ycuit7O5xsbG1JwWK7NtM71P29Z2XV2PpqamIFdVVdVqbLfLU269bI/XuXNnH9u21+W2ZdH7tOdF52xZ9H5srr6+PvV4sfrqdW2/s/WNqaur83GnTp1S92P3WVlZ2WosEtbJ5nSddP+xbFvosun9i4R1sG0RGw9i7RsT68u6nLFrIJazbB/t0qWLj/OMObq+sT4S60/2fMb6iC6L7Vv6HNrzotvCbqfrsHHjxiCny2bLotvbtpFeN097Wrqstk6xsUMfM9YvYuW2fTlWJ71Pez579+7t4/Xr1we5fv36tbqePUZsrND3PXuMWbNmpR7P1l3Xt63rWG9r19W5WN+ura0Ncs8995yP7bnW29n66rHKjnF6PIy1YYwdU/V+bC42/up2sm2v6xtrs9h8wdLtIvLhPqvp/rt06dLU7WLXS2y+YO8vuh72XOtzaOsQG3N0+8bmPHn6gV43zzxK7zM2b7RsWfR9yY5xM2fO9PHAgQODnL7uJ0yYkPn4sT6a9T5hz6c+vu2Del27nRbbztZHl8Wes9j4buur92v3s2rVKh/bttd9dOTIkUHuN7/5jY/fe++9IBd7ltLHmD17dpCLzSWyXoO2Ley6aWyf1P3Vnhd9vNjcLDb+xcYfkfDeYI+h17X10/3L3l9029g+GruWdFnt+dTjWuz5NzY2xu4hsefYWJmt2PgTez6LjYf2nMXGnFhZs96XYu07YMCAIHfGGWf4+Kabbgpyy5cvTy1LbPzTfS02VuWh65fnGSy2biwXuyZjx9DvBUTizyi6b9v7ix1/08TmfzGxe4+l28WOI/r4sXcyseeV2LsOm4+9j8tzr9Xrxp7XbH1j126MXjfP+wVNj00iIqeddpqPX3755SA3duxYH99yyy1BLva+KDbHiz3T27bQ47+d08aus9h5SduHSNhuth/G+oEuZ2y7to6v6TEgNq+w9dNjgD0vsevM0m2a9X4tkv1dVrH7jF2fsXmr3S7rc/see+wRLH/yk5/08Y033hjkYs/NsXdesbEpVrY+ffoEy4sXL05dFwAAAAAAAAAAAMDmwS8lAQAAAAAAAAAAAAAAAAAAAAAAAMiFj5IAAAAAAAAAAAAAAAAAAAAAAAAA5MJHSQAAAAAAAAAAAAAAAAAAAAAAAABycUmSZF/ZuWUiMq98xQEAAAAAAAAAAAAAAAAAAAAAAADQgQxPkmQr+8dcHyUBAAAAAAAAAAAAAAAAAAAAAAAAQMXmLgAAAAAAAAAAAAAAAAAAAAAAAACALQsfJQEAAAAAAAAAAAAAAAAAAAAAAADIhY+SAAAAAAAAAAAAAAAAAAAAAAAAAOTCR0kAAAAAAAAAAAAAAAAAAAAAAAAAcuGjJAAAAAAAAAAAAAAAAAAAAAAAAAC58FESAAAAAAAAAAAAAAAAAAAAAAAAgFz4KAkAAAAAAAAAAAAAAAAAAAAAAABALnyUBAAAAAAAAAAAAAAAAAAAAAAAACAXPkoCAAAAAAAAAAAAAAAAAAAAAAAAkMv/B5i1aOn5Kk2YAAAAAElFTkSuQmCC\n", - "text/plain": [ - "<Figure size 4320x1440 with 1 Axes>" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "plt.figure(figsize=(60, 20))\n", - "plt.title(sentence)\n", - "plt.imshow(data.squeeze(0).numpy(), cmap='gray')\n", - "plt.xticks([])\n", - "plt.yticks([])\n", - "plt.show()" - ] - }, - { - "cell_type": "code", - "execution_count": 26, - "metadata": { - "scrolled": true - }, - "outputs": [ - { - "data": { - "text/plain": [ - "('becane gool big alls at boasty<eos>', 0.31098294258117676)" - ] - }, - "execution_count": 26, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "model.predict_on_image(data)" - ] - }, - { - "cell_type": "code", - "execution_count": 36, - "metadata": {}, - "outputs": [], - "source": [ - "data, target = dataset[0]\n", - "sentence = convert_y_label_to_string(target, dataset) " - ] - }, - { - "cell_type": "code", - "execution_count": 37, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "([], [])" - ] - }, - "execution_count": 37, - "metadata": {}, - "output_type": "execute_result" - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAABG0AAABCCAYAAADt2ys3AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAABIzklEQVR4nO29eZRcV3Xv/zk1dfU8qOeWWmrNrcHWiOVRsrFsYvNiDCHMYMgA/IITlvNLXiDv9wJJSCB5ITxeGJ6DDQuDbUwIGGyDIyPLkyQktSW1rLFbaqnnearq7qqu6r6/P6rO5dTtW9XdGoxs9metXl11hzPdc++q/b1776Msy0IQBEEQBEEQBEEQBEG4uvD8phsgCIIgCIIgCIIgCIIgzEREG0EQBEEQBEEQBEEQhKsQEW0EQRAEQRAEQRAEQRCuQkS0EQRBEARBEARBEARBuAoR0UYQBEEQBEEQBEEQBOEqREQbQRAEQRAEQRAEQRCEqxARbQRBEARBEARBEARBEK5CRLQRBEEwUErdqJRqUkqFlVLveJ3rXqKUspRSvjT7P6uU+tZFlHteKXX7pbdwTnV9Ryn192n2ZSulfqaUGlFK/fD1aM/lRim1QynVPsdj71NKvXyl2+Soc87tEwRBEARBEK5+RLQRBOGy4CYM/CaM1svA3wL/ZllWnmVZP/lNN8bEsqx/sCzrD3/T7bgEfg+oABZYlvXuiy1EKfU5pdT3Ll+zBEEQBEEQBOHqREQbQXgToJSqeCOWPR+UUt7XqarFwPE0bVBKqTf1czOdl89lYjFwxrKs+BWsQ7hIrvS9rpQqU0qpK1mHIAiCIAjCm403tfEhCG9mlFJFSqlPKqUOAN8xtv93pVSHUiqklDqtlHprcnuWUuorSqnO5N9XlFJZyX2lSqmnlFLDSqlBpdRLhjjxHaXUAaXUJ5RSRZfY5mql1I+UUn1KqRal1J9mOPY7SqlvKKWeUUqNAbcqpeqVUnuS7TyulPpd4/i7lFInkv3uUEr9v8a+tyuljiTP26uUuiZNnWeBpcDPkuFRWcn6vqCUegUYB5YqpW5QSh1MhvkcVErdYJSxRyn198l6wslwoAVKqe8rpUaTxy+ZZag+oJRqVUr1K6X+2ijb9jBRSgWVUt9TSg0k+3VwFqN7a3J8hpRS31ZKBZPl7FBKtSfnTTfwbaWURyn1V0qps8nyn1BKlRjt+KFSqjvZ/xeVUmvTjGe+Uup5pdRXlVKfB/4n8J7kuPyBUmqZUmp3so7+5BgVGefPmMtKqbcBnzXKOZqm7vuUUi8rpf5Xss8tSqnfMfZXK6V+mpzvzUqpPzL2pb1XXOr50+S4Lkwz7kop9W/JsTqlkvdjckehUuohpVRXsp9/r5Li5BzaX5K8jp3J/T9xVPrnSqneZNkfTdM2lFJ+pdS9SqmfAs2OMfhfyXnYo5T6plIq29j/R8lxG0yOY7XurFLqX5N1jyqljiml1iVP+xjQopT6vFKqLl2bBEEQBEEQhF8joo0gvIFIGtN3KKUeAy4AdwBfAH43uX8V8Clgq2VZ+cCdwPnk6X8NbAM2ANcCbwH+R3LfnwPtQBmJ8JXPAlZy3+8C/5As64JS6lGl1E41T4+T5PE/A44CNcBbgU8rpe7McNr7k/3LB36VPP+/gHLgfuD7yT4DPAR8PNnvdcDuZL0bgYeBjwMLgP8L/NTNCLcsaxnQCvy3ZHhUNLnrQ8AfJ9sRAp4Gvpos78vA00qpBUZR702eUwMsA/YB3wZKgJPA38wyXDcBq5Jj9D+VUvUux3wEKAQWJdvxCWAiQ5kfIHENlwEr+fW1B6hMtm1xsp/3A+8AtgPVwBDwNeP4nwMrSFyHV4HvOytLjscvgVcsy/pTy7L+hsQ8+kFybB8CFPCPyTrqk335XPJ817lsWdYvHOVcm6HP1wGngVLgn4CHlLI9PR4nMeerSYRt/YNS6rbkvkz3itnH/wncB2y3LCtdHpnrgLPJNvwN8J+GAPYdIA4sBzaSuJ//0HFuuvY/AuQAa0lch381zqskMTdqgD8AvqaUKna0fb1S6stAB/CXJOb0IuOQL5KYJxuS7ashIbqRHKd/BH4fqCLxLHo8ed4dwC3JcwuTxwwAWJb1JRL3RjlwKCnofUgplZNm7ARBEARBEATLsuRP/uTvDfBHwoBtJWEk/ylQ6nLMcqAXuB3wO/adBe4yvmsjGBJ5XJ4Els/ShtJk3a8m2/IpY995IAwMG3/jwMvJ/dcBrY7yPgN8O01d3wG+a3y/GegGPMa2x4DPJT+3khBmChzlfAP4O8e20yQMbbd6zwO3G9/3AH9rfP8QcMBxzj7gPuP4vzb2/Qvwc+P7fwOOpKl7CQmxbKGx7QDw3uTnzwHfS37+GLAXuGYOc+c88Anj+13A2eTnHcAkEDT2nwTeanyvAmKAz6XsomSbC43r9jDwGvAXjmPt9qdp5zuAw3OYyxnLSR5zH9BsfM9JtrOShDgxBeQb+/8R+M4c7pUdJISOLwMv635naEMnoBzX80MkxNEokG3sex/w/BzaXwVMA8Uude4gId75jG29wLbk59uAQ0AbCfFrpUsZChgDlhnbrgdakp8fAv7J2JeXnB9LkuWfISF6eTKMTRYJQecZYBD41mzzWP7kT/7kT/7kT/7k77fxTzxtBOGNQx1QDBwh4a0y4DzAsqxm4NMkjNpepdTjOmyBhEfBBePwC8ltAP9MIjTiv5RS55RSf5WmDQNAY7INxck2mbzDsqwi/Qf8P8a+xUC1SoTyDCulhkl49GQK6WkzPlcDbZZlTTv6UJP8/C4SYsQFpdQLSqnrjXr/3FHvIqPvc8HZjguO/WY7AHqMzxMu3/Nmqa/b+Dye5vhHgGeBx5MhMv+klPJnKNPsg3ntAfosy4oY3xcDPzbG6yQJkaNCKeVVSn1RJUKnRvm1J1epcf7dQDbwzQztQSlVkZyjHcmyvqfLmWUuO8u5WSVCpcJKKTMfkT2OlmWNJz/mJfs+aFlWyDEm+hpmulcgIVT9MfCPlmWNZOoj0GFZlmV812UtBvxAlzHO/5eEF8ps7V+UbP9QmjoHrNS8QeYcKichiL1G4jnS6nJ+GQmRqMFo2y+S28ExPpZlhUk8G2osy9oN/BsJz6xepdSDSqkCZwVWwotNP0smSXjHCYIgCIIgCA5EtBGENwiWZf05idCW14D/QyI3xN8ppVY4jnvUsqybSBiFFvCl5K7O5DZNbXIblmWFLMv6c8uylpIIh3rAkXtjhVLq74AW4H8Dx4ClyTbNlTYSb+qLjL98y7LuytRt43MnsMgRllVLwusBy7IOWpZ1Dwmj9CfAE0a9X3DUm2NZ1mPzaLuzHYsd++12vF5YlhWzLOvzlmWtAW4A3g58OMMpZuiLfe11cY5j24DfcYxZ0LKsDhIha/eQ8IApJOFdAQnvDM2/kzDyn1FK5WZo0z8k615vWVYB8EGznAxzOaW9lmW9ZCVCpfIsy3LNr+OgEyhRSuUb28xrmPZeSTJEYry/rZS6cZa6aoyQJrOsNhKeNqXGGBfMsf1tyfYXzeHYFCzLepyEt84jJEKnOpVS/66Uusk4rJ+EuLjWaFuhZVla+EkZn+Q1XsCv78WvWpa1GVhDIkzqL4xjFyilPqUSubh2A17gVsuyts23L4IgCIIgCL8NiGgjCG8gLMvqtSzry5ZlXUPCs6QI2KeUehgSeUCUUrcl87VESBhe2jPlMeB/qMQKLqUk8lPopLZvV0otTxqXIyS8KqaT+x4mEf5TBLzTsqxrLcv6V8uy+ubZ/ANASCWSy2YnPTbWKaW2zvH8X5HwGPhLlUieuoNEqNHjSqmAUuoDSqlCy7JiwKjR738HPqGUui6ZJDVXKXW3w2CfD88AK5VS71dK+ZRS7yFhnD51keVdFEqpW5N5Sbwk+hvj131240+UUguT+VT+GvhBhmO/CXxBKbU4WVeZUuqe5L58EmLDAAlvjH9IU8anSISh/UwZCWwd5JMIqRtRStWQatxnmss9wJL55lXSWJbVRiK07B9VIqHzNSQEDL2MeNp7xShjD4k8Qf+plHpLhurKgT9Nztl3k8jd84xlWV0k8jP9i1KqQCXyVS1TSm2fQ/u7SOQV+rpSqjhZ9i3z6H8kKYjdQSJnz3kSAtTZ5P5pEvfNvyqlygGUUjXq1/mnHgM+qpTakLw+/wD8yrKs80qprcl7zU8ixCrCr58lf5CsazvweWCRZVn/3bKsk3NtuyAIgiAIwm8bItoIwhsUy7IaLMu6n0Sogg5DySKRQLSfRGhFOYm8MQB/TyKXRSMJT5lXk9sgkVT2ORIG9D7g65ZlPZ/c902g2rKs+y3LevUS2jtFwjthAwmPnX7gWyS8NeZy/iQJkeZ3kud+HfiwZVmnkod8CDifDLP5BAmDGsuyDgF/RCJkY4hEGNh9l9CPgWQ//pyEcPGXwNsty+q/2DIvkkrgP0gINieBF0h4T6TjURIiwTkSOVv+PsOx/xv4KYlwuRCwn0ROIoDvkgiN6QBOJPfNIBkS9Mckkv0+qZKrVTn4PLCJhFD4NPCfxr5Mc/mHyf8DSqmLnZPvI+El1An8GPgby7KeS+7LdK/YWJa1i0RuoZ8ppTalqedXJO6vfhJJtX8vOYcg4RkVIDGOQySuZ9Uc2/8hEkLdKRI5az49x/NSsCyrzbKsL1iWtYJEcmvNfydxr+xP3lPPkUiOTXKc/j/gR0AXCQ/A9ybPKyAh+AyRmCcDJMIvIfFsWWxZ1rsty3o6+UwQBEEQBEEQMqBSQ+0FQRAEQRAEQRAEQRCEqwHxtBEEQRAEQRAEQRAEQbgK8f2mGyAIgiAIggCQXP3LmegboI9fr14l22W7bJftsv3Kbf+4ZVnfd9kuCMJvCAmPEgRBEARBEARBEARBuAqZl6eNUspKXbnUHa/Xi8fjIR6PczlFIcuymK3+uRwz3/MutszfJEqpyzr2lxOllN2+i2mj2/WY7RpdytzR2y5HW83vl6PNbxQu5prNpQzht4O5PM+u1uedIAiCIAiCIMyRfsuyZnjAzVe04dprryUQCNDb20tnZyeQ+mPZ4/Gwdu1a6urqeOaZZ4jH4/Mq31ne5WS+wszVYFTm5eVxww03MD09zYsvvsjk5OSMY5wGjRZFzHbp/Xrf9PS0/d0sw1mO27XweDwXLbgAXHvttWzcuJH29naee+652U+4jMx2jcw+e71eiouL+eAHP0hRUREPPfQQHR0d9tjNtR7nmM61LfNp9+tJunsFuKxtvBIC7MXgvGeuFOnuTXO/3mfeg3rO6m1u5znvVbf71+2a6jLdjtP7zXHJNC+cYqVlWUxPT6f023we6c8ejwe/308sFrPPcatD7xcEQRAEQRCENygX3DbOOxFxZWUl69evp6amBpgpsPj9fsrLy6mrq8Pj8czbeHIKDvNhth/ssxnr8zl+vuVfDDk5OdTW1rJ8+XLbe8mtDjfDyNxnjovbd/O/W7lu2y/WOPJ4PBQVFVFaWsro6OhFlXEpzMXbRhMIBKiurqa9vZ1wOExtbS1FRUWz1uEc33R1znW+KKXweC5/zvDZrqHH4yEnJwefL1XbTXevXO75f7HlXS3i1sXgdm9qnIKf83O6+1Jvd97r+pq5za9Mc8Msz+354dzn9kyanp627w3zL11f1q9fT3l5+Yy5mKkeQRAEQRAEQXgzMG9LUClFIBDA5/PNMNSUUni9XnJzc8nPz7e3mcfNZtxdbAiKLvtKcyWM03QsWLCA1atXMzw8TFtbm+ub/kyeQ5mMuHTnuOHmuXMp+P1+vF4vQ0NDl1TOlSYrK4tFixZx+vRpXnvtNRYuXMiCBQtmvf7O+e7cdrWQSUzyer1ce+21VFVVEQwGX+eWXV1cikh5Oeo2P5veKk7xJZ0A6zy2pKSEqqoqCgsLZ52fbn13evikO36ugo7zXnHi9XrZuHEjJSUleL1e1/G52u4tQRAEQRAEQbhczFu0GRsbY2xsjFgs5uqGb/6lc/NPRyahIRPpXPJnK/P1MsTSecDMdk5JSQlLly6lqamJc+fOzQg1c453pjoyiQZzHfPLNV5erxev18v4+PhlKc/kcrVRKWWLNp2dnRw/fpzS0lIKCgquqIGYyVNiPvh8PvLz82cYuXMlJyeHm2++mcLCwjl7zF2Osc/KysLv919yOZebK/GscBNMZruf0+13iqrpvGpKS0tZt24dW7ZsYdWqVQQCAYAUQXg2Lxu3Op0Cjlv7p6enXYVnt/rM8fD5fFxzzTVkZ2fbc3E2oUcQBEEQBEEQ3izMe8nvSCRCVlYWgUAg7VvYqakpOwlxuvwDVxI3Y+Zi63WGt7iFJThzNcylzHRlm98DgQB5eXmMjY0RiUTSluMW2uDxeJiampq1LXPB7Odc+5gOj8dDXl4eBQUFrn26WEyD9nIRDAZZtGgRExMTRKNRsrOzCQaDl60Ot7G8XGUvWLCA5cuXc+zYMdcwtEzX0ev1smjRIoqKimhtbSUUCs3p/rkcba+srCQSidDT0zPnczL1Jd39+5sy9s08LZkEEBNnm3U402w5dtzCoLZv3866desoKCigt7eXvr4+zp8/7/qsdmvLbH0zxRvzuZhOrEkn2Ohypqen8fl8rFmzhmg0aj/TLqfnnyAIgiAIgiBczczb08bn8xEMBgkEAq5vUn0+n/223CkmzMWodsuvMBecBsBcPGsuxnAzPXdM42IuOSDMet3GwmmIaNFmYGAgRaBxGm+mUaT3OQWb+XprZAqVulh0mbFYjPHx8csq2ujxvFy5XwKBALm5uQBEo1Gmp6cJBoMEg8HL5glyJYWD0tJStm3bxsKFC9OGzaTD7/dz3XXX8dprrxGJRC7JKJ7PuR6PhxtvvJG1a9fOq45M45jOG/A3gQ47y8vLY/Xq1SmCks7xoj/rv3TjNzU1ZYuzWsDRn01hSHtaWZaF1+tl+fLlHDhwgK997Wv86le/YufOnTPOMzGTA+s+mGQ6z9l3t3vT3K7bodsLCQGxsLCQcDjM4ODgjGTE5n8RbwRBEARBEIQ3I/O2cCcnJ/H5fPj9fvsHt/mj2xRs9H+3H+PpSPdWdjYyhQuY+zPlYHBrS6YQsHR1p2uXW9iC87Nbe2OxmGuYix4rbbA5c0Xoc7TxZhpNs4kbmbww0olOmdBzQOezmZqaIhaLzfn8uaL7PZeQnkx9yM/Pp6Kigmg0itfrZXp6mry8PHw+3xVfRehicF7PSCTC4OAg5eXl9r5M94fepq9RfX09p0+fdl2tLB2Xw2jWHnrOhLNXkktt91yFQsuyyMnJYenSpWzevNnersddXxPnve6cp6aQop9RWrgx7039fNDfly5dSktLC62trbS1tXHmzBmqqqrw+XwZ72stiDiflaawlEmINkOanAKRLjOdEJqTk8NNN93E3r17GRsbS+mPKWRfTNJ7QRAEQRAEQXgjMG/LSAs2pnu/+T8YDGJZFiMjIzO8PS41VOlyMJ9wFGd75yICzHX/bAa0x+OxjalMS9mab+ednjrT09O24JDuvPliGmnzQRtnWVlZtqeWZrYlxGerzymMmfXNBaeYp5SisLCQ6upqQqEQU1NTtlhztb7Nd7YrFosxOjrKxMREyna/309ZWRlr1qyhurqa06dPc/ToUaLRqF2ODmEbHR1NK1Dpa6YNZh0WeanebBc7vy6FS61rPvMsPz+fJUuWpAitzjL0uKarQ4uSzrmuxR7T+0+pXy+lvWbNGgYGBgiFQrbI4fV6ycnJIRQKpdRnCkfONpqefpDq5eL0qDE9AHW/3ISXqampGV6LOqn9tddeyyuvvGKLvOnqEgRBEARBEIQ3Ixcl2pjeMNow0D+28/PzycrKIhKJ2D/OTUFB/1i/3N4Kutzi4mKGh4ddw4Mu1jhzGin681z6MN+QJLMOj8djixuzhbg4jT6nEOHmYXQxzDfUxnme3+8nOzubrKysOQsszvG/XP3LNCeCwSD5+fm2YObz+WyD80rMXbc2mx4Mei5YlsXk5OSMMXOeOzk5STgcprCwEEgYwJWVldTW1lJRUUEwGGRsbMzOTxWLxZiamrLFwpKSEqLRqKsIEwwGWb58OdXV1eTn5+P3+5mammJ4eJhDhw4xNDR0UfNLKUVOTo69zLj2urmaSefBl+6a5uXlUVZWRktLywyh0OPxkJ2dTW1tLYFAgP7+fgYGBlLCCE2R1rlNf9btMdvm9XqpqqqycxRNTU3Z1zs7O5twOGy3I9Pc0l57boK927nmWJgeX6bwkuk54PP5KC0tpbm5+Yp45gmCIAiCIAjC1c68RRu/3088Hk9Zycj0dMjPzycQCNgGn/nG1+m6f7kNMr/fz44dO9i1a1fa5KlmqJZpgPv9fnw+H7FYbMYqTZrs7GyKi4spLy8nGAzS2tpKb2/vFTMunWETMPeVXZxluO1/vQ1iM/zGDIdwGq/p2mZ6H5keJJfSj0wimPYMUCqxkpQZcnKp6HtCz6n8/Hwsy6Kjo4Px8XG7XcFg0J5z+fn5eDweJiYmGBwcZHBwkJGRkZT2mN4PU1NTLFiwAJ/Px/Lly6mrqyM7O5toNMqFCxcYHBy0jzPRwpo5r/UYlJeXU1tbS3l5OdnZ2bZHhtfrpbS0lK1bt3Lw4EFGR0ddPW8y4fP5yM7OtvNluYkh80ELTEop4vF4Ro+1+ZTpts28x5weKebzLxgMUlRURHd3tz2/gsEgBQUFFBQU2KJOOBwmHA6neJLM9tx0u4/0Z6UU5eXldi4pfd3TJSt38y7Mzs6mrKyMuro6pqen6evr48KFCyl5j2bztEr3jDL7Z27XK8319fVlbKuERgmCIAiCIAhvVuYt2gSDQXt1KI3pzp+Xl4ff72doaCjFbV67umtRZ3R01E6wa/7ovhSjyufz8b73vY+DBw8yNjaWstKI1+u1jSId6jI5Ocng4CDRaJScnBxqamoYGBigv79/hoEQCASoqKigvr6edevWkZeXx9GjR9m7dy/9/f22QTgX48E04sw+m2/ITTKJBG7jpo3InJwcCgoK7DL0cu2QWFrZ5/OljNPrgdnXuRpaWkDQnhilpaWcP3/+inpiRKNRQqGQLZTk5eWlrIp2Kej7JSsri+rqapYvX87ChQuZmpri8OHDvPbaa0xPT5Odnc3ChQtZuXIltbW19r0Tj8dpb2/n7NmzNDc309fXN6MOpRR+v5/c3FwWLVrEjTfeiN/v5+jRoxw9ejRFGHJ6Pvh8vhSDXo97dXU1GzduZPny5Zw+fZoDBw7Q399PJBLB6/VSU1PDe97zHiKRCI2NjbZw46wjHT6fz/bkSyeczgUtjhQXF1NVVYXX6yUcDtPT08Pg4KCrp5Y+L13Yp95vhvn4fD4CgYAdMmpZFtFolGg0aucCco6t9uAKhUL4fD6KioooLS2lpqaG8vJyAC5cuEBTUxOTk5MzcgppDz83kcgt5Mp81ujyzWsSj8cJh8MpnpNu+P1+qqqq2Lx5M9dddx0TExN0dHTw05/+lL6+PtsLRj9nQ6HQDG+gTKF2fr/f9vjSnmR+v59gMEgkEkmZr26IaCMIgiAIgiC8WZm3aOPxeFK8UbQgoikuLsbr9TI0NGQnnAXIzc3l1ltv5aabbqKmpobGxka+/OUvzyvR6WzE43E7cazzTXNBQQHvec97qKurs708hoeHefLJJ2lsbMTn8/HBD36QxsZGdu/eTW9vb8r5S5YsYe3atQQCAX7yk58wPDzMfffdRzAYZPfu3XR1dV2W3BhmGfpteLpy3YxMU6C6+eabuemmm4hEIoTDYRoaGti7dy9TU1OsXbuW2tpannvuOUZHR6+I+OHsj2UlkipHIhFisZhrv9zasW7dOnp7e+np6SErK4sbbriBeDxOa2urPb8u99v28fFxhoaGWLt2LdPT0xQUFDAxMTEjREN74sw13415jerr61mzZg3j4+M0NjaSlZXFRz/6UT772c8SiUTYuHEjW7duxe/388gjj9Df34/H46GgoICtW7dy/fXXs3nzZh566CHb88gUCLRw9+lPf5qXX36ZF154gf7+/hltdXqEeDweQqGQ7UHh9/vZsGEDt912GwBf//rXU3JWaVGgubmZRx55hAceeIBIJMLx48cJh8MopWyxKdM4abFIh2pdbEJyn8/Hxo0b+fCHP0x5eTlFRUUMDQ3x/PPP861vfYtIJDLj+aBFtOzsbFuAjcfjTE5Ozsiz4/V6ycrKorS0lLVr11JeXk5eXh6WZdHV1UVTU5MdzmMKRPq87OxsAKqqqrjrrrvIzc2lpaWFhoYG2tvbmZiYcJ1Tzs9OUUYLI2biX1Ngmp6etq+B1+slOzub/Px82/PGvP7OeiorK7n55pupqanh4YcfpqOjg49//OPU19cTi8UYGBgAEgLre97zHh599FEmJibsUCotdjo9eLQgunTpUoqKihgdHeX06dNMTEyQl5dHRUUF4XDYfl6kC7+6mvNNCYIgCIIgCMKlMC/RRhubJSUlKX/XXXcd5eXlnDp1inXr1hEOh+nr67PDj/Ly8vjABz6Az+dj9+7dDA0N8dGPftQ1V8mlMDU1xcTEhJ1/BLDzc3z4wx/m+eef59FHH2V8fJy8vDzWrVvHJz/5ST7zmc8wPDzMrl27uPXWWyktLeWb3/ymLUxVVlZy9913c+rUKZ599lnb6HvyySf50Ic+xJEjR+jt7Z2zx0q6/rqFQZnhXG44ywgGg6xYsYKdO3dy9OhRPve5zxGPx9myZQtLlizhjjvu4KWXXuLd7343u3fvnncIy3xwiiher5fJyUk7se9s6Bwyg4ODQMLbaWBggJdeeon777+fL33pS7ZX1GyCjTZinSttmdt1nUopIpEIY2NjlJSUYFkWS5YsYWJiIiUUxOv1cvvtt5Obm8sLL7xgG65zGZdt27aRl5fH888/T09PD5ZlUVhYyDvf+U6ys7NZtGgRt912GwMDA3z3u9+1w6CmpqYYGBjgueeeY2hoiDvuuIN3vetdPProoymGt/aMeOc738mDDz7IU089xcTEhOu1NsURv99PXl6eLYh5PB7e9ra3sWXLFkZHR/nqV786Q2g1yzx//jzf+973WLt2LZOTkzQ2NpKdnc3tt9+OUopnnnmGaDQ6QzSZnp6msLAwJV/KXDGvZ0FBAW9961t5//vfz9e+9jUaGxuJRqPcdNNNfPjDH6avr48f/vCH9rm5ubksW7aM7du3s2HDBoaHh+nt7bWfIX19fezZs4fm5mZ7PLZt28Ytt9zCkiVLOHr0KHv27GFoaIj8/Hxuu+023vKWt9Da2sojjzxiJ/jVolBubi7FxcXccsstLF++nD179nD69GlGRkaIx+MpIqTpTaPzgpkirR47t/lrJgCGxL0TDAZTPCIrKytn3D/aG6ikpASfz0dnZyc+n4/bb7+d8fFxHn/8cTo7O5menubIkSPU1NRw4cIFhoeHycrKYsmSJZSVldltNUNQzaXJ9Tytr6/nrrvuorq6mi9+8Yt2+JbH4yE3N5fKykq6urrwer3289hcxtwpVIlwIwiCIAiCILzZuChPmy1btnDPPfcwNjbG+Pg4zc3NHDhwgJaWFq655ho8Hg/xeNzOR3DjjTfaLv/hcJi6ujr7jT/MTHRpkimHg0YbLvqHvWmwVFRU8Hu/93scPHiQEydO2Ll2RkdHuXDhAp2dnWzfvp2f//znHDp0iNraWqqrq9m8eTOHDh0C4O6776azs5OzZ8+meDSYxsNsIUxmrhGzn+n6pxPPZmdnzyjbNHi1IaQ/b9y4kVWrVvHcc89x5swZO9SgoaGBvLw8Nm3aRGtrK1lZWfT09NhhR87rcCmkK296etr20vJ6veTn59urFrmhzz9//nzK946ODl566SXuvPNO/uu//ove3t4Z4+gmzJhhJOlypuh9WrQpLCwkOzub+vp6zp07x8DAQIpXw69+9Sve9773kZubayfANg3pYDAIYBvkgUCA+vp6br/9dh566CFb3NReGIWFheTk5PD2t7+d1tZWDh8+7JqfaXp6mnPnzrFv3z7e/va38+ijj9r9VkpRWlpKfX09+/fv52c/+1lawcZ53bKysliwYAHnzp3DsizKy8t5y1vewtjYGD/4wQ9cw5bM621ZFk1NTdx+++20trbi9XqJRqMcOHCAL33pS5w+fZqWlhZb/NKeMcFgkNLSUqampuw5Yc7t2doNUFJSwtatW/nIRz7C5z//eZqammyvlYaGBuLxOJ/4xCf46U9/SjQaJRAIsGPHDq699lpOnjzJ008/zcjIiC1K5efns3jxYnbu3Mndd9/Ngw8+SH19PUuWLKGxsZHvf//7jIyMEIlEiMfjDA0N8fjjj1NbW8uGDRv47Gc/yxe+8AVbpCorK2PZsmUsW7aMI0eO8Nhjj9HV1WULmabAYQoveizM+Wlu0wKPMxxVf9bjG41G7fOrq6vZsGEDDQ0NMwSekpISbrnlFhYuXMg3vvENqqurKS8v58yZM3R3dxOPx/F4PCxatIiJiQmmpqbs+3nZsmX2inemN6b5jNbzffPmzWzatAmPx8O//Mu/EAqFiMVi9v5AIEBhYSHd3d14vV474bUpaJuCjSAIgiAIgiC8GZn3eqlZWVmMjIzwy1/+ki9/+ct86Utf4uGHH+all16iu7vbFhvMsKjVq1fT1NREZ2cneXl59htqp8s8pIoYcxFs9HHwa68awDaYS0pKKCgo4NixYzOSI4+OjnLo0CGqq6tRSjE+Ps7+/ftpaWnhhhtusJc2X7x4MV1dXfT19aUIAVu2bKG7u9sWcjLlacnUTyf6LXW6cAB9TEFBAe9617vIzc21jaiKigoikYhtGGuDLBqNcv78eVpbW7nhhhuoqqpKCaNwa7dSihUrVrBx40aqqqpmvQ76nHRYVmJZ6HA4zOjoqL2yUbpjdb91fhW9LRaL8corr1BcXMzSpUspLi7OeL55zdPVpduux3xycpLx8XHi8TgFBQWsWLGCzs5OhoeHU84dGxvD4/FQXFxsh72Yda1atYr6+noWLFgAJDyhtm/fzquvvsrQ0JBt2GZnZ7N06VI714nOr9Te3u4a+mFZFqFQiLa2Nvr6+lKuY0lJCatXr2bVqlW8+uqrcxJsNMFgkEWLFjE+Po7H42H9+vWMjY3R1NRkewRlQilFKBTi0KFDdHZ22td8cHCQvXv38ra3vY3y8nJ7jufl5bFz504+85nP8Ed/9EesW7fOziOU6f5weqUFAgFWr17N1q1beeSRR2hubmZiYsIWQkZHR2lpaSEcDlNaWmonAA4GgwwPD7N//346OjoYHBwkFAoRCoXo7e3l5MmTvPDCC5SXl3P77bezbt06urq6OHHiBP39/XbojvaCGhsb4/z58xw5coSJiQl27txp58Pavn0711xzDS0tLezZs4fOzk6i0egModmZfNjtOaBFQ7ck1Ga4kA5NMpPHZ2VlUVJSQmlpKSdPnkypUye6PnfuHNPT01RXV7Ny5UogETI4PT2Nz+cjNzeXzZs309XVlfL8Ky0ttQUjZ9tNcbugoICNGzdSVFTE008/zcDAgB0Wp/uhn999fX226OlWrgg2giAIgiAIwpuZeYs2Y2NjDA0NcebMGfbu3cuhQ4c4ffo0vb299o96vdywz+dj9erVhMNhhoaGgEQeh+rqao4ePZrWGJ3NyE7bmWQokfYeKSgooKSkhP7+foaHh1Pq08b/wMAA5eXl9lvb9vZ2uru7KSoqspOYFhYW2slFtdDh8/nYtGkTTU1NjI6OzmjLbPla3PI76M/6jbTz7bnTWPX7/axcuZLKykr8fj/XXnstwWCQ5uZmwuHwjHHs7++ns7OTpUuXMjo6SmVlpR2m5BaqtWDBArZv385b3/pWWwybK+mu3fT0NMPDwwwNDVFTU5P2/HTjpw2//v5+GhsbWblyJUuWLLGXRk9Xf7p55laHKdyEQiGWLVtmJ1d1rpQzNTXF0NBQSliIWZ/Oy5GXl2d7sqxcuZLjx4/bZfl8PsrLy9m0aROHDh0iEAiQlZXF+Pg4Y2NjMwxTM7dKIBCwl2vW9erkxjk5ObZwMhe091hRUZEtTi1btoyRkRHa2trSrr7kHMt4PM6xY8fsMBp9r+3du5fy8nIqKyvJycmxx6+3t5fjx4/boUk6x5LbnNT3uNO7YunSpdTW1jI4OMjLL79sJ9g2hYuxsTE6OzspKCiwBSO90t34+HhK/hp9zujoKOfOnePQoUPccssthMNh2tvbU5KP6+ujRZTx8XE6Ozs5dOgQN910E0VFRWzZssVedamtrY3Ozk7Gx8dTcvdkEn3NOeW8r908ytzGTwsz2nNGr0Bm1u3xeIhEInR1ddmhrVqIDwQCtti1adMmAFu01t4+OqGwz+dLEZHM0C4tsi1YsACPx0Nra2tKviPzPjfzG5l9NO8BQRAEQRAEQXgzMy/RxrIs+vr67LwD2hNBGyv6bbM2Nr1eL2vWrKGrq4t4PM7ChQtZuHAh8XjcDnkxy3aKGvPF5/OleJcUFRVRUlJCW1uba4iRXtFKh7AARCIR+vr66Ovro76+PuWNvN/vtwWbsrIyioqKaG5utsNXnH2YzxtgnSy0pKSE8vJySkpKUjw33DxilFLk5eXZx65YsQKApqamGSu3QOJN+ejoKHl5eZw5c4a6ujqysrJmtNvj8VBSUsLmzZvZsWMHGzZssFehMstzYzaxTc+h3t5elixZknGMMu2bnp7m0KFD5ObmsmTJEntlnEvB2e7JyUn6+/u54YYbGB0dJRwOp4ST6XOGhoZYuHAhJSUlroZ3MBi0RSWfz0dFRYW93LYe62XLllFaWsq+ffts4VEbrG4opSgqKqKurm6Gx1p5eTk1NTW2QDaXvuoyA4EAeXl5tnBSWFjI+Ph4SojWbMKCZSUS8prLkVtWImyqu7ubyspK2/MoEolw5MgRnnjiCY4ePWoLwM5+FxQUsHTpUtatW8fatWtZtWoVtbW19spNq1evprCwkEOHDtHd3T1DAICEmNTZ2Ulubi5er5fKykri8Ti9vb12cnJn3yzLYmJigpdeeom6ujq6uroYHBx0TVCsn4GWZTE2NsapU6eoq6tjxYoVrFq1CstKLOmuV1tKJ9hk8iBxCjTprqUTM+Ro2bJlFBcXc+LECdv7xhSA4vE44+PjtqDb09NDbm4upaWllJaWUldXx/XXX8/Ro0cZGBiwn7nxeJxQKERpaSnZ2dm2SGR6Dpp9MMUZ879uy+TkJKOjo/bz2SnWzXcMBEEQBEEQBOGNyLw9bXSyVf3WVb/ZNX9M67wU09PTLFiwAMuyKCoq4oYbbqCiooJ9+/bNSEY6FzL9MNdeDJFIxP6ek5NDfn6+65LIkAgTWLRokZ1fR/elq6uLw4cPs2nTJvsNellZGQUFBbbQs23bNhobGxkcHHTN8zGbV41pbOi219TU2ElON2zYYIckmXkszHJ0W+LxODk5Ofj9fiYmJlw9f8z2RKNRjh49yqpVq2wDVhtAehni66+/nne+851UVlYSiUTIyclJGetLoa+vj7a2NhYvXpzyRt7JbPWEQiGOHTtmJ5XOlLD5YohGo7S1tfGOd7yDw4cPp4gQZht1klndF7PdgUBghtCmDWCfz0dOTg7r1q1jzZo17Nu3j9bWVkZHR+28Km590klaly9fzubNm2d4wOhcSIFAwE6E68Q5tkr9OgFtaWmp3e6RkREAO1QwUzidiXPOaw+UV155hbKyMqqrq23PC53/JDc3l7GxsRTPIU1dXR333nsvf/AHf8B9993H+9//fnbu3ElOTg5FRUWUlZUxMTHB0aNH7fqd7YnFYvT39+Pz+cjKymL58uV0dnayZ88exsfH7ba6iQs6d41ektwUlUwPEi0+eDweioqKmJiYYMeOHYyMjHD+/Hk6Ojro7++f8fwzRQtn+93EYLOds10Ty7JsL8iKigpqa2uxLIvDhw+n5MoxRSct8GhPI4/Hw4oVK9ixYwd33HEHxcXF/OQnP7ETKOvre+7cOaqrq6msrLSXqNfjo0OfPB4P4+Pjdr6h7Oxs177o5cjLysrIzc0lKysLIEXsmmvyd0EQBEEQBEF4ozLvRMT6Dazf77cNDUj8eM7JySErKysl2eTY2Bj19fXs2LGDWCzGoUOHUpJfOsn05jSTce/1eqmoqGBsbCwlmWw4HCY/P3/GOXrp5GuuuYZdu3alCC/hcJju7m6WL18OwL59+9iwYQPj4+NEo1GWLFnCe9/7Xv7qr/4qxdhzC+dw9kUfZ+7z+/0sXbqURx55hJMnT3L27FkCgYAd9uM0IvVn7aXR39+P1+ult7fX9qxwvqm3LIvi4mKqqqpoa2tj//79/Mmf/AmLFi1ieHiYsbExO5fKAw88wIYNG/jCF75AeXk5tbW1LFq0KOO1mM+b7lAoxMjICDU1NSxcuHCG15Um3RwxxYCDBw+yaNEiVqxYwb59+1I8S9J5K8yGPk/nBlqzZg0nT560Vw1z9rW4uJjh4WHC4fCMkL+amhpGR0ftfEqRSITXXnuNP/7jP+Yb3/iGnS+opaWF5557DkgIo4FAgMrKSkpKSujt7U3pR3l5OXfccQcrV64kHA4zPj6eEhbT1tZGd3c3119//Zz77/f7uf7663n/+9/PHXfcQVZWFseOHaOpqYlbb70Vn89Ha2sroVAoJe+I6fWRLmG2uf306dPccsstVFRUkJ2dzfj4uD1eFRUVtLS02Pl0TIO8p6eHH/3oR0QiEdsLSHv2veUtb2FoaIgLFy7MWNnKbKNepru7u5vFixfj9/uZnJy0k+m6tV8/W3QoYV5enu2Vo9vtTC4eCASoqamxE0Tfe++9nDp1Ckh4+5w+fTplfMyyzPKcXjhO4df5PHDifP6Mj49z7733cvDgQfbt25cSQmbWo73XCgsLGRwcxOv1curUKd773vfy7ne/m//4j//gn//5nwmFQrb3i2VZRKNRzpw5w49//GM+8YlPcPjwYRoaGmhtbbXz3ujn7PT0NKFQiKqqKm699VaefPLJlFBNpRRDQ0M0NTXx6U9/mpycHA4ePMj58+cJhUIpeYAEQRAEQRAE4c3MvEWbqakpOxxDizN6KVyv18vw8LC9ig7AoUOH+PnPf86uXbt4+OGH2b9/P5AwbPRqT3r1kWAwSE5ODh6Px056OlcxwOv1snTpUjo7O20Dob+/n6amJjvprmlclZaWsnr1agoKCti7dy+QCKeanJy03/yPjo5SUVHBs88+y4oVK/jIRz7CH/7hH2JZFo899pgdhmEmTTUNOWeSUNOw1f+9Xi/V1dXcd999fPWrX+Wpp55ibGyM0tJStm/fzk033eRqGOvxCgQCDA4OkpOTY393Eyv8fj/XXHMNW7Zs4cc//jFjY2M8+OCDPPDAA3R1dREKhVi4cCFLly7liSee4O/+7u/svDNve9vbuPPOO/nP//xP29ugoKCA4uJicnJyiMVinD9/3tXDx4nuQ0dHB0899RQPPPAAf/mXf5mSK0aj+1FSUsLExERKwlO9X+ff8Pv91NTUMDw8PCN8ZL6YYoNe6UrPUT3ntTDo8/m48cYb+dGPfkR3d/eMsgKBAG1tbfT399seG1/5yld4+OGH+da3vkVraytPPfUUu3fvtoWKWCzGd7/7XX7nd36H6upqdu3axcjICFVVVezcuROfz8f+/fs5ceKE7S1lcurUKX7xi1+watUqvvjFL/LFL36RM2fOpKzU5QyHW758OX6/n/3799PX18c//dM/MTw8zL59+8jLy+Otb30rd955J7t27eKFF16gq6srxbNE/+n7zFy62qzLXDZde62Y+ZtGRkYIh8MzPCh6enpcRVCdgPvChQu0tLTMWHHJPFaHnPX29jIxMWE/b7QIo4VmXYf21qurq+OTn/wkP/rRj1iwYAErV660lwPX80I/AwsLC9m0aRPbtm2jr6+Pffv2sWPHDtavX08oFKK9vX2GWOM2T00Pq7k+B51LamuUUnYC8MLCQvr6+mYkuNZtCAaDbNiwgQ984AMcOHCALVu2UFNTg2VZfPvb36awsJDrr7+ez3/+8/ziF7/g6NGjDA4OEo1GmZqaIhQK8fjjj/Piiy9SX1/P9ddfz6233sr4+Ditra2cPXuWzs5OYrEYu3btwu/386lPfYrt27fz4osv0t7eTiwWs0M/Fy1aRHt7OxMTE/zZn/0ZWVlZNDc3c/DgQQ4ePEhvb6/dDzePR0EQBEEQBEF4ozNv0QYgJyeHnJwcfD6fbQjq3A8vvPACk5OT+Hw+qquruf/++2lsbKS2tpZ7772X+vp6+vr6mJiYoLCwkIqKCgoLC4nFYnR3d3Py5EkaGhpcDRXtWu806LThU1xcTE9PD1NTU9TU1BAMBhkdHbVX+NFhLMuXL2f79u2sXbuWH/zgB7bBoT0bQqEQw8PD9PT0kJ2dTV9fHz/84Q/52Mc+xl133UUsFuNjH/sYO3bsoK2tzV4FSOdg0G+FzeSt6d6Ea8Oxp6eHp59+2vZk0Ibk0NBQSkJV02ugqKjIzq+hl5PWRqj5Jtrr9bJ27VoqKyvp7u62jdsXX3yRiYkJ6uvrycrK4siRIxw/fpz29nZGR0eZnp6ms7OThoYGKioq+NznPkdJSQkej4doNGqvfBOLxWhububBBx90vWZOLwzLSqzcdfDgQW6//XbuueceXnjhBQYGBmaspBMMBvn93/99Ghoa7HmhjVJ9XDgcZmJiwnUVqfmi2+vxeCgvL+euu+7iscce47bbbrO9QPQ1CAQC3HjjjQQCAfr6+lJCkbSgU1JSYs8N3fdIJEJ7ezsVFRUsXLiQ973vfWzbto0jR45w7Ngx+vv7aWlp4dlnn2Xp0qXceuutdg6V3bt309HRwcjICBUVFUxMTNhJvjXxeJyGhgb+9m//lptvvpm/+Iu/oKenh4GBAcbHx+nr6+PYsWOcOHHCnls9PT22YFJfX297kEWjUfbs2UNTUxMrVqxg6dKl3H///YyMjNjedvF4nGg0SiQSsYWDn//855w7dy7FC0gnqvX5fExNTdnL0WsBTAs6Om+QOW+c4ULm58nJSYaGhhgdHU3xSHEeNz09zcDAAD09PUxMTNDd3c2WLVtYvXo1zz//vJ04WV+39evXs3nzZqqrq9m/fz979uwhLy+Pu+++my1bttDQ0EBHRwfxeJzS0lLWr19PRUUFoVCIl19+mVOnTjE5OcnPfvYzPv3pTzM2NmYL2n6/37Vveu5lEmn0/HPeB2a/zfP9fj9r166lrKyMiooK6urqaGpq4sKFC/bxOtfSjh07WLp0Kc8++yy7d+9OEYG0V+CePXvsufu7v/u7HDt2jJMnT9Lb2wskhLn29nYGBgbw+/3k5eWxYMECqqurueaaa+wkx729vTz55JO89tprbNq0iY0bN7J582b72dvX10dzczMvv/wyo6OjvPTSS5SVlbFo0SKWL1/O1q1b+fGPf5yyEqEgCIIgCIIgvNmYt2hjWZYdkuH8oRyLxTh37pz9lnT16tU0NDTw2GOPUVxcTGFhIfn5+axatco2zoeHh2lubmZoaIjBwUH7LTi4hxeZuWc02oApKCigsbHRzoVQVVXFtm3bbDFH57DxeDwMDw/zzDPP2MdDYiUUHSK0devWlGWaq6qq6Orq4gc/+AHnzp0DsMUmn89ne2WEw2HbgDfFBxPzezweZ2JigoGBAUZHR+0x1WEY+k2yOQ46hKG8vJz9+/fbXg6nTp3Csiw7BEqPy7Jly9i4cSOxWIyDBw/auSRCoRCvvvoqbW1t+Hw+RkZG6OvrSwkxmZyc5MyZM0QiEcrKysjPz7e9kczlis08JG7hG+b80XNlaGiIJ554go0bN/Lud7+bSCRCLBazr/PZs2d57bXXqKmp4cyZM8Cvw9q2bt3KsWPHGBkZITs7m9zc3LTJZOeDKdisXr0av9/P97//fXbu3MnmzZsJBAL09/eTk5PDxo0bueeee9i1a5edbNvsb3l5OWNjY4yNjRGLxfB4POTk5LB+/XoOHDhAR0eHLUxpD6l169YRi8W4cOEC/f39DA4OEgwGbe+Vnp4eO6wlFArR3d09437Q9+ipU6cYGRnh+PHj9rLpOlmsNpz18XplLD03zPxUOrxrYGCApqYmysrK7HmvhRW97LVeMW7lypWMjY3R0tKSck1WrFhhe7GZQpb2tnGKdm5zx0lhYaEtIJnhh6YXi5mvSa8E19jYCMCaNWv42Mc+Zof7+P1+YrEYY2NjtLa2cvDgQc6ePWuHwL3yyitUV1ezcOFC1qxZY68yNjIyQnt7O319fXR1dTE0NIRSiqamJvbs2cPmzZvZunWrfc/rhMtmn93udfN/ppAoZ+ik1+uluLiYTZs2sWHDBnbv3k1xcTELFy7kAx/4AOfOnaOrq4uxsTGWLFlCbW0tsViMY8eOcejQIYaHh1M8yyBx/4VCIQYGBujq6iIvL4/BwUFbGNdt1c9CSKw4GAqF6Onpwefz2QKkXr1vfHycgYEBCgsL8Xq99jzU944OOxwaGqK/v5+Ojg7OnTtHTU2N/Yy+FM86QRAEQRAEQbiambdoE41GOXDgAAMDA3ZuC9NYCIfD9kpIAwMDnD9/nrNnz5KdnU1VVRVVVVUUFRXZOUO6u7tpb29nZGTE9ngxDRa3N+zOxK4670RRUREXLlwgFosxOjrK4OAg4XCYvLw8CgsL7ZCsnp4eTpw4QWNjox1OAzA8PGwbbQsWLODcuXMMDw9TUlKCz+fj2LFjdHd309rait/vp6KigkAggN/vZ2pqirGxMdtbQYd4uIU/mP2Kx+MMDQ3ZOS/09tzcXAKBAENDQ6791slUjxw5YocHnDhxgqKiIrKysmxPkMWLF3PdddcBcO7cOVpaWlLGVHsowMwcMrqdw8PDhEIh20tCew+ZHkCZvAMyCTcNDQ1kZ2ezcOHClOWCIWGMQyLMbWxszD53enqaoqIi7rzzTsLhMIsXLwagoaHBtf754PP57HFbtWoVjY2NHD58mJKSEioqKti6dSvj4+O2kNfa2srevXvtJeVNAoEAx44ds/fp8ZyYmGDfvn2cO3eOqakpgsGgHW6ml0GORCKMjIykCHnOcQyHw5w/f95eAcxEJ4Y9e/YsLS0t9rUDbM8f8xxzOXtnQlztHRSNRunr6+PMmTMp3h66Pu3NFolE7ATkzntYh7t0d3fbHnO6HHNFJl23E7c8MIODg1hWYhlpZ04bfYx5X+t6u7q6gMQ8XL9+vb06XDQaZXR01A7n0R5z+u/48eP09fVRW1tLZWUlU1NT9opzfX199ipj+poPDw/zy1/+kvHxcerq6qiuruaGG25gYGCAsbEx21ttdHSUnp4e29suE+Yzwfk8tCyLYDBIWVkZdXV11NXV0dvby4EDB/D7/WzZsoVVq1axbNkyampqGBgYID8/n0gkwpkzZzhx4gTd3d0p3n3mNYnFYoyMjDAyMuKaa8fMrwTY4u7IyIidO8gUBcPhMM3NzSlhic6cSeaxY2NjDAwM0NHRQVlZmeS2EQRBEARBEN7UXFQiYp0w1W1ZadPQ379/f4qhOjw8zMmTJ+3jTfd+pzjjlvDVWZdGL8utw4z02+umpiba2tooLCy038hOTEzQ399vv70122xZFoODg4RCIc6fP09hYSEDAwNUVFTQ1NTE0NCQbVDppYLN880lgN3Gxg2dx+PIkSMpx+nws56eHnub2U69HPWFCxfsetva2giFQnY4QlFREZs2baKmpobnnnuOkydPMjk5OcMLwem15CayxONx2yPJNBT1sXPtr7NcHX7j9/vJyckhOzubYDBoCwyRSIRDhw6l5FAJhUIcP36cj3/842RnZxOPxzl27BgdHR1zqjcTeXl5FBcXU1tbS05ODk888QTj4+Ps2rWL2267jeXLl1NUVEQ8Hqe5uZnvfOc7jIyMuPY5FArR0NCQ4lEyMTGREv6nBRHtpQGpRqs5Vk7MFXgyoQ1lM6eNxrzeWoDRiXnTeUw5l6o2y1JKcfLkSXJzc+0QK12Gx+MhFovZooC5z7ISS6drMclNpDLrMcfk8OHDxONxO0eO23gppfD7/WRnZ6csLd3R0UFnZycvv/yyvax9NBq181qZzybd50gkQltbG+3t7bZQZebwcQpPsViMpqYmurq6WLRoERs2bGDlypXE43EGBweBhGjW2dlJJBKxV3rS5zv7m+67+TkYDFJaWkpVVRU9PT0899xz9sp6fX19vPbaa2zYsIHq6mri8ThHjhzhxIkTDA0N2TllnNfBrMdcCcr5/DDbbop6Pp/PFiCdq6KZ803Pf12+Hl9zrmovHZ3rSLxsBEEQBEEQhDcraq4GNoDH47F0GMKcCk/+kNY/7E2RxvyBnqauOb9BDQaD1NbW8o53vIOvfOUrKUsg6x/0pqFg7jMNhHSGaFZWlm2omh4yuj9Ow1f3ca5j5BR5PB4Pd955JzU1NTQ0NHD48GHX89zq0kuHX3fdddTV1dHY2MjLL79sh2s5PRXmc/3d2qDLuRyYxtds4+fxeMjOzmbBggXEYjFCoZDrUtFzRV/HtWvXsnHjRqampjh69ChnzpxJGTefz2fPI3MlHKfwOFcyiZMX0/75Hm/mK9HLiN9222189atfTevtYZ6Tbr+uw008cbYDEnmXtm3bxpkzZ+ykzekw7xlt1Ovv6dqclZVFZWUl99xzD1//+tdneHPoNujQHmcf3Tx89DlmCJEz+bgeZ6fAoZ8pOiwMUj2gnJ5O6cbXbUz1Pqe4qoVQs51ObylzXujxdatLt90p7Hm93pRnvXm+UorJyUk755auR6lEomQtujlXzTLH0qzLPFaHVckS4IIgCIIgCMIbmAbLsrY4N15UThtTCHEzrvR+/aNch9GkW1HGjdmWe9bGgfbWaG5uThFsnIaMM4GxaQyYxolbfXq5Zmd5ptGoj9XGoGn0uQkzZvluY6GXKdfJQtO1zVlmNBrl/Pnzdk4b0ytCt1OPg9Mgmw9z7d9cSechkY7p6Wk758XlQM+H06dP09zcbOfc0G0yjVV9vO6zXnFovmiDdT4i32ztn01QcR5v1qsFCL2kdqbyMgl2mfqi7xvzGaLFln379s2hp6n3jFMoyUQ8HufEiRMAdjJkc/6azye3Z4VeIcqc9+aqWfpcfY7zPtMinw4FsyxrRkiYOU66HlPY0H0222Xe325jYbbfTQxy1m3OR9MDxixP990pzJh91G3T11kLPZD6/Nb3kHk99PGmqGTOHZ3/SAtFFyuaCoIgCIIgCMLVzkWtHjXbG3bzODM5qymUpDPyZxMA3IxN02U+XdtmMyRNA8lNzHC+aTfLMw15wH6TbBr78zHKtVEXDodTViQy68hkmJsGjHm807jTxpL55toNNw8Y5/inO3euHiCXGt5wOYQjLew539ZrozCdyHWxxqJbUm0n8/VmytT/2a6FNpz1Kkpmeena4QxNyVS/NtjdQm/MNl4O49spYExOTtLd3W0v0+30Qkl3r+vv+rngtkqRvrdMAUP31xRGzHvIeT+a5ehjNG7hcqbgZwpYznaZbTFX5TL367nt9DA0hRnTK8Yp1prPTS28mHXpY3Xiai0kp3vOmuNi3h9Ogc4Usy71+SEIgiAIgiAIVysXJdqkw/njezYjOlMYRbry0/04zyQaZNpv1p/O60EbgE4hxGlU6P9uhp2b0evcpusJBAJAQoSai7t/ulAOZ7vcxuByjH0mLrcxlU48cOvDfAUPpxGcbhzdPl+s0PB6egfM5VrolY0y4ZwL8xWUMl3DyzUemYSV2epx9s8tQbfzu9v9oesxw6ecoonz+WE+N2fzaMz0bHW2xSkiK6VsEc05v511uj2n3YQf/d8UgvWx+llpel86PS/1fh12ZXrbaNHJ2V4tGEkyYkEQBEEQBOHNyrxFm3SeAW4GC6S6tOvtbgbfXAzs+QgA6QxsZ3mmwZDu+Ezf3Qx6MxFnuna7rbakv3u9Xjs/y1zEidk8XTJtm6uoNdsxrxfON/zOfZdadqbt6eq7UuMyX9HpUpmammJyctJVtMkkWM2VuRw7m+fRpdQ127zXAoDb/ZTuHjPno9vxmUTadNvSCb5u7Z6r4Krb6fRumW2cMl2PdN5K5vPUKdxAqieg2T+niGXW4SYciVAjCIIgCIIg/DZwUZ42s4kn5o9up9u9c7+bsHEpRttsRoabh4Rbf5xvnWdrvz7WbXsm3Mqampqit7d3zp4HlyJezPfYy+05czFlzvWc+c6jTOElvw3E43Gi0SiRSGRW8eRSxuY3Oa5Og3+2e9Y51zJ5vjjPM+tyChmZws2com8m5ioEuWGG52V6BprClFPodhO5zDxPZlt0SJTbmJr1OMPXnG1xa6eERwmCIAiCIAhvVi7K00Yz1zfubvvc8jRcCs433m51Otuuj8m0Uko648JZt8bNiHHzDsn0ljgajdLW1jaHXmfmchszcxHr5lvfxbbv9TT8L5f3x3y4HKLIfMc2Ho8zPj5Odnb2nPp8uefXb0LMcRMknOGQGuc9bD4f3EKfNHpfJqHIzVvHWY557sXMSS2cuInXZohROq9IZzvcPArN55t5rs514/f7U8bbeb5Zjr4GZpJis35nO0W4EQRBEARBEN6MXNTqUfPxIkmXkyWT90267ZneCps/9NPhXEo23Vtb0wgwz9FvjzOtbJVOjJrPuE1NTTE0NJTxmLkYzJdqxKTzIsrUjosVDObSnyv1Rn22ctN5W10JwcI0XDMdOxcDf77EYjGGh4cZHx9/04eeuIX16Ps93UpykPoMcRNnM3n56WPSMdszIl04kls5mTAFKdO7T9fhHBPzv3NeOFfIMsM+M3nFmM9ZXa4WjpwJk93EIeezWVaPEgRBEARBEN6sqPn80FVK9QEXrlxzBEEQBEEQBEEQBEEQfutYbFlWmXPjvEQbQRAEQRAEQRAEQRAE4fUhfRyAIAiCIAiCIAiCIAiC8BtDRBtBEARBEARBEARBEISrEBFtBEEQBEEQBEEQBEEQrkJEtBEEQRAEQRAEQRAEQbgKEdFGEARBEARBEARBEAThKkREG0EQBEEQBEEQBEEQhKsQEW0EQRAEQRAEQRAEQRCuQkS0EQRBEARBEARBEARBuAoR0UYQBEEQBEEQBEEQBOEq5P8HWVjnahOypUIAAAAASUVORK5CYII=\n", - "text/plain": [ - "<Figure size 1440x1440 with 1 Axes>" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "plt.figure(figsize=(20, 20))\n", - "plt.title(sentence)\n", - "plt.imshow(data.squeeze(0).numpy(), cmap='gray')\n", - "plt.xticks([])\n", - "plt.yticks([])" - ] - }, - { - "cell_type": "code", - "execution_count": 38, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "('he rove fron his breakfait-nook bench<eos>', 0.6715805530548096)" - ] - }, - "execution_count": 38, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "model.predict_on_image(data)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.1" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} |