Merge branch 'master' of github.com:aktersnurra/text-recognizer

1 files changed, 82 insertions, 626 deletions

diff --git a/src/notebooks/Untitled.ipynb b/src/notebooks/Untitled.ipynb
index f114ed9..ca0b848 100644
--- a/src/notebooks/Untitled.ipynb
+++ b/src/notebooks/Untitled.ipynb
@@ -2,7 +2,7 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": 2,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -26,7 +26,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": 3,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -36,89 +36,104 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": 4,
    "metadata": {},
    "outputs": [],
    "source": [
-    "from text_recognizer.models import VisionTransformerModel, TransformerEncoderModel\n",
-    "from text_recognizer.datasets import IamLinesDataset\n",
-    "from text_recognizer.datasets.transforms import Compose, AddTokens"
+    "from text_recognizer.models import TransformerModel\n",
+    "from text_recognizer.datasets import IamLinesDataset"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 15,
+   "execution_count": 5,
    "metadata": {},
    "outputs": [],
    "source": [
-    "target_transform = Compose([torch.tensor, AddTokens(init_token=\"<sos>\", pad_token=\"_\", eos_token=\"<eos>\")])\n",
-    "dataset = IamLinesDataset(train=False, init_token=\"<sos>\", pad_token=\"_\", eos_token=\"<eos>\", target_transform=target_transform)\n",
+    "dataset = IamLinesDataset(train=False,\n",
+    "        init_token=\"<sos>\",\n",
+    "        pad_token=\"_\",\n",
+    "        eos_token=\"<eos>\",\n",
+    "        transform=[{\"type\": \"ToTensor\", \"args\": {}}],\n",
+    "        target_transform=[\n",
+    "            {\n",
+    "                \"type\": \"AddTokens\",\n",
+    "                \"args\": {\"init_token\": \"<sos>\", \"pad_token\": \"_\", \"eos_token\": \"<eos>\"},\n",
+    "            }\n",
+    "        ],\n",
+    "    )\n",
     "dataset.load_or_generate_data()"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": 41,
    "metadata": {},
    "outputs": [],
    "source": [
-    "config_path = \"../training/experiments/VisionTransformerModel_IamLinesDataset_CNNTransformer/1102_221553/config.yml\"\n",
+    "config_path = \"../training/experiments/TransformerModel_IamLinesDataset_CNNTransformer/1116_082932/config.yml\"\n",
     "with open(config_path, \"r\") as f:\n",
     "    experiment_config = yaml.safe_load(f)"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": 42,
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'CNNTransformer'"
+      ]
+     },
+     "execution_count": 42,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
    "source": [
-    "dataset_args = experiment_config.get(\"dataset\", {})\n",
-    "datasets_module = importlib.import_module(\"text_recognizer.datasets\")\n",
-    "dataset_ = getattr(datasets_module, dataset_args[\"type\"])\n",
-    "\n",
-    "network_module = importlib.import_module(\"text_recognizer.networks\")\n",
-    "network_fn_ = getattr(network_module, experiment_config[\"network\"][\"type\"])"
+    "experiment_config[\"network\"][\"type\"]"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 8,
+   "execution_count": 43,
    "metadata": {},
    "outputs": [
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "2020-11-03 07:32:07.256 | DEBUG    | text_recognizer.models.base:load_weights:457 - Loading network with pretrained weights.\n"
+      "2020-11-18 20:31:23.104 | DEBUG    | text_recognizer.models.base:load_weights:432 - Loading network with pretrained weights.\n"
      ]
     }
    ],
    "source": [
-    "model = VisionTransformerModel(network_fn=network_fn_, dataset=dataset_, dataset_args=dataset_args)"
+    "model = TransformerModel(network_fn=experiment_config[\"network\"][\"type\"], dataset=experiment_config[\"dataset\"][\"type\"], dataset_args=experiment_config[\"dataset\"])"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 9,
+   "execution_count": 59,
    "metadata": {},
    "outputs": [
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "2020-11-03 07:32:10.285 | DEBUG    | text_recognizer.models.base:load_from_checkpoint:404 - Loading checkpoint...\n"
+      "2020-11-18 20:34:49.381 | DEBUG    | text_recognizer.models.base:load_from_checkpoint:379 - Loading checkpoint...\n"
      ]
     }
    ],
    "source": [
-    "checkpoint_path = \"../training/experiments/VisionTransformerModel_IamLinesDataset_CNNTransformer/1102_221553/model/last.pt\"\n",
-    "model.load_from_checkpoint(checkpoint_path)"
+    "ckpt_path = \"../training/experiments/TransformerModel_IamLinesDataset_CNNTransformer/1116_082932/model/best.pt\"\n",
+    "model.load_from_checkpoint(ckpt_path)"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 10,
+   "execution_count": 60,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -127,17 +142,17 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 24,
+   "execution_count": 126,
    "metadata": {},
    "outputs": [],
    "source": [
-    "data, target = dataset[0]\n",
+    "data, target = dataset[1]\n",
     "sentence = convert_y_label_to_string(target, dataset) "
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 25,
+   "execution_count": 127,
    "metadata": {},
    "outputs": [
     {
@@ -146,7 +161,7 @@
        "torch.Size([98])"
       ]
      },
-     "execution_count": 25,
+     "execution_count": 127,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -157,71 +172,68 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 26,
+   "execution_count": 128,
    "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "([], [])"
-      ]
-     },
-     "execution_count": 26,
-     "metadata": {},
-     "output_type": "execute_result"
-    },
-    {
-     "data": {
-      "image/png": "iVBORw0KGgoAAAANSUhEUgAABG0AAABCCAYAAADt2ys3AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAABIzklEQVR4nO29eZRcV3Xv/zk1dfU8qOeWWmrNrcHWiOVRsrFsYvNiDCHMYMgA/IITlvNLXiDv9wJJSCB5ITxeGJ6DDQuDbUwIGGyDIyPLkyQktSW1rLFbaqnnearq7qqu6r6/P6rO5dTtW9XdGoxs9metXl11hzPdc++q/b1776Msy0IQBEEQBEEQBEEQBEG4uvD8phsgCIIgCIIgCIIgCIIgzEREG0EQBEEQBEEQBEEQhKsQEW0EQRAEQRAEQRAEQRCuQkS0EQRBEARBEARBEARBuAoR0UYQBEEQBEEQBEEQBOEqREQbQRAEQRAEQRAEQRCEqxARbQRBEARBEARBEARBEK5CRLQRBEEwUErdqJRqUkqFlVLveJ3rXqKUspRSvjT7P6uU+tZFlHteKXX7pbdwTnV9Ryn192n2ZSulfqaUGlFK/fD1aM/lRim1QynVPsdj71NKvXyl2+Soc87tEwRBEARBEK5+RLQRBOGy4CYM/CaM1svA3wL/ZllWnmVZP/lNN8bEsqx/sCzrD3/T7bgEfg+oABZYlvXuiy1EKfU5pdT3Ll+zBEEQBEEQBOHqREQbQXgToJSqeCOWPR+UUt7XqarFwPE0bVBKqTf1czOdl89lYjFwxrKs+BWsQ7hIrvS9rpQqU0qpK1mHIAiCIAjCm403tfEhCG9mlFJFSqlPKqUOAN8xtv93pVSHUiqklDqtlHprcnuWUuorSqnO5N9XlFJZyX2lSqmnlFLDSqlBpdRLhjjxHaXUAaXUJ5RSRZfY5mql1I+UUn1KqRal1J9mOPY7SqlvKKWeUUqNAbcqpeqVUnuS7TyulPpd4/i7lFInkv3uUEr9v8a+tyuljiTP26uUuiZNnWeBpcDPkuFRWcn6vqCUegUYB5YqpW5QSh1MhvkcVErdYJSxRyn198l6wslwoAVKqe8rpUaTxy+ZZag+oJRqVUr1K6X+2ijb9jBRSgWVUt9TSg0k+3VwFqN7a3J8hpRS31ZKBZPl7FBKtSfnTTfwbaWURyn1V0qps8nyn1BKlRjt+KFSqjvZ/xeVUmvTjGe+Uup5pdRXlVKfB/4n8J7kuPyBUmqZUmp3so7+5BgVGefPmMtKqbcBnzXKOZqm7vuUUi8rpf5Xss8tSqnfMfZXK6V+mpzvzUqpPzL2pb1XXOr50+S4Lkwz7kop9W/JsTqlkvdjckehUuohpVRXsp9/r5Li5BzaX5K8jp3J/T9xVPrnSqneZNkfTdM2lFJ+pdS9SqmfAs2OMfhfyXnYo5T6plIq29j/R8lxG0yOY7XurFLqX5N1jyqljiml1iVP+xjQopT6vFKqLl2bBEEQBEEQhF8joo0gvIFIGtN3KKUeAy4AdwBfAH43uX8V8Clgq2VZ+cCdwPnk6X8NbAM2ANcCbwH+R3LfnwPtQBmJ8JXPAlZy3+8C/5As64JS6lGl1E41T4+T5PE/A44CNcBbgU8rpe7McNr7k/3LB36VPP+/gHLgfuD7yT4DPAR8PNnvdcDuZL0bgYeBjwMLgP8L/NTNCLcsaxnQCvy3ZHhUNLnrQ8AfJ9sRAp4Gvpos78vA00qpBUZR702eUwMsA/YB3wZKgJPA38wyXDcBq5Jj9D+VUvUux3wEKAQWJdvxCWAiQ5kfIHENlwEr+fW1B6hMtm1xsp/3A+8AtgPVwBDwNeP4nwMrSFyHV4HvOytLjscvgVcsy/pTy7L+hsQ8+kFybB8CFPCPyTrqk335XPJ817lsWdYvHOVcm6HP1wGngVLgn4CHlLI9PR4nMeerSYRt/YNS6rbkvkz3itnH/wncB2y3LCtdHpnrgLPJNvwN8J+GAPYdIA4sBzaSuJ//0HFuuvY/AuQAa0lch381zqskMTdqgD8AvqaUKna0fb1S6stAB/CXJOb0IuOQL5KYJxuS7ashIbqRHKd/BH4fqCLxLHo8ed4dwC3JcwuTxwwAWJb1JRL3RjlwKCnofUgplZNm7ARBEARBEATLsuRP/uTvDfBHwoBtJWEk/ylQ6nLMcqAXuB3wO/adBe4yvmsjGBJ5XJ4Els/ShtJk3a8m2/IpY995IAwMG3/jwMvJ/dcBrY7yPgN8O01d3wG+a3y/GegGPMa2x4DPJT+3khBmChzlfAP4O8e20yQMbbd6zwO3G9/3AH9rfP8QcMBxzj7gPuP4vzb2/Qvwc+P7fwOOpKl7CQmxbKGx7QDw3uTnzwHfS37+GLAXuGYOc+c88Anj+13A2eTnHcAkEDT2nwTeanyvAmKAz6XsomSbC43r9jDwGvAXjmPt9qdp5zuAw3OYyxnLSR5zH9BsfM9JtrOShDgxBeQb+/8R+M4c7pUdJISOLwMv635naEMnoBzX80MkxNEokG3sex/w/BzaXwVMA8Uude4gId75jG29wLbk59uAQ0AbCfFrpUsZChgDlhnbrgdakp8fAv7J2JeXnB9LkuWfISF6eTKMTRYJQecZYBD41mzzWP7kT/7kT/7kT/7k77fxTzxtBOGNQx1QDBwh4a0y4DzAsqxm4NMkjNpepdTjOmyBhEfBBePwC8ltAP9MIjTiv5RS55RSf5WmDQNAY7INxck2mbzDsqwi/Qf8P8a+xUC1SoTyDCulhkl49GQK6WkzPlcDbZZlTTv6UJP8/C4SYsQFpdQLSqnrjXr/3FHvIqPvc8HZjguO/WY7AHqMzxMu3/Nmqa/b+Dye5vhHgGeBx5MhMv+klPJnKNPsg3ntAfosy4oY3xcDPzbG6yQJkaNCKeVVSn1RJUKnRvm1J1epcf7dQDbwzQztQSlVkZyjHcmyvqfLmWUuO8u5WSVCpcJKKTMfkT2OlmWNJz/mJfs+aFlWyDEm+hpmulcgIVT9MfCPlmWNZOoj0GFZlmV812UtBvxAlzHO/5eEF8ps7V+UbP9QmjoHrNS8QeYcKichiL1G4jnS6nJ+GQmRqMFo2y+S28ExPpZlhUk8G2osy9oN/BsJz6xepdSDSqkCZwVWwotNP0smSXjHCYIgCIIgCA5EtBGENwiWZf05idCW14D/QyI3xN8ppVY4jnvUsqybSBiFFvCl5K7O5DZNbXIblmWFLMv6c8uylpIIh3rAkXtjhVLq74AW4H8Dx4ClyTbNlTYSb+qLjL98y7LuytRt43MnsMgRllVLwusBy7IOWpZ1Dwmj9CfAE0a9X3DUm2NZ1mPzaLuzHYsd++12vF5YlhWzLOvzlmWtAW4A3g58OMMpZuiLfe11cY5j24DfcYxZ0LKsDhIha/eQ8IApJOFdAQnvDM2/kzDyn1FK5WZo0z8k615vWVYB8EGznAxzOaW9lmW9ZCVCpfIsy3LNr+OgEyhRSuUb28xrmPZeSTJEYry/rZS6cZa6aoyQJrOsNhKeNqXGGBfMsf1tyfYXzeHYFCzLepyEt84jJEKnOpVS/66Uusk4rJ+EuLjWaFuhZVla+EkZn+Q1XsCv78WvWpa1GVhDIkzqL4xjFyilPqUSubh2A17gVsuyts23L4IgCIIgCL8NiGgjCG8gLMvqtSzry5ZlXUPCs6QI2KeUehgSeUCUUrcl87VESBhe2jPlMeB/qMQKLqUk8lPopLZvV0otTxqXIyS8KqaT+x4mEf5TBLzTsqxrLcv6V8uy+ubZ/ANASCWSy2YnPTbWKaW2zvH8X5HwGPhLlUieuoNEqNHjSqmAUuoDSqlCy7JiwKjR738HPqGUui6ZJDVXKXW3w2CfD88AK5VS71dK+ZRS7yFhnD51keVdFEqpW5N5Sbwk+hvj131240+UUguT+VT+GvhBhmO/CXxBKbU4WVeZUuqe5L58EmLDAAlvjH9IU8anSISh/UwZCWwd5JMIqRtRStWQatxnmss9wJL55lXSWJbVRiK07B9VIqHzNSQEDL2MeNp7xShjD4k8Qf+plHpLhurKgT9Nztl3k8jd84xlWV0k8jP9i1KqQCXyVS1TSm2fQ/u7SOQV+rpSqjhZ9i3z6H8kKYjdQSJnz3kSAtTZ5P5pEvfNvyqlygGUUjXq1/mnHgM+qpTakLw+/wD8yrKs80qprcl7zU8ixCrCr58lf5CsazvweWCRZVn/3bKsk3NtuyAIgiAIwm8bItoIwhsUy7IaLMu6n0Sogg5DySKRQLSfRGhFOYm8MQB/TyKXRSMJT5lXk9sgkVT2ORIG9D7g65ZlPZ/c902g2rKs+y3LevUS2jtFwjthAwmPnX7gWyS8NeZy/iQJkeZ3kud+HfiwZVmnkod8CDifDLP5BAmDGsuyDgF/RCJkY4hEGNh9l9CPgWQ//pyEcPGXwNsty+q/2DIvkkrgP0gINieBF0h4T6TjURIiwTkSOVv+PsOx/xv4KYlwuRCwn0ROIoDvkgiN6QBOJPfNIBkS9Mckkv0+qZKrVTn4PLCJhFD4NPCfxr5Mc/mHyf8DSqmLnZPvI+El1An8GPgby7KeS+7LdK/YWJa1i0RuoZ8ppTalqedXJO6vfhJJtX8vOYcg4RkVIDGOQySuZ9Uc2/8hEkLdKRI5az49x/NSsCyrzbKsL1iWtYJEcmvNfydxr+xP3lPPkUiOTXKc/j/gR0AXCQ/A9ybPKyAh+AyRmCcDJMIvIfFsWWxZ1rsty3o6+UwQBEEQBEEQMqBSQ+0FQRAEQRAEQRAEQRCEqwHxtBEEQRAEQRAEQRAEQbgK8f2mGyAIgiAIggCQXP3LmegboI9fr14l22W7bJftsv3Kbf+4ZVnfd9kuCMJvCAmPEgRBEARBEARBEARBuAqZl6eNUspKXbnUHa/Xi8fjIR6PczlFIcuymK3+uRwz3/MutszfJEqpyzr2lxOllN2+i2mj2/WY7RpdytzR2y5HW83vl6PNbxQu5prNpQzht4O5PM+u1uedIAiCIAiCIMyRfsuyZnjAzVe04dprryUQCNDb20tnZyeQ+mPZ4/Gwdu1a6urqeOaZZ4jH4/Mq31ne5WS+wszVYFTm5eVxww03MD09zYsvvsjk5OSMY5wGjRZFzHbp/Xrf9PS0/d0sw1mO27XweDwXLbgAXHvttWzcuJH29naee+652U+4jMx2jcw+e71eiouL+eAHP0hRUREPPfQQHR0d9tjNtR7nmM61LfNp9+tJunsFuKxtvBIC7MXgvGeuFOnuTXO/3mfeg3rO6m1u5znvVbf71+2a6jLdjtP7zXHJNC+cYqVlWUxPT6f023we6c8ejwe/308sFrPPcatD7xcEQRAEQRCENygX3DbOOxFxZWUl69evp6amBpgpsPj9fsrLy6mrq8Pj8czbeHIKDvNhth/ssxnr8zl+vuVfDDk5OdTW1rJ8+XLbe8mtDjfDyNxnjovbd/O/W7lu2y/WOPJ4PBQVFVFaWsro6OhFlXEpzMXbRhMIBKiurqa9vZ1wOExtbS1FRUWz1uEc33R1znW+KKXweC5/zvDZrqHH4yEnJwefL1XbTXevXO75f7HlXS3i1sXgdm9qnIKf83O6+1Jvd97r+pq5za9Mc8Msz+354dzn9kyanp627w3zL11f1q9fT3l5+Yy5mKkeQRAEQRAEQXgzMG9LUClFIBDA5/PNMNSUUni9XnJzc8nPz7e3mcfNZtxdbAiKLvtKcyWM03QsWLCA1atXMzw8TFtbm+ub/kyeQ5mMuHTnuOHmuXMp+P1+vF4vQ0NDl1TOlSYrK4tFixZx+vRpXnvtNRYuXMiCBQtmvf7O+e7cdrWQSUzyer1ce+21VFVVEQwGX+eWXV1cikh5Oeo2P5veKk7xJZ0A6zy2pKSEqqoqCgsLZ52fbn13evikO36ugo7zXnHi9XrZuHEjJSUleL1e1/G52u4tQRAEQRAEQbhczFu0GRsbY2xsjFgs5uqGb/6lc/NPRyahIRPpXPJnK/P1MsTSecDMdk5JSQlLly6lqamJc+fOzQg1c453pjoyiQZzHfPLNV5erxev18v4+PhlKc/kcrVRKWWLNp2dnRw/fpzS0lIKCgquqIGYyVNiPvh8PvLz82cYuXMlJyeHm2++mcLCwjl7zF2Osc/KysLv919yOZebK/GscBNMZruf0+13iqrpvGpKS0tZt24dW7ZsYdWqVQQCAYAUQXg2Lxu3Op0Cjlv7p6enXYVnt/rM8fD5fFxzzTVkZ2fbc3E2oUcQBEEQBEEQ3izMe8nvSCRCVlYWgUAg7VvYqakpOwlxuvwDVxI3Y+Zi63WGt7iFJThzNcylzHRlm98DgQB5eXmMjY0RiUTSluMW2uDxeJiampq1LXPB7Odc+5gOj8dDXl4eBQUFrn26WEyD9nIRDAZZtGgRExMTRKNRsrOzCQaDl60Ot7G8XGUvWLCA5cuXc+zYMdcwtEzX0ev1smjRIoqKimhtbSUUCs3p/rkcba+srCQSidDT0zPnczL1Jd39+5sy9s08LZkEEBNnm3U402w5dtzCoLZv3866desoKCigt7eXvr4+zp8/7/qsdmvLbH0zxRvzuZhOrEkn2Ohypqen8fl8rFmzhmg0aj/TLqfnnyAIgiAIgiBczczb08bn8xEMBgkEAq5vUn0+n/223CkmzMWodsuvMBecBsBcPGsuxnAzPXdM42IuOSDMet3GwmmIaNFmYGAgRaBxGm+mUaT3OQWb+XprZAqVulh0mbFYjPHx8csq2ujxvFy5XwKBALm5uQBEo1Gmp6cJBoMEg8HL5glyJYWD0tJStm3bxsKFC9OGzaTD7/dz3XXX8dprrxGJRC7JKJ7PuR6PhxtvvJG1a9fOq45M45jOG/A3gQ47y8vLY/Xq1SmCks7xoj/rv3TjNzU1ZYuzWsDRn01hSHtaWZaF1+tl+fLlHDhwgK997Wv86le/YufOnTPOMzGTA+s+mGQ6z9l3t3vT3K7bodsLCQGxsLCQcDjM4ODgjGTE5n8RbwRBEARBEIQ3I/O2cCcnJ/H5fPj9fvsHt/mj2xRs9H+3H+PpSPdWdjYyhQuY+zPlYHBrS6YQsHR1p2uXW9iC87Nbe2OxmGuYix4rbbA5c0Xoc7TxZhpNs4kbmbww0olOmdBzQOezmZqaIhaLzfn8uaL7PZeQnkx9yM/Pp6Kigmg0itfrZXp6mry8PHw+3xVfRehicF7PSCTC4OAg5eXl9r5M94fepq9RfX09p0+fdl2tLB2Xw2jWHnrOhLNXkktt91yFQsuyyMnJYenSpWzevNnersddXxPnve6cp6aQop9RWrgx7039fNDfly5dSktLC62trbS1tXHmzBmqqqrw+XwZ72stiDiflaawlEmINkOanAKRLjOdEJqTk8NNN93E3r17GRsbS+mPKWRfTNJ7QRAEQRAEQXgjMG/LSAs2pnu/+T8YDGJZFiMjIzO8PS41VOlyMJ9wFGd75yICzHX/bAa0x+OxjalMS9mab+ednjrT09O24JDuvPliGmnzQRtnWVlZtqeWZrYlxGerzymMmfXNBaeYp5SisLCQ6upqQqEQU1NTtlhztb7Nd7YrFosxOjrKxMREyna/309ZWRlr1qyhurqa06dPc/ToUaLRqF2ODmEbHR1NK1Dpa6YNZh0WeanebBc7vy6FS61rPvMsPz+fJUuWpAitzjL0uKarQ4uSzrmuxR7T+0+pXy+lvWbNGgYGBgiFQrbI4fV6ycnJIRQKpdRnCkfONpqefpDq5eL0qDE9AHW/3ISXqampGV6LOqn9tddeyyuvvGKLvOnqEgRBEARBEIQ3Ixcl2pjeMNow0D+28/PzycrKIhKJ2D/OTUFB/1i/3N4Kutzi4mKGh4ddw4Mu1jhzGin681z6MN+QJLMOj8djixuzhbg4jT6nEOHmYXQxzDfUxnme3+8nOzubrKysOQsszvG/XP3LNCeCwSD5+fm2YObz+WyD80rMXbc2mx4Mei5YlsXk5OSMMXOeOzk5STgcprCwEEgYwJWVldTW1lJRUUEwGGRsbMzOTxWLxZiamrLFwpKSEqLRqKsIEwwGWb58OdXV1eTn5+P3+5mammJ4eJhDhw4xNDR0UfNLKUVOTo69zLj2urmaSefBl+6a5uXlUVZWRktLywyh0OPxkJ2dTW1tLYFAgP7+fgYGBlLCCE2R1rlNf9btMdvm9XqpqqqycxRNTU3Z1zs7O5twOGy3I9Pc0l57boK927nmWJgeX6bwkuk54PP5KC0tpbm5+Yp45gmCIAiCIAjC1c68RRu/3088Hk9Zycj0dMjPzycQCNgGn/nG1+m6f7kNMr/fz44dO9i1a1fa5KlmqJZpgPv9fnw+H7FYbMYqTZrs7GyKi4spLy8nGAzS2tpKb2/vFTMunWETMPeVXZxluO1/vQ1iM/zGDIdwGq/p2mZ6H5keJJfSj0wimPYMUCqxkpQZcnKp6HtCz6n8/Hwsy6Kjo4Px8XG7XcFg0J5z+fn5eDweJiYmGBwcZHBwkJGRkZT2mN4PU1NTLFiwAJ/Px/Lly6mrqyM7O5toNMqFCxcYHBy0jzPRwpo5r/UYlJeXU1tbS3l5OdnZ2bZHhtfrpbS0lK1bt3Lw4EFGR0ddPW8y4fP5yM7OtvNluYkh80ELTEop4vF4Ro+1+ZTpts28x5weKebzLxgMUlRURHd3tz2/gsEgBQUFFBQU2KJOOBwmHA6neJLM9tx0u4/0Z6UU5eXldi4pfd3TJSt38y7Mzs6mrKyMuro6pqen6evr48KFCyl5j2bztEr3jDL7Z27XK8319fVlbKuERgmCIAiCIAhvVuYt2gSDQXt1KI3pzp+Xl4ff72doaCjFbV67umtRZ3R01E6wa/7ovhSjyufz8b73vY+DBw8yNjaWstKI1+u1jSId6jI5Ocng4CDRaJScnBxqamoYGBigv79/hoEQCASoqKigvr6edevWkZeXx9GjR9m7dy/9/f22QTgX48E04sw+m2/ITTKJBG7jpo3InJwcCgoK7DL0cu2QWFrZ5/OljNPrgdnXuRpaWkDQnhilpaWcP3/+inpiRKNRQqGQLZTk5eWlrIp2Kej7JSsri+rqapYvX87ChQuZmpri8OHDvPbaa0xPT5Odnc3ChQtZuXIltbW19r0Tj8dpb2/n7NmzNDc309fXN6MOpRR+v5/c3FwWLVrEjTfeiN/v5+jRoxw9ejRFGHJ6Pvh8vhSDXo97dXU1GzduZPny5Zw+fZoDBw7Q399PJBLB6/VSU1PDe97zHiKRCI2NjbZw46wjHT6fz/bkSyeczgUtjhQXF1NVVYXX6yUcDtPT08Pg4KCrp5Y+L13Yp95vhvn4fD4CgYAdMmpZFtFolGg0aucCco6t9uAKhUL4fD6KioooLS2lpqaG8vJyAC5cuEBTUxOTk5MzcgppDz83kcgt5Mp81ujyzWsSj8cJh8MpnpNu+P1+qqqq2Lx5M9dddx0TExN0dHTw05/+lL6+PtsLRj9nQ6HQDG+gTKF2fr/f9vjSnmR+v59gMEgkEkmZr26IaCMIgiAIgiC8WZm3aOPxeFK8UbQgoikuLsbr9TI0NGQnnAXIzc3l1ltv5aabbqKmpobGxka+/OUvzyvR6WzE43E7cazzTXNBQQHvec97qKurs708hoeHefLJJ2lsbMTn8/HBD36QxsZGdu/eTW9vb8r5S5YsYe3atQQCAX7yk58wPDzMfffdRzAYZPfu3XR1dV2W3BhmGfpteLpy3YxMU6C6+eabuemmm4hEIoTDYRoaGti7dy9TU1OsXbuW2tpannvuOUZHR6+I+OHsj2UlkipHIhFisZhrv9zasW7dOnp7e+np6SErK4sbbriBeDxOa2urPb8u99v28fFxhoaGWLt2LdPT0xQUFDAxMTEjREN74sw13415jerr61mzZg3j4+M0NjaSlZXFRz/6UT772c8SiUTYuHEjW7duxe/388gjj9Df34/H46GgoICtW7dy/fXXs3nzZh566CHb88gUCLRw9+lPf5qXX36ZF154gf7+/hltdXqEeDweQqGQ7UHh9/vZsGEDt912GwBf//rXU3JWaVGgubmZRx55hAceeIBIJMLx48cJh8MopWyxKdM4abFIh2pdbEJyn8/Hxo0b+fCHP0x5eTlFRUUMDQ3x/PPP861vfYtIJDLj+aBFtOzsbFuAjcfjTE5Ozsiz4/V6ycrKorS0lLVr11JeXk5eXh6WZdHV1UVTU5MdzmMKRPq87OxsAKqqqrjrrrvIzc2lpaWFhoYG2tvbmZiYcJ1Tzs9OUUYLI2biX1Ngmp6etq+B1+slOzub/Px82/PGvP7OeiorK7n55pupqanh4YcfpqOjg49//OPU19cTi8UYGBgAEgLre97zHh599FEmJibsUCotdjo9eLQgunTpUoqKihgdHeX06dNMTEyQl5dHRUUF4XDYfl6kC7+6mvNNCYIgCIIgCMKlMC/RRhubJSUlKX/XXXcd5eXlnDp1inXr1hEOh+nr67PDj/Ly8vjABz6Az+dj9+7dDA0N8dGPftQ1V8mlMDU1xcTEhJ1/BLDzc3z4wx/m+eef59FHH2V8fJy8vDzWrVvHJz/5ST7zmc8wPDzMrl27uPXWWyktLeWb3/ymLUxVVlZy9913c+rUKZ599lnb6HvyySf50Ic+xJEjR+jt7Z2zx0q6/rqFQZnhXG44ywgGg6xYsYKdO3dy9OhRPve5zxGPx9myZQtLlizhjjvu4KWXXuLd7343u3fvnncIy3xwiiher5fJyUk7se9s6Bwyg4ODQMLbaWBggJdeeon777+fL33pS7ZX1GyCjTZinSttmdt1nUopIpEIY2NjlJSUYFkWS5YsYWJiIiUUxOv1cvvtt5Obm8sLL7xgG65zGZdt27aRl5fH888/T09PD5ZlUVhYyDvf+U6ys7NZtGgRt912GwMDA3z3u9+1w6CmpqYYGBjgueeeY2hoiDvuuIN3vetdPProoymGt/aMeOc738mDDz7IU089xcTEhOu1NsURv99PXl6eLYh5PB7e9ra3sWXLFkZHR/nqV786Q2g1yzx//jzf+973WLt2LZOTkzQ2NpKdnc3tt9+OUopnnnmGaDQ6QzSZnp6msLAwJV/KXDGvZ0FBAW9961t5//vfz9e+9jUaGxuJRqPcdNNNfPjDH6avr48f/vCH9rm5ubksW7aM7du3s2HDBoaHh+nt7bWfIX19fezZs4fm5mZ7PLZt28Ytt9zCkiVLOHr0KHv27GFoaIj8/Hxuu+023vKWt9Da2sojjzxiJ/jVolBubi7FxcXccsstLF++nD179nD69GlGRkaIx+MpIqTpTaPzgpkirR47t/lrJgCGxL0TDAZTPCIrKytn3D/aG6ikpASfz0dnZyc+n4/bb7+d8fFxHn/8cTo7O5menubIkSPU1NRw4cIFhoeHycrKYsmSJZSVldltNUNQzaXJ9Tytr6/nrrvuorq6mi9+8Yt2+JbH4yE3N5fKykq6urrwer3289hcxtwpVIlwIwiCIAiCILzZuChPmy1btnDPPfcwNjbG+Pg4zc3NHDhwgJaWFq655ho8Hg/xeNzOR3DjjTfaLv/hcJi6ujr7jT/MTHRpkimHg0YbLvqHvWmwVFRU8Hu/93scPHiQEydO2Ll2RkdHuXDhAp2dnWzfvp2f//znHDp0iNraWqqrq9m8eTOHDh0C4O6776azs5OzZ8+meDSYxsNsIUxmrhGzn+n6pxPPZmdnzyjbNHi1IaQ/b9y4kVWrVvHcc89x5swZO9SgoaGBvLw8Nm3aRGtrK1lZWfT09NhhR87rcCmkK296etr20vJ6veTn59urFrmhzz9//nzK946ODl566SXuvPNO/uu//ove3t4Z4+gmzJhhJOlypuh9WrQpLCwkOzub+vp6zp07x8DAQIpXw69+9Sve9773kZubayfANg3pYDAIYBvkgUCA+vp6br/9dh566CFb3NReGIWFheTk5PD2t7+d1tZWDh8+7JqfaXp6mnPnzrFv3z7e/va38+ijj9r9VkpRWlpKfX09+/fv52c/+1lawcZ53bKysliwYAHnzp3DsizKy8t5y1vewtjYGD/4wQ9cw5bM621ZFk1NTdx+++20trbi9XqJRqMcOHCAL33pS5w+fZqWlhZb/NKeMcFgkNLSUqampuw5Yc7t2doNUFJSwtatW/nIRz7C5z//eZqammyvlYaGBuLxOJ/4xCf46U9/SjQaJRAIsGPHDq699lpOnjzJ008/zcjIiC1K5efns3jxYnbu3Mndd9/Ngw8+SH19PUuWLKGxsZHvf//7jIyMEIlEiMfjDA0N8fjjj1NbW8uGDRv47Gc/yxe+8AVbpCorK2PZsmUsW7aMI0eO8Nhjj9HV1WULmabAYQoveizM+Wlu0wKPMxxVf9bjG41G7fOrq6vZsGEDDQ0NMwSekpISbrnlFhYuXMg3vvENqqurKS8v58yZM3R3dxOPx/F4PCxatIiJiQmmpqbs+3nZsmX2inemN6b5jNbzffPmzWzatAmPx8O//Mu/EAqFiMVi9v5AIEBhYSHd3d14vV474bUpaJuCjSAIgiAIgiC8GZn3eqlZWVmMjIzwy1/+ki9/+ct86Utf4uGHH+all16iu7vbFhvMsKjVq1fT1NREZ2cneXl59htqp8s8pIoYcxFs9HHwa68awDaYS0pKKCgo4NixYzOSI4+OjnLo0CGqq6tRSjE+Ps7+/ftpaWnhhhtusJc2X7x4MV1dXfT19aUIAVu2bKG7u9sWcjLlacnUTyf6LXW6cAB9TEFBAe9617vIzc21jaiKigoikYhtGGuDLBqNcv78eVpbW7nhhhuoqqpKCaNwa7dSihUrVrBx40aqqqpmvQ76nHRYVmJZ6HA4zOjoqL2yUbpjdb91fhW9LRaL8corr1BcXMzSpUspLi7OeL55zdPVpduux3xycpLx8XHi8TgFBQWsWLGCzs5OhoeHU84dGxvD4/FQXFxsh72Yda1atYr6+noWLFgAJDyhtm/fzquvvsrQ0JBt2GZnZ7N06VI714nOr9Te3u4a+mFZFqFQiLa2Nvr6+lKuY0lJCatXr2bVqlW8+uqrcxJsNMFgkEWLFjE+Po7H42H9+vWMjY3R1NRkewRlQilFKBTi0KFDdHZ22td8cHCQvXv38ra3vY3y8nJ7jufl5bFz504+85nP8Ed/9EesW7fOziOU6f5weqUFAgFWr17N1q1beeSRR2hubmZiYsIWQkZHR2lpaSEcDlNaWmonAA4GgwwPD7N//346OjoYHBwkFAoRCoXo7e3l5MmTvPDCC5SXl3P77bezbt06urq6OHHiBP39/XbojvaCGhsb4/z58xw5coSJiQl27txp58Pavn0711xzDS0tLezZs4fOzk6i0egModmZfNjtOaBFQ7ck1Ga4kA5NMpPHZ2VlUVJSQmlpKSdPnkypUye6PnfuHNPT01RXV7Ny5UogETI4PT2Nz+cjNzeXzZs309XVlfL8Ky0ttQUjZ9tNcbugoICNGzdSVFTE008/zcDAgB0Wp/uhn999fX226OlWrgg2giAIgiAIwpuZeYs2Y2NjDA0NcebMGfbu3cuhQ4c4ffo0vb299o96vdywz+dj9erVhMNhhoaGgEQeh+rqao4ePZrWGJ3NyE7bmWQokfYeKSgooKSkhP7+foaHh1Pq08b/wMAA5eXl9lvb9vZ2uru7KSoqspOYFhYW2slFtdDh8/nYtGkTTU1NjI6OzmjLbPla3PI76M/6jbTz7bnTWPX7/axcuZLKykr8fj/XXnstwWCQ5uZmwuHwjHHs7++ns7OTpUuXMjo6SmVlpR2m5BaqtWDBArZv385b3/pWWwybK+mu3fT0NMPDwwwNDVFTU5P2/HTjpw2//v5+GhsbWblyJUuWLLGXRk9Xf7p55laHKdyEQiGWLVtmJ1d1rpQzNTXF0NBQSliIWZ/Oy5GXl2d7sqxcuZLjx4/bZfl8PsrLy9m0aROHDh0iEAiQlZXF+Pg4Y2NjMwxTM7dKIBCwl2vW9erkxjk5ObZwMhe091hRUZEtTi1btoyRkRHa2trSrr7kHMt4PM6xY8fsMBp9r+3du5fy8nIqKyvJycmxx6+3t5fjx4/boUk6x5LbnNT3uNO7YunSpdTW1jI4OMjLL79sJ9g2hYuxsTE6OzspKCiwBSO90t34+HhK/hp9zujoKOfOnePQoUPccssthMNh2tvbU5KP6+ujRZTx8XE6Ozs5dOgQN910E0VFRWzZssVedamtrY3Ozk7Gx8dTcvdkEn3NOeW8r908ytzGTwsz2nNGr0Bm1u3xeIhEInR1ddmhrVqIDwQCtti1adMmAFu01t4+OqGwz+dLEZHM0C4tsi1YsACPx0Nra2tKviPzPjfzG5l9NO8BQRAEQRAEQXgzMy/RxrIs+vr67LwD2hNBGyv6bbM2Nr1eL2vWrKGrq4t4PM7ChQtZuHAh8XjcDnkxy3aKGvPF5/OleJcUFRVRUlJCW1uba4iRXtFKh7AARCIR+vr66Ovro76+PuWNvN/vtwWbsrIyioqKaG5utsNXnH2YzxtgnSy0pKSE8vJySkpKUjw33DxilFLk5eXZx65YsQKApqamGSu3QOJN+ejoKHl5eZw5c4a6ujqysrJmtNvj8VBSUsLmzZvZsWMHGzZssFehMstzYzaxTc+h3t5elixZknGMMu2bnp7m0KFD5ObmsmTJEntlnEvB2e7JyUn6+/u54YYbGB0dJRwOp4ST6XOGhoZYuHAhJSUlroZ3MBi0RSWfz0dFRYW93LYe62XLllFaWsq+ffts4VEbrG4opSgqKqKurm6Gx1p5eTk1NTW2QDaXvuoyA4EAeXl5tnBSWFjI+Ph4SojWbMKCZSUS8prLkVtWImyqu7ubyspK2/MoEolw5MgRnnjiCY4ePWoLwM5+FxQUsHTpUtatW8fatWtZtWoVtbW19spNq1evprCwkEOHDtHd3T1DAICEmNTZ2Ulubi5er5fKykri8Ti9vb12cnJn3yzLYmJigpdeeom6ujq6uroYHBx0TVCsn4GWZTE2NsapU6eoq6tjxYoVrFq1CstKLOmuV1tKJ9hk8iBxCjTprqUTM+Ro2bJlFBcXc+LECdv7xhSA4vE44+PjtqDb09NDbm4upaWllJaWUldXx/XXX8/Ro0cZGBiwn7nxeJxQKERpaSnZ2dm2SGR6Dpp9MMUZ879uy+TkJKOjo/bz2SnWzXcMBEEQBEEQBOGNyLw9bXSyVf3WVb/ZNX9M67wU09PTLFiwAMuyKCoq4oYbbqCiooJ9+/bNSEY6FzL9MNdeDJFIxP6ek5NDfn6+65LIkAgTWLRokZ1fR/elq6uLw4cPs2nTJvsNellZGQUFBbbQs23bNhobGxkcHHTN8zGbV41pbOi219TU2ElON2zYYIckmXkszHJ0W+LxODk5Ofj9fiYmJlw9f8z2RKNRjh49yqpVq2wDVhtAehni66+/nne+851UVlYSiUTIyclJGetLoa+vj7a2NhYvXpzyRt7JbPWEQiGOHTtmJ5XOlLD5YohGo7S1tfGOd7yDw4cPp4gQZht1klndF7PdgUBghtCmDWCfz0dOTg7r1q1jzZo17Nu3j9bWVkZHR+28Km590klaly9fzubNm2d4wOhcSIFAwE6E68Q5tkr9OgFtaWmp3e6RkREAO1QwUzidiXPOaw+UV155hbKyMqqrq23PC53/JDc3l7GxsRTPIU1dXR333nsvf/AHf8B9993H+9//fnbu3ElOTg5FRUWUlZUxMTHB0aNH7fqd7YnFYvT39+Pz+cjKymL58uV0dnayZ88exsfH7ba6iQs6d41ektwUlUwPEi0+eDweioqKmJiYYMeOHYyMjHD+/Hk6Ojro7++f8fwzRQtn+93EYLOds10Ty7JsL8iKigpqa2uxLIvDhw+n5MoxRSct8GhPI4/Hw4oVK9ixYwd33HEHxcXF/OQnP7ETKOvre+7cOaqrq6msrLSXqNfjo0OfPB4P4+Pjdr6h7Oxs177o5cjLysrIzc0lKysLIEXsmmvyd0EQBEEQBEF4ozLvRMT6Dazf77cNDUj8eM7JySErKysl2eTY2Bj19fXs2LGDWCzGoUOHUpJfOsn05jSTce/1eqmoqGBsbCwlmWw4HCY/P3/GOXrp5GuuuYZdu3alCC/hcJju7m6WL18OwL59+9iwYQPj4+NEo1GWLFnCe9/7Xv7qr/4qxdhzC+dw9kUfZ+7z+/0sXbqURx55hJMnT3L27FkCgYAd9uM0IvVn7aXR39+P1+ult7fX9qxwvqm3LIvi4mKqqqpoa2tj//79/Mmf/AmLFi1ieHiYsbExO5fKAw88wIYNG/jCF75AeXk5tbW1LFq0KOO1mM+b7lAoxMjICDU1NSxcuHCG15Um3RwxxYCDBw+yaNEiVqxYwb59+1I8S9J5K8yGPk/nBlqzZg0nT560Vw1z9rW4uJjh4WHC4fCMkL+amhpGR0ftfEqRSITXXnuNP/7jP+Yb3/iGnS+opaWF5557DkgIo4FAgMrKSkpKSujt7U3pR3l5OXfccQcrV64kHA4zPj6eEhbT1tZGd3c3119//Zz77/f7uf7663n/+9/PHXfcQVZWFseOHaOpqYlbb70Vn89Ha2sroVAoJe+I6fWRLmG2uf306dPccsstVFRUkJ2dzfj4uD1eFRUVtLS02Pl0TIO8p6eHH/3oR0QiEdsLSHv2veUtb2FoaIgLFy7MWNnKbKNepru7u5vFixfj9/uZnJy0k+m6tV8/W3QoYV5enu2Vo9vtTC4eCASoqamxE0Tfe++9nDp1Ckh4+5w+fTplfMyyzPKcXjhO4df5PHDifP6Mj49z7733cvDgQfbt25cSQmbWo73XCgsLGRwcxOv1curUKd773vfy7ne/m//4j//gn//5nwmFQrb3i2VZRKNRzpw5w49//GM+8YlPcPjwYRoaGmhtbbXz3ujn7PT0NKFQiKqqKm699VaefPLJlFBNpRRDQ0M0NTXx6U9/mpycHA4ePMj58+cJhUIpeYAEQRAEQRAE4c3MvEWbqakpOxxDizN6KVyv18vw8LC9ig7AoUOH+PnPf86uXbt4+OGH2b9/P5AwbPRqT3r1kWAwSE5ODh6Px056OlcxwOv1snTpUjo7O20Dob+/n6amJjvprmlclZaWsnr1agoKCti7dy+QCKeanJy03/yPjo5SUVHBs88+y4oVK/jIRz7CH/7hH2JZFo899pgdhmEmTTUNOWeSUNOw1f+9Xi/V1dXcd999fPWrX+Wpp55ibGyM0tJStm/fzk033eRqGOvxCgQCDA4OkpOTY393Eyv8fj/XXHMNW7Zs4cc//jFjY2M8+OCDPPDAA3R1dREKhVi4cCFLly7liSee4O/+7u/svDNve9vbuPPOO/nP//xP29ugoKCA4uJicnJyiMVinD9/3tXDx4nuQ0dHB0899RQPPPAAf/mXf5mSK0aj+1FSUsLExERKwlO9X+ff8Pv91NTUMDw8PCN8ZL6YYoNe6UrPUT3ntTDo8/m48cYb+dGPfkR3d/eMsgKBAG1tbfT399seG1/5yld4+OGH+da3vkVraytPPfUUu3fvtoWKWCzGd7/7XX7nd36H6upqdu3axcjICFVVVezcuROfz8f+/fs5ceKE7S1lcurUKX7xi1+watUqvvjFL/LFL36RM2fOpKzU5QyHW758OX6/n/3799PX18c//dM/MTw8zL59+8jLy+Otb30rd955J7t27eKFF16gq6srxbNE/+n7zFy62qzLXDZde62Y+ZtGRkYIh8MzPCh6enpcRVCdgPvChQu0tLTMWHHJPFaHnPX29jIxMWE/b7QIo4VmXYf21qurq+OTn/wkP/rRj1iwYAErV660lwPX80I/AwsLC9m0aRPbtm2jr6+Pffv2sWPHDtavX08oFKK9vX2GWOM2T00Pq7k+B51LamuUUnYC8MLCQvr6+mYkuNZtCAaDbNiwgQ984AMcOHCALVu2UFNTg2VZfPvb36awsJDrr7+ez3/+8/ziF7/g6NGjDA4OEo1GmZqaIhQK8fjjj/Piiy9SX1/P9ddfz6233sr4+Ditra2cPXuWzs5OYrEYu3btwu/386lPfYrt27fz4osv0t7eTiwWs0M/Fy1aRHt7OxMTE/zZn/0ZWVlZNDc3c/DgQQ4ePEhvb6/dDzePR0EQBEEQBEF4ozNv0QYgJyeHnJwcfD6fbQjq3A8vvPACk5OT+Hw+qquruf/++2lsbKS2tpZ7772X+vp6+vr6mJiYoLCwkIqKCgoLC4nFYnR3d3Py5EkaGhpcDRXtWu806LThU1xcTE9PD1NTU9TU1BAMBhkdHbVX+NFhLMuXL2f79u2sXbuWH/zgB7bBoT0bQqEQw8PD9PT0kJ2dTV9fHz/84Q/52Mc+xl133UUsFuNjH/sYO3bsoK2tzV4FSOdg0G+FzeSt6d6Ea8Oxp6eHp59+2vZk0Ibk0NBQSkJV02ugqKjIzq+hl5PWRqj5Jtrr9bJ27VoqKyvp7u62jdsXX3yRiYkJ6uvrycrK4siRIxw/fpz29nZGR0eZnp6ms7OThoYGKioq+NznPkdJSQkej4doNGqvfBOLxWhububBBx90vWZOLwzLSqzcdfDgQW6//XbuueceXnjhBQYGBmaspBMMBvn93/99Ghoa7HmhjVJ9XDgcZmJiwnUVqfmi2+vxeCgvL+euu+7iscce47bbbrO9QPQ1CAQC3HjjjQQCAfr6+lJCkbSgU1JSYs8N3fdIJEJ7ezsVFRUsXLiQ973vfWzbto0jR45w7Ngx+vv7aWlp4dlnn2Xp0qXceuutdg6V3bt309HRwcjICBUVFUxMTNhJvjXxeJyGhgb+9m//lptvvpm/+Iu/oKenh4GBAcbHx+nr6+PYsWOcOHHCnls9PT22YFJfX297kEWjUfbs2UNTUxMrVqxg6dKl3H///YyMjNjedvF4nGg0SiQSsYWDn//855w7dy7FC0gnqvX5fExNTdnL0WsBTAs6Om+QOW+c4ULm58nJSYaGhhgdHU3xSHEeNz09zcDAAD09PUxMTNDd3c2WLVtYvXo1zz//vJ04WV+39evXs3nzZqqrq9m/fz979uwhLy+Pu+++my1bttDQ0EBHRwfxeJzS0lLWr19PRUUFoVCIl19+mVOnTjE5OcnPfvYzPv3pTzM2NmYL2n6/37Vveu5lEmn0/HPeB2a/zfP9fj9r166lrKyMiooK6urqaGpq4sKFC/bxOtfSjh07WLp0Kc8++yy7d+9OEYG0V+CePXvsufu7v/u7HDt2jJMnT9Lb2wskhLn29nYGBgbw+/3k5eWxYMECqqurueaaa+wkx729vTz55JO89tprbNq0iY0bN7J582b72dvX10dzczMvv/wyo6OjvPTSS5SVlbFo0SKWL1/O1q1b+fGPf5yyEqEgCIIgCIIgvNmYt2hjWZYdkuH8oRyLxTh37pz9lnT16tU0NDTw2GOPUVxcTGFhIfn5+axatco2zoeHh2lubmZoaIjBwUH7LTi4hxeZuWc02oApKCigsbHRzoVQVVXFtm3bbDFH57DxeDwMDw/zzDPP2MdDYiUUHSK0devWlGWaq6qq6Orq4gc/+AHnzp0DsMUmn89ne2WEw2HbgDfFBxPzezweZ2JigoGBAUZHR+0x1WEY+k2yOQ46hKG8vJz9+/fbXg6nTp3Csiw7BEqPy7Jly9i4cSOxWIyDBw/auSRCoRCvvvoqbW1t+Hw+RkZG6OvrSwkxmZyc5MyZM0QiEcrKysjPz7e9kczlis08JG7hG+b80XNlaGiIJ554go0bN/Lud7+bSCRCLBazr/PZs2d57bXXqKmp4cyZM8Cvw9q2bt3KsWPHGBkZITs7m9zc3LTJZOeDKdisXr0av9/P97//fXbu3MnmzZsJBAL09/eTk5PDxo0bueeee9i1a5edbNvsb3l5OWNjY4yNjRGLxfB4POTk5LB+/XoOHDhAR0eHLUxpD6l169YRi8W4cOEC/f39DA4OEgwGbe+Vnp4eO6wlFArR3d09437Q9+ipU6cYGRnh+PHj9rLpOlmsNpz18XplLD03zPxUOrxrYGCApqYmysrK7HmvhRW97LVeMW7lypWMjY3R0tKSck1WrFhhe7GZQpb2tnGKdm5zx0lhYaEtIJnhh6YXi5mvSa8E19jYCMCaNWv42Mc+Zof7+P1+YrEYY2NjtLa2cvDgQc6ePWuHwL3yyitUV1ezcOFC1qxZY68yNjIyQnt7O319fXR1dTE0NIRSiqamJvbs2cPmzZvZunWrfc/rhMtmn93udfN/ppAoZ+ik1+uluLiYTZs2sWHDBnbv3k1xcTELFy7kAx/4AOfOnaOrq4uxsTGWLFlCbW0tsViMY8eOcejQIYaHh1M8yyBx/4VCIQYGBujq6iIvL4/BwUFbGNdt1c9CSKw4GAqF6Onpwefz2QKkXr1vfHycgYEBCgsL8Xq99jzU944OOxwaGqK/v5+Ojg7OnTtHTU2N/Yy+FM86QRAEQRAEQbiambdoE41GOXDgAAMDA3ZuC9NYCIfD9kpIAwMDnD9/nrNnz5KdnU1VVRVVVVUUFRXZOUO6u7tpb29nZGTE9ngxDRa3N+zOxK4670RRUREXLlwgFosxOjrK4OAg4XCYvLw8CgsL7ZCsnp4eTpw4QWNjox1OAzA8PGwbbQsWLODcuXMMDw9TUlKCz+fj2LFjdHd309rait/vp6KigkAggN/vZ2pqirGxMdtbQYd4uIU/mP2Kx+MMDQ3ZOS/09tzcXAKBAENDQ6791slUjxw5YocHnDhxgqKiIrKysmxPkMWLF3PdddcBcO7cOVpaWlLGVHsowMwcMrqdw8PDhEIh20tCew+ZHkCZvAMyCTcNDQ1kZ2ezcOHClOWCIWGMQyLMbWxszD53enqaoqIi7rzzTsLhMIsXLwagoaHBtf754PP57HFbtWoVjY2NHD58mJKSEioqKti6dSvj4+O2kNfa2srevXvtJeVNAoEAx44ds/fp8ZyYmGDfvn2cO3eOqakpgsGgHW6ml0GORCKMjIykCHnOcQyHw5w/f95eAcxEJ4Y9e/YsLS0t9rUDbM8f8xxzOXtnQlztHRSNRunr6+PMmTMp3h66Pu3NFolE7ATkzntYh7t0d3fbHnO6HHNFJl23E7c8MIODg1hWYhlpZ04bfYx5X+t6u7q6gMQ8XL9+vb06XDQaZXR01A7n0R5z+u/48eP09fVRW1tLZWUlU1NT9opzfX199ipj+poPDw/zy1/+kvHxcerq6qiuruaGG25gYGCAsbEx21ttdHSUnp4e29suE+Yzwfk8tCyLYDBIWVkZdXV11NXV0dvby4EDB/D7/WzZsoVVq1axbNkyampqGBgYID8/n0gkwpkzZzhx4gTd3d0p3n3mNYnFYoyMjDAyMuKaa8fMrwTY4u7IyIidO8gUBcPhMM3NzSlhic6cSeaxY2NjDAwM0NHRQVlZmeS2EQRBEARBEN7UXFQiYp0w1W1ZadPQ379/f4qhOjw8zMmTJ+3jTfd+pzjjlvDVWZdGL8utw4z02+umpiba2tooLCy038hOTEzQ399vv70122xZFoODg4RCIc6fP09hYSEDAwNUVFTQ1NTE0NCQbVDppYLN880lgN3Gxg2dx+PIkSMpx+nws56eHnub2U69HPWFCxfsetva2giFQnY4QlFREZs2baKmpobnnnuOkydPMjk5OcMLwem15CayxONx2yPJNBT1sXPtr7NcHX7j9/vJyckhOzubYDBoCwyRSIRDhw6l5FAJhUIcP36cj3/842RnZxOPxzl27BgdHR1zqjcTeXl5FBcXU1tbS05ODk888QTj4+Ps2rWL2267jeXLl1NUVEQ8Hqe5uZnvfOc7jIyMuPY5FArR0NCQ4lEyMTGREv6nBRHtpQGpRqs5Vk7MFXgyoQ1lM6eNxrzeWoDRiXnTeUw5l6o2y1JKcfLkSXJzc+0QK12Gx+MhFovZooC5z7ISS6drMclNpDLrMcfk8OHDxONxO0eO23gppfD7/WRnZ6csLd3R0UFnZycvv/yyvax9NBq181qZzybd50gkQltbG+3t7bZQZebwcQpPsViMpqYmurq6WLRoERs2bGDlypXE43EGBweBhGjW2dlJJBKxV3rS5zv7m+67+TkYDFJaWkpVVRU9PT0899xz9sp6fX19vPbaa2zYsIHq6mri8ThHjhzhxIkTDA0N2TllnNfBrMdcCcr5/DDbbop6Pp/PFiCdq6KZ803Pf12+Hl9zrmovHZ3rSLxsBEEQBEEQhDcraq4GNoDH47F0GMKcCk/+kNY/7E2RxvyBnqauOb9BDQaD1NbW8o53vIOvfOUrKUsg6x/0pqFg7jMNhHSGaFZWlm2omh4yuj9Ow1f3ca5j5BR5PB4Pd955JzU1NTQ0NHD48GHX89zq0kuHX3fdddTV1dHY2MjLL79sh2s5PRXmc/3d2qDLuRyYxtds4+fxeMjOzmbBggXEYjFCoZDrUtFzRV/HtWvXsnHjRqampjh69ChnzpxJGTefz2fPI3MlHKfwOFcyiZMX0/75Hm/mK9HLiN9222189atfTevtYZ6Tbr+uw008cbYDEnmXtm3bxpkzZ+ykzekw7xlt1Ovv6dqclZVFZWUl99xzD1//+tdneHPoNujQHmcf3Tx89DlmCJEz+bgeZ6fAoZ8pOiwMUj2gnJ5O6cbXbUz1Pqe4qoVQs51ObylzXujxdatLt90p7Hm93pRnvXm+UorJyUk755auR6lEomQtujlXzTLH0qzLPFaHVckS4IIgCIIgCMIbmAbLsrY4N15UThtTCHEzrvR+/aNch9GkW1HGjdmWe9bGgfbWaG5uThFsnIaMM4GxaQyYxolbfXq5Zmd5ptGoj9XGoGn0uQkzZvluY6GXKdfJQtO1zVlmNBrl/Pnzdk4b0ytCt1OPg9Mgmw9z7d9cSechkY7p6Wk758XlQM+H06dP09zcbOfc0G0yjVV9vO6zXnFovmiDdT4i32ztn01QcR5v1qsFCL2kdqbyMgl2mfqi7xvzGaLFln379s2hp6n3jFMoyUQ8HufEiRMAdjJkc/6azye3Z4VeIcqc9+aqWfpcfY7zPtMinw4FsyxrRkiYOU66HlPY0H0222Xe325jYbbfTQxy1m3OR9MDxixP990pzJh91G3T11kLPZD6/Nb3kHk99PGmqGTOHZ3/SAtFFyuaCoIgCIIgCMLVzkWtHjXbG3bzODM5qymUpDPyZxMA3IxN02U+XdtmMyRNA8lNzHC+aTfLMw15wH6TbBr78zHKtVEXDodTViQy68hkmJsGjHm807jTxpL55toNNw8Y5/inO3euHiCXGt5wOYQjLew539ZrozCdyHWxxqJbUm0n8/VmytT/2a6FNpz1Kkpmeena4QxNyVS/NtjdQm/MNl4O49spYExOTtLd3W0v0+30Qkl3r+vv+rngtkqRvrdMAUP31xRGzHvIeT+a5ehjNG7hcqbgZwpYznaZbTFX5TL367nt9DA0hRnTK8Yp1prPTS28mHXpY3Xiai0kp3vOmuNi3h9Ogc4Usy71+SEIgiAIgiAIVysXJdqkw/njezYjOlMYRbry0/04zyQaZNpv1p/O60EbgE4hxGlU6P9uhp2b0evcpusJBAJAQoSai7t/ulAOZ7vcxuByjH0mLrcxlU48cOvDfAUPpxGcbhzdPl+s0PB6egfM5VrolY0y4ZwL8xWUMl3DyzUemYSV2epx9s8tQbfzu9v9oesxw6ecoonz+WE+N2fzaMz0bHW2xSkiK6VsEc05v511uj2n3YQf/d8UgvWx+llpel86PS/1fh12ZXrbaNHJ2V4tGEkyYkEQBEEQBOHNyrxFm3SeAW4GC6S6tOvtbgbfXAzs+QgA6QxsZ3mmwZDu+Ezf3Qx6MxFnuna7rbakv3u9Xjs/y1zEidk8XTJtm6uoNdsxrxfON/zOfZdadqbt6eq7UuMyX9HpUpmammJyctJVtMkkWM2VuRw7m+fRpdQ127zXAoDb/ZTuHjPno9vxmUTadNvSCb5u7Z6r4Krb6fRumW2cMl2PdN5K5vPUKdxAqieg2T+niGXW4SYciVAjCIIgCIIg/DZwUZ42s4kn5o9up9u9c7+bsHEpRttsRoabh4Rbf5xvnWdrvz7WbXsm3Mqampqit7d3zp4HlyJezPfYy+05czFlzvWc+c6jTOElvw3E43Gi0SiRSGRW8eRSxuY3Oa5Og3+2e9Y51zJ5vjjPM+tyChmZws2com8m5ioEuWGG52V6BprClFPodhO5zDxPZlt0SJTbmJr1OMPXnG1xa6eERwmCIAiCIAhvVi7K00Yz1zfubvvc8jRcCs433m51Otuuj8m0Uko648JZt8bNiHHzDsn0ljgajdLW1jaHXmfmchszcxHr5lvfxbbv9TT8L5f3x3y4HKLIfMc2Ho8zPj5Odnb2nPp8uefXb0LMcRMknOGQGuc9bD4f3EKfNHpfJqHIzVvHWY557sXMSS2cuInXZohROq9IZzvcPArN55t5rs514/f7U8bbeb5Zjr4GZpJis35nO0W4EQRBEARBEN6MXNTqUfPxIkmXkyWT90267ZneCps/9NPhXEo23Vtb0wgwz9FvjzOtbJVOjJrPuE1NTTE0NJTxmLkYzJdqxKTzIsrUjosVDObSnyv1Rn22ctN5W10JwcI0XDMdOxcDf77EYjGGh4cZHx9/04eeuIX16Ps93UpykPoMcRNnM3n56WPSMdszIl04kls5mTAFKdO7T9fhHBPzv3NeOFfIMsM+M3nFmM9ZXa4WjpwJk93EIeezWVaPEgRBEARBEN6sqPn80FVK9QEXrlxzBEEQBEEQBEEQBEEQfutYbFlWmXPjvEQbQRAEQRAEQRAEQRAE4fUhfRyAIAiCIAiCIAiCIAiC8BtDRBtBEARBEARBEARBEISrEBFtBEEQBEEQBEEQBEEQrkJEtBEEQRAEQRAEQRAEQbgKEdFGEARBEARBEARBEAThKkREG0EQBEEQBEEQBEEQhKsQEW0EQRAEQRAEQRAEQRCuQkS0EQRBEARBEARBEARBuAoR0UYQBEEQBEEQBEEQBOEq5P8HWVjnahOypUIAAAAASUVORK5CYII=\n",
-      "text/plain": [
-       "<Figure size 1440x1440 with 1 Axes>"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    }
-   ],
+   "outputs": [],
    "source": [
-    "plt.figure(figsize=(20, 20))\n",
-    "plt.title(sentence)\n",
-    "plt.imshow(data.squeeze(0).numpy(), cmap='gray')\n",
-    "plt.xticks([])\n",
-    "plt.yticks([])"
+    "data = data * (data > 0.1)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 129,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from torchvision import transforms"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 130,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "ra = transforms.RandomAffine((-1.1, 1.1), scale=(0.5, 1))"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 32,
+   "execution_count": 131,
    "metadata": {},
    "outputs": [],
    "source": [
-    "    def make_len_mask(inp):\n",
-    "        return (inp == 79).transpose(0, 1)"
+    "data = ra(data)"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 34,
+   "execution_count": 132,
    "metadata": {},
    "outputs": [
     {
      "data": {
+      "image/png": "iVBORw0KGgoAAAANSUhEUgAADSUAAACECAYAAADcSq9PAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAAArFElEQVR4nO3de7SVVbkw8GcK3k0xEfvQvGQqKh39DC8pKVkZpEkUnmPHPOVJK80ir2hfmnnJW6WEhWYePQfNS6ThEfWIQ/GG6CC8tFXwbpSXvCECIgjz+2MtXte7Dmu71nZvtsDvNwajZ67nnXM+633XbQ/H05tyzgEAAAAAAAAAAAAAAADQrFW6uwAAAAAAAAAAAAAAAABg+aIpCQAAAAAAAAAAAAAAAGiJpiQAAAAAAAAAAAAAAACgJZqSAAAAAAAAAAAAAAAAgJZoSgIAAAAAAAAAAAAAAABaoikJAAAAAAAAAAAAAAAAaImmJAAAAACAFVxK6ZSU0uXdXUezUkqfTinN6Mb9L0spnd5Ja01KKR1ajQ9KKd3S5LxHUkqDOqOG5V3tOQQAAAAAAAA+ODQlAQAAAADwgZJzvivnvE0zx6aUBqWU/tbRvVJK30wp3d3R+a3IOV+Rc96nyWO3zzlP6uKSAAAAAAAAADpMUxIAAAAAQJ2U0kbL49rwXlJKPbu7hu6SKjr830W6+r2bUtowpZS6cg8AAAAAAADoTJqSAAAAAAAiIqXUK6V0eErp/oi4rObxkSmlv6eU3kwpzUgpfbb6+OoppfNTSs9X/52fUlq9muudUrohpTQrpfRaSumummaIy1JK96eUvptS6tVOPSeklJ6q7vtoSmlYTe6bKaW7U0o/Tym9nlJ6JqU0pCa/RUrpjurciRHR+z2e+9CU0oMppdnVPQdXHz8kpfRYdZ2nU0rfqZkzKKX0t5TS8Smlf6SUXkgpfTml9MWU0uPV5/2jmuNXqXlOr6aUrkkpfbhBPaW7H6WUnk0pHZtSejil9EZK6eqU0hoppbUj4qaI6JtSmlP917e9a1O3z7YRcWFEfKo6d1ZNev2U0oTqc78vpbRlzbx+KaWJ1ec4I6X0z+2d35p5xV2ZUkpjUko/r8uPTykdXfOcP1eNT6mer/+q1vNISmlAzbydUkoPVHN/qJ6f0+uu08iU0osRcel7nZ+U0mEppSerz+/6lFLfmlxOKR2RUnqiut9pKaUtU0qTq6+fa1JKqzVxLrZIlffHKtXxxSmlf9Tkx6aUflgzZbOU0j3VPW9JKfWuOXa36v6zUkoPpZQG1eQmpZTOSCndExHzIuJjrVy/lNKqKaVhKaXrI+LJmsdXT5X3319TSi+llC5MKa35XucwVZyXKu+Z2Smlv6SU+len/XtEPJNS+mlKaYv3OocAAAAAAADQ3TQlAQAAAAArrVRplNknpXRlRDwXEftExBkRsX81v01EHBkRO+ecPxQRX4iIZ6vT/19E7BYRO0bEDhGxS0T8uJo7JiL+FhEbRsRGEfGjiMjV3P4R8bPqWs+llH6fUvp8+t93cHkqIj4dEetFxE8j4vKU0v+pye8aETOi0nB0TkRcklJxl5XfR8Sfq7nTIuIb7ZyDXSLivyLiuIjoFRF71jzHf0TEfhGxbkQcEhHnpZR2qpn+kYhYIyI2joiTI+LiiPh6RHyyWvtJNc0V34+IL0fEXhHRNyJej4hfN6prKf45IgZHxBYR8U8R8c2c89yIGBIRz+ec16n+ez7avzaFnPNjEfHdiLi3OrdXTfrAqJz39aPSjHJGRESqNEJNjMo57lM97jcppe1aeC4REVdGxL8suWYppfWj8vq7qsHx+1dzvSLi+oi4oDpvtYi4LiqNdB+urjusbu5HqrnNIuLb0c75SSntHRFnRuV8/5+ovC/qa/pCVK7xbhFxfET8NirX/aMR0T8ivvZeTz7n/ExEzI6I/1t9aM+ImJMqjWIRldfJHTVT/jUqr8E+EbFaRBxbrXfjiJgQEadXn+OxEfHHlNKGNXMPrj7vD0XEy9HE9UspfSKl9MuI+Hv1OU6oPr8lzoqIraNyDj8e774H3usc7lN9rltH5b39zxHxavWcnF2tp09ETE0p3Z5SOjiltFZ75xIAAAAAAAC6i6YkAAAAAGCllFI6MirNN2dFxL0RsWXOeVjOeXzOeWH1sEURsXpEbJdSWjXn/GzO+alq7qCIODXn/I+c88tRaWA5uJpbGJVmhM1yzgtzznflnHNERHX8p5zzsIjYMiKmRMTZEfFstaaoHveHnPPzOefFOeerI+KJqDSPLPFczvninPOiiPjP6n4bpZQ2jYidI+KknPPbOec7I+K/2zkV34qI/8g5T6zu9fec8/RqDRNyzk/lijsi4paoNBstsTAizqier6ui0gQ1Kuf8Zs75kYh4NCpNLxGV5p//l3P+W8757Yg4JSKGp5R6tlNbrV9Vz8dr1eezYzvHtndtmnVdzvn+nPM7EXFFzX77RcSzOedLc87v5JwfiIg/RsQBLa5/V1Qa1Zacz+FRaY56vsHxd+ecb6xe77Hx7nndLSJ6RuX8LMw5XxsR99fNXRwRP6m+Ht6K9s/PQVF5PUyrXqcTo3Inqc1r1jsn5zy7eo3bIuKWnPPTOec3onLnqv8bzbkjIvZKKX2kOh5XHW8RlUa4h2qOvTTn/Hi1/mvi3evx9Yi4sXpuFuecJ0bE1Ij4Ys3cy3LOj1Sv5eBo5/qllPZOKU2NiBsjYn5EDMw5fyrnfFHOeVb1mBSVJqejcs6v5ZzfjEqj4YFNnMOFUWmO6hcRKef8WM75hSWF5pyn5JwPj0rj3pioNHj9LaX0uybPKQAAAAAAACwzmpIAAAAAgJXVFlG5C86DUWl+eLX+gJzzkxHxw6g00PwjpXRVSqlvNd03KndAWeK56mMREedG5e46t6SUnk4pndCghlcj4uFqDetXa4qIiJTSv6WUHkwpzUopzYrKHWh618x9sabOedVwnWoNr1fvIlRbWyMfjcpdmf6XlNKQlNKUlNJr1Rq+WFfDq9UmmYiIt6r/+1JN/q1qTRGVu/RcV/N8HotK09dG7dRW68WaeF7NukvT3rVpVqP9NouIXZc8j+pzOSgqdyNqWrVJ7ap4965C/xqV5qdm61mj2tDVNyL+vqTprWpm3dyXc87za8btnZ9SLuc8Jyqv041rjq+/xo2u+Xu5IyIGReXOQXdGxKSo3CFpr4i4K+e8uObY9q7HAXXXY2BUmvSWqD0f73X9+kTlzkdtUflc+OtS6t4wItaKiD/XrHFz9fGIds5hzvm2qNzl6tdR+Uz5bUpp3foNqs1MSz4bFkTl/Q8AAAAAAAAfKJqSAAAAAICVUs75mKjcqagtIkZHxDMppdNSSlvVHff7nPPAqDQz5Kjc1Sgi4vnqY0tsWn0sqncKOibn/LGI2D8ijk4pfXbJgSmlrVJKp0XEMxExKiL+EhEfq9YUKaXNIuLiiDgyIjbIOfeq1pmaeGovRMT6KaW162prZGb1PJSklFaPyh1kfh4RG1VruLHJGhrtMyTn3Kvm3xo55793cL0l8lIea3htmpzfnpkRcUfd81inenebVl0ZlbtFbRYRu0blfLfqhYjYuHr3niU+WndM/XNs7/yUctXX0QYR8X6v09LcEZU7RQ2qxndHxB5RaUq6o8k1ZkbE2LrrsXbO+ayaY+obthpev5zzVVFpUBoblbuIPZ9SujilNLBmjVei0ny1fc0a6+WclzRKtXsOc86/yjl/MiK2i4itI+K4mmM3SCkdmVK6PyJui4geEfGZnPNuTZ4PAAAAAAAAWGY0JQEAAAAAK62c8z9yzr/MOf9TRHw1InpFxL0ppf+IiEgpbZNS2rvaoDM/Ko0IS+7ecmVE/DiltGFKqXdEnBwRl1fn7ZdS+ni1UeSNqNwRaHE19x8RcW91r6/knHfIOZ+Xc365prS1o9JI8XJ1ziHR5J1Scs7PRcTUiPhpSmm1ajPFl9qZcklEHJJS+mxKaZWU0sYppX4RsVpErF6t4Z2U0pCI2KeZGhq4MCLOqDbgRPW8DX0f6y3xUkRskFJar+axhtemwfxNUkqrNbnfDRGxdUrp4JTSqtV/O6eUtm218JzzA1FpcPldRPxPznlWq2tE5bW0KCKOTCn1rJ7TXd5jTnvn58qovB52rL7ufxYR9+Wcn+1AbZFSyimlQUvL5ZyfiMp76utRaRSaHZXr8dVovinp8oj4UkrpCymlHimlNVJKg1JKmzQ4/j2vX855frUZcZ+I2CEino2IS1NKT1Xzi6PSNHheSqlP9XlunFL6QnWJhuewuteuKaVVI2JuVD5Xlnw2fKu6114R8dOI+GjOeWTO+bEmzwUAAAAAAAAsU5qSAAAAAAAiIuf855zz9yOib1QaaCIqTTlnRaVx5MWI6BMRJ1Zzp0el+efhqNzpaFr1sYiIrSLi1oiYE5Wmkd/knG+v5i6MiL455+/nnKc1qOXRiPhFde5LEfGJiLinhafzr1G5885rEfGTiPivdp73/RFxSEScF5UGqjsiYrOc85sR8YOIuCYiXq+ueX0LNdQbVZ1/S0rpzYiYUq3xfck5T49KE8jTKaVZKaW+0f61qXdbRDwSES+mlF5pYr83o9KcdWBU7ojzYlTunrV6B5/C7yPic9X/bVnOeUFEfCUqd/WZFZUGnxsi4u12pjU8PznnWyPipKjctemFqNxF68CO1JZS+mhEvFndo5E7IuLVnPPMmnGq1vSeqvOGRsSPotJANzMqdx5a6n//aPX65Zxn5pzPyDlvFRHfqEmNjIgnI2JKSml2VN7v21TntHcO141KQ9PrEfFcRLwaEedWc/dG5b13QM55Qs55UTPnAAAAAAAAALpLyjl3dw0AAAAAAEAnSSndFxEX5pwv7eY6vh4R2+ecT3zPgwEAAAAAAIDljqYkAAAAAABYjqWU9oqIGVG5o9dBUbkb18dyzi90a2EAAAAAAADACq1ndxcAAAAAAAC8L9tExDURsXZEPB0RwzUkrZhSSo9ExGZLSb0cERt63OMe97jHP3CPfyfnfMVSHgcAAAAAWCG4UxIAAAAAAAAAAAAAAADQklW6uwAAAAAAAAAAAAAAAABg+aIpCQAAAAAAAAAAAAAAAGhJz1YOTinlrioEAAAAAAAAAAAAAAAA+MB5Jee8Yf2D7pQEAAAAAAAAAAAAAAAANPLc0h7UlAQAAAAAAAAAAAAAAAC0pGd3FwAAAADQEfvtt18R33DDDd1YCQAAAAAAAAAArHzcKQkAAAAAAAAAAAAAAABoiaYkAAAAAAAAAAAAAAAAoCUp59z8wSk1fzAAAKygdthhh9L4oYce6qZKAFYu48ePL41vu+22Il599dVLuXPOOWeZ1AQAAAAAAAAAACuBP+ecB9Q/6E5JAAAAAAAAAAAAAAAAQEs0JQEAAAAAAAAAAAAAAAAt0ZQEAAAAAAAAAAAAAAAAtKRndxcAAADLg/vvv7+IJ0yYUMr169eviK+++uplVhPAymC33XYr4t///velXO1n7qhRo5ZZTQAAAAAAAAAAgDslAQAAAAAAAAAAAAAAAC3SlAQAAAAAAAAAAAAAAAC0pGd3FwAAsLIbMWJEaTxq1Kim5l1zzTWl8ezZs4v40EMPff+FreROPfXU0niXXXZpeOwvfvGLIr766qu7rCaAldEWW2xRxA888EDD4x5++OFlUQ4AAAAAAAAAAFDlTkkAAAAAAAAAAAAAAABASzQlAQAAAAAAAAAAAAAAAC3RlAQAAAAAAAAAAAAAAAC0pGd3FwB0XP/+/Yu4ra2tGysBWrHbbrsV8ZQpU7qxkvYNGDCgiKdOndqNlayYhg0bVsT77bdfKTdq1Kim1pg4cWJpPGfOnCKufZ1FfLBfax8ktdfiT3/6U9PzXnnllS6o5oNr3333LeIJEyZ0YyXAymCbbbYp4iuvvLLhcW+88cayKAcAAAAAAAAAAKhypyQAAAAAAAAAAAAAAACgJZqSAAAAAAAAAAAAAAAAgJb07O4CWHkNGDCgiPfYY49SrlevXkWcUirlTjnllK4sa7nyjW98o4inT59eyl1yySXLuhygSVOmTOnuEpry9a9/vYjnz59fyrW1tS3rclY4e++9dxHPmjWrQ2sMGjSoNL7rrruKeM6cOR1ac2W3aNGiIp42bVrT83r2XLl+Vtf+jpswYUI3VgKsDDbZZJOmjnvllVe6uBIAAAAAAAAAAKCWOyUBAAAAAAAAAAAAAAAALdGUBAAAAAAAAAAAAAAAALREUxIAAAAAAAAAAAAAAADQkp7dXQArjwMPPLA07tOnTxGPGjWq4bw99tijy2paVvr3718at7W1dcq666+/fhHXnk+AztC3b98i7qzPLd611157FfGNN97YoTVWWaXcX96vX78ivvDCCztW2EquV69eHZr39ttvd24hH3ATJkzo7hKAlci6667b1HGLFy/u4kq63mGHHVYa1/4ee/7550u53r17F/G1115bys2YMaMLqgMAAAAAAAAAgDJ3SgIAAAAAAAAAAAAAAABaoikJAAAAAAAAAAAAAAAAaEnP7i6AFdspp5yy1LgV99xzT2n89NNPF/HHPvaxDq25LFx11VVFPHny5FLukEMOKY2nTZtWxFdccUXTe7z++utFXHte6HwDBw4sjXfeeecinjlzZik3a9asIr711lu7tK4PmtGjRxfxpptuWsq9+OKLRfyd73xnmdVEx2255Zadsk7//v2L+N///d9LuYULFxbxM888U8rdfffdRdzW1tYptXyQrLXWWkU8Y8aMpufVfh7Vn7M333zz/Re2kps9e3ZTx+24446l8fz585uaN2DAgNJ46tSpTc2LiDj22GOL+Oc//3nT87pCK3V/UAwaNKg0njRpUrfUsazUfvZGRGy//fZFvNFGG5Vy999/fxFPmTKlQ/sNGzasNL7uuus6tA4sTbPfbxtvvHHD3O9+97vSuPY3/IIFC0q52tfv9OnTm9r7/Tj//POL+N577y3lLr744qbWGDFiRGncym8LAAAAAAAAAADoKHdKAgAAAAAAAAAAAAAAAFqiKQkAAAAAAAAAAAAAAABoiaYkAAAAAAAAAAAAAAAAoCU9u7sAVixnnXVWaXzCCSd0+h4vvPBCEffv37+U+/KXv1zEp59+eqfv3YpTTz21iB999NF2jz3//POLeOrUqaXcjBkzGs7r3bt3ET/yyCMtVvjBN3z48CL+9Kc/Xco9/PDDRbzjjjuWcmPGjCni9zr3tbbbbrvS+NBDDy3iF198sZQ755xzGq6z++67F/Fpp51Wyp100klN17M8OPfcc0vjiy66qIjb2tqannfcccd1bmF0itdee61D84499tjSeKuttiriG2+8sZQbP358w3VqP0frtff6Wl706dOniC+99NKm562zzjpFvGjRolJu4sSJTa+z5557FvHQoUNLublz5xbxySef3PSaK4IJEyY0ddzAgQNL49tuu62peT17ln9+77rrrkV83333tTt3/fXXb2oPlu6MM84oja+88soivuCCC5peZ8CAAaVx/W+3zjBs2LAifvrpp0u5hx56qOG82t/iDzzwQCn31FNPFfGDDz5Yym2xxRZFPHLkyFLu/vvvL+Lbb7+94d6bb755wxwdU3st/vu//7uUa+U37sqk/m+G2r8nat/zERHjxo0r4vr39T777FPE06dP78wSl+qll14q4quvvrpDayxYsKCzygEAAAAAAAAAgKa5UxIAAAAAAAAAAAAAAADQEk1JAAAAAAAAAAAAAAAAQEt6dncBLP922mmnIn722We7fL8NNtigiNva2kq5733ve12+fyMHH3xwaTx27Nim595www1FfOCBB5ZyP/3pTxvO23777Yu4/lysCHr06FHEI0aMaHjc8OHDS+NHH320qfUHDx5cGg8ZMqQ0bm/P9kyePHmpcUTE+eefX8Q//OEPO7T+D37wg9L4V7/6VYfW6ajRo0cX8UUXXVTKtfc6rP2sePnllzu/sBXE4YcfXsRjxoxZpnsfeeSRpfG8efOamnfiiSeWxquvvnpp/J3vfKdD9Zx88slFfPbZZ5dyI0eO7NCaHyRvvvlmh+allIq49jsxImLq1KlNr7PvvvsW8THHHFPK1X4eHnDAAaXcH/7wh6b3WJH17t27NG72e3jKlCml8YABA5rec/78+U0fy/9W+3srIuKCCy7o0Dq1v78iIubMmVPE06dPb3qdoUOHFvHmm29eytW+TrbbbrtS7l/+5V+KeIsttijlzjjjjCJu5bfhjBkzivjmm28u5Q466KAiHjRoUCk3adKkIj7vvPOa3o/m1H7ONPv7dkWVc27quNq/HyLK37Xjxo1rOK/+87X+M74jfvKTn5TG7f1dd+aZZ3Zoj8742wIAAAAAAAAAAN4Pd0oCAAAAAAAAAAAAAAAAWqIpCQAAAAAAAAAAAAAAAGiJpiQAAAAAAAAAAAAAAACgJT27uwCWf5/4xCeK+P777+/y/V5++eWGuY022qjL92/k85//fGk8duzYpufeeuutRfztb3+74XFDhgwpjefMmdP0HsuDE088sTQ+88wzm5o3bty4pvfYaaedinjo0KGl3OGHH970Oh01d+7cIu7fv38p19bW1tQaRx11VGk8derUIp48efL7qG7pPvvZz5bGtc+h2ZojIvbdd98iPu20095/YcuRYcOGlcYzZswo4kcffbSU+/jHP75MalqaHXfcsTSeN29ew2Nrr2ffvn1Lue9///udWldExFprrdXpay5rI0eOLI1vu+22Dq0zc+bMIv7qV79ayh1xxBEN59V/5r322msNj+3Tp08RP/PMM62WuMKqfd0/8cQTnbJm7Wf4e1lzzTU7Zc+VyXbbbVfEDz/8cKes+c4775TG6623XlPzttlmm9K49vt01KhRDefdd999pfHOO+/c8NhWvpebdcUVVxTxmDFjSrlJkyZ1+n68a/311+/uEj4w3nrrrYa52r8h/vjHP5Zyzb4/d9hhh9K4M777evfu/b7XqHfJJZeUxvXvSQAAAAAAAAAAWNbcKQkAAAAAAAAAAAAAAABoiaYkAAAAAAAAAAAAAAAAoCU9u7sAln/vvPNOEU+bNq3L93v88ccb5t54440u37+RVVddtVPWefrppxvmPvnJT5bGTzzxRKfs2Z1GjBhRxDfddFOX73fIIYcU8fnnn9/l+9Vbd911i7itra3peUOHDi3it99+u5Tbeuuti3jy5Mnvo7ql22WXXUrjO+64o6l5xx9/fGk8derUTqtpebPhhhuWxrNnzy7i1VZbrZRba621lklNSwwaNKiIH3744VJuwIABDeftuuuuRXzRRRd1el31evTo0eV7dLX99tuvNL7zzjs7tM7ChQuLeN68eU3Pq7+eEyZMaHhs7fX9z//8zxaqW/7tv//+RXz99deXcvvuu28RH3HEEcuspiVmzZrV6WsefPDBRTx27NiGxw0fPrw0HjduXKfX0hVWWeXd/w+G2u/giPJ36/jx45tes/43X7O/AY8++ujS+Kqrrmp6z1pz584t4jXXXLNDa3TUiy++WBrvuOOORfzggw8u01pWBi+88EJ3l9CyAw44oDT+wx/+0CnrLlq0qIg/85nPlHK9evUq4ltuuaWUq/0+a8+Xv/zl0rj+eTTrmmuuKeJTTjml6Xm33357Ec+ZM6eUO+OMM4r4W9/6VofqAgAAAAAAAACAruJOSQAAAAAAAAAAAAAAAEBLNCUBAAAAAAAAAAAAAAAALdGUBAAAAAAAAAAAAAAAALSkZ3cXwPJv3rx5Rbz//vuXctdff/37Xv/HP/5xaTxz5syGx77yyitFPHDgwFLu7rvvft+1tOf111/vlHUmTZpUGh977LFFfPrpp5dyo0aN6pQ9l6VBgwaVxrNnzy7iBx98sMv3X2211Yp4xowZXb7fdtttVxr36NGjQ+vsvffeRXzNNdeUcptvvnmH1mzWrFmzSuNNNtmk4bHf+973irj+ud50002dWtfy5OWXXy6Na89N/ev+7bffXhYlFQYPHlzEJ5xwQik3evTohvOeeeaZIl68eHHnFxYRO+20UxHXvneXJ7XvicmTJ5dyCxcu7NCaX/rSl4r4qaeeanpezrk0fuutt4p42LBhpdy1117bodpWBF/72teKeNNNNy3l7rnnnmVdTklKqdPX7Nu3b1PH7b777qXxuHHjmppX/1tl/vz5RVz/ebf22msX8THHHNPU+u+lra2tiI844ohSrvZzZfz48U2vudlmm5XGl112WcNj99xzzyK+7bbbSrnbb7+96T1rnXTSSUU8ceLEDq3RUeuvv35pXPsdtttuu5VyU6ZM6dJaDjvssNK4d+/eRXzmmWd26d4REZ/73OdK41tvvbXT96h9TsuLrqq59rdT/W/fkSNHNpxX+/fTI488UspNnz69iA844IAO1TVixIjS+Mknnyzi+t9H/fv3L+IBAwaUcgsWLCjiuXPnlnJd/V4CAAAAAAAAAID3w52SAAAAAAAAAAAAAAAAgJZoSgIAAAAAAAAAAAAAAABa0rO7C2D5d9111xXxL3/5y1Lu+uuv79Cav/jFL4r40UcfLeU22GCDhvPGjRtXxKeeemopd/nllxfxI488UsrNnz+/4X7Nuvnmmzs0LyLioIMOKuLPfOYzpdzPf/7zhvNWXXXVDu/ZXfbff//S+Oijj16m+8+ZM2eZ7nfSSSeVxl/72tc6tE7fvn2LeMSIEaXc6NGjm1pj3333LY179nz3K2D8+PEN5911112l8cUXX1zE2267bSk3c+bMIj7zzDObqmtl8NJLL5XGG264YcNjP/zhD3dpLf379y+Na69ZvSeffLKIBw4cWMpdeumlRbzNNtt0UnVlP/zhD4v4N7/5TZfs0dUGDx5cxD/5yU9KuV122aWpNeqv2TrrrFPEjz32WNO11B9b+z35+OOPl3LHHXdc0+uuaB588MEiPv3000u5+s/RZe2pp57q0LwddtihiL/73e+WcocffnjDebXv+7feeqvp/b73ve8Vcf13Vnv222+/Ij7//PNLudrPg1bstNNORfzCCy+UcqeddlqH1lywYEHTx955551FfNttt5VyTzzxRBFPmzatlKut+6ijjirlHnrooSK+8cYbS7ndd9+9iCdPntx0nTvuuGMR77PPPqVc7Xlr7zrUvz9qX2s33HBDKffKK68U8fbbb1/K1b4Opk6dWsrV/q6q/T0SETF9+vQiXha/Qfbaa6/S+NZbb+30Pdra2op4+PDhpVzt3z0fJP369euUdYYMGVIaz5s3r4gXL17c9Dq176X6eR39XKn9Xt50001LuWOOOaaI699Ltb/na69tRMQXvvCFIj733HNLuTvuuKOIx4wZU8pdddVVzZYNAAAAAAAAAABdwp2SAAAAAAAAAAAAAAAAgJZoSgIAAAAAAAAAAAAAAABaoikJAAAAAAAAAAAAAAAAaEnP7i6AFcvMmTNL49/97ndFPGHChFJu++23L+JtttmmlBs7dmwR33LLLaXc8OHDG+5/3333FfGIESNKuf3226+IBw0aVMqtvfbaRbzBBhuUch/60IeK+I033ijlzj333CK+/vrrS7k77rijiGfPnl3KLVy4sDSufY6HHnpoNGvixIlNH/tBsckmm3Tr/muuuWaX7/HHP/6xiEePHt0pa77zzjsNc/fcc08R/+lPfyrl1l133SJ++OGHS7lZs2YVcZ8+fUq5iy++uIjrX6+9e/cu4n/7t38r5QYPHtywzpXZ/PnzS+Pa61Kv9rPj0ksvLeX+9re/FXH9e6lHjx5FXH9dau27776l8dlnn91ULdtuu20pV/uZesMNNzRc470MGDCgiI888shSbty4cUU8ZcqUDu+xLP3mN78pjX/9618X8bRp00q5rbfeuqk1TzzxxNL4oIMOKuJvf/vbTdf2kY98pDTef//9i3jnnXduep0V3dSpU4u4V69eHVpj6NChpfHmm29exJtuumkpt9pqqxVx/eft0UcfXRrXvieGDRtWyq2xxhpFfOWVV5ZyDz30UBEffvjh7ZVecvfddy81fi+1r/tW1H6WdPRzZeDAgaVxbd1f/OIXO7TmYYcdVhrfeOONHVrnlFNOabhu/ff8ggULivjggw9ueo9LLrmkiOs/H1566aWlrh8R8fe//72Ix48fX8rNmDGjqb1POumk0niPPfYo4j333LOU69u3bxE/99xzpdyQIUOa2q/emDFjirj+s7j+99/rr79exHPnzi3lcs5FfPLJJzfcr/7vgq5Q+16q/77+1Kc+VcRPP/10KbfVVlsV8eLFi0u5RYsWLTWOiLjqqquK+MEHH2y6zm9961tFfNNNNzU9r17t66T2My0iYuTIkQ3n1X4efuUrXynlPvvZzxbxXnvtVcpddNFFRfzoo4+WcqNGjWq4X1tbWxEfc8wxDY+r/zu2Wccdd1xp3K9fvyI+8MADS7n/+Z//KeLHHnuslPvLX/5SxA888EApV/+bBAAAAAAAAAAAOsqdkgAAAAAAAAAAAAAAAICWaEoCAAAAAAAAAAAAAAAAWqIpCQAAAAAAAAAAAAAAAGhJyjk3f3BKzR8MEbHnnnsW8Sc/+clS7rHHHivim2++eZnVRPe67LLLSuNx48YV8Q033NDl+w8ePLiIf/SjH5Vyl19+eWn87LPPFvHGG29cym233XZFvPfee5dyTz31VBHPmzevlJs8eXIR//a3v22y6ojjjz++iM8555ym53WGQw45pDS+9957i3jXXXct5b773e8W8dy5c0u5l156qYjXWGONUq72PF177bWl3HXXXddixR98Q4cOLeLx48c3PK5///6l8XrrrVfE99xzT9Pz2traivib3/xmKVf/nmzWaaedVsQbbbRRKffQQw81nLfuuuuWxltuuWXDWu6+++4O1basTZw4sYg///nPNz2v9juy/jOm9lpfeOGF76O6jqm9vhtssEEp98QTTxTxX/7yl1Lu1ltv7drCusDIkSNL47PPPrubKqGr/exnPyuN638HsPIYMGBAEdf/JllllXf/fzvuvPPOhmv84Ac/KI1/9atfdVJ1zan9LfrhD3+4lOvo9+fo0aOL+IILLijlZsyY0XDeUUcdVcTnnXde0/sNGTKkNO7Vq1cRX3nllU2vs/vuuxdx7W/tlc0ee+xRGtd+f2+66aalXO3fOcvibzAAAAAAAAAAAFYIf845D6h/0J2SAAAAAAAAAAAAAAAAgJZoSgIAAAAAAAAAAAAAAABaknLOzR+cUvMHAzThiiuuKOJtt922lLvzzjuL+N577y3lFixYUMTrrbdeKbfOOus03O+vf/1rET/++OOl3KBBgxqu8+qrr5Zyffr0KeIJEyaUcm1tbQ33r91j0qRJDY8DVi477bRTaTxt2rRuqgRWXGeddVZpfMIJJ3RTJawIhg8fXhqPGzeumyrpGueee25pvHjx4iJea621Srnvf//7HdqjX79+pfH06dM7tA4AAAAAAAAAALBM/DnnPKD+QXdKAgAAAAAAAAAAAAAAAFqiKQkAAAAAAAAAAAAAAABoiaYkAAAAAAAAAAAAAAAAoCU9u7sAYOV20EEHFXG/fv1KuenTpy/TWpbFfpMmTeryPYDlz7Rp07q7BFjhLVq0qLtLYAWy6qqrdncJXeq4447r8j2W9W99AAAAAAAAAACg87lTEgAAAAAAAAAAAAAAANASTUkAAAAAAAAAAAAAAABAS3p2dwEAS0yfPr27SwAAVlA9evTo7hJYgay++urdXQIAAAAAAAAAAEC3c6ckAAAAAAAAAAAAAAAAoCWakgAAAAAAAAAAAAAAAICWaEoCAAAAAAAAAAAAAAAAWtKzuwsAAADoaj169OjuEliBXHbZZd1dAgAAAAAAAAAAQLdzpyQAAAAAAAAAAAAAAACgJZqSAAAAAAAAAAAAAAAAgJb07O4CAAAA2tO/f/8ibmtr69AaCxcu7KxyAAAAAAAAAAAAgHCnJAAAAAAAAAAAAAAAAKBFmpIAAAAAAAAAAAAAAACAlmhKAgAAAAAAAAAAAAAAAFrSs7sLAAAAaM9ee+1VxG1tbR1a4+233+6scgAAAAAAAAAAAIBwpyQAAAAAAAAAAAAAAACgRZqSAAAAAAAAAAAAAAAAgJb07O4CAAAA2vPII48U8cEHH1zKjR07tqk1Zs+e3ak1AQAAAAAAAAAAwMrOnZIAAAAAAAAAAAAAAACAlmhKAgAAAAAAAAAAAAAAAFqiKQkAAAAAAAAAAAAAAABoSc/uLgAAAKA9kyZNKuLDDz+8Q2ssWLCgk6oBAAAAAAAAAAAAItwpCQAAAAAAAAAAAAAAAGiRpiQAAAAAAAAAAAAAAACgJT27uwAAAIBmjRkzpjQ+/vjji/icc84p5QYPHlzEPXr06NrCAAAAAAAAAAAAYCXjTkkAAAAAAAAAAAAAAABASzQlAQAAAAAAAAAAAAAAAC3RlAQAAAAAAAAAAAAAAAC0JOWcmz84peYPBgAAAAAAAAAAAAAAAJZ3f845D6h/0J2SAAAAAAAAAAAAAAAAgJZoSgIAAAAAAAAAAAAAAABaoikJAAAAAAAAAAAAAAAAaImmJAAAAAAAAAAAAAAAAKAlmpIAAAAAAAAAAAAAAACAlmhKAgAAAAAAAAAAAAAAAFrSs8XjX4mI57qiEAAAAAAAAAAAAAAAAOADZ7OlPZhyzsu6EAAAAAAAAAAAAAAAAGA5tkp3FwAAAAAAAAAAAAAAAAAsXzQlAQAAAAAAAAAAAAAAAC3RlAQAAAAAAAAAAAAAAAC0RFMSAAAAAAAAAAAAAAAA0BJNSQAAAAAAAAAAAAAAAEBLNCUBAAAAAAAAAAAAAAAALdGUBAAAAAAAAAAAAAAAALREUxIAAAAAAAAAAAAAAADQEk1JAAAAAAAAAAAAAAAAQEv+PwM/aoaIbrClAAAAAElFTkSuQmCC\n",
       "text/plain": [
-       "torch.Size([98, 1])"
+       "<Figure size 4320x1440 with 1 Axes>"
       ]
      },
-     "execution_count": 34,
      "metadata": {},
-     "output_type": "execute_result"
+     "output_type": "display_data"
     }
    ],
    "source": [
-    "make_len_mask(target.unsqueeze(0)).shape"
+    "plt.figure(figsize=(60, 20))\n",
+    "plt.title(sentence)\n",
+    "plt.imshow(data.squeeze(0).numpy(), cmap='gray')\n",
+    "plt.xticks([])\n",
+    "plt.yticks([])\n",
+    "plt.show()"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 27,
+   "execution_count": 133,
    "metadata": {
     "scrolled": true
    },
@@ -229,10 +241,10 @@
     {
      "data": {
       "text/plain": [
-       "('to stel mire of a thar chishirchit<eos>', 0.20226626098155975)"
+       "('and Came came into Mr. I. I. \"Amering whin<eos>', 0.32183724641799927)"
       ]
      },
-     "execution_count": 27,
+     "execution_count": 133,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -243,153 +255,6 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 103,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "2020-10-31 16:35:40.255 | DEBUG    | text_recognizer.models.base:load_weights:457 - Loading network with pretrained weights.\n",
-      "2020-10-31 16:35:40.837 | DEBUG    | text_recognizer.models.base:load_from_checkpoint:404 - Loading checkpoint...\n"
-     ]
-    }
-   ],
-   "source": [
-    "target_transform = Compose([torch.tensor, AddTokens(pad_token=\"_\", eos_token=\"<eos>\")])\n",
-    "dataset = IamLinesDataset(train=False, pad_token=\"_\", eos_token=\"<eos>\", target_transform=target_transform)\n",
-    "dataset.load_or_generate_data()\n",
-    "\n",
-    "\n",
-    "config_path = \"../training/experiments/TransformerEncoderModel_IamLinesDataset_CNNTransformerEncoder/1031_150630/config.yml\"\n",
-    "with open(config_path, \"r\") as f:\n",
-    "    experiment_config = yaml.safe_load(f)\n",
-    "\n",
-    "\n",
-    "dataset_args = experiment_config.get(\"dataset\", {})\n",
-    "datasets_module = importlib.import_module(\"text_recognizer.datasets\")\n",
-    "dataset_ = getattr(datasets_module, dataset_args[\"type\"])\n",
-    "\n",
-    "network_module = importlib.import_module(\"text_recognizer.networks\")\n",
-    "network_fn_ = getattr(network_module, experiment_config[\"network\"][\"type\"])\n",
-    "\n",
-    "\n",
-    "checkpoint_path = \"../training/experiments/TransformerEncoderModel_IamLinesDataset_CNNTransformerEncoder/1031_150630/model/last.pt\"\n",
-    "\n",
-    "\n",
-    "model = TransformerEncoderModel(network_fn=network_fn_, dataset=dataset_, dataset_args=dataset_args)\n",
-    "model.load_from_checkpoint(checkpoint_path)\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 92,
-   "metadata": {
-    "scrolled": true
-   },
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "===============================================================================================\n",
-      "Layer (type:depth-idx)                        Output Shape              Param #\n",
-      "===============================================================================================\n",
-      "├─WideResidualNetwork: 1-1                    [-1, 256, 2, 60]          --\n",
-      "|    └─Sequential: 2-1                        [-1, 256, 2, 60]          --\n",
-      "|    |    └─Conv2d: 3-1                       [-1, 8, 28, 952]          72\n",
-      "|    |    └─Sequential: 3-2                   [-1, 16, 28, 952]         --\n",
-      "|    |    |    └─WideBlock: 4-1               [-1, 16, 28, 952]         3,632\n",
-      "|    |    └─Sequential: 3-3                   [-1, 32, 14, 476]         --\n",
-      "|    |    |    └─WideBlock: 4-2               [-1, 32, 14, 476]         14,432\n",
-      "|    |    └─Sequential: 3-4                   [-1, 64, 7, 238]          --\n",
-      "|    |    |    └─WideBlock: 4-3               [-1, 64, 7, 238]          57,536\n",
-      "|    |    └─Sequential: 3-5                   [-1, 128, 4, 119]         --\n",
-      "|    |    |    └─WideBlock: 4-4               [-1, 128, 4, 119]         229,760\n",
-      "|    |    └─Sequential: 3-6                   [-1, 256, 2, 60]          --\n",
-      "|    |    |    └─WideBlock: 4-5               [-1, 256, 2, 60]          918,272\n",
-      "├─Conv2d: 1-2                                 [-1, 97, 2, 60]           24,929\n",
-      "├─Linear: 1-3                                 [-1, 97, 96]              11,616\n",
-      "├─PositionalEncoding: 1-4                     [-1, 97, 96]              --\n",
-      "|    └─Dropout: 2-2                           [-1, 97, 96]              --\n",
-      "├─TransformerEncoder: 1-5                     [-1, 2, 96]               --\n",
-      "|    └─ModuleList: 2                          []                        --\n",
-      "|    |    └─TransformerEncoderLayer: 3-7      [-1, 2, 96]               --\n",
-      "|    |    |    └─MultiheadAttention: 4-6      [-1, 2, 96]               37,248\n",
-      "|    |    |    └─Dropout: 4-7                 [-1, 2, 96]               --\n",
-      "|    |    |    └─LayerNorm: 4-8               [-1, 2, 96]               192\n",
-      "|    |    |    └─Linear: 4-9                  [-1, 2, 2048]             198,656\n",
-      "|    |    |    └─Dropout: 4-10                [-1, 2, 2048]             --\n",
-      "|    |    |    └─Linear: 4-11                 [-1, 2, 96]               196,704\n",
-      "|    |    |    └─Dropout: 4-12                [-1, 2, 96]               --\n",
-      "|    |    |    └─LayerNorm: 4-13              [-1, 2, 96]               192\n",
-      "|    |    └─TransformerEncoderLayer: 3-8      [-1, 2, 96]               --\n",
-      "|    |    |    └─MultiheadAttention: 4-14     [-1, 2, 96]               37,248\n",
-      "|    |    |    └─Dropout: 4-15                [-1, 2, 96]               --\n",
-      "|    |    |    └─LayerNorm: 4-16              [-1, 2, 96]               192\n",
-      "|    |    |    └─Linear: 4-17                 [-1, 2, 2048]             198,656\n",
-      "|    |    |    └─Dropout: 4-18                [-1, 2, 2048]             --\n",
-      "|    |    |    └─Linear: 4-19                 [-1, 2, 96]               196,704\n",
-      "|    |    |    └─Dropout: 4-20                [-1, 2, 96]               --\n",
-      "|    |    |    └─LayerNorm: 4-21              [-1, 2, 96]               192\n",
-      "|    |    └─TransformerEncoderLayer: 3-9      [-1, 2, 96]               --\n",
-      "|    |    |    └─MultiheadAttention: 4-22     [-1, 2, 96]               37,248\n",
-      "|    |    |    └─Dropout: 4-23                [-1, 2, 96]               --\n",
-      "|    |    |    └─LayerNorm: 4-24              [-1, 2, 96]               192\n",
-      "|    |    |    └─Linear: 4-25                 [-1, 2, 2048]             198,656\n",
-      "|    |    |    └─Dropout: 4-26                [-1, 2, 2048]             --\n",
-      "|    |    |    └─Linear: 4-27                 [-1, 2, 96]               196,704\n",
-      "|    |    |    └─Dropout: 4-28                [-1, 2, 96]               --\n",
-      "|    |    |    └─LayerNorm: 4-29              [-1, 2, 96]               192\n",
-      "|    |    └─TransformerEncoderLayer: 3-10     [-1, 2, 96]               --\n",
-      "|    |    |    └─MultiheadAttention: 4-30     [-1, 2, 96]               37,248\n",
-      "|    |    |    └─Dropout: 4-31                [-1, 2, 96]               --\n",
-      "|    |    |    └─LayerNorm: 4-32              [-1, 2, 96]               192\n",
-      "|    |    |    └─Linear: 4-33                 [-1, 2, 2048]             198,656\n",
-      "|    |    |    └─Dropout: 4-34                [-1, 2, 2048]             --\n",
-      "|    |    |    └─Linear: 4-35                 [-1, 2, 96]               196,704\n",
-      "|    |    |    └─Dropout: 4-36                [-1, 2, 96]               --\n",
-      "|    |    |    └─LayerNorm: 4-37              [-1, 2, 96]               192\n",
-      "|    |    └─TransformerEncoderLayer: 3-11     [-1, 2, 96]               --\n",
-      "|    |    |    └─MultiheadAttention: 4-38     [-1, 2, 96]               37,248\n",
-      "|    |    |    └─Dropout: 4-39                [-1, 2, 96]               --\n",
-      "|    |    |    └─LayerNorm: 4-40              [-1, 2, 96]               192\n",
-      "|    |    |    └─Linear: 4-41                 [-1, 2, 2048]             198,656\n",
-      "|    |    |    └─Dropout: 4-42                [-1, 2, 2048]             --\n",
-      "|    |    |    └─Linear: 4-43                 [-1, 2, 96]               196,704\n",
-      "|    |    |    └─Dropout: 4-44                [-1, 2, 96]               --\n",
-      "|    |    |    └─LayerNorm: 4-45              [-1, 2, 96]               192\n",
-      "|    |    └─TransformerEncoderLayer: 3-12     [-1, 2, 96]               --\n",
-      "|    |    |    └─MultiheadAttention: 4-46     [-1, 2, 96]               37,248\n",
-      "|    |    |    └─Dropout: 4-47                [-1, 2, 96]               --\n",
-      "|    |    |    └─LayerNorm: 4-48              [-1, 2, 96]               192\n",
-      "|    |    |    └─Linear: 4-49                 [-1, 2, 2048]             198,656\n",
-      "|    |    |    └─Dropout: 4-50                [-1, 2, 2048]             --\n",
-      "|    |    |    └─Linear: 4-51                 [-1, 2, 96]               196,704\n",
-      "|    |    |    └─Dropout: 4-52                [-1, 2, 96]               --\n",
-      "|    |    |    └─LayerNorm: 4-53              [-1, 2, 96]               192\n",
-      "|    └─LayerNorm: 2-3                         [-1, 2, 96]               192\n",
-      "├─Linear: 1-6                                 [-1, 97, 81]              7,857\n",
-      "===============================================================================================\n",
-      "Total params: 3,866,250\n",
-      "Trainable params: 3,866,250\n",
-      "Non-trainable params: 0\n",
-      "Total mult-adds (M): 18.78\n",
-      "===============================================================================================\n",
-      "Input size (MB): 0.10\n",
-      "Forward/backward pass size (MB): 2.06\n",
-      "Params size (MB): 14.75\n",
-      "Estimated Total Size (MB): 16.91\n",
-      "===============================================================================================\n"
-     ]
-    }
-   ],
-   "source": [
-    "model.summary(experiment_config[\"train_args\"][\"input_shape\"], 4)"
-   ]
-  },
-  {
-   "cell_type": "code",
    "execution_count": 110,
    "metadata": {},
    "outputs": [],
@@ -454,415 +319,6 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 95,
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "[[1, 28, 952], [92]]"
-      ]
-     },
-     "execution_count": 95,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "experiment_config[\"train_args\"][\"input_shape\"]"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 99,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "=========================================================================================================\n",
-      "Layer (type:depth-idx)                                  Output Shape              Param #\n",
-      "=========================================================================================================\n",
-      "├─Sequential: 1-1                                       [-1, 158, 1, 28, 6]       --\n",
-      "|    └─Unfold: 2-1                                      [-1, 168, 158]            --\n",
-      "|    └─Rearrange: 2-2                                   [-1, 158, 1, 28, 6]       --\n",
-      "├─Linear: 1-2                                           [-1, 158, 512]            86,528\n",
-      "├─PositionalEncoding: 1-3                               [-1, 158, 512]            --\n",
-      "|    └─Dropout: 2-3                                     [-1, 158, 512]            --\n",
-      "├─Embedding: 1-4                                        [-1, 92, 512]             41,984\n",
-      "├─PositionalEncoding: 1-5                               [-1, 92, 512]             --\n",
-      "|    └─Dropout: 2-4                                     [-1, 92, 512]             --\n",
-      "├─Transformer: 1-6                                      [-1, 92, 512]             --\n",
-      "|    └─Encoder: 2-5                                     [-1, 158, 512]            --\n",
-      "|    |    └─ModuleList: 3                               []                        --\n",
-      "|    |    |    └─EncoderLayer: 4-1                      [-1, 158, 512]            3,150,848\n",
-      "|    |    |    └─EncoderLayer: 4-2                      [-1, 158, 512]            3,150,848\n",
-      "|    |    |    └─EncoderLayer: 4-3                      [-1, 158, 512]            3,150,848\n",
-      "|    |    |    └─EncoderLayer: 4-4                      [-1, 158, 512]            3,150,848\n",
-      "|    |    └─LayerNorm: 3-1                              [-1, 158, 512]            1,024\n",
-      "|    └─Decoder: 2-6                                     [-1, 92, 512]             --\n",
-      "|    |    └─ModuleList: 3                               []                        --\n",
-      "|    |    |    └─DecoderLayer: 4-5                      [-1, 92, 512]             4,200,960\n",
-      "|    |    |    └─DecoderLayer: 4-6                      [-1, 92, 512]             4,200,960\n",
-      "|    |    |    └─DecoderLayer: 4-7                      [-1, 92, 512]             4,200,960\n",
-      "|    |    |    └─DecoderLayer: 4-8                      [-1, 92, 512]             4,200,960\n",
-      "|    |    └─LayerNorm: 3-2                              [-1, 92, 512]             1,024\n",
-      "├─Sequential: 1-7                                       [-1, 92, 82]              --\n",
-      "|    └─LayerNorm: 2-7                                   [-1, 92, 512]             1,024\n",
-      "|    └─Linear: 2-8                                      [-1, 92, 512]             262,656\n",
-      "|    └─GELU: 2-9                                        [-1, 92, 512]             --\n",
-      "|    └─Dropout: 2-10                                    [-1, 92, 512]             --\n",
-      "|    └─Linear: 2-11                                     [-1, 92, 82]              42,066\n",
-      "=========================================================================================================\n",
-      "Total params: 29,843,538\n",
-      "Trainable params: 29,843,538\n",
-      "Non-trainable params: 0\n",
-      "Total mult-adds (M): 118.22\n",
-      "=========================================================================================================\n",
-      "Input size (MB): 0.10\n",
-      "Forward/backward pass size (MB): 2.73\n",
-      "Params size (MB): 113.84\n",
-      "Estimated Total Size (MB): 116.68\n",
-      "=========================================================================================================\n"
-     ]
-    }
-   ],
-   "source": [
-    "model.summary(experiment_config[\"train_args\"][\"input_shape\"], 4)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 61,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "t=[12,1,1,1,1,1,4,4,4,4,4]"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 62,
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "1"
-      ]
-     },
-     "execution_count": 62,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "t[t!=79]"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 63,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "x = torch.arange(10)\n",
-    "value = 5\n",
-    "x = x[x!=value]"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 64,
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "tensor([0, 1, 2, 3, 4, 6, 7, 8, 9])"
-      ]
-     },
-     "execution_count": 64,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "x"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 7,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "t = torch.rand(98)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 8,
-   "metadata": {
-    "scrolled": true
-   },
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "tensor(1.7656e-43)"
-      ]
-     },
-     "execution_count": 8,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "t.cumprod(dim=0)[-1]"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 24,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "pred_tokens = torch.Tensor([1,2,21,31, 89, 89])"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 25,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "pred_tokens = torch.stack([pred_tokens, pred_tokens])"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 26,
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "tensor([[ 1.,  2., 21., 31., 89., 89.],\n",
-       "        [ 1.,  2., 21., 31., 89., 89.]])"
-      ]
-     },
-     "execution_count": 26,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "pred_tokens"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 27,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "eos_token_index = torch.nonzero(\n",
-    "            pred_tokens == 89, as_tuple=False,\n",
-    "        )"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 31,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "0\n"
-     ]
-    }
-   ],
-   "source": [
-    "if eos_token_index.nelement():\n",
-    "    print(eos_token_index[0][0].item())"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 32,
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "tensor([[0, 4],\n",
-       "        [0, 5],\n",
-       "        [1, 4],\n",
-       "        [1, 5]])"
-      ]
-     },
-     "execution_count": 32,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "eos_token_index"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 29,
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "8"
-      ]
-     },
-     "execution_count": 29,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "eos_token_index.nelement()"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 38,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from text_recognizer.models import accuracy"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 44,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "pred = torch.Tensor([1,2,21,31, 80, 80]).unsqueeze(0)\n",
-    "target = torch.Tensor([1,2,1,31, 80, 80]).unsqueeze(0)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 57,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "pred = torch.stack([pred, pred])\n",
-    "target = torch.stack([target, target])"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 115,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "target = torch.tensor([0, 1, 2, 3])\n",
-    "pred = torch.tensor([0, 2, 1, 3])"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 116,
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "0.5"
-      ]
-     },
-     "execution_count": 116,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "accuracy(pred, target)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 53,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "acc = (target.argmax(-1) == pred.argmax(-1)).float()\n",
-    "\n",
-    "# return float(100 * acc.sum() / len(acc))"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 54,
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "tensor([[1.],\n",
-       "        [1.]])"
-      ]
-     },
-     "execution_count": 54,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "acc"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 58,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "train_acc = (pred == target).sum().item()/target.shape[-1]"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 59,
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "3.3333333333333335"
-      ]
-     },
-     "execution_count": 59,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "train_acc"
-   ]
-  },
-  {
-   "cell_type": "code",
    "execution_count": null,
    "metadata": {},
    "outputs": [],