diff options
Diffstat (limited to 'src')
26 files changed, 418 insertions, 920 deletions
diff --git a/src/notebooks/00-testing-stuff-out.ipynb b/src/notebooks/00-testing-stuff-out.ipynb index 9d265ba..0294394 100644 --- a/src/notebooks/00-testing-stuff-out.ipynb +++ b/src/notebooks/00-testing-stuff-out.ipynb @@ -22,36 +22,94 @@ }, { "cell_type": "code", - "execution_count": 68, + "execution_count": 4, "metadata": {}, "outputs": [], "source": [ - "from text_recognizer.networks.residual_network import IdentityBlock, ResidualBlock, BasicBlock, BottleNeckBlock, ResidualLayer, Encoder, ResidualNetwork" + "from text_recognizer.networks.residual_network import IdentityBlock, ResidualBlock, BasicBlock, BottleNeckBlock, ResidualLayer, ResidualNetwork" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "IdentityBlock(\n", + " (blocks): Identity()\n", + " (activation_fn): ReLU(inplace=True)\n", + " (shortcut): Identity()\n", + ")" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "IdentityBlock(32, 64)" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "ResidualBlock(\n", + " (blocks): Identity()\n", + " (activation_fn): ReLU(inplace=True)\n", + " (shortcut): Sequential(\n", + " (0): Conv2d(32, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", + " (1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", + " )\n", + ")" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "ResidualBlock(32, 64)" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "BasicBlock(\n", + " (blocks): Sequential(\n", + " (0): Sequential(\n", + " (0): Conv2dAuto(32, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", + " (1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", + " )\n", + " (1): ReLU(inplace=True)\n", + " (2): Sequential(\n", + " (0): Conv2dAuto(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", + " (1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", + " )\n", + " )\n", + " (activation_fn): ReLU(inplace=True)\n", + " (shortcut): Sequential(\n", + " (0): Conv2d(32, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", + " (1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", + " )\n", + ")\n" + ] + } + ], "source": [ "dummy = torch.ones((1, 32, 224, 224))\n", "\n", @@ -62,9 +120,39 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "BottleNeckBlock(\n", + " (blocks): Sequential(\n", + " (0): Sequential(\n", + " (0): Conv2dAuto(32, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", + " (1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", + " )\n", + " (1): ReLU(inplace=True)\n", + " (2): Sequential(\n", + " (0): Conv2dAuto(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", + " (1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", + " )\n", + " (3): ReLU(inplace=True)\n", + " (4): Sequential(\n", + " (0): Conv2dAuto(64, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", + " (1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", + " )\n", + " )\n", + " (activation_fn): ReLU(inplace=True)\n", + " (shortcut): Sequential(\n", + " (0): Conv2d(32, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", + " (1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", + " )\n", + ")\n" + ] + } + ], "source": [ "dummy = torch.ones((1, 32, 10, 10))\n", "\n", @@ -191,7 +279,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 9, "metadata": {}, "outputs": [], "source": [ @@ -200,7 +288,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 10, "metadata": {}, "outputs": [], "source": [ @@ -218,7 +306,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 11, "metadata": {}, "outputs": [], "source": [ @@ -227,7 +315,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 14, "metadata": {}, "outputs": [ { @@ -505,7 +593,7 @@ "===============================================================================================" ] }, - "execution_count": 8, + "execution_count": 14, "metadata": {}, "output_type": "execute_result" } diff --git a/src/notebooks/04a-look-at-iam-lines.ipynb b/src/notebooks/04a-look-at-iam-lines.ipynb index 093920a..d64b391 100644 --- a/src/notebooks/04a-look-at-iam-lines.ipynb +++ b/src/notebooks/04a-look-at-iam-lines.ipynb @@ -32,7 +32,7 @@ }, { "cell_type": "code", - "execution_count": 54, + "execution_count": 3, "metadata": {}, "outputs": [ { @@ -40,8 +40,8 @@ "output_type": "stream", "text": [ "IAM Lines Dataset\n", - "Number classes: 81\n", - "Mapping: {0: '0', 1: '1', 2: '2', 3: '3', 4: '4', 5: '5', 6: '6', 7: '7', 8: '8', 9: '9', 10: 'A', 11: 'B', 12: 'C', 13: 'D', 14: 'E', 15: 'F', 16: 'G', 17: 'H', 18: 'I', 19: 'J', 20: 'K', 21: 'L', 22: 'M', 23: 'N', 24: 'O', 25: 'P', 26: 'Q', 27: 'R', 28: 'S', 29: 'T', 30: 'U', 31: 'V', 32: 'W', 33: 'X', 34: 'Y', 35: 'Z', 36: 'a', 37: 'b', 38: 'c', 39: 'd', 40: 'e', 41: 'f', 42: 'g', 43: 'h', 44: 'i', 45: 'j', 46: 'k', 47: 'l', 48: 'm', 49: 'n', 50: 'o', 51: 'p', 52: 'q', 53: 'r', 54: 's', 55: 't', 56: 'u', 57: 'v', 58: 'w', 59: 'x', 60: 'y', 61: 'z', 62: ' ', 63: '!', 64: '\"', 65: '#', 66: '&', 67: \"'\", 68: '(', 69: ')', 70: '*', 71: '+', 72: ',', 73: '-', 74: '.', 75: '/', 76: ':', 77: ';', 78: '?', 79: '_', 80: '<blank>'}\n", + "Number classes: 80\n", + "Mapping: {0: '0', 1: '1', 2: '2', 3: '3', 4: '4', 5: '5', 6: '6', 7: '7', 8: '8', 9: '9', 10: 'A', 11: 'B', 12: 'C', 13: 'D', 14: 'E', 15: 'F', 16: 'G', 17: 'H', 18: 'I', 19: 'J', 20: 'K', 21: 'L', 22: 'M', 23: 'N', 24: 'O', 25: 'P', 26: 'Q', 27: 'R', 28: 'S', 29: 'T', 30: 'U', 31: 'V', 32: 'W', 33: 'X', 34: 'Y', 35: 'Z', 36: 'a', 37: 'b', 38: 'c', 39: 'd', 40: 'e', 41: 'f', 42: 'g', 43: 'h', 44: 'i', 45: 'j', 46: 'k', 47: 'l', 48: 'm', 49: 'n', 50: 'o', 51: 'p', 52: 'q', 53: 'r', 54: 's', 55: 't', 56: 'u', 57: 'v', 58: 'w', 59: 'x', 60: 'y', 61: 'z', 62: ' ', 63: '!', 64: '\"', 65: '#', 66: '&', 67: \"'\", 68: '(', 69: ')', 70: '*', 71: '+', 72: ',', 73: '-', 74: '.', 75: '/', 76: ':', 77: ';', 78: '?', 79: '_'}\n", "Data: (7101, 28, 952)\n", "Targets: (7101, 97)\n", "\n" @@ -56,16 +56,16 @@ }, { "cell_type": "code", - "execution_count": 55, + "execution_count": 4, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "(97, 81)" + "(97, 80)" ] }, - "execution_count": 55, + "execution_count": 4, "metadata": {}, "output_type": "execute_result" } @@ -76,7 +76,7 @@ }, { "cell_type": "code", - "execution_count": 56, + "execution_count": 5, "metadata": {}, "outputs": [ { @@ -85,7 +85,7 @@ "'A MOVE to stop Mr. Gaitskell from'" ] }, - "execution_count": 56, + "execution_count": 5, "metadata": {}, "output_type": "execute_result" } @@ -99,7 +99,7 @@ }, { "cell_type": "code", - "execution_count": 57, + "execution_count": 6, "metadata": {}, "outputs": [ { @@ -251,620 +251,116 @@ }, { "cell_type": "code", - "execution_count": 106, + "execution_count": 46, "metadata": {}, "outputs": [], "source": [ - "data, target = dataset[10]" + "data, target = dataset[0]\n", + "sentence = convert_y_label_to_string(target) " ] }, { "cell_type": "code", - "execution_count": 107, + "execution_count": 47, "metadata": {}, "outputs": [], "source": [ - "text = convert_y_label_to_string(dataset.targets[10])" + "h, w, s = 28, 18, 4" ] }, { "cell_type": "code", - "execution_count": 108, + "execution_count": 48, "metadata": {}, "outputs": [], "source": [ - "data1, target1 = dataset[110]" - ] - }, - { - "cell_type": "code", - "execution_count": 60, - "metadata": { - "scrolled": true - }, - "outputs": [ - { - "data": { - "text/plain": [ - "\"Griffiths resolution. Mr. Foot's line will\"" - ] - }, - "execution_count": 60, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "text" - ] - }, - { - "cell_type": "code", - "execution_count": 89, - "metadata": {}, - "outputs": [], - "source": [ - "S = 30\n", - "S_min = 10\n", - "N = 16\n", - "C = 20" - ] - }, - { - "cell_type": "code", - "execution_count": 92, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "tensor([[ 6, 10, 2, 17, 8, 18, 16, 4, 14, 14, 6, 9, 4, 14, 10, 10, 12, 4,\n", - " 11, 18, 19, 4, 10, 8, 13, 2, 18, 4, 17, 9],\n", - " [ 8, 13, 8, 5, 5, 6, 4, 10, 9, 14, 19, 6, 7, 6, 10, 6, 5, 3,\n", - " 14, 10, 1, 18, 3, 3, 13, 13, 16, 12, 5, 6],\n", - " [ 1, 9, 18, 10, 10, 10, 10, 6, 7, 7, 14, 6, 12, 12, 3, 9, 14, 16,\n", - " 11, 14, 3, 10, 9, 15, 19, 8, 13, 5, 12, 15],\n", - " [10, 2, 6, 11, 14, 1, 13, 1, 7, 2, 19, 1, 1, 17, 6, 16, 18, 12,\n", - " 3, 18, 19, 17, 9, 12, 14, 15, 3, 8, 1, 9],\n", - " [12, 7, 14, 5, 2, 12, 1, 16, 9, 16, 18, 17, 6, 11, 2, 7, 5, 8,\n", - " 16, 6, 19, 13, 12, 17, 11, 13, 17, 12, 5, 1],\n", - " [13, 11, 14, 18, 15, 8, 17, 13, 18, 5, 10, 6, 15, 3, 4, 11, 12, 5,\n", - " 4, 1, 17, 12, 7, 5, 5, 9, 19, 15, 4, 5],\n", - " [12, 1, 4, 5, 6, 13, 19, 1, 1, 15, 3, 14, 8, 19, 7, 5, 19, 9,\n", - " 5, 11, 14, 10, 11, 1, 12, 19, 14, 13, 19, 15],\n", - " [16, 3, 7, 10, 12, 15, 9, 18, 9, 10, 16, 2, 14, 4, 18, 3, 8, 12,\n", - " 16, 19, 11, 5, 9, 19, 11, 14, 7, 16, 4, 3],\n", - " [ 1, 4, 2, 10, 10, 2, 3, 5, 9, 6, 16, 3, 11, 6, 14, 19, 3, 11,\n", - " 6, 3, 19, 17, 2, 12, 12, 4, 5, 15, 19, 1],\n", - " [18, 9, 12, 1, 1, 1, 3, 8, 9, 7, 9, 19, 5, 12, 10, 17, 15, 14,\n", - " 18, 15, 8, 4, 1, 3, 14, 14, 2, 5, 9, 4],\n", - " [15, 5, 15, 14, 8, 9, 8, 15, 12, 15, 18, 2, 7, 13, 19, 12, 1, 16,\n", - " 12, 11, 1, 12, 17, 16, 18, 3, 6, 11, 9, 11],\n", - " [18, 17, 14, 11, 5, 15, 3, 10, 10, 16, 14, 9, 12, 7, 8, 13, 18, 11,\n", - " 6, 9, 16, 10, 14, 6, 5, 19, 11, 13, 7, 14],\n", - " [18, 5, 11, 13, 15, 9, 7, 10, 3, 19, 8, 10, 13, 4, 11, 5, 14, 17,\n", - " 2, 16, 18, 8, 7, 16, 15, 19, 8, 13, 13, 9],\n", - " [14, 14, 4, 10, 6, 14, 14, 1, 12, 1, 3, 6, 7, 6, 19, 9, 2, 19,\n", - " 13, 9, 1, 2, 11, 2, 2, 10, 3, 11, 1, 15],\n", - " [ 6, 16, 19, 14, 14, 17, 18, 10, 18, 18, 2, 19, 15, 15, 1, 3, 12, 11,\n", - " 17, 7, 15, 6, 10, 2, 13, 17, 13, 6, 8, 14],\n", - " [ 9, 19, 9, 13, 12, 9, 5, 18, 4, 7, 14, 14, 12, 1, 16, 19, 16, 2,\n", - " 15, 4, 14, 9, 15, 2, 9, 13, 12, 1, 12, 11]])" - ] - }, - "execution_count": 92, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "torch.randint(low=1, high=C, size=(N, S), dtype=torch.long)" - ] - }, - { - "cell_type": "code", - "execution_count": 94, - "metadata": {}, - "outputs": [], - "source": [ - "T = 50 # Input sequence length" - ] - }, - { - "cell_type": "code", - "execution_count": 125, - "metadata": {}, - "outputs": [], - "source": [ - "target_lengths = torch.randint(low=1, high=T, size=(N,), dtype=torch.long)\n", - "target = torch.randint(low=1, high=C, size=(sum(target_lengths),), dtype=torch.long)" - ] - }, - { - "cell_type": "code", - "execution_count": 129, - "metadata": {}, - "outputs": [], - "source": [ - "input_lengths = torch.full(size=(N,), fill_value=T, dtype=torch.long)" - ] - }, - { - "cell_type": "code", - "execution_count": 130, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "tensor([50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50])" - ] - }, - "execution_count": 130, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "input_lengths" - ] - }, - { - "cell_type": "code", - "execution_count": 126, - "metadata": { - "scrolled": true - }, - "outputs": [ - { - "data": { - "text/plain": [ - "tensor([20, 28, 20, 32, 30, 34, 14, 15, 21, 3, 20, 13, 28, 40, 15, 27])" - ] - }, - "execution_count": 126, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "target_lengths" - ] - }, - { - "cell_type": "code", - "execution_count": 128, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "tensor([17, 17, 5, 5, 15, 5, 16, 19, 12, 3, 6, 15, 6, 13, 10, 1, 19, 12,\n", - " 2, 13, 19, 13, 7, 4, 19, 9, 19, 1, 3, 16, 1, 12, 11, 8, 17, 6,\n", - " 10, 8, 15, 15, 18, 11, 2, 6, 17, 8, 1, 12, 3, 15, 10, 14, 3, 3,\n", - " 17, 4, 18, 15, 13, 18, 19, 12, 1, 17, 18, 9, 10, 8, 2, 3, 9, 4,\n", - " 7, 9, 12, 11, 9, 5, 12, 10, 4, 15, 8, 6, 17, 9, 7, 7, 18, 15,\n", - " 16, 16, 14, 17, 11, 14, 13, 9, 10, 19, 7, 13, 12, 5, 19, 3, 7, 18,\n", - " 7, 6, 5, 1, 6, 11, 1, 19, 18, 15, 6, 4, 13, 14, 12, 19, 18, 4,\n", - " 15, 14, 12, 1, 14, 18, 1, 4, 1, 7, 12, 6, 3, 9, 8, 19, 7, 13,\n", - " 1, 4, 14, 1, 14, 8, 19, 2, 6, 11, 19, 11, 3, 13, 14, 17, 3, 3,\n", - " 10, 10, 18, 2, 11, 10, 8, 2, 18, 9, 2, 1, 16, 2, 5, 9, 1, 4,\n", - " 16, 18, 12, 11, 12, 13, 13, 18, 2, 3, 2, 7, 18, 8, 2, 16, 12, 18,\n", - " 10, 15, 16, 12, 3, 5, 6, 2, 14, 3, 10, 2, 12, 14, 3, 14, 11, 14,\n", - " 6, 11, 5, 4, 6, 9, 17, 1, 7, 1, 6, 13, 7, 7, 2, 4, 4, 15,\n", - " 1, 11, 10, 12, 10, 4, 3, 3, 7, 19, 5, 19, 18, 17, 1, 11, 14, 12,\n", - " 18, 16, 17, 16, 12, 5, 5, 5, 5, 4, 12, 18, 1, 16, 3, 12, 9, 8,\n", - " 13, 18, 6, 7, 17, 1, 9, 2, 17, 10, 2, 3, 8, 11, 3, 3, 17, 11,\n", - " 17, 19, 6, 12, 4, 11, 18, 3, 18, 16, 7, 9, 6, 15, 10, 17, 1, 17,\n", - " 2, 7, 7, 3, 7, 5, 15, 8, 3, 15, 6, 4, 16, 12, 4, 11, 1, 15,\n", - " 14, 4, 14, 17, 16, 14, 18, 10, 17, 4, 2, 19, 8, 10, 11, 2, 18, 2,\n", - " 19, 10, 15, 15, 14, 10, 3, 16, 14, 5, 15, 4, 7, 13, 16, 14, 3, 14])" - ] - }, - "execution_count": 128, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "target" - ] - }, - { - "cell_type": "code", - "execution_count": 111, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "tensor([16, 53, 44, 41, 41, 44, 55, 43, 54, 62, 53, 40, 54, 50, 47, 56, 55, 44,\n", - " 50, 49, 74, 62, 22, 53, 74, 62, 15, 50, 50, 55, 67, 54, 62, 47, 44, 49,\n", - " 40, 62, 58, 44, 47, 47], dtype=torch.uint8)" - ] - }, - "execution_count": 111, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "target[target < 79]" - ] - }, - { - "cell_type": "code", - "execution_count": 122, - "metadata": {}, - "outputs": [], - "source": [ - "ts = torch.stack([target, target1])\n", - "\n", - "targets = torch.Tensor([])\n", - "target_lengths = []\n", - "for t in ts:\n", - " t = t[t < 79]\n", - " targets = torch.cat([targets, t])\n", - " target_lengths.append(len(t))\n", - "\n", - "targets = targets.type(dtype=torch.long)\n", - "target_lengths = torch.Tensor(target_lengths).type(dtype=torch.long)" - ] - }, - { - "cell_type": "code", - "execution_count": 123, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "tensor([42, 41])" - ] - }, - "execution_count": 123, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "target_lengths" - ] - }, - { - "cell_type": "code", - "execution_count": 124, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "tensor([16, 53, 44, 41, 41, 44, 55, 43, 54, 62, 53, 40, 54, 50, 47, 56, 55, 44,\n", - " 50, 49, 74, 62, 22, 53, 74, 62, 15, 50, 50, 55, 67, 54, 62, 47, 44, 49,\n", - " 40, 62, 58, 44, 47, 47, 47, 44, 54, 55, 40, 39, 62, 37, 60, 62, 55, 43,\n", - " 40, 62, 16, 50, 57, 40, 53, 49, 48, 40, 49, 55, 74, 62, 18, 48, 48, 40,\n", - " 39, 44, 36, 55, 40, 47, 60, 62, 22, 53, 74])" - ] - }, - "execution_count": 124, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "targets" - ] - }, - { - "cell_type": "code", - "execution_count": 115, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "tensor([42., 41.])" - ] - }, - "execution_count": 115, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "target_lengths" - ] - }, - { - "cell_type": "code", - "execution_count": 99, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "torch.Size([430])" - ] - }, - "execution_count": 99, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "target.shape" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "tensor([10, 22, 19, 13, 15, 23, 28, 14, 22, 21, 16, 14, 22, 28, 22, 26])" - ] - }, - "execution_count": 14, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "torch.randint(low=S_min, high=S, size=(N,), dtype=torch.long)" + "from einops.layers.torch import Rearrange\n", + "slide = nn.Sequential(nn.Unfold(kernel_size=(h, w), stride=(1, s)), Rearrange(\"b (c h w) t -> b t c h w\", h=h, w=w, c=1))" ] }, { "cell_type": "code", - "execution_count": 77, + "execution_count": 49, "metadata": {}, "outputs": [], "source": [ - "targets = torch.stack([target, target])" + "patches = slide(data.unsqueeze(0))" ] }, { "cell_type": "code", - "execution_count": 83, + "execution_count": 50, "metadata": {}, "outputs": [ { - "ename": "RuntimeError", - "evalue": "stack expects each tensor to be equal size, but got [0] at entry 0 and [42] at entry 1", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mRuntimeError\u001b[0m Traceback (most recent call last)", - "\u001b[0;32m<ipython-input-83-692d40f2bb6a>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[0mts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTensor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mt\u001b[0m \u001b[0;32min\u001b[0m \u001b[0menumerate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtargets\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstack\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mts\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mt\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mt\u001b[0m \u001b[0;34m<\u001b[0m \u001b[0;36m79\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", - "\u001b[0;31mRuntimeError\u001b[0m: stack expects each tensor to be equal size, but got [0] at entry 0 and [42] at entry 1" + "name": "stdout", + "output_type": "stream", + "text": [ + "A MOVE to stop Mr. Gaitskell from\n" ] - } - ], - "source": [ - "ts = torch.Tensor()\n", - "for i, t in enumerate(targets):\n", - " torch.stack([ts, t[t < 79]])" - ] - }, - { - "cell_type": "code", - "execution_count": 78, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "tensor([16, 53, 44, 41, 41, 44, 55, 43, 54, 62, 53, 40, 54, 50, 47, 56, 55, 44,\n", - " 50, 49, 74, 62, 22, 53, 74, 62, 15, 50, 50, 55, 67, 54, 62, 47, 44, 49,\n", - " 40, 62, 58, 44, 47, 47, 16, 53, 44, 41, 41, 44, 55, 43, 54, 62, 53, 40,\n", - " 54, 50, 47, 56, 55, 44, 50, 49, 74, 62, 22, 53, 74, 62, 15, 50, 50, 55,\n", - " 67, 54, 62, 47, 44, 49, 40, 62, 58, 44, 47, 47], dtype=torch.uint8)" - ] - }, - "execution_count": 78, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "targets[targets<79]" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [], - "source": [ - "from text_recognizer.networks import LineRecurrentNetwork" - ] - }, - { - "cell_type": "code", - "execution_count": 52, - "metadata": {}, - "outputs": [], - "source": [ - "crnn = LineRecurrentNetwork(encoder=\"ResidualNetworkEncoder\",\n", - " \n", - " encoder_args={\n", - " \"in_channels\": 1,\n", - " \"num_classes\": 80,\n", - " \"depths\": [2, 2],\n", - " \"block_sizes\": [64, 128],\n", - " \"activation\": \"leaky_relu\",\n", - " \"stn\": False,},\n", - " patch_size=[28, 14], \n", - " stride=[1, 6])" - ] - }, - { - "cell_type": "code", - "execution_count": 53, - "metadata": {}, - "outputs": [], - "source": [ - "output = crnn(data)" - ] - }, - { - "cell_type": "code", - "execution_count": 54, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "torch.Size([157, 1, 80])" - ] - }, - "execution_count": 54, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "output.shape" - ] - }, - { - "cell_type": "code", - "execution_count": 55, - "metadata": {}, - "outputs": [], - "source": [ - "# output = output.unsqueeze(0)\n", - "targets = target.unsqueeze(0).type(torch.long)" - ] - }, - { - "cell_type": "code", - "execution_count": 56, - "metadata": {}, - "outputs": [], - "source": [ - "input_lengths = torch.full(\n", - " size=(output.shape[1],), fill_value=output.shape[0], dtype=torch.long,\n", - ")\n", - "target_lengths = torch.full(\n", - " size=(output.shape[1],), fill_value=targets.shape[1], dtype=torch.long,\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": 57, - "metadata": {}, - "outputs": [], - "source": [ - "ctc = nn.CTCLoss(blank=0)" - ] - }, - { - "cell_type": "code", - "execution_count": 58, - "metadata": {}, - "outputs": [ + }, { "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAABG0AAAAqCAYAAAAJfKYAAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/d3fzzAAAACXBIWXMAAAsTAAALEwEAmpwYAABwDklEQVR4nO19d3hU1db+O30ymfTee0IgtAQIJYCUIE2QIkVERURFBUXFci0gnxW5dhRRFJGmIkUIJXRICBAglARIQnrvPZlk2u+P3L2/MzPnTIlcL/f7nfd5eMjMnLXW3vvsstbaa68t0Ov14MGDBw8ePHjw4MGDBw8ePHjw4HFvQfifLgAPHjx48ODBgwcPHjx48ODBgwcPU/BOGx48ePDgwYMHDx48ePDgwYMHj3sQvNOGBw8ePHjw4MGDBw8ePHjw4MHjHgTvtOHBgwcPHjx48ODBgwcPHjx48LgHwTttePDgwYMHDx48ePDgwYMHDx487kHwThsePHjw4MGDBw8ePHjw4MGDB497EXq93up/APR/5d/d4gOg5t/BSyKR6AHoxWKxXi6X6+3s7PRSqZSVTiqV6h0dHfVisfi/pn7/F9/f//X++f9L/e6lMvH1+79dP1dXV4N5m/nP0dFR7+PjQ3//b6zff/v7+/9lzuPr999dv3upTHz9+Pr9t9RPIBBYRSOVSvVKpbJH9evVqxfr9xKJRB8TE6MXCoWc9XN1ddW7urqa0AoEAr27u7s+ICCAs0z/V98fX7//bP2Y//4rIm0EAoHxV0V3kT3l9a+GhkajgUqlgq+vLyZOnAgHBwd4eHhQAqlUChcXF6jVamg0GmvL3KMyWQMLsu5mW/1l/KusPS7TX2nXu0nLxksgELDWz5Jc8ru15bMg2+KzPZXLgMX39xf7f094W9WnrGyjv2XM2Nj+/5Y57x7C3zLnubu7Y/bs2YiJiYFQaLr8RUZGYty4cejVq5ctxbEG/Pv7G/FvWCesnl/+U2vUX8Q99f7+Dbin6vdX9SAWWM3rb+xjNulBtvC6R8DP6T3g5ezsbBVBV1cXWltbeyTs9u3brN/v378fN27cgFKpNCgTE/X19aivrzf5Pjo6GjU1NXjllVd6VCZza4NEIoFMJjNLy/W9TCaDWCxm+9ni+5NKpZBKpZYeY4VQKIRMJoNIJOoR/V2Yh+7FMXM3wVq/u+a0uVsLARsf4kz5dy829vb2AACFQgE3Nzfk5eXh5s2b8PLyQkJCAn1OLpfDz88PQUFBsLOzY+VFymwLeqrw3Y32+SvKpi10PWmXu0V/N2nZeHHxt1Yu8zlzbWpJNvNdmpNNfuOitRXWyv2rYPZ3W8vKRvt3jlU2uf/Otvq/ClvabNCgQRg0aBACAgLg5eWFsrIy+Pn5GThtnJycEBAQgEuXLmHr1q1wc3P7dxSbx9+E/+Q68Z+SzeN/IRAI4O3tDQcHhx7Ry+Vy+Pr6QiKR2EQnFAoREBBAdUkucL1npVIJLy8vLiPMKlhal/5TfYzv2zwAoKGhwepnPT09MWjQIDg6Otos55lnnsHcuXPp54SEBFy/fh0bNmxAV1eXWdqEhAQ888wz1MEklUoxYcIEbNiwASkpKTaXBeBeG5RKJSZPnozp06dzjl02OoFAgKCgIMydOxd9+vSxWR91dHTEjBkzMGXKFJvnG5FIhOjoaMyePRshISFWyTZ+xtHR0SqHz39yI+Q/uInCibvmtLlbE7I1Rua/CyqVCkD3i/Ly8kK/fv1QVVWFsLAw3Lp1CwAgk8kQHByMyspKyOVy9O/f36yHlAtczqm/Q+G727L5xfjeg7UOIHO0bNFFtiiEzGcJbU8mdy46a/odl1yu/m5ttNJfGSvWyL0bzrO74Xz7O+Vaw78nEIvF8PHxQZ8+fZCVlYWKigqcPHnSIEpSIBBAJBKhV69ecHNzw+nTp+9WsXn8l+Ov9uf/RoXzXlRWbUFgYCAWLVqE+++/32bHi1gsxsiRI/Hkk08iIiLCJtrIyEg8//zzGD9+vFVRrkwoFApMnDgRixYtgr+/v020TDg4OJjdmf87Nud4mOI/1X7/ze9NKBSib9++BicdrIWbmxveeecd+nnu3Lk4fvw4li5dSm09LkgkErz44ovw9vYG0D02lyxZgqVLl+L333+3uSzm4O7ujsWLF+P111+HXC63mk4oFCIuLg6vvfYaZsyYAScnJ5veta+vL5577jmsXLkSQUFBNtFKpVIkJCTg1Vdfxfjx46FQKDjphUIhYmJicN999xk84+npCT8/P7NyHRwcMGLECJvnYaDbsRQbG2si11q4ublh+PDh8PPzs5kW6A4C+XeMvR45bZjK+X/rAsAmv7OzEwMGDEBbWxuKiopQW1uLsLAw3L59G9nZ2QC6nTZRUVHo378/fHx80NzcjL59+1ot825Ew7AZeFzPs+Hf7WT5T7/bvxP/jjFgCz9LPAQCAef7Nj6ew+bY6GkZjOXa0ucsldecXGt5EXrjfz2RzTUmudqF2d7GconzQCqVQiKRQCQSsR7jMSdXLBbTsFWhUGh1f2KTbUtfFIvFkEqlNsvlKosxeuKIdHZ2xvnz53HixAlcuXIFANC7d2/Mnj3bwJhrbGxEYWEhQkJCzIZv99SZ9Vfg4OCAXr162aTQEQiFQvj4+CA0NNRmWicnJ/Tp06dHu/5isRgBAQE9VnjuJfxfiQ79u2j/0xAIBAgODsaoUaMwbdo0zmhoLojFYgwdOhSJiYno37+/1eH/AoEAvXr1wn333YcHH3zQrLOIrX3t7OyQkJCAyZMnIzIy0mDeZ84hlt6Ns7Mz55i9Vzfn/n/QGfkIJ9tRWVmJpKQkVFZW2ky7du1aPP/88/Tzd999h4yMDKtoT548iccffxylpaUAgLa2NgNeTISEhNhcNiZaWlrQ0dGBkJAQuLq6Wk2n1+tRU1MDOzs7jBw5ElFRUTbJbWhoQFdXF8LDwzF79mxOPZMNGo0G9fX1cHJywvjx4+Hj48P5rJubGxYvXoyXXnrJQIZcLsfkyZM55YpEIvTr1w+vvfYaxo4da7Me4uXlheeffx7Lly+3eQ2QSqUYOnQoVqxYgdjYWJvahsDd3b3HR8/MweaSCAQCCIVCBAUFITIyElKplNWZYG4SViqVcHZ2tskJYYvhYc13bBOZo6MjvLy84O7uDn9/f5SXl0OpVCI6Ohrjx48HALS2tiI5ORnnzp1DfX096urqOEPpjetmbsEUCoUICwuDu7s7Z92MkiUZ8HZwcKDHtWxdAIVCIcLDw+Hi4mJT5xQIBHB1dYWvry/tB9bQMCGTyaz2EN/t6AHynSUD0xwtMXLlcjk1dNno2T4LBAJIpVLIZDJ6LtWa9ieyiXFOZDPrYWmhJnKJkW2N04Gr/1proHP1XWvl/iuJHQSC7nO8/fv3NznmYk4u8327u7tj4MCBUCqVFhVZY3q5XI7BgwfDy8uLOlyYv9vZ2WHKlCnw9fU14CMSiWjEh0KhMGkPkUiEoKAgPPvss/joo4+watUqPPPMM3jggQcM+IjFYla5CoUCs2bNwnvvvYePP/4Yr7zyCh5++GHEx8fD0dHRbJuLxWL06dMHK1aswNq1a/HWW2/hqaeewuTJkxEaGmrRcHF1dcUHH3yAjz76CCtWrMDcuXMRGxtrdj7i+t7DwwNTpkxBUFBQjxZLJuRyOVxcXDBq1ChUV1fT70mdFi1aBKB7Z37EiBFISUlBSUkJ5s2b16PdPTY4OTn1mFYul2PWrFlYvXo1Zx4eLggE3UdE3nvvPSxatAguLi5W0yoUCjz55JNYtWoVgoODbZ53w8LCsGbNGsyePdvqIypsMqyNkrjbDsK/g5bHvwdkXpRIJOjVq5fNDksyp7q6uiI6OtrqoxlkLpdKpejXrx+Cg4NtLTp1skZHR0OhUJjwtgZ2dnYICAiwWfbdhK3j4t9xRPvvpCX0PCxDIBCYHFti9lfjY03V1dVoa2sz4SMUCrFnzx76+fXXX8e1a9fouFOr1Th58iT9PTMz00AHMOZ17do1rF+/nn53/vx5mlPHmNfrr7+O4cOHAwAqKiqsqjcXmpqacP36dej1epucNjqdDsXFxcjLy0NMTAyGDx9u8VgmE/X19cjIyIBUKsW8efMQEBBg9caoRqNBQUEBysvLER8fj0GDBrE6KIiNExQUhIiICAP9RSQSYcaMGfD29ua0P5ydnREZGQkvLy+bcucQuaGhoQgPD+/RMVdXV1eEh4fD1dW1R3qoTCazWa416NHBWTc3NzzyyCMICQnB2rVrkZOTY/IMWfjYFhtfX18kJibiu+++g1arNSurJ5M/kWuNscAsX2trKxwcHBAeHo7a2lpMmzaNGsMHDx6ERCLBpEmT6PMikQiFhYUGg9lYhrW7wwEBAfjyyy+hVqvx4IMPmi0z4ctUEiZPnownn3wSX3/9NZKSkjjble2dBAcHY8OGDSgsLMQLL7yA9vZ2znIyYW9vjzlz5mDcuHH4+OOPkZGRAZ1OZ1BeoVAIoVAIrVbL2hcCAwPxxBNPYM2aNawhi1KpFBqNhvJl8rZ0zIQ4I4yPQRBIJBJ4enoiMTERJ06cMOAjk8k45QoEAtjb26NPnz6YNGkSQkNDIRaLUVFRgaKiInzxxRdmyyWRSODj44PJkycjLi4Ojo6OaG1tRUFBATIzM3Hx4kVUV1dDrVab8BGJRHBwcEBcXBwSExMRGBgIgUCAkpISFBQUID09Hbdu3UJHR4eBQ4DZVp6envjoo49gb29PIwyuXr2Ky5cvo76+3qDvkMnPx8cHlZWV6OrqMmgTPz8/TJ48GRcvXkRubi5UKpWJY8bZ2RlKpRJlZWVmHTfGTl+SO6qoqAharRY6nQ56vR5isRiTJ0/GO++8gy1btmD9+vUmZ5RFIhFcXFwgEolQU1Nj4OBwcHDAmjVrEB0djVdeeYVGYCgUCri4uNDdFWZZCL1YLMaCBQuwcuVKfPzxx9i8eTN1uBDjfNq0aXjjjTcwZ84cAz4+Pj744IMPIJfL8dZbbyE3NxdA9zhycnKCRCLB6tWrMXXqVAiFQmg0Gmi1Wmg0Guzbt4/ycXBwQGNjIwQCASQSCZydnaFWq/HYY4/h5ZdfhlKppHRarRalpaV48803cerUKZNxKBQK4e3tDXd3d7z77rsYNWoUHTNarRadnZ04duwYVq5cicbGRoP6MPkEBARgyZIlVK5Go0F+fj6WLVuG69evs9Jx9U9/f39s2rQJO3fuxNtvv42WlhZOuUx6trlNIBBgyZIlkEqliImJwdGjRwEAmzZtQmdnJwICArBo0SKcPXsWDg4OVNa+fftYnS3G87k5ueYclFxrI9tvHh4eiIqKQkJCAm7cuIHOzk5WOjZ6kUiEiIgIeHp6IiIiAunp6Sb14Cqzn58fjRzIz8+3io5AJpPRaIEjR47QSFVz9Fzn/ZuamsweK+xp9MB/ivZuwFz/+XfQ3Q36vyrbFlRXV6OjowN+fn6YMGECrl69ajWtTqdDUVERpFIpevXqhZCQEDQ2NlpV9tLSUqjVari6uuL+++9HXl6eRR2XoKurC6WlpZDL5YiLi8OBAwd6lIhVLBZj+PDhdN38O2D8bm15z8b6HNe6YIvsnvQ1W+SyPWOLXOO51tb2uhtj8O+Sy0bHdE5cvHjRoJ9u2bIFer0ewcHBeOihh3D06FHW8avT6bB48WL6+euvv8amTZtQX1+PO3fuQK/XW32sRqfTYfz48ejq6sKSJUvw/vvvY+7cuay2HdCt9zo4OCA1NRXPPvusVTKYYOoOWq0WN2/eRGdnp82bFC0tLcjJycHw4cMxZ84cpKamml3jmfQajQbXrl2DXq9HUFAQli1bhjfffNPADjPHo7a2FsXFxejbty8WL16M9PR05OXlGbxvvV6Prq4u6PV6ODo6mkQPRkdH45FHHsFnn31moMOTtbW9vR0ymQwODg6QSCQWdR8mfVdXFwQCAZycnGBvb4/m5mar+69Op0NnZyfkcjmUSiXEYjHnpUPM+jD5i0Qi2NnZWZzDbR1XPc52JpPJEBcXh/DwcBQUFECtVrM6StgMbIlEgrFjx2Lnzp2or69nVW65FF1jXlzPW+sxZILsRvv7+8Pf35/SaDQaJCYmGhxVIL8NHz4cer2eMweCOblMJ4BarYaXlxeCgoKs4mH8nVAoRK9evZCQkID09HRUVVUZGNdctEzZvr6+iIuLw/nz56nDQCQSITQ01MShQjqaTCajXt68vDyqYBPjePLkyVAoFNizZw+r4kPC0Hr37o3r16/TgSESiRAeHo5nn30WmzdvxtWrV02MTa62FQqFcHFxwUMPPYTa2locPHjQwIkBdBsDU6dOxSuvvILo6GiMGDGC/mZnZ4d33nkHv/zyC27fvm0i19/fH2+88QZmzJgBhUIBnU4HuVwOsVgMnU7H6rQhbebg4ICHH34Yy5YtQ1BQEDQaDYRCIQ3f6+zsRH5+Pj799FNs2bLFhL5Xr154/fXXMWnSJMhkMuh0OtjZ2UEkEkGr1aKxsRFHjx7Fp59+iqysLHR2dkIkEpk4WmbNmkUjdQCgo6MDubm5+PDDD3HkyBG0tbVBr9dDKpVi9uzZWLBgAT788EOcO3fOgJe3tze+/vpr1NbWYvfu3Vi/fj3y8/OhVqshEong6uqKDz/8EJ2dnVi9ejVqamroe2OGLDo5OUGr1dL3JJfL8fTTT2P69OlYvnw5MjMzKZ27uztmzpwJtVqNo0ePmkymAoEAISEhWLVqFe7cuYMvv/ySOjkAYODAgXjwwQexZcsWmqvK3t4eb7/9NhQKBZYtW0Z5yWQyA+eZn58f5s2bh8rKSiQnJwPo7quDBg3CE088gdOnT2PgwIE4dOgQioqKDMoVHx+PkSNH4osvvkBxcTF1/q1duxYdHR1ITk6GQqHA+++/j1u3bqGzsxPOzs4m0XfNzc0AuufRiRMnYsqUKThw4ADi4uLw66+/IiUlBU1NTZBKpfDw8EBJSQkuX77MqsAHBgZi8eLFyMrKQmtrK9566y3k5eVRw8PR0RHHjx+nMgH2+be2thaPPvooWlpaIJfL4enpiVu3bhmMH+N5nmuhKioqwuuvv46jR48aLHjMqCJjMOdSYwVeIpEgLy8PBQUFmDp1KkQiEXXeCAQCBAQEYO7cuRCLxaitrcWlS5eoomEsg+07LrkEXE5wrs0DJjQaDYqKitDR0YGoqCirFBcmL6LURUVFwdfX16yCwPy+q6sLubm5GDt2LHr37g2RSMSqtHDNwcQZTCJXc3JyONuO+bfxM3Z2dmhpaWHtu8y2t1QfS3LN0XG1ma1yuWCuH3DJ7omMu+Fk+iv01tLeDeOwubkZHR0dUCqVmDFjBtauXWs1vU6nQ3V1NUQiEfr27Yu+ffsiMzOT6rjm5NbW1kKtVsPR0REzZ87Etm3b0NDQYNV7VavVqKurg0QiwbBhwxAZGUkdL8bOYoC7PcViMaZNm4Y9e/ZQfcxaWkvtw0XbE+eKJXpry8g27/67+6kt9fgrz9lCa824sdbpfrflGkOn0yE8PBzz58/HoUOHcOnSJfobWS9JFPukSZPoesJE//79ce3aNfq5tbWV6gtKpRKOjo64ePEihgwZYlWZampqAHSPH6VSiT///BMDBw7EnTt3TJ594YUXoNfr8fnnn7Me27HUJsa/VVRUoKOjwyo7lQm1Wo2KigoIBAL07t0biYmJyM7ONtDXuCAQCFBaWoquri44OTkhMTERv//+Oy5evGjVWtHR0YGamhqaW2f8+PGUH7P+dnZ2UCqVkEgkBk4pvV4PZ2dnjB8/HklJSbh586aBXOJwsbOzg4ODAxQKhVkHiHGbKxQK6nDx9PS0KSKKbI7L5XI4OjpCKpVy5kESCAQ0NUBHR4fB925ubrRfGZeT6LJSqZTajmTTk2xQs6FHThuZTAZnZ2c4Ojqid+/eSElJgVqttri7yCy0t7c3wsPDqVcQ+F+lnA3m+FqSy7ZoGdMxfyOODPIyjh07Bq1Wi8mTJ9NniXNly5YtaGpqgqenp81ymfVtbW1FVVWV1ecSCW+hUAidToeGhgaoVCqMHDkS+/bt4wwDFAgEJtEwzc3NqKmpQa9evTBjxgxcuXIFGo0Ger0eIpEICxcuxG+//WbCR61Wo76+njq1jhw5YuDNDAkJwcMPP4za2lqcOnXKZKce6DZKvLy8MH78eNy6dcvAafPkk09i3LhxuHLlCq5fv04Vd4FAQPsfFwYMGIDZs2fj+vXrOH36tMFgkslkmDJlCl5++WVkZ2dj69atdCcZ6N7ZHjNmDC5evIjs7GyDvuXm5oYvvvgCYWFh2LhxI40ymzFjBiIiIsx69u3s7PDoo49iyZIlSE1NxYYNG5CXl4cBAwZgxowZ8PX1pbfYfPLJJzh06JABvUQiwSeffAJXV1d89dVXyM7Ohp2dHQ1tDAsLg0wmw6RJk+Di4oL3338fly9fNonYKS0txcKFCzF06FBMnDgRAQEBkMvlCA0NxTfffIPVq1dj+/btNOpALpfDw8MDEydOREZGhkHkA5nIlEolFixYAGdnZ3zyySe4ffs2tFotfVcuLi4YMGAATpw4AZ1OB7FYbPD+goKCEBcXh927d1OHkUKhgIeHB6ZPn47s7Gyo1Wp6lC8qKgppaWmoqqpi3fESCoVwdnbGgAEDEBERgcuXL1MDftKkSVCr1Th9+jQ6OzshEHSHwo8ePRrFxcUGvJycnNDY2Eida/3790dAQAC2bdtG+7NUKkVUVBQGDBiAkydP4osvvkBNTY1BO4lEIowdOxaNjY24dOkSnV88PDwwbtw4pKSkIDc3Fy+99BLq6uponbRarYnBSsavXC5Hnz590KtXL+zZswerV69GVVWVwVxMxjHXIuzg4AAHBwekp6ebjFGmbC4HMEFFRQWOHDlilVzmgsVWpoaGBvz8888mcpmLnTG4vq+pqcHXX3+N4OBg+Pj4IDU1FcOHD8fSpUvxzTff0GeOHj2Khx56iI6VRx55BNu2bTPhZ+uuLtlJYwPXmsXkpdPpUFpaiqamJvj5+dl8RrqrqwvZ2dno378/PDw8aNSjJaVSp9OhoKAAbW1tCAwMpDtNxoabUCiEQqEwiYZqb29Hfn4+AgMD4eLiYvLuyN9SqRSOjo5QKBQQi8Umjk5yfJOUmUkrFAqhVCrh4OAAqVQKnU6HpqYmNDc3m+2vAoGAKmPE2d7W1obGxkaqOHM54AicnJyoXI1Gg8bGRrS0tFjsD8aQyWRwcXFBY2MjVCqVVbJ7AnN8euokISDRtHq93ubIDjIPEHprHUvGdIBhHdVqNaqqqtDR0YHAwEBWerLxYyxXIBCgpaUF1dXV8PX1RUJCAjIyMpCfn29iNIhEIgNdTq1Wo7y8HIGBgQgPD8ekSZOQnJyM5uZmqNVq6HQ6s+3d1NSEuro6eHl5ITExEfn5+SgpKUFnZ6eBIi8UCiESiWg0sbFjuHfv3hgzZgzOnDmDlpYWk6hhZr40nU5HHVLG8y353VguMVTVajV1Klnz3phyiYFiqzORXDVMokHvhjPyP4n/VNn/Lrls6//OnTvh7++PlpYWiMVi1kiGvLw8vP/++5x8NRoNfHx8WI1xkkCYCRcXF3R0dFhMQvztt9/i22+/NfiO2KxEFpn/uSJjbG1blUrFeRrBHEhEiFarhUQiweOPP47U1FScO3fO4g1ZRC5xFgQGBuLpp59GQUGBiaOBDVqtFl1dXdBqtZDJZFi+fDnOnz+PrKwsul4LhUL06dMHvr6+EIvFBhu1ZL6JiYnB3Llz8dlnnxnonwqFAqNGjaKbj66urqiurmadP6VSKY3GB7rn5cGDB8PV1ZXedEWOoAHdc4i9vT00Gg1rf3B2dqZHvnx9feHg4IDm5mYqWyKRwM7ODl1dXfDz88OUKVMwatQovPHGG5SHQCBAeHi4gQ1JdBYSVTV48GBMnz4dYWFh0Gg0uH37Nq5fv45z586Z6EIENjttBAIBzUMiFosxbNgw/P777wYKizHYdhlcXV0xdOhQZGRkUEWZS3m1tKPG9ZuxcWBOOQZg4DElia8CAgLg4uKCuLg4A3ri9XVycsKgQYNw7tw51jJZIxf4X+eLLckmjT12IpEIkZGRiI6ORkZGhkl0iTnZxEgeOnQovL29UVBQQHmy5Vph1kskEqFPnz4IDQ1FQUEBdfYQ76OPjw+90YBLAY6NjYWrqyvKy8sN5CoUCgQHB1PFnZSXOHnY2oQpOyAgwGCiIJPvxIkTcevWLbz//vsoKSkxUHoEgu6IGH9/f4MoFaFQiMmTJ2PgwIF46623sHfvXmrMHzp0CP7+/li1ahXnuwoLC0NiYiKOHz+OTz/9FHV1ddDpdEhJScHWrVsxbdo09OvXD3fu3MG8efNMkns5OzsjODgYb775Jo4ePQqtVguRSIQDBw4gKCgIq1atwtWrV+Ho6Ih58+bh9ddfx+rVq5GZmWkwgdfV1eHo0aM4c+YMfvzxRzzyyCPw8PBAQUEBnnrqKbz77ruQSqX48ccf0d7ejqtXr6KqqgoJCQlwdnY2MNCKiopw8OBBXL9+HcHBwZgyZQoA4N1330VRURHa2tpw8uRJLFmyBP369UNqaiq6urrg7OxscvxkzZo1EIvF2LlzJzo7O3Hu3DlMnToV48ePxzfffIPm5mZIpVL07dsXdnZ2OHXqlMFEymzr+vp6XLhwgTrTbty4AY1GAzc3NyQmJiIzMxOXL18G0D3uvb294erqarJYOTo6Qq1Wo7W1FXZ2dhg0aBC6urpw/PhxdHV1UaPV19cXjo6O8PPzo/09KyuL8iEh61evXqU7ChKJBP7+/nBxcYFSqURoaCiampqgUChQU1PDaXyS8eXg4EAXlICAALS2tkKpVKK2thZ1dXUm4aZsO6XkWBc5AqBUKlFdXY3W1lbWeZvJjwniyK+vr+eUy6Qz57jhcnYb8zSuD9vc4unpiaeffhoA6M1RTU1NcHd3x9KlS1FcXEwN5hMnTtCdu59//tnkfLktcknZBQIBjRbhqqM5eolEQueJ4OBg2Nvbo76+nvV5NtlSqRRFRUVwc3ODt7e3yW6QObnl5eWQSCQICwujCgpzLnR0dERMTAzGjx+P1atXm/CorKykR+/EYrFBv5BIJAgMDMTIkSMxYcIExMTEwNHRERMnTjTgI5PJYGdnZxDlQByyAwcORGJiIkaOHAlvb290dnYiJSUFGzduxLVr10yc1UKhkJ5zHz16NO6//35ERkZCqVSioKAAO3fuxG+//Yb6+noTBxHzHTk6OuKdd97B8OHD4enpidbWVhw/fhzfffcdcnNzTQxzc0Z6ZGQk1q1bh23btmHfvn10PmPKJn9bgvGzlpwxpG/6+PggLy/P5DdLMsViMVxdXeHp6QkPDw9IJBKkpaVZLCfhr1Ao4OPjA2dnZ7i6ukKn06G1tRUVFRWoqKhgNTrIRpqHhwfc3d3h6elJnSUNDQ0oLi6m8+a5c+fQu3dvxMbGGvAgjn8SSajVatHa2orS0lJUVVVBIBCgsbERZ8+exYwZMzB9+nR4eXnh2LFj+OmnnygfR0dHxMXFoaamBiUlJWhtbYVWq8WxY8cQEhKCPn364L333kN8fDxSU1Nx+/ZtGmHp6+uLzMxMk/qVlpbiwoULeOCBB/Dkk08iLCwMx44dw40bN1BQUID6+nq4uroiNDQUvr6+8Pb2hkgkMogk0mg0CA4OxkcffYQ///wT58+fx+3bt1FSUgKJRIKAgACEhITAw8ODOgyzs7PR3t6OGzduUD7u7u6IiopCYWEhmpub4eHhgbCwMHh5ecHHxwd6vR5FRUUoLCyEVqtFZWUl57FusViM/v37U7mOjo6ora1FdnY2urq60N7ejoKCArS3t1vsd56enpg/fz4KCwtRVlYGjUaD0tJS1NXVGWzssa1fzO8UCgUCAwNRVFRkcKybDbaMQy5aY3qm/m6rbLb5hY2HuXYwR2euLNbINQeZTIYVK1YgNzcXn3/+OYYOHQovLy+UlZUBAM1vo1KpDPQxYwfNrVu38PPPP2PhwoUAuvPUEZ35/PnztC+4uLjAyckJM2bMQGpqKj0WZOxgJtH2ZWVlBlE98fHxyMrKwtq1a7F8+XI4OTnh5Zdfxn333WeSt9BaGOs1arUaarXa4hEcY+h0OqhUKrS2tqKtrQ1eXl5YuXIl3n77bXrsn0uuXq+HSqWCRqOhm/zjx4/Hww8/jF9++QX19fUQCoWcjinitGlqaqLO8bfeegtvv/02PZ7m6emJSZMm0Zw0xnm6qqurIZFI8OCDDyI3Nxd79+5Fa2srTUI8ZcoU6PV6+Pr6ws/Pj0bsGvdrHx8fiMVi+t78/Pwwbdo02NvbQywWIy4uDrt27aI0fn5+ePzxx5Gbm4sjR46YtFFsbCwSEhIAdKcPcXFxQUVFBfT67hQJI0aMwOjRo3H58mXMnj0bo0ePNrHfBQIBYmJicOTIEfpeg4ODsWjRIly4cAEODg5YvXo1JBIJdDodlEolJkyYAJFIhI0bN2L58uWs7d4jpw3ziFBMTAyCg4NRUVFhVfgogVKpRFxcHFxcXFBbW8tqnDA7GJfDxtyOrTX0TEilUgQGBpp43uLj4zlp+vfvb5anNY4molwrlUqrymnMVygU0mS0MpmM5mgpLi62ql2lUim9nszb2xvjxo3DL7/8QiMbyD9jkDN7EomEZgK/dOkSGhoaqPNEIpEgJCQEkZGRyM7ORmdnp8G7IrxDQkIwePBgHD58mB6tIYZBfHw8PDw8DAZNbGysidJDeJE+IZPJ0K9fP4SEhKCiooI6WDw9PeHt7Y2DBw+irKyM9SiEq6srYmNjaf8kRnZ8fDyqqqpw6tQpg4WeKB7r1q1jbW+RSISAgAAolUqkpqaitraW7jIRb+/+/fuRnJyMxsZGFBcXmyg/CoUCxcXFOH/+PG1HtVqNzs5OtLe346OPPsKdO3fg5eUFhUKBqVOnYvbs2SgpKTFY/MjOmlqthkqlws6dO9HV1YW2tjY0NDTgnXfewZNPPombN2/i1KlTKC8vR2VlJaKiohAeHo6SkhLKq6OjAx999BG9eYdMPDdu3MCGDRvQ2dlJj8iEhYXB1dUVDQ0NGDZsmMHCWFlZSRPD3rx5E+np6SgoKEBtbS369u2LsLAwZGZmwsfHB0OGDEFubi6ys7Oh0+mow5OJjo4O3LlzB1KpFOHh4XBwcEB7eztGjRoFPz8/fPfdd2hpaaH9ZNy4cdQQYEImkyEoKAg5OTkICQnBgAEDcOHCBRQXF0Ov19OcMOPGjUNgYCBWrFiBxsZG1NbW4pFHHqF8HBwc4OTkhOTkZLS3t9MjcdOmTYODgwPGjh2L2NhYNDY2Ij8/HwcOHMCBAwfQ0NBg0p+AbiU4JCQEQ4YMQUREBN555x0q99KlS9i2bZvBzgIXRCIRpk6dioSEBNTX1yMnJwe7d++m5eSC8ZwbEBCARx55BDt37jQ5UmhMx4Sl8hk7fWylZ6JPnz54+OGHUVFRgerqamRlZeHWrVu4c+cOIiIi0NbWhokTJ6K5uRnNzc0m84utirter4eLiwtmzJiBTZs2WV1OAqlUioEDB2Lw4MHUwLI2KSrQnXtuxowZCA4OpvmaZDKZyRxsDDs7OwwfPhx9+vSh0SiOjo50Y0YoFMLDwwOzZs3C3LlzTdZLiUSCWbNmoXfv3vTSARKRotd3H6nt27cvlixZgt69e0Mul8PNzY0e9WSChKq3tbVBq9VCKBTC19cXDz/8MGbOnImWlhb4+PhAoVDAwcEBCxcuhFKpxOuvv24SNadQKDB48GAsWbIEAQEBNAJQo9EgNjYWkZGR6OrqwtatW6nDgE238PPzQ2xsLLy9vekG1tKlS2FnZ4d//OMfdMwyjRpzhllCQgKCgoLQ0tKCpKQkg00srv5vzIsZccJmWLHRSSQSjB8/Hg888ACWLFliwM+4zGwOIX9/f8ybNw8DBw6Eq6srpFIpdZAywSZbKpViypQpmDx5MnWekDU/MzMTu3btwrFjx1h5RUVFYc6cOYiOjoarqyskEgmkUimamppw+PBh7Nu3D6WlpThz5gweeughkznDxcUFb731Fh1PJDQ9IyMDv/32Gy5evIiCggKcP38e06dPh1wux8iRIxEfH4/jx49TPr6+vli3bh2qq6uxb98+HDhwANXV1Thy5AgeeeQRujYsWrQIs2fPxu3bt3Hy5EmIxWL07dsXM2fOpLzs7e2hVqtx69YtXLt2DVOnToWdnR3uv/9+3HfffaipqUFKSgqysrIQERGBqKgomqtBJpMZOG2IjhQSEoJnn30WjzzyCK5du4ZTp07BwcEBAwYMgJeXF+zs7KBQKODo6Egjppll8vPzww8//IDTp0+jsLAQvXr1omupvb09pFIpHBwcqD6QlpaGTZs24cqVKya6i5eXF95//314enpSuc7OztQIr6+vx5YtW7B7925qQHKNGXd3d7zyyiuQSqWorKyEVqvF8ePH8dNPP+HmzZsmugBx9JF5hCAkJARffvklfvrpJyQlJZmMW+B/dWtyIxc5gmMcucSUxRwjJKLYyckJHR0daG5uptEN5BkuEP2aXNrClG2JFuieO/39/aFSqdDc3EyjtZi05mwnrk0ba2wUc2sl2QxxcHBATEwMWltb4eLiQi9CmTFjBgQCAXJzcw2iXWfOnEmjY4HuTczDhw9jwYIFAIDHHnsMiYmJALo3gskRIRJR/ssvv6CsrAwnTpxAfHy8wREid3d3pKSkAAC+/PJLvPDCC/S3w4cPw9vbG7/++ivlBQCvvPIKPv/8c4ttYdwuxhtXQHfUtEgksjqfKAHRI6urq7Fnzx6MGDEC48aN48yNajyfEydKWloaMjIy8NJLL+GZZ56BTqfD6dOnIZfLOaNuSHqFkpIS7NmzB7Nnz8a0adOgUqnwww8/oKGhAWPHjsXQoUNx7do1DB482CRR8unTp1FRUYElS5bgueeeg0AgwOXLl+Ho6IjXXnsN7e3tyMrKQlhYGEJCQpCamkpTPjAdXK6urnjqqaewdetWtLW1Yfr06YiIiMCFCxdw33330cs4yBgfMmQI5s2bh+zsbIMjdkD3XPzUU0/R6NmIiAj4+fkhOzsbGo2Gnm4YPnw4+vfvj/79++PPP//E8ePHDWwjkUiEAQMG0Hw6AoEAo0ePxuzZs+Hv749evXpBq9Xio48+Qk1NDUJCQhAaGgp3d3fWtY+2O+cvHCBOG/LSPT09MW7cOFy9etUgWaglyGQy9OnTBzExMTh79qxFjy1XFA/bzqct9Ey0t7dDqVSafJ+bm4uQkBAaiaNUKqFUKg2uoeOKkLFGLgmZcnV1tTnxHNkVJYpxV1cXRo4cifHjx2PXrl2oq6uzKNvBwQEuLi40EfO0adNw6dIl5ObmIjw8HLGxsfj1119N5EqlUhotodFoMH78eJw6dQpHjx6lDpOAgAC4u7tjypQpuHjxIsrKygwWOzJReXp60uiX8vJyREZGYsCAAZBIJOjduzfGjx+P33//HZ2dnRCLxay5f8i7I4qJp6cn3NzcMHPmTNy8eZMOWEdHR3pUgG1xsrOzg1wux8CBAzF06FAkJyejq6sLEomE3qRCnFjMfqvT6Uy82yR8mWQjd3Z2ps4l5ljR6/UGUUbZ2dkmOwEymYxTdmdnJ65cuULDEouKiqhBvmPHDpOJl7wDtVpNo6PEYjEyMzPR1taGgIAATJkyBWlpaWhpacHNmzcxatQoTJw40SA5m1arpcnMKioqUF5eDplMhrlz52LHjh00WVlhYSEiIiIQGRmJ1tZWPP300wYLIwlb7dWrF0aNGoXr16+jvr4e169fR2xsLCZOnIjy8nJMmjQJUVFR2Lx5M8rLy1nHFzFgCgoKUFlZSfuhnZ0dnnvuOaSlpeHIkSO0fUkEjUgkMlDKCa8xY8agsbERDz74IJycnLBp0yaan0kkEsHDwwMxMTEoKCjAtWvXYGdnR52ZBO7u7jh79ixSUlKoXIVCgWHDhkGtVuPEiRNQq9VwdnaGTCaDm5sbvZGNbV4jTsDQ0FDcunUL2dnZcHR0hEwmg5eXF1X0jBdpY2i1WnpkzcnJCUqlkkZGMGFpDtPr9fDy8jKJqLPWCc0FW+kt7VoyF+jGxkY4OztDJBLBzc0N/v7+CA8Pp3yYkVK2lJOp8MfHx+O9996z6LQxphcIuo9iPvroo5g1axZVrqZMmWJSLjbZAoEACQkJeO+99wB0z419+vRBfHw8Ll++jKamJs5y+Pv745lnnsHIkSOhVCqh0WgwZcoUHDt2DCUlJZDL5Zg5cybmzp2Lw4cP48CBAwY8HB0d8T//8z80kmPw4MG4fPkybty4gdbWVoSGhuKll16Cvb09PvnkE7S1tSEkJARqtRq1tbUGvMRiMZ1zysvL4eDggMcff5zOaydOnKBROk5OTpg3bx4SExPx448/GihQZINp+fLlaG9vx3vvvUejDlQqFRISEjB//nyaH4okL2dDbW0tXn75ZfTp04dGqjz66KOYMWMGNmzYgKamJovHCQkqKiqwf/9+TJkyBWPGjMHp06dZHbXG/dr4MzleYBzBZTx+mJ+lUikGDBjA6RjmkguAbn54enri8uXLdKPGXC4FpmyZTIZBgwahsrISaWlp0Ol0cHR0xIABAzBu3DhERUUhNTXVhJ7Me/b29jh37hxaW1shEAgQGhqKESNGYOnSpVAoFNi4cSPq6uqwf/9+DB482ICPQqFAUVERUlNTodPp4ObmhkGDBmH69OmIjo7GsmXL6KUAly5dwoQJEwAA+fn5BmtyXV0dUlJSMHLkSKxYsQICgQA7duxAVVUV/vjjD/Tr1w9CoRDNzc0oKytDREQE4uLicPXqVezdu9egTL6+vtDr9SgpKcGFCxeQlZVFNwuzs7PppQ91dXU4dOgQfvzxRzQ3N0Mmk5nk7CBH5QUCAerq6lBbW4tBgwZhxIgRSE9Px969e6keFhwcjIULF0KlUqGoqMhg3lepVBAKhVi4cCHKy8tx4MABfPvtt2hra4NcLsfo0aMxb9483Llzh+pZDg4OeO2111BSUmIS1bJ//35UVlZCIBAgIiICS5YsQXl5OaqqqjB27Fi89tpr0Ol02Lx5M3X6kOMJzA2e8vJynDhxAuPGjUN2djaCgoLw2GOPUdnkiAWJyIuOjkZMTAyioqLw9ddfUz4dHR2IiYnBypUr0dHRgX379tH3SzZwBwwYgKioKISFhUEul6O6uhqZmZk4ceKEwZF6Zt8m60mfPn3Qu3dvREVFwcfHB62trcjLy0NqaiouXbpEo1nZxig5bk0SYZOLLm7cuIETJ06gtLTUYK5ns4GIU7GpqQk5OTk4ffo0rl27RjcbbXHCuLq6Qi6X041TY7nGMGf3EBvH29sbs2fPNvldrVbjypUrFm9SCgkJwdatWw2+27p1q0m0JoGHhwcaGhrw9ddfm+SFCwoKQk1NjUnUBYFQKERAQACOHj1K6zZs2DCbb5hk6vtMuLu7QyQSWZ1ol0AikcDV1RVNTU04d+4cjR6///77LdIKBAKa1iMvLw+///47lEolHn30Ubz00kuYOnUqJBIJqyMeAN3Mqaurw8mTJ1FUVIQXXngBs2bNQmRkJOrr66FQKJCSkoK0tDQEBwfTHJqkDbKzs3Hw4EHY29vjoYcewmuvvYa8vDwoFAqo1Wp8/fXX8PHxwcqVKxEREQEHBweo1WpIJBIDp01HRwfmzp2LmJgYmldx//791FkUHByMkJAQuj6R23MHDx5sslHk4+MDlUqF7777DrGxsXj44YcRHh6OtLQ0tLa2UmeVh4cHnJ2d0dTUhIMHDyIlJcXA6abX6xESEgJ/f3/cvn2bOoBJHlc3NzccPnwY+/fvp2NCLBZDIpGY9QP0yGkjl8shl8tx+/ZtREZGYtq0adixYwfu3LljlbKi1Wpx4cIFmpjzypUrnMkGLYEsMraGlQHsTp6bN2+afGdvb28QaaLValFUVITGxkZ4eXkB4HbasE1cbBMaMda4Jg1ztCTha3FxMUpKSjBs2DAsX76cJoG1RO/m5gaJRIIjR44gJCQEcXFxePHFF5GZmYmhQ4fSnDnG9SL5CG7fvo3W1lYMGzYML774Ivz9/eHq6oohQ4bg0qVLiImJwf3334/z589j69atJrswx44dQ2RkJEaPHg2VSoXi4mKMHDkS9fX12LdvH2bNmoX58+fjxo0bdPe7ubnZ5Ao4cuzJz8+POgMTEhIwY8YMpKWlYe/evdDpdDTMPywsDE5OTibe7a6uLhw9ehQTJkzArFmzkJ2djfz8fOj1ejQ3NyMyMhJRUVEmZ2n1er1JSDc5vwj8rxIUGRmJ48ePG+x4EwcOUZIdHBxMeGm1Wnh4eCAkJMTACaPX62m9mA4iuVxusDPDBhJ1QxZg5u4zMT6IY0alUmHSpEl47bXXDHgQerKAiEQiNDU1USduc3Mzbty4gYULF2LUqFHw8fGBj48PqqqqKA+xWAwnJyca8UPa4+LFi3j88cfxwAMPIDc3F3PmzEFGRgaSk5NpODpbIjitVouamhrk5+dj3LhxGD58OEaMGAEHBwe88cYbNPKK1FssFqO0tBT//Oc/DfgIhUJMmjQJHR0dmDBhAo4cOYLz588b7JaJRCJ0dXXhyJEj2Lx5Mz1zb9w/1q9fj6qqKupYI3ILCgpoMmI7Ozvo9Xo0NjayXnPJhEgkQmtrK/744w8kJSVBKpVCIpGgra0NtbW1rBFIxlCpVPj444/R1dUFuVwOnU6H+vp61hB3gNsxXl5eji+//JImpuM61kVgaafPGlpry9fU1EQTDw8cOBBhYWEAupPIr1mzBkD3jo+bmxuAbqevUqnkvKqbrcxsu5JkjmTbCLBUbjKO7ezs4OTkROe65557ziSpKpdscusCyUMxZMgQvPvuuzh+/Di+/fZbVgWRyFUoFFSuVCrFSy+9hHHjxmHHjh2oqKjA1KlTkZ6eji1bthiMY8KDOO9EIhESExMREBCA5ORkbNiwAbNnz0bv3r2xatUqJCcnQ6PR4OzZs6z9RiQS4dVXX8XYsWPx888/QyKRYPLkyUhKSsKWLVvQ0tJCFSJnZ2d4e3vj8ccfR0hIiAmfWbNmwd3dHatXr8a5c+do5I5IJEJWVhbGjh0Lb29veHh4mD3TX1dXh6amJmRmZkIoFMLd3R29e/fGpEmTEB4eznrshbSL8buqr6/H5s2bMWLECPj5+cHZ2ZnzpiJz/V4mk3HmROCKTuvs7MTu3butut3CmF6v1yMvLw8//PADampq6FFsS3kUiGyVSoXt27ejoaEB9fX10Ol0kMlkCAkJQXt7OyZPnmxyVbxer6drUUFBAaqrq2mUrJubG90pHjp0KJKSkpCVlYVjx47Rq3kJamtrsXHjRpogmBy1ffHFFzF69GiMGjUK+fn5KC8vx/79+xEREYGQkBBkZWUZHHOsq6vDl19+idu3b+P555/HyJEjkZycjOLiYuzduxfDhw/HhAkT0NzcjNTUVAwbNgzV1dX48ssvTW6rVCgU6N+/P3bu3ImcnBzs27cPQUFB8PHxQUZGBs3fcfjwYXz++ecoLCykkcPG77yhoQGnTp3CqFGjUF1djZSUFEyePBkNDQ34+OOPkZaWRsd+REQE9Ho9MjMzUVxcbLDJ19bWhrS0NNx///34448/sHHjRpSXl9Nj2RKJBNnZ2UhOToanpyeWLVuGhIQEREREoLy83EDvqKiowJYtW2j/GDhwIDQaDY06PnfuHF599VVMmDABe/fuRV1dHRwdHTF9+nTMmjULDzzwAOXV2NiIEydOICUlBWfPnkVQUBBeffVVTJw4Ed988w2ampogkUgQHR2NhQsXYuTIkQgMDER7ezs2btxI+ZSXl+Pzzz/H0qVLMXLkSJw4cQLNzc30Eovp06cjPDycHhkWi8Xw9fVFS0sLBg8ebGDMSqVSqNVqekHLggULMHToUPj5+aG5uRm1tbUICAiAQqHAuHHj8OGHH+LEiROsupmHhwe+/vprmqOwqqoK9vb28Pb2Rn19Pfr27YtVq1YZHCVjG/eOjo4IDQ2Fv78/JBIJRo8ejXfeeQcZGRkGthbbBgtbdOHs2bPx8ccfGxyvtYbWGERfq6ysREpKisGarNfrkZSUhOjoaHh6eqJfv34QCAQGG+QExjk/fv75ZzQ2NkIqleL777/HE088gVWrVtFNRpIjcv369fj+++8Nogt1Oh3eeustzJ8/H4MHD0ZiYiJiY2PpZSAqlYrmuQkLC4NWq6WGd0/BdIBHRERAJBJZPLrMpAO6542QkBBUVVWhvLwcRUVFqK+vN7hVi4teIOi+vUmn0yErKws1NTXYtGmTwc3JN27cQHl5OSsfJycn+Pj4oKioCLW1tcjJyUF1dTWWLFmC8PBwtLe3488//8TRo0fR0dGB27dvm/ST7OxsFBUV4dtvv0VNTQ0SExMhk8lw9epV/P7778jLy8OIESNQX1+P/v37w9fXl+aQZb7/8vJyHDlyBP7+/qirq8ORI0dw5swZ6HQ65OTkoH///hgzZgxu3rxJb7Z0dHSEs7OzSX9ta2vDxx9/jJKSEtjZ2aGtrQ3x8fHYv38/VCoVvLy8EBkZSTf3z549i6KiIpMTG52dnXBzc8PgwYORl5cHe3t7REVFwc7OjgZoHDt2DO3t7fSUkjV5iHrktHF2doa/vz/UajU6OjoQGhqKSZMmYePGjdDr9RYdKCR5G8mJc+DAAWRmZlodpcOEo6Mj9Rz3xHFjDYqLi+Hp6UkdN0RB4VLqmbC020VA8qecOXPGLD82WpI3pqSkBL/88gvs7OwwduxYvPLKKyZOGzb6oKAg6HQ6nDx5EqdPn8aHH36IadOmYfTo0Th9+jQ1No3rpVAo4Orqivz8fBw+fBheXl7o378/li9fjpqaGmzfvh2pqal45JFHsGjRIixbtgyXL1/GxYsXKR+dTofjx48jMzMTL7/8MmbPno2WlhYkJydj+/btEIvFiI+Px+DBgzFnzhxUVFSgubkZhw4dYo2QWbZsGW2H69evQyQS4YEHHsCrr75Kz4WXl5cjIyMD9913HyorK7Fz506DXZPKykps2bIFsbGxNBHy1q1boVKpqDL23HPPQSQS4fz58zR0ny1ajCREKy0tRU5ODu7cuYMHH3wQzc3N2L17Nz0mxTRWhEIhcnNzTZKONjY2wtXVFUuXLoVYLEZGRgba29vpuCFHF9ra2pCeno6JEyfi1KlTJrvqYrHYJLyQGGs3b95EamoqYmNjcebMGeoQyszMxNWrVw2UJ2ZfEIlEqK+vx/nz55GQkICDBw9Sp1RLSwsuXLiABx98EAsWLIBUKsWmTZsMnBLt7e24ePEivLy8aNJgrVaLS5cu4erVqxg1ahT+8Y9/oK2tDbt27UJlZSWnQ4K0RW1tLS5evIhx48Zh6dKlNEkyiQokY9PFxQWurq7o6Ogw2Tnp6OhAbGwsfHx8UFhYiIMHD6KxsdEg0Zqfnx8kEgndjXFxcaHH+QhIxBF5V+SICckD4+XlBbFYDB8fH/j5+SEsLAyHDx+miZmNQa6NJ8Z5UFAQ3N3d4e7ujsjISEgkEnz66ac0fw4XdDodvLy8IJfL4e3tTXcktm3bhkOHDpk4b7icLA4ODnjppZfQ0tKCr7/+2iRHBhesne+tcZRwQaFQ0HPptbW1VEHctm0bFi1ahMbGRqSnp5vsTpEk4z0pM+mDV65csSlih0nf2NiIy5cvY9KkSXB1dYVAILAqnw3B+fPnkZOTQxUzlUqFtrY2lJWVQa/Xcx61qqysREZGBoYOHUqP7NbX16OtrQ1VVVUIDg6Gr68vvRnHeBy2trYiPz8f4eHh0Ov1aGtro854kUiE0aNHo7GxkR6jIGs3mwEKdBuhJEHr2LFj4eTkhLS0NAPnrl6vR1NTE/Ly8tDW1kZvgiQQCoUYOnQoqqqqaMJ7IlckEqGsrAx5eXlwcnKCXC4360wkcyL5vaGhAdnZ2Rg3bhx1VhFYE6FWUlKC4uJiSKVSqxNNG5ePRHRaI5eZ9Le4uNjqa2aN5ba1tdGbS8RisVU5GUi5SKJrkoRUq9VCpVIhNzcXqampGDNmDGe0H4kkIfXWaDT0WOiNGzcQGhpKHaXV1dX4+OOPDfiQo8xkt7a9vR23bt3C2bNnMWLECHh5eUEoFEKlUuHYsWPQ6XR47LHHDG4wJCCRQhMnToSDgwPdta2srMSaNWtQXFyMyMhIGmFy5swZpKenmzjKyK63UChEa2srfvvtN3R0dGDRokW4evUq4uLi0NDQgIMHD6KoqMhgw+fUqVMm9Xv33XexePFiKBQKGnly6NAhmuuQGO2FhYX0SAEAg/m+o6MDEokEZWVlOHr0KM0zRDZykpOTaYQsOY4WGxsLDw8Pk8SyJME3mStu3bqF3NxctLW1QSAQYP/+/XjwwQfpUbmOjg489dRTeOSRRwzy7JC+Q24E7ezsREVFBdLS0tCrVy94enpCLBYjLCwMq1atgru7O+rq6hAREYG6ujqDCzq6urpw6NAhTJ48GW5ubrC3t4dOp8PLL7+MmTNn4tKlS9SJe+7cORw9ehT9+/fHokWLMGvWLAOnDemrsbGxePvttyEUClFXV4eoqCiUlZVh3bp1sLOzw8yZM3HfffdhyJAhdOfeGB4eHqitrYWfnx8UCgUOHz6MS5cuIT4+HosXL8asWbPoJRbmxrlGo8EHH3wANzc3zJs3DwkJCfQmNPIejCPzuCCRSDB9+nT8+OOPJsdOzdGzzaNkg7ulpQWZmZkICAhAbm4uSktLAXTrSuRoT2JiItLS0lgjQxsaGlBYWIj58+cD6O7LmzdvhqOjI+bMmQN3d3e8/PLLBrkhP/vsM8THx2PIkCEGThuSuuG7774DAHz33XdYs2aNwWZ1cHAwtm/fTvM5ubi4cLaXObBtugcFBUEoFJqNtGF71/b29vD19UVWVhbq6+vR0dGBK1eusCZnNqYXi8UIDw+HVqvFnTt30NXVRe0h4uiqr69n7aPED+Du7o6LFy+ipaWF6vClpaVQKpVobW1FTU0N2traIJPJDOw+oHscE2dHcXExNm/ejAMHDkCv16Ouro46kEtKSlBQUIB+/frR49RPPfWUgWOqtbUVb7/9Nuzt7WkSeZVKBblcjrS0NAwZMgTjx4/HkSNHMGzYMEyePBkymYzaGUxUV1ejuroaQqEQ+fn5qKqqwpAhQxAWFgYfHx8sXboUvr6+dKP8+vXrqK6uNuFD9JCEhARcvnwZ48aNw7Bhw+j6W1tbi/Pnz3MeteSCzU4bmUyGXr16oVevXnTwSSQSPPfcc3B1dcWePXuQkZFhthASiQRjxoyhoZ1vvPEGtm/fjuTkZHpzjDG4Jidvb2/MmjULe/bsQVZWllXXXDO/Y8riUmIHDBhgEtVhHFnDFRZsTeghua6b7HjYCgcHB4SGhqKsrAwZGRl46623MGfOHPj7+1ukJUmEOzs7cePGDVRWVkKlUmHcuHEoKCjAn3/+ibKyMtbJhOxspqWl4erVq3j11Vfx0EMPwcnJCQcPHqQ7Olu3bkX//v0xfPhwfPbZZwbXaxMvb1paGpqbm6lxlZSUhPr6ekgkEvzzn//ERx99hPnz56O6uhq//fYba+Lnuro6JCUlISkpiRoE69evR58+fRAdHY1169bhzTffRGFhIXbs2IHAwEA8++yzGDlyJB599FHKR6VS4fLly9i0aRNWrlyJp59+GvX19UhOTsbJkyfxww8/YPHixVi7di2OHz+OU6dO4c6dO6ipqYFYLKYJ1YBug/H111/H//zP/6CsrAxbt27FypUr8corr2DcuHFISkpCZmYmysvLoVKpIBKJ4OLigrCwMJNEy83Nzfjtt98wb948REVFITk5GWfPnkVBQQHq6uogk8kgFAoRGBiImJgY5OXlITk52cTQCwwMxOjRo3Hz5k2ai4oojSEhIfDy8sLevXuRkZFBPb+NjY04cOAApk6dasBLLBbDy8uLOg6io6Nx+fJlHDt2jIb/qtVq3L59G1evXsVDDz2E8+fPY8+ePQZj1d7eHkqlEvv27cOtW7eo8lhfX49du3YhMTER/v7++Oqrr3DlyhWzOSeIQdLZ2YnLly8jLy8Po0ePxsmTJ7F7926aqZ+MTVdXVzg4OCAqKgobN27Enj17KK/GxkbY29vTPn379m3aXkS+r68v3NzcsGjRIixYsIAm7j59+rQBH1IuUmaSeygwMBA//fSTQdLvwsJC7N+/n3M+I04bHx8frFixgkYckSN/e/fuZc3VZAx7e3v8/PPP1JEnFotx8+ZNTlrmzhrzdx8fH8yYMQPbtm3jjNIhdMafrXWCWKJlCxEHusfNpUuXDG7/A4Bp06bhxx9/hFarhbe3NwICAnDixAmEhoYiODiYsx7m5Br/Vl1dje3bt5utG5OG+bdKpcLFixdx+/ZtjBgxApWVlVixYoVFPuQfUcBWr16Nrq4u7Nmzh95sJhAIDJICEhAny6lTpzBt2jT06tULZWVleO6551BSUoKOjg66AxwREUETuTINNLVajT///BMvvfQS2tra8OOPP2Lbtm30OmOFQgGJRIKIiAiaz470c2PFp7OzE0uXLkV5eTk6OzsxceJESKVS9O7dG5cuXUJ7ezsEAgHd/be3t0dnZyers5Lk9QkMDDSIIiRRguTIMzGwzb1vYqTpdDqIRCI4OjpS459EOlnrZCQ54crKyugxI+MdbEu8yHEYNrD1S4FAgKCgILz44ovo378/Ro0aZUBjTi4J877//vvxxBNPwNnZGdnZ2fjtt99w/vx5VvlMvgJB960ar732GsLDw9HR0YHk5GSkp6cjNDQUM2fOZA0RJxFcs2fPxuzZs6FUKnH9+nX8+uuvsLe3x/DhwxEfH08NBwA0SS0TQUFBdFc+KSkJ2dnZiIqKwuzZs6HVanH27FnqlGtsbERycjIyMjJMdFM7OztMmDABY8aMQZ8+fegmDNA9BvLy8vD5559DoVAgIiIC06dPR+/evREQEGASzajVaukxMZ1Oh9raWvzxxx84e/YsmpqaEBMTA7lcjtjYWFy8eJEmJ2eLgiY61QcffACJRILhw4dDo9EgLi4Ovr6+aG1tpVFtnZ2d1GHq5uZm0ObkxitnZ2f07dsXN27coDKJXJJclBxBunHjBm7evGnicCPrC4m+bW9vp+sVSbFgZ2eHI0eOoL6+HsOGDcPMmTORlpaGTz/91KRPNTY20mT8/fv3R2RkJM6dO4ecnBwolUrMnz8fwcHB+PTTT1FcXIz33nsPsbGxNDIe6B53o0aNokmr29rakJiYiBkzZuDgwYP49ttvcf/99+O9997DlClT4OHhQW92ZB69BLrnKW9vb8ybNw9yuRz//Oc/oVKp8MUXXyAqKgqvvvoqLl++DLFYjKamJpNbHploaGjAmjVrMHfuXKxYsQJz585FUFAQjSwuKSlBU1OTxbVToVDg5Zdfxo0bNyASiVBbW4uamhqLN5ixgdSZHEe0Zm4j847x88Y37Z48eRIajYauIUuXLsWnn36KkJAQzJgxg+aT+uOPPwz4aLVaDBs2jEbhrFmzBj/88AOWLFmCrKwsjBkzBn/++SfS0tKogy06OhrDhg0zKatKpcKGDRsQGBiIJ554AqtXr8batWuxZs0aDBs2jG5Ss9GygW3+5JqfySUOXV1dnE5vLvuV5Jdqbm5GR0cHHdNcNw8xYWdnBxcXF6hUKrr5otfrUV9fb2AzsL1bklZDKpWisbGR5kvq7OxEQUGBCa1arcbRo0cN9FmdTofGxkYqt6amhka4Mjehy8rKcOPGDYwYMQLr1q2DSqUysLEIL6Zc0se7urpw8OBBvPDCCxg+fDj27NmDlpYWnDhxAnq9Hn369DGJbiIR+Hq9Hnfu3EFBQQF69+6NDRs2oKurC4WFhfjhhx/w/PPPo7q6Gvn5+azXtZPTIJMmTcKQIUPQ2NiI/fv3Y9asWTQRfU/Go81OG2KEqdVqKJVKmp8jICAAixcvxpUrV3D16lWrJhSBoPse9okTJyIvLw+nTp2yKns8EzKZDAsXLsSFCxdw8+ZNqyrPZgAA4PScMh005KgHcTycO3cOffv2NQnltUYu+SyVStGrVy+0t7ebdEbms4CpgUDaMDAwEBkZGWhoaEBFRQWys7PNloPwkslkiImJQX19PaqqqtDY2Ih9+/YhKSkJOp2OTiJsYevMa9haWlpw48YN6mggO5kCgQDZ2dn46KOP8M0332Do0KEGfDQaDerq6qhDYteuXVSZBrp3Q5KSkhAcHIwVK1Zg1apVmDp1KjZt2oSdO3ca8CovL8ebb75pIPv69et455138P3339Oz8vv378fPP/+MVatW4eWXX8awYcMMssATZe33339HaGgoFixYgE8//RQXL17E999/jw0bNqC8vBzLly/H/PnzMW/ePKhUKmqYDxw40IDXtGnTEBMTg927d2Pnzp34xz/+gTfeeAODBg3C8OHDoVKp0N7ejra2NoOF3DiRmFwux1dffYXq6mosXrwYTz75JBYtWkQnXYFAQHfHcnJy8Pnnn+Py5csmuQ6USiXWrl1Ls86rVCqa3Eur1eLXX3/Fr7/+ipaWFoOd7MOHD9NjJgQRERHYtm0bzTWUkZGBjz76CLm5uVRBEwgEKCsrw4EDB+Dn54dff/2VJvIl8PT0xPbt22m4PnMne9++fXjooYeg1Wqxd+9eOkcww0uN+zeRnZOTg4MHD0Iul2Pz5s10kiT9XyAQoKqqCoWFhWhvbzeJEGloaMDJkydRXV2NkydP0qg+ZlRAbm4u7ty5g7y8PDQ2NqK8vBw3b940OIfPvGaZyC0pKUFhYSH9p9FoqAOvoKAAt27d4ryeUq1WIycnB9nZ2cjLy6NXLBOnErkO3RLa2tpw5MgRdHR00EWEJOc1t2tuPMcWFRXhiSeewMWLF80eLTGex2yBNbRscz9p11u3bmHChAkYNGgQANDkuqQ/XLhwAXZ2dvD396dONVvKxnTkke80Gg3r1eHm6kX+12q1yMzMxOrVqzFs2DBkZGQgPT2dlQebbLVajZ9++gl1dXWQy+U4fPgw7ZNisdjkRium3PPnz+P1119HbGwszp49i+vXr1PHSEFBAcrKyjB//nxotVokJSXRs/SEfsOGDSgqKoJarUZycjKNyJBIJEhJScHcuXPxyiuv4Pvvv6c3TURFRZncPkRupiGys7Ky0NDQgKVLl9IcVCQfSnx8PEaMGIEtW7bQaD1mmS5evIgpU6ZgxYoV2Lx5M8rKyiAQdOe6GTp0KFpbWw2OMBq/VwI3Nzf4+flBp9PBw8MDQ4cORUxMDD799FNkZWWZXGNsrr+6urri2WefRXFxMX755RdqjHPJNtYfCDo7O00MBXO0AoEAU6dOxfz5862KGDaW6+bmhieeeAITJ06EXq/HoEGDMGPGDIwbN84sLdA9X8+fPx9z586lN2cMGzYMXV1ddNPj7bffZr1xLTg4GI8//jg9tj1o0CDMnTuXRtwkJyfjs88+o8d49HrTkHMXFxcMHToUWq0Wo0ePhlqtpjePLF++HBcuXDC4PpYkJjeOZoqMjMS2bdvQ0dGBX3/9FRs3bqQJfck7If2e3EY1f/58fPXVVzh06BDefPNNyouMFSJXq9WioaEBDQ0NEIvF+OmnnzB27Fg8/vjjiI6OxtmzZ3Hnzh20tbXBw8MDv//+u0EdOzo6UFpaSnWCxMRETJgwAZs2bcKBAweQm5uLsrIyyOVy+Pr6IioqCvHx8Xj22WcpD7VajW3btmH48OFYvnw5Bg4ciAsXLuDOnTvUSUGScvfp0we3b9/Gxx9/jJycHJM8ESEhIZgzZw5yc3NRWVkJpVIJHx8fREdHY+jQobC3t8cvv/yCX375BSqVCgMGDIBCocCePXtMnG4ODg6YOnUqgoKCMGjQIERERCA9PR1fffUVysrK4O3tjfvuuw/Z2dk4c+YM6uvr8cEHH2D16tUGunxISAgWLFiAn376Cb///jvUajWGDx9OdaCysjLs2bMHnp6eWLJkCSZOnEiPdBnflqfRaODq6oq4uDhcuHCB5qx56623sGbNGsTExCA6OhpVVVVYv3499u/fz3mcsb6+Hk1NTdi2bRvc3d0xZ84cTJ8+HRqNBsePH8c777xjcmsmm5NArVYjNjYWAwYMQH5+Pj744AOkpaVxzk3meDU2NmLlypUmThs2e4b5mW0OI/pBWFgYHQM7duzA+PHjaa7JuXPn4ocffoCdnR0SExOxZ88ekwif2NhY/PDDD3jttdewe/dumtNz4MCB8Pb2phGaCxcupDd9Tp8+neqjTAwYMADx8fH47LPPMHfuXLi7u+OZZ55BXFwcbt++DaBbz7Ozs8PSpUuxbt061stZmOBybLHNh2KxGA0NDZw6Fxct0dlra2sNdEVrbGgit66ujh4VNabl4iMQCGgOHuZNpVx11mg0uHnzpsFV7CQFA5dcEplXX1+PK1euoLa2Fh0dHXS8GsOYD9G/rl+/jmvXriEmJgY3b97Ehg0bcOvWLXh4eCAiIoJ1c47IrqqqopH+7e3tdP709/fHggUL0NraSjeA2C5Dyc7ORkBAAC5evIiff/4ZxcXFGDx4MNzd3VFVVUWDIWzxedjstCFXIJ85cwb9+/dHUFAQNZLI7oIlx0lzc7OBEaPVaultObYUHuiubEdHB/V820rPBFdyRiZIZyWRN+S8tLkEfOZ4AaDe0l9//ZU1rM0crUAgoEcxTpw4QUPGrd3tdnZ2hlwux9atW1FXV0cnDeOdduN2JXlTcnJycPnyZXpkjMg1XjzOnTuH1157Da+//roBn7q6OrqDwCWrqqoKP/30ExQKBWbMmAE/Pz/WkDKdTmdyzTlJ8rpixQq89tpr9HrStrY2FBcX4+mnn4aXl5dB2Kxer6fh/OvXr4dYLMb48eMREBBAbwDYsWMHkpKSEBUVhZEjR6JXr16QSqUmUS2lpaW4c+cOnJ2d4ejoiLa2NhQUFODhhx+mNyGRs7EdHR3Iy8vDlStXkJOTY2J0kx247777Dr/99hv69OmDkSNHIiwsDCKRCNXV1bh27RquXbuGwsJCmivAuJ1yc3Px9ttvY9CgQXB2dkZbWxuNhCHKHBmLzB3w1tZWLFu2zIAXuWY3IyMDN27coNeDErnMqIGkpCS6c2js3c7Pz6eKK1MumdgXL14Mvb47pJBpUJGyGYMoDCqVCps3b8auXbtQX19vkkdIIBCgtLQUixYtYuWlVquxePFiaDQaNDQ00N9J+To7O3Ho0CGD3SLCn+3sN/lbp9MhPz8f06dPp23FdNgRhylTwWKira0N27Ztw2+//UbnXgIyhpnzKdecnJeXh2effZbuLpB3RniQxZNtx8hYqTt48CCVzyaXy3jl2rnjkstFy1a/qqoqJCUl4Y033qBKjUajgVgshkAgwKuvvoovvvgCBQUFdO5NSUnBE088YdbYZpPLVk+ym2QN2BRclUqFlJQUpKamGowLNlo22TU1NfSaYvJuyfXbXEet9Ho9deYlJydTucTZeP36dezfvx+PPfYYFi9ejMcffxyhoaEGPEpKSvDDDz/Q/kvkdnZ2YvPmzYiIiEC/fv3wz3/+ExqNhm4EzZgxw4BPW1ubQeL2c+fO4dChQ5g9ezZeeOEFPPfcc3SHr76+Htu3b8euXbtM2lyn02Hr1q0ICQnB0KFDER8fT68sVavVuHnzJl5++WWTm8/Y+oC3tzd2795N6auqqrBx40YcPnzYQHFmc6IY91FyFOeNN96gCeHZ6K1xcnIZXVy0x48fR2hoqEn+H0u0xAmxfft2GgIuFovR0dFh1mHL5L1//356ZbVUKoVOp0NFRQUuXryIU6dOobCw0KSvE2Nr27Zt1PAnFxncuHEDKSkpuHz5Mt29ZUaeMVFXV4fDhw9DKpXSyIW0tDScOXMGpaWl1JgmfU6n+98bCpnrQ0tLC9atW4eTJ0/i6tWr1AHMvGRAr//fo8PfffcdGhsbMWfOHDz88MMGTpuWlha6EcBcA4hRlpmZiRdffBFPPvkkBg0ahGHDhtEIM71eb2LEMN9fdXU1PvzwQ5SWlmLmzJlYtGgRdZaRdb2zsxN5eXkG+qder6fO28WLF2P06NH0dh4yxxP9bf/+/TQfj07XndyZGQEkkUiwZMkSA7nkmPHt27fx66+/4siRI+js7IS9vT0cHByg1WrR3t5uYsgGBwfjgw8+oE6t7du349dff6XJeRUKBRQKBdrb26FSqaBSqXDq1CnMnTvXwBGoUqmwevVqnDp1Cl1dXXBycoKjoyPUajVaW1uhVqtRVVWFzz77DMePH0ffvn1RXFxML2cw7psKhQJyuZwe2+ro6MChQ4dw7do13HfffZDL5TQaiEQ4sYH0ucLCQrz77rs4cOAAoqKicPv2bVy4cAEdHR1WHanIzc3FCy+8AL1ej5SUFBQWFqKzs9Ni7lC2dZQco7Rko7H9xra2t7e3Q6FQGFwG8f333+O5554zSBZ9/vx5nD9/HnK5HPb29gY6lVarhb29PdLT0/HSSy9Bo9HQtQ7o3tAnSaEB4B//+AdOnz6N1NRUk1tCu7q6DJLkLlu2DMuWLYObmxs8PT3h6OiILVu24MEHH8T333+P0NBQPP/88za1A/M35rxKdBGS89IcjDeHyK2MZN4xJ9tYLjmK29HRQf+21n4mGzBEZ7VGdmdnp8FGpl6vp0dzzfUbrVaLAwcO0A2b6upqizkTmd+1t7djwYIF8PLyQnZ2Njo6OugtheSqbi5earUaP//8M86cOYPy8nKqM4nFYmRnZ9MIKbYxpdFo8Pjjj8PBwQH5+fnUcXjt2jX07duXprawNZevwBYnh0Ag0Lu6umLKlCkoLS3FgAED8NhjjyEgIABSqRTNzc1YvHgxjh07ZlKQfy1gAgBQKBT6nTt3IjY2lh5J+uyzz/DFF18YeN64oNfrL+v1+kEAEBcXp9+xYweeeuoppKSk2HQ27F9GCuUVFBSkZ7veu7m5mUbStLS0wNHRkd4ycufOHQDdu7RFRUUC0k7WKvxAt+NkyJAhuHr1KnGc0DJZ4kWuy/b390daWpqJ04TZVgKBQG+cN8DV1RVDhw7F2bNnafgvV+cn708gEOglEgnCwsKgUCiQmZlpYOSx1Zd4dV1dXVFeXk7LJBQKaZm43jtJFmlvbw8/Pz/o9d1n8f91Ba2AIcekrZhONg8PD3h5eaGqqgpVVVUGToV/LZQG74/IdXBwgL+/Pzo7O1FcXEwnVzIJEv7kvba3t9P6iUQifUBAANzc3FBRUUGvDydyyTWnJM8T08nyL8cZrZBQKKTlIkolaVdGHSh/pnPD+P1JpVJOucwdION3YjxmRCKRXiaTUblMPsYGCHPRMC6TpX7A5ihgwrifG0ceMJ5jdQYwnzcef5YWJPK7sZHwrzbhnBOM5RrPXUbvwKR+TLlMWjZjxajtDXiRK2+5ZLO9O8ZztH5s7WSsKLD1J676cRmhzO8s1c/X11f/9NNPo6ioCLt27YJAIMC4cePg6elJ54Lm5mbk5eWZHJ9av349qquraf2MZVsyio3az6B+xm3AVRdreHHJN2fsMxx0BvUzJ5f8LhAIoFQqMWTIEHpL38qVKy2WiXxHEnqOGTOG3oxCdrRSU1OhVqtZ53Qim5wTj4mJoQn9cnJykJGRgdLSUnr80XjOk8vlCA4OxujRoxEYGAiBQIDy8nJcuXIF2dnZ9DIEDiXOoE+99NJLaGxspM5uku/DGoOIOafb29vrmQnjbdHHjOtnjWOHgLSlRCKBRCJBS0uL1ToH+Y0kqSbrkFarJZtGFtdkpmzyHTE+mNEmbPMLuWGDyGXSca1dzPcnFAr1JFcTWSeZtGz0RDazTGTtMyeXOceLxWLY29vTW//27Nljdp1h0gKgCSwjIiLotd0ajQY5OTnYvHmzyftj0pPjf0FBQfRmTqA76uHy5csoKChAQ0MD2XChbS4Wi2FnZwdPT096+5WzszN1dF65coVGhpBIIXLNdV1dHS2TUqnUz5o1CzExMdDru2/KTE9PR1lZGc3FQdpeoVBgyZIlePTRR/HPf/4Tu3btgkqlorz8/Pz0CQkJ9IbJ5uZmaDQaKtvHxwc//PADWlpa8OabbyI3N5e257/6Fn1/TH3Y3t4eb7zxBqZMmYJ//OMfOHLkiIGz2thxx9SDhEKhvm/fvvj888+Rk5NDnWSk/UlfZTo9jNZoysvBwUHPvOGJ6JhMR43x+s5cL5ljhjgimHMLk9Z4nTWaf1htBy69zsLcRXm5u7vrPTw88OCDD+LMmTNwcnJCfHw8vv76a5PbA4HuiNiHHnoIAQEBBmuyWCzWazQampfIGCKRCNOmTcPu3btRU1ODZcuWYdGiRbj//vvJleCcazLQHQl07NgxBAcH4+zZs3jmmWeQlZWFL7/8EuXl5Zg3bx6io6MxYsQIXLp0yWDOY7YPaXO2v4Fu58miRYtoJL1Go2HVOdj0DXJzGnFGEfuPS+dg0srlcjz//PNobW3Fli1bDCLYjfuE3sjOEolEiImJweTJk3HkyBFcv37d4Cp6Nt3DWOdwd3fXE8eqsZ5pTE+O/ZPfhEIhurq6WHVGNrlMXZkgLi4Ojz76KF5//XU0NTVxrlnM27LJZxJA4O7ujp07d9I8gcZrMjM6k/w/fPhwzJs3D1u3bqVRnWxgvj8mbI60UavVyM3NRV5eHq5du4b09HSMGTMGgYGBKC4uxvnz5y0qLp2dnXjhhReQkJCAuLg46PV6/Pnnn8QI56oAq0JZV1eHjRs3Wsyjw9UZmTD2nhPcuXMH/fr1A9C9Mz1w4ECauK22tpYmE7Ukl0sRamxsNDl2wsXH+LNWq0V+fr5BIl3SebgUcCbq6+tpVnVboNFokJ2dbVImNrl6/f/ebsSW0NiSokocAk1NTTQaylrllqmUlZeX07BIY3o2fkQu2dFhPsfs4yRKgotHaWkpZySYLecZjWUTWi7Z5voc13lqc7Rc9TM+wsNGa8mZSvoI12/m6MwZGMyjUGz82BRta+UT2WwOD2sMKLa+zzbOucAll61+5srDdRSE2TbM77n6ja1yjWnYvmeTa+14YSIoKIjmY6qoqEBSUhKmTJmCW7duQS6XIy4uDgDoDr5SqTRRIBkKDGeZe1I2Ji3zM1OutfTWtqel57jkkvq3t7fj9OnTOHPmjEm+Ny655G+NRoOKigps27aN0jINZ3O89Pr/jQI6evQoNaSYUWlcdevq6kJBQQHy8/OpXKYhZO3uXWVlJVatWmWVXEtzAVNp5ZLNtu6bAxcd8ztSXhLhZK7cXH2HJA9mfmdNPYxlG9NYml+IgW4sjzk+mQo+Gx+2Y/jMudya9YtEeJmTy+wXarUaTU1NaG1tNclXR55l06dIeTo6OlBRUYHKykqcPXvWbB2N6bu6utDY2EiP9+/evZvScm20kDq2t7fTm0lJglKmw8uYVqPRmCTPb29vx86dOw3KzJTLbF+SkLqxsRFxcXFISkoy0DEqKiqwd+9eVkeZRqNBc3Mzrl+/jtjYWERGRqKgoABqtdpgk47IZH7u6OjAzZs3MXbsWAwbNgwnTpywKjKFyK+vr0dOTg58fX0REBBAb6gk7WT8PIHxO1QqlTSKj9AZR+WYo2fC+GigreuKMa21cs2hvr4ey5YtQ3l5Oe7cuUPXXgAmm+EAMGLECAQEBADgttFCQ0PRr18/nDt3jkbNOzg4YPfu3QC6N9UvXLiAqKgoGqnEhLOzM+Li4nD8+HH63fr16xEcHAy9Xo+1a9eiubkZe/fuxfXr15Geno7s7Gx89dVXnPU0ni+59KOuri78/PPPrP3E+HnyN+FdXFyMTz75hPYVS+/EeD5Zv3497WtsugEXL52uO29Wbm4uzWdjSfc05mVsM5tbP403JS3Z+sxnme3K7POXL19GZmYm5y2tXPq9TqdDS0sLduzYAYFAYBC9bwy2aFFyNTvZXGKW2ZoxabPTpq2tDVeuXDHIA3Dp0iWQ3VprrqzS6XQoKyvDb7/9RhcPMqlygasyJSUl+Oabb8weB2LS2zpR+fv707OSAGi+koqKClRVVaG6uhoODg6c12ayye2JgmfOkGNOCpbC69jombLZvMFsEwGzQzPpLMk0J9u44xorPz3FX+FhDY2lZ6xxWPQUlmjN9Slb5FpjLFiCJeclmxxj44HLqdATucb0tjixmLS2Ki/G8iy1i7nxZ26yt2bxtfQb1xi0xkFgDX8uXlwOEkuOG+PvWltbkZqaCgD0Fon+/ftDIpGgqqoKs2bNorfmjR07FocOHYKHhwfGjx8Pe3t7q+th6xzD1q5sfYBZX0vjmK2tuPhaWz4uucY3z3GVi6tfkvVUrVbDXIQdG5hrPdMItGYutEauNXxI4l8uR4/x+7U053HR2sKPa47ges4a5ZeL3to274lsS3OwOf3J+G+uOlhy0Nny7rjeFdt6w/zHxY9NLqGxJZLcmC9xsHDNqebqxyWXi5bNSWHtTYR6fXcC0OrqagwdOhT9+vUzSOpPeHHRkwsI7rvvPgwbNgzp6elW5XcjhmhTUxNGjRpFL1UxLhuXXHIbUlxcHAYPHoybN2+aHNO0Zrw4OjrC09PT5NZBLtnW6CDWrNXWzHu2yuVCe3s7GhsbMWXKFABAVlYW2tvb0bdvXwCGTpuCggJoNBoMHDjQIO+op6cn6urq8Mcff8DX1xcjRozAzZs3qdNGpVJh3bp1CAkJwaxZs3DkyBGsWbMGixYtwrPPPot169ZRXkqlEgMHDjRw2uzatQs3btzA8uXL8csvv8DFxQUrVqzA7NmzMWvWLDz66KNITk7m7FuW9Erm92yJbLlojdcCcmTMlnWeKZeAbYybG6PGznNz5be0jrB9b+3zbODiy4RxOgNzYJaJzInmbvoyR8tMut0TWLbwWQpgHI5KrlBkXiVoCaTgJAmrNfls2H4nO/1WhiVbVTYmyEJnPHkD3eF35CaNnsjtiQHKRstGb8mRwiWbSW/8uzEs0VkLQm9cZmsWmp7AloXKnFxbaXsq1xKvvwN3w2Fzt+hsqb8t9NZO3sZ9nW1RtURrS3nNlYX5tzmnjyW+5p6xZgHkovsrz5gzONnmCza0tLSgq6sL165dQ0lJCUpKSpCeno7+/ftjwoQJEAgE9IgG2U2+cuUK2traOJPL2zK39qTf/5Wx9nfR/tW5jEnDZkjaItsW+r8i15iPLQb0X52zmbS2vifj+aqn7/mvoCey/1vREz3270JP1z1b+FnL09wz5eXluHjxIkQiEb3S2Vp6rVaLa9eu4c6dOxgwYAAGDx5sctsrF/Lz83H+/HkolUrMmzcPHh4eVvdZlUqFK1euoKysDCNHjkTv3r1NbtGyBiKRCDNnzoSrq6tNRvh/Yt3oqVyBQICGhga0tLQgKCgIV65cwZAhQ3D27FkazQV05y8KDw/H1atX0dbWZpA71NfXF1VVVbhw4QJ8fX2xcuVK5OTkUP6fffYZWltbsWnTJpw4cQInT57EsmXL8Omnn+KTTz4xKE9paamBE2fevHkYNGgQ1q1bh/b2drz00kvw8fHBN998g08++QRjx46lxx1tSUhsjc5ijtba57mcRNby6QmMdTPj72zVGdloLTkmueTaOu9ZI7unc5w1cs3BZqeNMWOuRrHmJVnbmLYoPFxlsWaw6PV6k9s5kpOT0dzczHp9tru7O702ValUsvLjksscQMb/LIGLn7En0JKziEljTG8NjMts7n1aKo+53y3Rm4O1g91WudZMCkzYuktuDayRbcmB0NMxamsZjctjzVhkc4YYjxGuMWOtXOMxwAWu8ck1hizhbjjBrHUO3E2D0ZIsc3S2Pscll8uxy8WntrYWM2bMgEwmg1wuR1VVFc2hsH37duTl5aGsrAzp6ekYMmQIvUmO68irJbnGRrI1dTP+zdp+ycWXq2/8Fbm2KpDm5Nra/22R3ZP5kEn7VwwYIsPW+pIxyrWT2FOF25Lj05a1nq28ts4r1sruqS5grYL/V8tnq1xr+0NPDAxrymaNbGtorQXXrr21tO3t7dizZw9SU1PRu3dvm2RrtVoUFRVh+/bt6OzsxMiRI2kyWnPQ6/VobW3Ftm3bkJ6ejoEDB3Je8cxWF3JDzp49eyCTyTBmzBhWu8DcvAp0O3/i4uIMbiG1JJuNj/H3luRawl+xTYzlHjlyBJ6engZXp/fp0wcLFiyAVCrFwoULMWbMGHh5edFk2UB3hI4xAgICsG7dOkyfPh0ffvghjh49Cnt7eyxYsABHjhwBAGRmZiIrKwtTpkyBs7OzCY+wsDAEBgbi6NGjWLp0KUaMGIHCwkJ6HGvz5s1wcXHBpEmTAAAJCQnw9/fH3Llz4eHhYXM72Gp3Gc//bHaXLe+HzWYzt1Zw8eGS3dN546/M3T2huxvl7Ok6yNX2lnjZ7ga2wJgoHT1RLrjorOFp6RlryuTo6Ijo6GiD78LDw3HhwgUkJCSw0qSnpyM6Opru1toi968qiJYcQuZomc+xGaJsz1mSYdzxmPxtcWpY867NwZJs5ne2OKqYz/bkvbHJtVUZt0X2X2lDrgm8Jwu38Tsw1x/MyeUy3CzNRcafLRmTlpRmWyZYY97m5HLJY4MlI5bL8DNnEDJpLb1Da8rF5GX8Dq0dd+bKbM337u7uqK6uho+PD4qKigAA06ZNg1KpxHfffYeGhgbMnj0bKSkpmDZtGtLT0+kNOMZHfW2ZK5jP9WSusNVBYotsS2uStXJtrR9Tri20XOORi9YaZ5klWuO+asv6YPydJYeMMa3xPGXtmGErMxsfW2DLXGftPGyOtqd93fi9WbveWFKabVlrzcm1Zn7nkmuLHvVXZHP1X0tyrTXabJFLbvX66quvcP/993PKJWVkQvevW0TPnDmDjo4O6HQ6sw54Jj+tVouCggKsXbsWiYmJuHLlitX9geS6SEpKQmNjI6qqqkzyZbAZzcaoqKjA7t27kZmZabZP2NIX2P7uiQ5irtzW1A3oXpNv3LiBvLw86pAjazK5ahsAMjIyqJOEHBUzdpCEh4fjhx9+gK+vL77//nvY29tj5syZSElJgb29Pd555x189dVXePTRRw025I0dYs7Ozvjjjz9QWFgIhUKBhQsXoq2tDb/99hsEAgEuXbqEhQsXAgC2bNkCoVCIEydO4OzZswaOJzYYvytLawEXLOk7lvjaOoZtKYs52eb6Q0/obO33xm1vTbks2QHm7BE2WLMuWNsneuy0saZwPaEzfonWLubmOqo5BY1J19bWhunTp9NzjdeuXYO3tzcSEhJMwhzDwsKg13cfFWOG89kq9686AMwp31yLMPmd65m7oUD1ZMBZUqAs0ROwyWYbEJYmE67P1ixc5nhx0dviELGG1pq2tpW2J4Z7T8vF9n1P55e/Ui5rZTP7L1t/68miaYtRayzXeBzZMkcaf8e18HGVg00uV3tYq2hYan82PnV1dQCADz/8EFqtFomJiXBycsLGjRtRVVWFF154AevXr0dQUBDy8/ORnp4OrVaL77//3mrDoycKGBdsMeZ6wrOnihlzTmf+Zm1ft9S3zbWfOWNCIOBOHmsMrv7ZE93CWrl/ZS21tny2yv0rTqiePMMlm2vdscZRYG6+YjOSLIFLrqX2t1ZuTww0Y13NWn3DWtnGugTX3GNuLNtSL+P3bolWo9GguLgYmzdv5uRlTlZzczNOnz5N9XRroVarUVhYiM2bNxvkrbBmLBMH0eHDh00Sqxu/UwLjfqNSqbBjxw6DTQOutrPVIWc8b7M9aw6W2sAaPVkikWDlypUAQBPCkzW5rq4OMpkML774Ih544AFcv34dx44dQ1dXF77//nu4uroa8JJKpcjKyqKb7bt378auXbugUCjg6uoKjUaDzs5OfPjhh/j+++8xZ84chIaG4syZM/TmYqDbQTRnzhykpqYiPj4eb7/9NjZu3IjW1lYEBQWhvr4eW7ZswUMPPYRt27YBALZu3YodO3aA7cZhtjnE2DayBoTO+B0yv2eTy2XvmJsLrNHHzIGrf7N9trbMXLD2WTbePV2LjWl7svZx0dqio9l65XcNgCKrCQwRpNfrPe4Cn3uV171YprvJ654u093kxdfvb+N1L5bpbvK6F8t0N3ndi2W6m7zuxTLdTV73dJnuJi++fn8br/9v6ncPlelu8roXy3Q3ed2LZbqbvO7FMt1NXvd0me4mL75+fxuve75+TNjktOHBgwcPHjx48ODBgwcPHjx48ODx96BHiYh58ODBgwcPHjx48ODBgwcPHjx4/HvBO2148ODBgwcPHjx48ODBgwcPHjzuQfBOGx48ePDgwYMHDx48ePDgwYMHj3sQvNOGBw8ePHjw4MGDBw8ePHjw4MHjHgTvtOHBgwcPHjx48ODBgwcPHjx48LgHwTttePDgwYMHDx48ePDgwYMHDx487kHwThsePHjw4MGDBw8ePHjw4MGDB497ELzThgcPHjx48ODBgwcPHjx48ODB4x4E77ThwYMHDx48ePDgwYMHDx48ePC4B/H/ABvxjjSjIiKOAAAAAElFTkSuQmCC\n", "text/plain": [ - "torch.Size([1, 97])" + "<Figure size 1440x1440 with 60 Axes>" ] }, - "execution_count": 58, "metadata": {}, - "output_type": "execute_result" + "output_type": "display_data" } ], "source": [ - "targets.shape" + "# remove batch size\n", + "n = 60\n", + "patches = patches.squeeze(0)\n", + "fig = plt.figure(figsize=(20, 20))\n", + "print(sentence)\n", + "for i in range(n):\n", + " ax = fig.add_subplot(1, n, i + 1)\n", + " ax.imshow(patches[i].squeeze(0), cmap='gray')\n", + " ax.set_xticks([])\n", + " ax.set_yticks([])" ] }, { "cell_type": "code", - "execution_count": 59, + "execution_count": 44, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "torch.Size([157, 1, 80])" + "torch.Size([234, 1, 28, 18])" ] }, - "execution_count": 59, + "execution_count": 44, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "output.shape" + "patches.shape" ] }, { "cell_type": "code", - "execution_count": 60, + "execution_count": 13, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "tensor(6.9447, grad_fn=<MeanBackward0>)" + "24.0" ] }, - "execution_count": 60, + "execution_count": 13, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "ctc(output, targets, input_lengths, target_lengths)" - ] - }, - { - "cell_type": "code", - "execution_count": 64, - "metadata": {}, - "outputs": [], - "source": [ - "from einops.layers.torch import Rearrange\n", - "slide = nn.Sequential(nn.Unfold(kernel_size=(28, 14), stride=(1, 5)), Rearrange(\"b (c h w) t -> b t c h w\", h=28, w=14, c=1))" - ] - }, - { - "cell_type": "code", - "execution_count": 65, - "metadata": {}, - "outputs": [], - "source": [ - "patches = slide(data.unsqueeze(0))" - ] - }, - { - "cell_type": "code", - "execution_count": 68, - "metadata": {}, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAABH4AAAFeCAYAAADzIdiIAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/d3fzzAAAACXBIWXMAAAsTAAALEwEAmpwYAAAqdUlEQVR4nO3dfbCcZXk/8Os+LyGBRt5fEyC2w2jRIiittuIM0p8KDlOs2lY7LdTBgkUHUSul1lbLH0idouM4TGfSqQY7BZUKCkhRxmKVzg+qoZTwUopYaHkXeZEEkpw9e//+8KS/kCbkvnN2z+65n89nhsnJnotr7332e559zpVnn0055wAAAACgPROjXgAAAAAAw2HwAwAAANAogx8AAACARhn8AAAAADTK4AcAAACgUQY/AAAAAI2aWsg7Syn57HhGJuecRnXfss+IPZ5z3n9Udy7/jJJ9P10l+3SY4x46a0f7/gUd/ACMm5TKj4tzXrSv4/ePegGLUUeyAQCtcdwD25jXW71SSiemlO5OKf0gpXTeoBYFi4H801WyT1fJPl0m/3SV7NOCXR78pJQmI+LiiDgpIo6MiHemlI4c1MJgnMk/XSX7dJXs02XyT1fJPq2Yzxk/vxQRP8g5/zDnvDkivhgRpwxmWTD25J+ukn26SvbpMvmnq2SfJsxn8LMiIv57q78/MHcbdIH801WyT1fJPl0m/3SV7NOEoV/cOaV0RkScMez7gXEj+3SZ/NNVsk9XyT5dJv+Mu/kMfh6MiEO3+vvKudueJ+e8OiJWR/hoO5qy0/zLPo2y76erZJ8uc9xDV9n304T5vNXrexFxRErpxSmlJRHxjoi4ajDLgrEn/3SV7NNVsk+XyT9dJfs0YZfP+Mk591JK74uIb0TEZER8Lud8x8BWBmNM/ukq2aerZJ8uk3+6SvZpRcp54c5Ec9obo5RzTqO6b9kfXymVx2Ih95cDtjbnfOyo7nyx5r8j2WiefT9dJft0mOMeOmtH+/6hX9wZYJz5hZ0dkQ0AGA+Tk5PFtbOzs0NcCeOkJhcR3c7GfK7xAwAAAMAYM/gBAAAAaJTBDwAAAECjDH4AAAAAGmXwAwAAANAogx8AAACARhn8AAAAADTK4AcAAACgUQY/AAAAAI0y+AEAAABolMEPAAAAQKOmRr0AAADoopRSVX3OeUgrYZzU5KIrmZidnR31EsaCbDyfXJRzxg8AAABAowx+AAAAABpl8AMAAADQKIMfAAAAgEYZ/AAAAAA0yuAHAAAAoFEGPwAAAACNMvgBAAAAaJTBDwAAAECjDH4AAAAAGjU16gUAAEAX5ZxHvQTGkFywI7LBrnLGDwAAAECjDH4AAAAAGmXwAwAAANAogx8AAACARhn8AAAAADTK4AcAAACgUQY/AAAAAI0y+AEAAABolMEPAAAAQKMMfgAAAAAaZfADAAAA0KipUS8AABh/S5Ysqarv9XrFtf1+v3Y5jAm5mJ+Jibp/g10s2ySlFNPT00W1NZmIWDzbYD5qctGF7cH/JxvdVPq8v9Bz7owfAAAAgEYZ/AAAAAA0yuAHAAAAoFEGPwAAAACNMvgBAAAAaJTBDwAAAECjDH4AAAAAGmXwAwAAANAogx8AAACARhn8AAAAADRqatQLAABGI6VUXNvr9ap69/v92uUwJuRi4bS6PXLOxdlodRvMh23CjshGNw3ieXfGDwAAAECj5nXGT0rpvoh4JiJmI6KXcz52EIuCxUD+6SrZp6tkny6Tf7pK9mnBIN7q9fqc8+MD6AOLkfzTVbJPV8k+XSb/dJXss6h5qxcAAABAo+Y7+MkR8c2U0tqU0hmDWBAsIvJPV8k+XSX7dJn801Wyz6I337d6HZdzfjCldEBEXJ9S+vec83e2Lpj74fADQoteMP+yT8Ps++kq2afLHPfQVfb9LHop5zyYRil9PCLW55z/8gVqBnNnsAtyzuWfT1tpZ/mXfUZs7bAuRGjfv7jVfGx3TW3E+Hzk7LD2/S1nvwu56IJRH/dMTJS9sUAmGALHPXTWjvb9u/xWr5TSHiml5Vu+jog3RsTtu9oPFhP5p6tkn66SfbpM/ukq2acV83mr14ERceXcv/RMRcSlOefrBrIqGH/yT1fJPl0l+3SZ/NNVsk8TBvZWr6I7c9obIzTMU553RvYZsaGd8lxC/sdXF97SY99frwu56IJRZ99bvRghxz101o72/fO9uDMAsEjV/NI+Tr+cla57If9xqyWLMRe1AyjZGL5xyIZcAC0YxHHPfD/OHQAAAIAxZfADAAAA0CiDHwAAAIBGGfwAAAAANMrgBwAAAKBRBj8AAAAAjTL4AQAAAGiUwQ8AAABAowx+AAAAABpl8AMAAADQKIMfAAAAgEZNjXoBAMBo9Pv9US9hl+ScR72Epi3GXMgE2yMXQAsGsS9zxg8AAABAowx+AAAAABpl8AMAAADQKIMfAAAAgEYZ/AAAAAA0yuAHAAAAoFEGPwAAAACNMvgBAAAAaJTBDwAAAECjDH4AAAAAGjU16gUAAMBCmpgo/7fPfr8/xJUwTuQC2pZSKq7NOQ+td82+JiJixYoVRXWPPPLIju+z6h4BAAAAWDQMfgAAAAAaZfADAAAA0CiDHwAAAIBGGfwAAAAANMrgBwAAAKBRBj8AAAAAjTL4AQAAAGiUwQ8AAABAowx+AAAAABpl8AMAAADQqKlRLwAAABZSv98f9RIiImL58uVV9aeffnpR3aWXXrory+m8xZiL0kxscc011xTX3nvvvVW9c85V9dSTjfmpWUdKqar39PR0ce0JJ5xQ1fv8888vqjv11FN3+D1n/AAAAAA0yuAHAAAAoFEGPwAAAACNMvgBAAAAaJTBDwAAAECjDH4AAAAAGmXwAwAAANAogx8AAACARhn8AAAAADTK4AcAAACgUVOjXgAAALQipVRc++53v7uq9/nnn19U90//9E9VfRm+YeWiNBNb9Pv94trPfvazVb2HqWb75ZyHuJLBqnlcEbKxrdrtN8xsHHTQQcW1f/zHf1zVe9myZUV1ExM7Pq/HGT8AAAAAjdrp4Cel9LmU0mMppdu3um2flNL1KaV75v7ce7jLhNGQf7pK9ukq2afL5J+ukn1aV3LGz5qIOHGb286LiG/lnI+IiG/N/R1atCbkn25aE7JPN60J2ae71oT8001rQvZp2E4HPznn70TEE9vcfEpEXDL39SUR8ZbBLgvGg/zTVbJPV8k+XSb/dJXs07pdvcbPgTnnh+e+fiQiDhzQemAxkH+6SvbpKtmny+SfrpJ9mjHvT/XKOeeU0g4vj51SOiMizpjv/cA4eqH8yz4ts++nq2SfLnPcQ1fZ97PY7eoZP4+mlA6OiJj787EdFeacV+ecj805H7uL9wXjpij/sk+D7PvpKtmnyxz30FX2/TRjVwc/V0XEaXNfnxYRXxvMcmBRkH+6SvbpKtmny+SfrpJ9mlHyce6XRcT/jYiXpJQeSCmdHhEXRsQbUkr3RMT/mfs7NEf+6SrZp6tkny6Tf7pK9mndTq/xk3N+5w6+9asDXguMHfmnq2SfrpJ9ukz+6SrZp3Xzvrgz7MzExK6+o3DnpqbKIjwzMzO0NbBrhpWL0kxsMTs7W1zb7/ereue8w2sAsgPjsL/YQjYYhJRSUZ1MdNMPf/jDqvrS/ZI8LW41uah5rYqIePnLX15cOz09XdV78+bNVfXUk435qTnOrD0mffWrX11cu+eee1b1vvrqq4vqnn766R1+b3hH2AAAAACMlMEPAAAAQKMMfgAAAAAaZfADAAAA0CiDHwAAAIBGGfwAAAAANMrgBwAAAKBRBj8AAAAAjTL4AQAAAGiUwQ8AAABAowx+AAAAABo1NeoFUG5iYnhzuunp6eLagw46qKr361//+uLaX/7lX67qfcghhxTVfeADH6jqu5jIxfOVZmKLxx9/vLj2xhtvrOp95ZVXFtc+8cQTVb0Xk+np6eJ8HHfccVW9X/3qVxfXHnzwwVW9a7Jx0003VfW+5pprimuffPLJqt6Mj3322SdOOumkotprr722qrdctGHt2rVV9aWvFb1eb1eWw5ioyUXt8cPP//zPF9cuXbq0qvfMzExVPfVkY35qfm+anJys6r333nsX1+acq3oPYt/vjB8AAACARhn8AAAAADTK4AcAAACgUQY/AAAAAI0y+AEAAABolMEPAAAAQKMMfgAAAAAaZfADAAAA0CiDHwAAAIBGGfwAAAAANGpq1AsYhZTSqJewS6anp4trjznmmKre5513XnHt6173uqreS5YsKa595plnqnrnnIvqdt9996q+wzCs3MnF85VmYouNGzcW177qVa+q6v2Lv/iLxbXnnntuVe9hbpNBe8lLXhJXXXVVUW1NniMiNmzYUFzb7/erem/atKm49uijj67qXfOz+LGPfay4dv369VXrGHU2Wrf//vvHmWeeWVRbu3/+8z//8+JauZifYR43PvLII1X1f/Inf1JU99BDD+3KcqgwLrkozcQWd911V3FtzWvssNVs73HYhw0rH7LxfLXbuSYbtb1nZ2eH1vuQQw4pqnuh42hn/AAAAAA0yuAHAAAAoFEGPwAAAACNMvgBAAAAaJTBDwAAAECjDH4AAAAAGmXwAwAAANAogx8AAACARhn8AAAAADTK4AcAAACgUVOjXsAo5Jyr6icmyudjKaWq3itWrCiu/fCHP1xce+qpp1at48c//nFx7dVXX13V+4Ybbiiuvemmm6p677XXXkV19957b1XfYSjNUU0mIuRiW6WZ2GLlypXFtb/2a79W1fuoo44qrv3t3/7tqt6rV68urq3d5w3aT37yk7juuuuKar/73e9W9f7e975XXFubjUMOOaS49qSTTqrq/fKXv7y49u1vf3tx7SWXXFK1jlFno3XPPfdcrFu3rqj2ZS97WVXvt73tbcW1X/jCF6p6y8X81BwLzs7OVvW+/PLLi+p6vV5VX4ZvWLkozcQW/X6/uLZ2X1D7e1CNVvdLtdtMNua3jqmp8vHH5ORkVe977rmnuHZmZqaq90tf+tKiuqVLl+7we874AQAAAGiUwQ8AAABAowx+AAAAABpl8AMAAADQKIMfAAAAgEYZ/AAAAAA0yuAHAAAAoFEGPwAAAACNMvgBAAAAaJTBDwAAAECjDH4AAAAAGpVyzgt3ZykN7c5SSsW1ExN1867Jycni2te85jVVvT/72c8W1y5fvry49gtf+ELVOtasWVNc++ijj1b1rnluhmXjxo0xOzs7soUsX748v/KVryyqrcnEXO/iWrmYnwMOOKCq/hOf+ERx7ebNm6t6n3HGGcW1mzZtWptzPrbqDgZoYmIi77bbbkW1izEXEfXZ+NM//dPi2pmZmeLac845p2odtbkbFzXHLjnnkYVqt912ywcddFBR7Uc/+tGq3jW5+OAHP1jVezHmovZ4dpj7mprjxtp1l9b3+/2RZj+llMdhf96FXAwrQ7timNuv5ve3Xq830uOemvzX5CJCNrZV+3t9TX1t7wMPPLC49tJLL63qXbqWd73rXXHXXXdtd2M74wcAAACgUTsd/KSUPpdSeiyldPtWt308pfRgSunWuf/ePNxlwmjIP10l+3SV7NNl8k9XyT6tKznjZ01EnLid2z+dcz567r9rB7ssGBtrQv7ppjUh+3TTmpB9umtNyD/dtCZkn4btdPCTc/5ORDyxAGuBsSP/dJXs01WyT5fJP10l+7RuPtf4eV9K6ba50+L23lFRSumMlNL3U0rfn8d9wbjZaf63zn7NRThhzFXv+xfyQwRgiKqzPzs7u5Drg2GqOu5Z6MXBEPmdlybs6uDnryLi5yLi6Ih4OCIu2lFhznl1zvnYUV5ZHQasKP9bZ396enoBlwdDs0v7/nH4ZBeYp13Kfu2ntcCYqj7uWcC1wTD5nZdm7NLgJ+f8aM55Nufcj4i/johfGuyyYHzJP10l+3SV7NNl8k9XyT4t2aXBT0rp4K3++usRcfuOaqE18k9XyT5dJft0mfzTVbJPS6Z2VpBSuiwijo+I/VJKD0TExyLi+JTS0RGRI+K+iDhzeEuE0ZF/ukr26SrZp8vkn66SfVq308FPzvmd27n5b4awFhg78k9XyT5dJft0mfzTVbJP63Y6+BmVYV4MdGqq7mH/zu/8TnHtRz7ykared955Z3HtRz/60eLae+65p2odw/zUqXF6Lkdl1apV8fnPf76otiYTEXKxrdpM1Hzqzo9+9KOq3pdddllx7e67717Vu9/vV9WPUs45er1eUe3ExHw+bPKF1V5ot2YbP/7441W9v/KVrxTX1mSj9lOkan4Oh/npbMPaH4z6E+V6vV78+Mc/Lqq94oorqnovW7asuFYuFrZ3zb7Dxe/rycX8jEvmao/XFtNxT0T5dq59XLLxfMPMRe3r2zPPPFNc++CDD1b1PuaYY4rqXmjbDe8IGwAAAICRMvgBAAAAaJTBDwAAAECjDH4AAAAAGmXwAwAAANAogx8AAACARhn8AAAAADTK4AcAAACgUQY/AAAAAI0y+AEAAABolMEPAAAAQKOmRr2AHUkpVdVPTk4W15588slVvf/sz/6suPbyyy+v6v3JT36yuHb9+vXFtTnnqnVMTIzHDLB23TMzM0PpO2hPPfVUXHnllUW1NZmIkIttlWZii5rH2O/3q3rffPPNxbXLli2r6j07O1tVP0oppap9dI1hZqNmzbXPx9q1a4tra7JRm9Eatc9hzc9W7Wt+ae+NGzdW9R20nHNxNm655Zaq3kuXLi2u7UIual+vhrlNavZLw1zHqJVmo/a5k4v5qfmZrV1HTX3t6+b09HRxba/Xq+o9aCml4vXWHpvLxvPV5CJiuPuPmt7/+Z//WdX7da97XVHdC615PH6rAwAAAGDgDH4AAAAAGmXwAwAAANAogx8AAACARhn8AAAAADTK4AcAAACgUQY/AAAAAI0y+AEAAABolMEPAAAAQKMMfgAAAAAaNbXQd5hSGkrfgw46qLj2wx/+cFXv2267rbj2E5/4RFXv5557rrh2dna2qneN/fbbr7j2LW95S1XvXq9XXHv11VdX9X7qqaeK6oa57Uo8/PDDxdmoyUSEXGyrNBMLYWZmZii1ERHT09PFtZs2barqPWj77rtvcT5qchERcd111xXXPv3001W9h2lY2ajJRa3DDz+8qv5Nb3pTce0RRxxR1XvFihVFdeeee25V32EoPe6p3QeMSy4OPfTQqvph5aI0E1ts3LixuHZqqu5w+T/+4z+Ka//hH/6hqve6deuK6mqPJQZtt912K85GTSYi5GJbpZnYombfkXOu6j0xUX5OQU1tRES/36+qH6VVq1bFBRdcUFRbk4sI2dhWbS4mJyeH1rum/s4776zqPYgZijN+AAAAABpl8AMAAADQKIMfAAAAgEYZ/AAAAAA0yuAHAAAAoFEGPwAAAACNMvgBAAAAaJTBDwAAAECjDH4AAAAAGmXwAwAAANCoqVEvYEdSSlX1J598cnHt8uXLq3p/7GMfK6599tlnq3r3er3i2omJ8jndihUrqtbx5S9/ubj2qKOOquo9OTlZXHv22WdX9T7rrLOK6tauXVvVd9D6/X5xNmoyESEX2yrNxBa33nprce3UVN0u85xzzimuXbVqVVXvP/qjPyqufeihh6p6D9qKFSviggsuKKqtyUVExHve857i2g996ENVvdetW1dcW5uNP/iDPyiuPfTQQ4trP/7xj1et47DDDiuuveiii6p6H3DAAcW1y5Ytq+pduk1KczcsExMTxY/tzDPPrOpdk4vzzz+/qvfKlSuLa2tzsf/++xfXLl26tLi2JssRETnn4tran+/NmzcX19Y+76X7vG9/+9tVfQdt1apVsXr16qLamkxEyMW2al4HIyKuu+664tra1+R+v19cW/u73mKyzz77xDve8Y6i2ppcRMjGQlqyZElV/TCPe2r2TTvijB8AAACARhn8AAAAADTK4AcAAACgUQY/AAAAAI0y+AEAAABolMEPAAAAQKMMfgAAAAAaZfADAAAA0CiDHwAAAIBGGfwAAAAANMrgBwAAAKBRUwt9hymlorqpqbqlnXDCCcW1Dz30UFXvW265pbi23+9X9a59nKV+4Rd+oar+hhtuKK796le/WtX7gx/8YHHtEUccUdX78ssvL6p7wxveUNV3GEqzMaxMRHQjF6WZ2OL3f//3i2vvuOOOqt7HH398ce3hhx9e1fvkk08url29enVV70F75JFH4pOf/GRR7Xvf+96q3j/7sz9bXLtmzZqq3meffXZx7d13313V+7jjjiuuXblyZXHtm970pqp1nHjiicW1e+yxR1XvCy64oLj2/vvvr+q95557FtX94Ac/qOo7aJOTk7HXXnsV1dZkIqIuF2984xuretfkaPfdd6/qfeGFFxbX1uSiNBNbTE5OFte+6EUvqupds73f+ta3VvU+/fTTi+puv/32qr6DNjExUZyNmkxEyMW2SjOxxT/+4z8W1+acq3rPzMwMrfcwj48H7b777ovf+73fK6qt3T/LxvMNMxcHH3xwVf3nP//54tojjzyyqnfp/vSFtoczfgAAAAAatdPBT0rp0JTSDSmlO1NKd6SU3j93+z4ppetTSvfM/bn38JcLC0f26TL5p6tkn66SfbpM/mldyRk/vYj4UM75yIh4TUS8N6V0ZEScFxHfyjkfERHfmvs7tET26TL5p6tkn66SfbpM/mnaTgc/OeeHc863zH39TETcFRErIuKUiLhkruySiHjLkNYIIyH7dJn801WyT1fJPl0m/7Su6mpIKaVVEXFMRNwcEQfmnB+e+9YjEXHgDv6fMyLijHmsEUZO9umy+eZ/+fLlC7BKGLz5Zn8xXYwUtjbf7B900EELsEoYjvnmv/bDEGAhFF/cOaX0MxHxlYg4J+f8k62/l3966e3tXn4757w653xszvnYea0URmQQ2S/9NDsYN4PIf+2n/sA4GET2JyZ8hgaLzyCyv/feLoPC4jSI/C9dunQBVgp1io5IUkrT8dMfgL/LOV8xd/OjKaWD575/cEQ8NpwlwujIPl0m/3SV7NNVsk+XyT8tK/lUrxQRfxMRd+WcP7XVt66KiNPmvj4tIr42+OXB6Mg+XSb/dJXs01WyT5fJP60refP5ayPidyNiXUrp1rnbPhIRF0bEl1NKp0fE/RHxm0NZIYyO7NNl8k9XyT5dJft0mfzTtJ0OfnLON0bEji5Q8quDXQ6MD9mny+SfrpJ9ukr26TL5p3Vj+3ETS5Ysqaqv+eSMBx54oKr3xo0bi2v7/X5V759eI6zM9PR0ce31119ftY5vf/vbVfXD6v2Zz3ymqvcrX/nKorpx+GSV0mzUZCJCLrZVmoktzj333OLa3/iN36jq/c///M/FtYcddlhV7/e9733FtatXr67qPWg/+tGPitdw4403VvW+8MILi2tf8YpXVPU+55xzimtPO+20nRdt5eabby6uXblyZXHtGWfUfZDgvvvuW1x70003VfW++uqrq+pr9Hq9orr169cPbQ0l+v1+PPvss0W1NZmIiFixYkVx7bvf/e6q3jW5qF33NddcU1VfqjQTW9QcF9RepPuRRx4prn3b295W1fvYY8s+K2XUF9V/9tlnY926dUW1w8pERDdyUZqJLY455pji2n/913+t6l2z/WZmZqp61/6ONUpPP/10fP3rXy+qrclFhGxsqzYXNT+3e+21V1Xvl7zkJUNZR0T5POKFfo/0cRMAAAAAjTL4AQAAAGiUwQ8AAABAowx+AAAAABpl8AMAAADQKIMfAAAAgEYZ/AAAAAA0yuAHAAAAoFEGPwAAAACNMvgBAAAAaJTBDwAAAECjphbyzlJKMTk5WVQ7NVW3tJRSce3KlSureu+///7FtQ888EBV79nZ2ar6UjXbIyKKn5ddsW7duuLa0047rar3tddeW1S3efPmqr7DUPqcDCsTNWvYYjHmojQTWxx44IFV9TUuu+yy4trjjz++qveKFSsqVzNa/X6/qO7OO++s6nvWWWcV137pS1+q6l2z76/193//98W1r33ta4trDznkkKp1zMzMFNd+9atfrepd+pxH1O/3Stedc67qO0pXXHFFVf2v/MqvFNfW5qLX6xXXfu1rX6vqXfNc12SoJsu19dPT01W977333uLa9evXV/Uu3S/VrnnQnnrqqeJs1P78y8Xz1b5WnX322cW1tcflNcfbtcekNc/7qPX7/eJtUZOLCNnYVm0uamYM99xzT1Xviy++uLi25vg1ImLJkiVV9dvjjB8AAACARhn8AAAAADTK4AcAAACgUQY/AAAAAI0y+AEAAABolMEPAAAAQKMMfgAAAAAaZfADAAAA0CiDHwAAAIBGGfwAAAAANGpqIe8s5xz9fr+o9tlnn63q/eijjxbXHn/88VW9Tz311OLaz3zmM1W9Z2dni2t7vV5xbUppaOuo7b106dLi2sMPP7yqd2lOSnM3LBMTE7HHHnsU1dY8FxFysa3afcc111xTXLtx48aq3vfdd19x7Zo1a6p6n3feeVX1o5ZzLqqbnp6u6rty5cri2ueee66q9ze+8Y3i2k2bNlX1/q//+q/i2ksvvbS49gMf+EDVOm6//fbi2ptuuqmqd802Kc3HFjMzM0PpO2g55+LtUJOJiIgvfvGLxbXvf//7q3rX5OLmm2+u6r158+bi2prnrzQTW9Tsa2pflx977LHi2osuuqiq9/77719UV3NcPAwbNmwozkZNJiLkYlulmdji05/+dHFt7farPRYcl97DUJqPmlxEyMZ8+9bsP2p/p7j88suLa3/rt36rqvd+++1XVPdCj88ZPwAAAACNMvgBAAAAaJTBDwAAAECjDH4AAAAAGmXwAwAAANAogx8AAACARhn8AAAAADTK4AcAAACgUQY/AAAAAI0y+AEAAABo1NRC32HOuaiu3+9X9b3qqquKa1/1qldV9X7Xu95VXPuyl72sqvff/u3fFtfef//9Vb1rHHLIIcW1L33pS6t6n3LKKcW1L37xi6t633DDDUV1GzZsqOo7aIcffnj8xV/8RVFtTSYi5GJbpZnY4lOf+lRx7caNG6t6p5SKa7/0pS9V9X7yySer6kdp3333jbe//e1FtW9+85ureq9ataq49rvf/W5V74svvri4tjYbExPl/+5y5ZVXFtc+9dRTVeu45ZZbimvXr19f1XtycrK4ttfrDaV37bHEoPX7/eJs1GQiQi7m0zeiLhs1+/La3jX7mRq1z8ug9fv94jXUPndyMT+bN28uri39vW2Lmv1Ybe/FJOdc/HzXvk7JxvzMzs4OrfcDDzxQXHv55ZdX9X7Pe95TVPdC284ZPwAAAACNMvgBAAAAaJTBDwAAAECjDH4AAAAAGmXwAwAAANAogx8AAACARhn8AAAAADTK4AcAAACgUQY/AAAAAI0y+AEAAABolMEPAAAAQKNSznnh7iylPDFRNmtKKVX1XrJkSXHt8ccfX9X74osvLq7dfffdq3qXbo9aU1NTQ+kbETE5OVlVf9tttxXXXnLJJVW9r7zyyqK6p556Knq9Xl2oBugVr3hF/uY3v1lUO6xMRHQjF6WZ2OK5554rrp2ZmanqXaP2uanJyYYNG9bmnI+tXdOgHHXUUfnrX/96Ue0dd9xR1fuyyy4rrr3mmmuqei/GbNT+HPb7/eLa2n1Tr9crrq09FpmdnS2uyzmPbN+fUsrD2u/KxfOVZmJX1G6/mmPYYR2Hz8zMRL/fH1n2Jycn87Jly4pqazIRIRfjrGZ71/6uNz09XVz73HPPjfS4Z2JiIpeut3Y7yMbz1eSidh21auYRBxxwQFXv1atXF9WdddZZcffdd293AzrjBwAAAKBROx38pJQOTSndkFK6M6V0R0rp/XO3fzyl9GBK6da5/948/OXCwpF9ukz+6SrZp6tkny6Tf1pXcp5wLyI+lHO+JaW0PCLWppSun/vep3POfzm85cFIyT5dJv90lezTVbJPl8k/Tdvp4Cfn/HBEPDz39TMppbsiYsWwFwajJvt0mfzTVbJPV8k+XSb/tK7qGj8ppVURcUxE3Dx30/tSSrellD6XUtp7B//PGSml76eUvj+/pcLozDf7TzzxxEItFQZO/ukqxz101Xyzv1gvQAsR8k+bigc/KaWfiYivRMQ5OeefRMRfRcTPRcTR8dPp6EXb+/9yzqtzzseO8srqMB+DyP4+++yzUMuFgZJ/uspxD101iOzXflIRjAv5p1VFg5+U0nT89Afg73LOV0RE5JwfzTnP5pz7EfHXEfFLw1smjIbs02XyT1fJPl0l+3SZ/NOykk/1ShHxNxFxV875U1vdfvBWZb8eEbcPfnkwOrJPl8k/XSX7dJXs02XyT+tKPtXrtRHxuxGxLqV069xtH4mId6aUjo6IHBH3RcSZQ1gfjJLs02XyT1fJPl0l+3SZ/NO0kk/1ujEitvdGxWsHvxwYH7JPl8k/XSX7dJXs02XyT+vSQl51PKWUSy92VXtRrOnp6eLa3Xbbrar3McccU1z71re+tar3CSecUFy7adOm4tp/+7d/q1rHk08+WVx7ww03VPX+l3/5l+La9evXV/UutXHjxuj3+yO70tphhx2W//AP/7CotiYTEXIxX/1+v7h2aqrkJMld611TW7uWDRs2rB3lhWb33HPP/JrXvKaodu3atVW9N2zYsCtLKlLz+libjdnZ2bFYR6/XK66tfV2emZmpqq9RupZerxc555Ht+ycnJ/OyZcuKamsyESEX811HzT53YqLqQ3CH+rpSamZmZqTHPRMTE7n0eHsc9hVbtJ6LiLp9zeTk5Nj0rnkuN23aNNLjnpRSLn0Oh3n8V2sxZmOYF9KufV0ufb2PqN9/HH744UV199xzTzz77LPb3Sh19wgAAADAomHwAwAAANAogx8AAACARhn8AAAAADTK4AcAAACgUQY/AAAAAI0y+AEAAABolMEPAAAAQKMMfgAAAAAaZfADAAAA0CiDHwAAAIBGpZzzwt1ZSnliYjizpsnJyeLa2jVMTU0V1y5ZsqSq97DWsWHDhqreNTmoWUdExObNm4tre71eVe/S531mZib6/X6qaj5AU1NT+UUvetGwehfXdiEXNfuCiLrHWLvvqFnL7OxsVe+adW/evHltzvnYqjsYoMnJybz77rsX1c7MzFT1HmY2+v3+0HrXZKlmHbWv6TXrrllHRF2ma3+2StfS6/Ui5zyyff/ExEQuPS4Y1jaI6EYuatdRU59SXYSGuf1Kt8moj3tqsl/7+icXz1e776jZ3rX7jpptUvu6WWPUxz0ppVz6+IZ5bCIb81vHMOcktc9N6f50/fr1MTs7u90H6YwfAAAAgEYZ/AAAAAA0yuAHAAAAoFEGPwAAAACNMvgBAAAAaJTBDwAAAECjDH4AAAAAGmXwAwAAANAogx8AAACARhn8AAAAADQq5ZwX7s5S+lFE3L+db+0XEY8v2EJGw2McrcNzzvuP6s5l32McsXHM/7hvs0HwGEdvHLMfMf7bbRBaf4zj/vhkf3Q8xtEbx/yP+zYbBI9x9HaY/QUd/OxISun7OedjR72OYfIY2Z4ubDOPke3pwjbzGNmRLmy31h9j649vWLqw3TxGtqcL28xjHG/e6gUAAADQKIMfAAAAgEaNy+Bn9agXsAA8RranC9vMY2R7urDNPEZ2pAvbrfXH2PrjG5YubDePke3pwjbzGMfYWFzjBwAAAIDBG5czfgAAAAAYsJEOflJKJ6aU7k4p/SCldN4o1zIsKaX7UkrrUkq3ppS+P+r1DEJK6XMppcdSSrdvdds+KaXrU0r3zP259yjXOO66kP0I+Wf7upB/2Wd7ZH9xkv35k/3FS/7nT/4Xp9ayP7LBT0ppMiIujoiTIuLIiHhnSunIUa1nyF6fcz56sX7023asiYgTt7ntvIj4Vs75iIj41tzf2Y6OZT9C/tlKx/Iv+/wP2V/U1oTs7zLZX/TWhPzvMvlf1NZEQ9kf5Rk/vxQRP8g5/zDnvDkivhgRp4xwPRTKOX8nIp7Y5uZTIuKSua8viYi3LOSaFhnZX8Tkf97kf5GS/XmT/UVK9udN9hcx+Z83+V+kWsv+KAc/KyLiv7f6+wNzt7UmR8Q3U0prU0pnjHoxQ3Rgzvnhua8fiYgDR7mYMdeV7EfIP/9bV/Iv+2xL9tsi++Vkvz3yX07+27Josz816gV0wHE55wdTSgdExPUppX+fmx42K+ecU0o+Lo4I+ae7ZJ+ukn26qnPZj5B//kfn8r/Ysj/KM34ejIhDt/r7yrnbmpJzfnDuz8ci4sr46el+LXo0pXRwRMTcn4+NeD3jrBPZj5B/tqsT+Zd9tkP22yL75WS/PfJfTv7bsmizP8rBz/ci4oiU0otTSksi4h0RcdUI1zNwKaU9UkrLt3wdEW+MiNtf+P9atK6KiNPmvj4tIr42wrWMu+azHyH/I1zLuGs+/7LPDsh+W2S/nOy3R/7LyX9bFm32R/ZWr5xzL6X0voj4RkRMRsTncs53jGo9Q3JgRFyZUor46ba+NOd83WiXNH8ppcsi4viI2C+l9EBEfCwiLoyIL6eUTo+I+yPiN0e3wvHWkexHyD/b0ZH8yz7/i+wvXrI/P7K/uMn//Mj/4tVa9lPOi+ZtaQAAAABUGOVbvQAAAAAYIoMfAAAAgEYZ/AAAAAA0yuAHAAAAoFEGPwAAAACNMvgBAAAAaJTBDwAAAECjDH4AAAAAGvX/AIFoz6xu0mScAAAAAElFTkSuQmCC\n", - "text/plain": [ - "<Figure size 1440x1440 with 6 Axes>" - ] - }, - "metadata": { - "needs_background": "light" - }, - "output_type": "display_data" - } - ], - "source": [ - "# remove batch size\n", - "patches = patches.squeeze(0)\n", - "fig = plt.figure(figsize=(20, 20))\n", - "for i in range(6):\n", - " ax = fig.add_subplot(1, 6, i + 1)\n", - " ax.imshow(patches[i].squeeze(0), cmap='gray')" + "32 * 0.75" ] }, { diff --git a/src/tasks/prepare_experiments.sh b/src/tasks/prepare_experiments.sh new file mode 100755 index 0000000..9b91daa --- /dev/null +++ b/src/tasks/prepare_experiments.sh @@ -0,0 +1,3 @@ +#!/bin/bash +experiments_filename=${1:-training/experiments/sample_experiment.yml} +python training/prepare_experiments.py --experiments_filename $experiments_filename diff --git a/src/tasks/prepare_sample_experiments.sh b/src/tasks/prepare_sample_experiments.sh deleted file mode 100755 index bc34f48..0000000 --- a/src/tasks/prepare_sample_experiments.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/bash -python training/prepare_experiments.py --experiments_filename training/experiments/sample_experiment.yml diff --git a/src/tasks/train_crnn_line_ctc_model.sh b/src/tasks/train_crnn_line_ctc_model.sh new file mode 100644 index 0000000..9831289 --- /dev/null +++ b/src/tasks/train_crnn_line_ctc_model.sh @@ -0,0 +1,3 @@ +#!/bin/bash +experiments_filename=${1:-training/experiments/line_ctc_experiment.yml} +exec ./prepare_experiments.sh experiments_filename=experiments_filename diff --git a/src/text_recognizer/character_predictor.py b/src/text_recognizer/character_predictor.py index b733a53..df37e68 100644 --- a/src/text_recognizer/character_predictor.py +++ b/src/text_recognizer/character_predictor.py @@ -15,6 +15,7 @@ class CharacterPredictor: """Intializes the CharacterModel and load the pretrained weights.""" self.model = CharacterModel(network_fn=network_fn) self.model.eval() + self.model.use_swa_model() def predict(self, image_or_filename: Union[np.ndarray, str]) -> Tuple[str, float]: """Predict on a single images contianing a handwritten character.""" diff --git a/src/text_recognizer/models/base.py b/src/text_recognizer/models/base.py index d23fe56..caf8065 100644 --- a/src/text_recognizer/models/base.py +++ b/src/text_recognizer/models/base.py @@ -77,9 +77,9 @@ class Model(ABC): # Stochastic Weight Averaging placeholders. self.swa_args = swa_args - self._swa_start = None self._swa_scheduler = None self._swa_network = None + self._use_swa_model = False # Experiment directory. self.model_dir = None @@ -220,15 +220,24 @@ class Model(ABC): if self._optimizer and self._lr_scheduler is not None: if "OneCycleLR" in str(self._lr_scheduler): self.lr_scheduler_args["steps_per_epoch"] = len(self.train_dataloader()) - self._lr_scheduler = self._lr_scheduler( - self._optimizer, **self.lr_scheduler_args - ) - else: - self._lr_scheduler = None + + # Assume lr scheduler should update at each epoch if not specified. + if "interval" not in self.lr_scheduler_args: + interval = "epoch" + else: + interval = self.lr_scheduler_args.pop("interval") + self._lr_scheduler = { + "lr_scheduler": self._lr_scheduler( + self._optimizer, **self.lr_scheduler_args + ), + "interval": interval, + } if self.swa_args is not None: - self._swa_start = self.swa_args["start"] - self._swa_scheduler = SWALR(self._optimizer, swa_lr=self.swa_args["lr"]) + self._swa_scheduler = { + "swa_scheduler": SWALR(self._optimizer, swa_lr=self.swa_args["lr"]), + "swa_start": self.swa_args["start"], + } self._swa_network = AveragedModel(self._network).to(self.device) @property @@ -280,21 +289,16 @@ class Model(ABC): return self._optimizer @property - def lr_scheduler(self) -> Optional[Callable]: - """Learning rate scheduler.""" + def lr_scheduler(self) -> Optional[Dict]: + """Returns a directory with the learning rate scheduler.""" return self._lr_scheduler @property - def swa_scheduler(self) -> Optional[Callable]: - """Returns the stochastic weight averaging scheduler.""" + def swa_scheduler(self) -> Optional[Dict]: + """Returns a directory with the stochastic weight averaging scheduler.""" return self._swa_scheduler @property - def swa_start(self) -> Optional[Callable]: - """Returns the start epoch of stochastic weight averaging.""" - return self._swa_start - - @property def swa_network(self) -> Optional[Callable]: """Returns the stochastic weight averaging network.""" return self._swa_network @@ -311,20 +315,32 @@ class Model(ABC): WEIGHT_DIRNAME.mkdir(parents=True, exist_ok=True) return str(WEIGHT_DIRNAME / f"{self._name}_weights.pt") + def use_swa_model(self) -> None: + """Set to use predictions from SWA model.""" + if self.swa_network is not None: + self._use_swa_model = True + + def forward(self, x: Tensor) -> Tensor: + """Feedforward pass with the network.""" + if self._use_swa_model: + return self.swa_network(x) + else: + return self.network(x) + def loss_fn(self, output: Tensor, targets: Tensor) -> Tensor: """Compute the loss.""" return self.criterion(output, targets) def summary( - self, input_shape: Optional[Tuple[int, int, int]] = None, depth: int = 5 + self, input_shape: Optional[Tuple[int, int, int]] = None, depth: int = 3 ) -> None: """Prints a summary of the network architecture.""" if input_shape is not None: - summary(self._network, input_shape, depth=depth, device=self.device) + summary(self.network, input_shape, depth=depth, device=self.device) elif self._input_shape is not None: input_shape = (1,) + tuple(self._input_shape) - summary(self._network, input_shape, depth=depth, device=self.device) + summary(self.network, input_shape, depth=depth, device=self.device) else: logger.warning("Could not print summary as input shape is not set.") diff --git a/src/text_recognizer/models/character_model.py b/src/text_recognizer/models/character_model.py index 64ba693..50e94a2 100644 --- a/src/text_recognizer/models/character_model.py +++ b/src/text_recognizer/models/character_model.py @@ -75,11 +75,7 @@ class CharacterModel(Model): # Put the image tensor on the device the model weights are on. image = image.to(self.device) - logits = ( - self.swa_network(image) - if self.swa_network is not None - else self.network(image) - ) + logits = self.forward(image) prediction = self.softmax(logits.squeeze(0)) diff --git a/src/text_recognizer/models/line_ctc_model.py b/src/text_recognizer/models/line_ctc_model.py index af41f18..16eaed3 100644 --- a/src/text_recognizer/models/line_ctc_model.py +++ b/src/text_recognizer/models/line_ctc_model.py @@ -98,16 +98,12 @@ class LineCTCModel(Model): # Put the image tensor on the device the model weights are on. image = image.to(self.device) - log_probs = ( - self.swa_network(image) - if self.swa_network is not None - else self.network(image) - ) + log_probs = self.forward(image) raw_pred, _ = greedy_decoder( predictions=log_probs, character_mapper=self.mapper, - blank_label=80, + blank_label=79, collapse_repeated=True, ) diff --git a/src/text_recognizer/networks/__init__.py b/src/text_recognizer/networks/__init__.py index d20c86a..a39975f 100644 --- a/src/text_recognizer/networks/__init__.py +++ b/src/text_recognizer/networks/__init__.py @@ -2,12 +2,14 @@ from .ctc import greedy_decoder from .lenet import LeNet from .line_lstm_ctc import LineRecurrentNetwork +from .losses import EmbeddingLoss from .misc import sliding_window from .mlp import MLP from .residual_network import ResidualNetwork, ResidualNetworkEncoder from .wide_resnet import WideResidualNetwork __all__ = [ + "EmbeddingLoss", "greedy_decoder", "MLP", "LeNet", diff --git a/src/text_recognizer/networks/line_lstm_ctc.py b/src/text_recognizer/networks/line_lstm_ctc.py index 5c57479..9009f94 100644 --- a/src/text_recognizer/networks/line_lstm_ctc.py +++ b/src/text_recognizer/networks/line_lstm_ctc.py @@ -1,9 +1,11 @@ """LSTM with CTC for handwritten text recognition within a line.""" import importlib +from pathlib import Path from typing import Callable, Dict, List, Optional, Tuple, Type, Union from einops import rearrange, reduce from einops.layers.torch import Rearrange, Reduce +from loguru import logger import torch from torch import nn from torch import Tensor @@ -14,40 +16,72 @@ class LineRecurrentNetwork(nn.Module): def __init__( self, - encoder: str, - encoder_args: Dict = None, + backbone: str, + backbone_args: Dict = None, flatten: bool = True, input_size: int = 128, hidden_size: int = 128, + bidirectional: bool = False, num_layers: int = 1, num_classes: int = 80, patch_size: Tuple[int, int] = (28, 28), stride: Tuple[int, int] = (1, 14), ) -> None: super().__init__() - self.encoder_args = encoder_args or {} + self.backbone_args = backbone_args or {} self.patch_size = patch_size self.stride = stride self.sliding_window = self._configure_sliding_window() self.input_size = input_size self.hidden_size = hidden_size - self.encoder = self._configure_encoder(encoder) + self.backbone = self._configure_backbone(backbone) + self.bidirectional = bidirectional self.flatten = flatten - self.fc = nn.Linear(in_features=self.input_size, out_features=self.hidden_size) + + if self.flatten: + self.fc = nn.Linear( + in_features=self.input_size, out_features=self.hidden_size + ) + self.rnn = nn.LSTM( input_size=self.hidden_size, hidden_size=self.hidden_size, + bidirectional=bidirectional, num_layers=num_layers, ) + + decoder_size = self.hidden_size * 2 if self.bidirectional else self.hidden_size + self.decoder = nn.Sequential( - nn.Linear(in_features=self.hidden_size, out_features=num_classes), + nn.Linear(in_features=decoder_size, out_features=num_classes), nn.LogSoftmax(dim=2), ) - def _configure_encoder(self, encoder: str) -> Type[nn.Module]: + def _configure_backbone(self, backbone: str) -> Type[nn.Module]: network_module = importlib.import_module("text_recognizer.networks") - encoder_ = getattr(network_module, encoder) - return encoder_(**self.encoder_args) + backbone_ = getattr(network_module, backbone) + + if "pretrained" in self.backbone_args: + logger.info("Loading pretrained backbone.") + checkpoint_file = Path(__file__).resolve().parents[ + 2 + ] / self.backbone_args.pop("pretrained") + + # Loading state directory. + state_dict = torch.load(checkpoint_file) + network_args = state_dict["network_args"] + weights = state_dict["model_state"] + + # Initializes the network with trained weights. + backbone = backbone_(**network_args) + backbone.load_state_dict(weights) + if "freeze" in self.backbone_args and self.backbone_args["freeze"] is True: + for params in backbone.parameters(): + params.requires_grad = False + + return backbone + else: + return backbone_(**self.backbone_args) def _configure_sliding_window(self) -> nn.Sequential: return nn.Sequential( @@ -69,13 +103,14 @@ class LineRecurrentNetwork(nn.Module): # Rearrange from a sequence of patches for feedforward network. b, t = x.shape[:2] x = rearrange(x, "b t c h w -> (b t) c h w", b=b, t=t) - x = self.encoder(x) + x = self.backbone(x) # Avgerage pooling. - x = reduce(x, "(b t) c h w -> t b c", "mean", b=b, t=t) if self.flatten else x - - # Linear layer between CNN and RNN - x = self.fc(x) + x = ( + self.fc(reduce(x, "(b t) c h w -> t b c", "mean", b=b, t=t)) + if self.flatten + else rearrange(x, "(b t) h -> t b h", b=b, t=t) + ) # Sequence predictions. x, _ = self.rnn(x) diff --git a/src/text_recognizer/networks/losses.py b/src/text_recognizer/networks/losses.py new file mode 100644 index 0000000..73e0641 --- /dev/null +++ b/src/text_recognizer/networks/losses.py @@ -0,0 +1,31 @@ +"""Implementations of custom loss functions.""" +from pytorch_metric_learning import distances, losses, miners, reducers +from torch import nn +from torch import Tensor + + +class EmbeddingLoss: + """Metric loss for training encoders to produce information-rich latent embeddings.""" + + def __init__(self, margin: float = 0.2, type_of_triplets: str = "semihard") -> None: + self.distance = distances.CosineSimilarity() + self.reducer = reducers.ThresholdReducer(low=0) + self.loss_fn = losses.TripletMarginLoss( + margin=margin, distance=self.distance, reducer=self.reducer + ) + self.miner = miners.MultiSimilarityMiner(epsilon=margin, distance=self.distance) + + def __call__(self, embeddings: Tensor, labels: Tensor) -> Tensor: + """Computes the metric loss for the embeddings based on their labels. + + Args: + embeddings (Tensor): The laten vectors encoded by the network. + labels (Tensor): Labels of the embeddings. + + Returns: + Tensor: The metric loss for the embeddings. + + """ + hard_pairs = self.miner(embeddings, labels) + loss = self.loss_fn(embeddings, labels, hard_pairs) + return loss diff --git a/src/text_recognizer/networks/residual_network.py b/src/text_recognizer/networks/residual_network.py index 1b5d6b3..046600d 100644 --- a/src/text_recognizer/networks/residual_network.py +++ b/src/text_recognizer/networks/residual_network.py @@ -278,7 +278,8 @@ class ResidualNetworkEncoder(nn.Module): if self.stn is not None: x = self.stn(x) x = self.gate(x) - return self.blocks(x) + x = self.blocks(x) + return x class ResidualNetworkDecoder(nn.Module): diff --git a/src/text_recognizer/networks/transformer.py b/src/text_recognizer/networks/transformer.py index 868d739..c091ba0 100644 --- a/src/text_recognizer/networks/transformer.py +++ b/src/text_recognizer/networks/transformer.py @@ -1 +1,5 @@ """TBC.""" +from typing import Dict + +import torch +from torch import Tensor diff --git a/src/text_recognizer/weights/CharacterModel_EmnistDataset_ResidualNetworkEncoder_weights.pt b/src/text_recognizer/weights/CharacterModel_EmnistDataset_ResidualNetworkEncoder_weights.pt Binary files differnew file mode 100644 index 0000000..9f9deee --- /dev/null +++ b/src/text_recognizer/weights/CharacterModel_EmnistDataset_ResidualNetworkEncoder_weights.pt diff --git a/src/text_recognizer/weights/CharacterModel_EmnistDataset_ResidualNetwork_weights.pt b/src/text_recognizer/weights/CharacterModel_EmnistDataset_ResidualNetwork_weights.pt Binary files differindex a25bcd1..0dc7eb5 100644 --- a/src/text_recognizer/weights/CharacterModel_EmnistDataset_ResidualNetwork_weights.pt +++ b/src/text_recognizer/weights/CharacterModel_EmnistDataset_ResidualNetwork_weights.pt diff --git a/src/text_recognizer/weights/LineCTCModel_IamLinesDataset_LineRecurrentNetwork_weights.pt b/src/text_recognizer/weights/LineCTCModel_IamLinesDataset_LineRecurrentNetwork_weights.pt Binary files differindex 9bd8ca2..93d34d7 100644 --- a/src/text_recognizer/weights/LineCTCModel_IamLinesDataset_LineRecurrentNetwork_weights.pt +++ b/src/text_recognizer/weights/LineCTCModel_IamLinesDataset_LineRecurrentNetwork_weights.pt diff --git a/src/training/experiments/iam_line_ctc_experiment.yml b/src/training/experiments/iam_line_ctc_experiment.yml deleted file mode 100644 index 141c74e..0000000 --- a/src/training/experiments/iam_line_ctc_experiment.yml +++ /dev/null @@ -1,94 +0,0 @@ -experiment_group: Sample Experiments -experiments: - - train_args: - batch_size: 24 - max_epochs: 128 - dataset: - type: IamLinesDataset - args: - subsample_fraction: null - transform: null - target_transform: null - train_args: - num_workers: 6 - train_fraction: 0.85 - model: LineCTCModel - metrics: [cer, wer] - network: - type: LineRecurrentNetwork - args: - # encoder: ResidualNetworkEncoder - # encoder_args: - # in_channels: 1 - # num_classes: 80 - # depths: [2, 2] - # block_sizes: [128, 128] - # activation: SELU - # stn: false - encoder: WideResidualNetwork - encoder_args: - in_channels: 1 - num_classes: 80 - depth: 16 - num_layers: 4 - width_factor: 2 - dropout_rate: 0.2 - activation: selu - use_decoder: false - flatten: true - input_size: 256 - hidden_size: 128 - num_layers: 2 - num_classes: 80 - patch_size: [28, 14] - stride: [1, 5] - criterion: - type: CTCLoss - args: - blank: 79 - optimizer: - type: AdamW - args: - lr: 1.e-03 - betas: [0.9, 0.999] - eps: 1.e-08 - weight_decay: false - amsgrad: false - # lr_scheduler: - # type: OneCycleLR - # args: - # max_lr: 1.e-02 - # epochs: null - # anneal_strategy: linear - lr_scheduler: - type: CosineAnnealingLR - args: - T_max: null - swa_args: - start: 75 - lr: 5.e-2 - callbacks: [Checkpoint, ProgressBar, WandbCallback, WandbImageLogger, SWA] # EarlyStopping, OneCycleLR] - callback_args: - Checkpoint: - monitor: val_loss - mode: min - ProgressBar: - epochs: null - # log_batch_frequency: 100 - # EarlyStopping: - # monitor: val_loss - # min_delta: 0.0 - # patience: 7 - # mode: min - WandbCallback: - log_batch_frequency: 10 - WandbImageLogger: - num_examples: 6 - # OneCycleLR: - # null - SWA: - null - verbosity: 1 # 0, 1, 2 - resume_experiment: null - test: true - test_metric: test_cer diff --git a/src/training/experiments/line_ctc_experiment.yml b/src/training/experiments/line_ctc_experiment.yml index c21c6a2..432d1cc 100644 --- a/src/training/experiments/line_ctc_experiment.yml +++ b/src/training/experiments/line_ctc_experiment.yml @@ -1,55 +1,46 @@ -experiment_group: Sample Experiments +experiment_group: Lines Experiments experiments: - train_args: - batch_size: 64 - max_epochs: 32 + batch_size: 42 + max_epochs: &max_epochs 32 dataset: - type: EmnistLinesDataset + type: IamLinesDataset args: - subsample_fraction: 0.33 - max_length: 34 - min_overlap: 0 - max_overlap: 0.33 - num_samples: 10000 - seed: 4711 - blank: true + subsample_fraction: null + transform: null + target_transform: null train_args: - num_workers: 6 + num_workers: 8 train_fraction: 0.85 model: LineCTCModel metrics: [cer, wer] network: type: LineRecurrentNetwork args: - # encoder: ResidualNetworkEncoder - # encoder_args: - # in_channels: 1 - # num_classes: 81 - # depths: [2, 2] - # block_sizes: [64, 128] - # activation: SELU - # stn: false - encoder: WideResidualNetwork - encoder_args: + backbone: ResidualNetwork + backbone_args: in_channels: 1 - num_classes: 81 - depth: 16 - num_layers: 4 - width_factor: 2 - dropout_rate: 0.2 + num_classes: 64 # Embedding + depths: [2,2] + block_sizes: [32,64] activation: selu - use_decoder: false - flatten: true - input_size: 256 - hidden_size: 128 + stn: false + # encoder: ResidualNetwork + # encoder_args: + # pretrained: training/experiments/CharacterModel_EmnistDataset_ResidualNetwork/0917_203601/model/best.pt + # freeze: false + flatten: false + input_size: 64 + hidden_size: 64 + bidirectional: true num_layers: 2 - num_classes: 81 - patch_size: [28, 14] - stride: [1, 5] + num_classes: 80 + patch_size: [28, 18] + stride: [1, 4] criterion: type: CTCLoss args: - blank: 80 + blank: 79 optimizer: type: AdamW args: @@ -58,40 +49,42 @@ experiments: eps: 1.e-08 weight_decay: 5.e-4 amsgrad: false - # lr_scheduler: - # type: OneCycleLR - # args: - # max_lr: 1.e-03 - # epochs: null - # anneal_strategy: linear lr_scheduler: - type: CosineAnnealingLR + type: OneCycleLR args: - T_max: null + max_lr: 1.e-02 + epochs: *max_epochs + anneal_strategy: cos + pct_start: 0.475 + cycle_momentum: true + base_momentum: 0.85 + max_momentum: 0.9 + div_factor: 10 + final_div_factor: 10000 + interval: step + # lr_scheduler: + # type: CosineAnnealingLR + # args: + # T_max: *max_epochs swa_args: - start: 4 + start: 24 lr: 5.e-2 - callbacks: [Checkpoint, ProgressBar, WandbCallback, WandbImageLogger, SWA] # EarlyStopping, OneCycleLR] + callbacks: [Checkpoint, ProgressBar, WandbCallback, WandbImageLogger] # EarlyStopping] callback_args: Checkpoint: monitor: val_loss mode: min ProgressBar: - epochs: null - log_batch_frequency: 100 + epochs: *max_epochs # EarlyStopping: # monitor: val_loss # min_delta: 0.0 - # patience: 5 + # patience: 10 # mode: min WandbCallback: log_batch_frequency: 10 WandbImageLogger: num_examples: 6 - # OneCycleLR: - # null - SWA: - null verbosity: 1 # 0, 1, 2 resume_experiment: null test: true diff --git a/src/training/experiments/sample_experiment.yml b/src/training/experiments/sample_experiment.yml index 17e220e..8664a15 100644 --- a/src/training/experiments/sample_experiment.yml +++ b/src/training/experiments/sample_experiment.yml @@ -2,7 +2,7 @@ experiment_group: Sample Experiments experiments: - train_args: batch_size: 256 - max_epochs: 32 + max_epochs: &max_epochs 32 dataset: type: EmnistDataset args: @@ -66,16 +66,17 @@ experiments: # type: OneCycleLR # args: # max_lr: 1.e-03 - # epochs: null + # epochs: *max_epochs # anneal_strategy: linear lr_scheduler: type: CosineAnnealingLR args: - T_max: null + T_max: *max_epochs + interval: epoch swa_args: start: 2 lr: 5.e-2 - callbacks: [Checkpoint, ProgressBar, WandbCallback, WandbImageLogger, EarlyStopping, SWA] # OneCycleLR] + callbacks: [Checkpoint, ProgressBar, WandbCallback, WandbImageLogger, EarlyStopping] callback_args: Checkpoint: monitor: val_accuracy @@ -92,10 +93,6 @@ experiments: WandbImageLogger: num_examples: 4 use_transpose: true - # OneCycleLR: - # null - SWA: - null verbosity: 0 # 0, 1, 2 resume_experiment: null test: true diff --git a/src/training/run_experiment.py b/src/training/run_experiment.py index 286b0c6..a347d9f 100644 --- a/src/training/run_experiment.py +++ b/src/training/run_experiment.py @@ -10,6 +10,7 @@ from typing import Callable, Dict, List, Tuple, Type import click from loguru import logger +import numpy as np import torch from tqdm import tqdm from training.gpu_manager import GPUManager @@ -20,11 +21,12 @@ import yaml from text_recognizer.models import Model +from text_recognizer.networks import losses EXPERIMENTS_DIRNAME = Path(__file__).parents[0].resolve() / "experiments" - +CUSTOM_LOSSES = ["EmbeddingLoss"] DEFAULT_TRAIN_ARGS = {"batch_size": 64, "epochs": 16} @@ -69,21 +71,6 @@ def create_experiment_dir(experiment_config: Dict) -> Path: return experiment_dir, log_dir, model_dir -def check_args(args: Dict, train_args: Dict) -> Dict: - """Checks that the arguments are not None.""" - args = args or {} - - # I just want to set total epochs in train args, instead of changing all parameter. - if "epochs" in args and args["epochs"] is None: - args["epochs"] = train_args["max_epochs"] - - # For CosineAnnealingLR. - if "T_max" in args and args["T_max"] is None: - args["T_max"] = train_args["max_epochs"] - - return args or {} - - def load_modules_and_arguments(experiment_config: Dict) -> Tuple[Callable, Dict]: """Loads all modules and arguments.""" # Import the data loader arguments. @@ -115,8 +102,12 @@ def load_modules_and_arguments(experiment_config: Dict) -> Tuple[Callable, Dict] network_args = experiment_config["network"].get("args", {}) # Criterion - criterion_ = getattr(torch.nn, experiment_config["criterion"]["type"]) - criterion_args = experiment_config["criterion"].get("args", {}) + if experiment_config["criterion"]["type"] in CUSTOM_LOSSES: + criterion_ = getattr(losses, experiment_config["criterion"]["type"]) + criterion_args = experiment_config["criterion"].get("args", {}) + else: + criterion_ = getattr(torch.nn, experiment_config["criterion"]["type"]) + criterion_args = experiment_config["criterion"].get("args", {}) # Optimizers optimizer_ = getattr(torch.optim, experiment_config["optimizer"]["type"]) @@ -129,13 +120,11 @@ def load_modules_and_arguments(experiment_config: Dict) -> Tuple[Callable, Dict] lr_scheduler_ = getattr( torch.optim.lr_scheduler, experiment_config["lr_scheduler"]["type"] ) - lr_scheduler_args = check_args( - experiment_config["lr_scheduler"].get("args", {}), train_args - ) + lr_scheduler_args = experiment_config["lr_scheduler"].get("args", {}) or {} # SWA scheduler. if "swa_args" in experiment_config: - swa_args = check_args(experiment_config.get("swa_args", {}), train_args) + swa_args = experiment_config.get("swa_args", {}) or {} else: swa_args = None @@ -159,19 +148,15 @@ def load_modules_and_arguments(experiment_config: Dict) -> Tuple[Callable, Dict] def configure_callbacks(experiment_config: Dict, model_dir: Dict) -> CallbackList: """Configure a callback list for trainer.""" - train_args = experiment_config.get("train_args", {}) - if "Checkpoint" in experiment_config["callback_args"]: experiment_config["callback_args"]["Checkpoint"]["checkpoint_path"] = model_dir - # Callbacks + # Initializes callbacks. callback_modules = importlib.import_module("training.trainer.callbacks") - callbacks = [ - getattr(callback_modules, callback)( - **check_args(experiment_config["callback_args"][callback], train_args) - ) - for callback in experiment_config["callbacks"] - ] + callbacks = [] + for callback in experiment_config["callbacks"]: + args = experiment_config["callback_args"][callback] or {} + callbacks.append(getattr(callback_modules, callback)(**args)) return callbacks @@ -207,11 +192,35 @@ def load_from_checkpoint(model: Type[Model], log_dir: Path, model_dir: Path) -> model.load_checkpoint(checkpoint_path) +def evaluate_embedding(model: Type[Model]) -> Dict: + """Evaluates the embedding space.""" + from pytorch_metric_learning import testers + from pytorch_metric_learning.utils.accuracy_calculator import AccuracyCalculator + + accuracy_calculator = AccuracyCalculator( + include=("mean_average_precision_at_r",), k=10 + ) + + def get_all_embeddings(model: Type[Model]) -> Tuple: + tester = testers.BaseTester() + return tester.get_all_embeddings(model.test_dataset, model.network) + + embeddings, labels = get_all_embeddings(model) + logger.info("Computing embedding accuracy") + accuracies = accuracy_calculator.get_accuracy( + embeddings, embeddings, np.squeeze(labels), np.squeeze(labels), True + ) + logger.info( + f"Test set accuracy (MAP@10) = {accuracies['mean_average_precision_at_r']}" + ) + return accuracies + + def run_experiment( experiment_config: Dict, save_weights: bool, device: str, use_wandb: bool = False ) -> None: """Runs an experiment.""" - logger.info(f"Experiment config: {json.dumps(experiment_config, indent=2)}") + logger.info(f"Experiment config: {json.dumps(experiment_config)}") # Create new experiment. experiment_dir, log_dir, model_dir = create_experiment_dir(experiment_config) @@ -272,7 +281,11 @@ def run_experiment( model.load_from_checkpoint(model_dir / "best.pt") logger.info("Running inference on test set.") - score = trainer.test(model) + if experiment_config["criterion"]["type"] in CUSTOM_LOSSES: + logger.info("Evaluating embedding.") + score = evaluate_embedding(model) + else: + score = trainer.test(model) logger.info(f"Test set evaluation: {score}") diff --git a/src/training/trainer/callbacks/__init__.py b/src/training/trainer/callbacks/__init__.py index c81e4bf..e1bd858 100644 --- a/src/training/trainer/callbacks/__init__.py +++ b/src/training/trainer/callbacks/__init__.py @@ -3,12 +3,7 @@ from .base import Callback, CallbackList from .checkpoint import Checkpoint from .early_stopping import EarlyStopping from .lr_schedulers import ( - CosineAnnealingLR, - CyclicLR, - MultiStepLR, - OneCycleLR, - ReduceLROnPlateau, - StepLR, + LRScheduler, SWA, ) from .progress_bar import ProgressBar @@ -18,15 +13,10 @@ __all__ = [ "Callback", "CallbackList", "Checkpoint", - "CosineAnnealingLR", "EarlyStopping", + "LRScheduler", "WandbCallback", "WandbImageLogger", - "CyclicLR", - "MultiStepLR", - "OneCycleLR", "ProgressBar", - "ReduceLROnPlateau", - "StepLR", "SWA", ] diff --git a/src/training/trainer/callbacks/lr_schedulers.py b/src/training/trainer/callbacks/lr_schedulers.py index bb41d2d..907e292 100644 --- a/src/training/trainer/callbacks/lr_schedulers.py +++ b/src/training/trainer/callbacks/lr_schedulers.py @@ -7,113 +7,27 @@ from training.trainer.callbacks import Callback from text_recognizer.models import Model -class StepLR(Callback): - """Callback for StepLR.""" +class LRScheduler(Callback): + """Generic learning rate scheduler callback.""" def __init__(self) -> None: - """Initializes the callback.""" - super().__init__() - self.lr_scheduler = None - - def set_model(self, model: Type[Model]) -> None: - """Sets the model and lr scheduler.""" - self.model = model - self.lr_scheduler = self.model.lr_scheduler - - def on_epoch_end(self, epoch: int, logs: Optional[Dict] = None) -> None: - """Takes a step at the end of every epoch.""" - self.lr_scheduler.step() - - -class MultiStepLR(Callback): - """Callback for MultiStepLR.""" - - def __init__(self) -> None: - """Initializes the callback.""" - super().__init__() - self.lr_scheduler = None - - def set_model(self, model: Type[Model]) -> None: - """Sets the model and lr scheduler.""" - self.model = model - self.lr_scheduler = self.model.lr_scheduler - - def on_epoch_end(self, epoch: int, logs: Optional[Dict] = None) -> None: - """Takes a step at the end of every epoch.""" - self.lr_scheduler.step() - - -class ReduceLROnPlateau(Callback): - """Callback for ReduceLROnPlateau.""" - - def __init__(self) -> None: - """Initializes the callback.""" super().__init__() - self.lr_scheduler = None def set_model(self, model: Type[Model]) -> None: """Sets the model and lr scheduler.""" self.model = model - self.lr_scheduler = self.model.lr_scheduler + self.lr_scheduler = self.model.lr_scheduler["lr_scheduler"] + self.interval = self.model.lr_scheduler["interval"] def on_epoch_end(self, epoch: int, logs: Optional[Dict] = None) -> None: """Takes a step at the end of every epoch.""" - val_loss = logs["val_loss"] - self.lr_scheduler.step(val_loss) - - -class CyclicLR(Callback): - """Callback for CyclicLR.""" - - def __init__(self) -> None: - """Initializes the callback.""" - super().__init__() - self.lr_scheduler = None - - def set_model(self, model: Type[Model]) -> None: - """Sets the model and lr scheduler.""" - self.model = model - self.lr_scheduler = self.model.lr_scheduler - - def on_train_batch_end(self, batch: int, logs: Optional[Dict] = None) -> None: - """Takes a step at the end of every training batch.""" - self.lr_scheduler.step() - - -class OneCycleLR(Callback): - """Callback for OneCycleLR.""" - - def __init__(self) -> None: - """Initializes the callback.""" - super().__init__() - self.lr_scheduler = None - - def set_model(self, model: Type[Model]) -> None: - """Sets the model and lr scheduler.""" - self.model = model - self.lr_scheduler = self.model.lr_scheduler + if self.interval == "epoch": + self.lr_scheduler.step() def on_train_batch_end(self, batch: int, logs: Optional[Dict] = None) -> None: """Takes a step at the end of every training batch.""" - self.lr_scheduler.step() - - -class CosineAnnealingLR(Callback): - """Callback for Cosine Annealing.""" - - def __init__(self) -> None: - """Initializes the callback.""" - super().__init__() - self.lr_scheduler = None - - def set_model(self, model: Type[Model]) -> None: - """Sets the model and lr scheduler.""" - self.model = model - self.lr_scheduler = self.model.lr_scheduler - - def on_epoch_end(self, epoch: int, logs: Optional[Dict] = None) -> None: - """Takes a step at the end of every epoch.""" - self.lr_scheduler.step() + if self.interval == "step": + self.lr_scheduler.step() class SWA(Callback): @@ -122,21 +36,32 @@ class SWA(Callback): def __init__(self) -> None: """Initializes the callback.""" super().__init__() + self.lr_scheduler = None + self.interval = None self.swa_scheduler = None + self.swa_start = None + self.current_epoch = 1 def set_model(self, model: Type[Model]) -> None: """Sets the model and lr scheduler.""" self.model = model - self.swa_start = self.model.swa_start - self.swa_scheduler = self.model.lr_scheduler - self.lr_scheduler = self.model.lr_scheduler + self.lr_scheduler = self.model.lr_scheduler["lr_scheduler"] + self.interval = self.model.lr_scheduler["interval"] + self.swa_scheduler = self.model.swa_scheduler["swa_scheduler"] + self.swa_start = self.model.swa_scheduler["swa_start"] def on_epoch_end(self, epoch: int, logs: Optional[Dict] = None) -> None: """Takes a step at the end of every training batch.""" if epoch > self.swa_start: self.model.swa_network.update_parameters(self.model.network) self.swa_scheduler.step() - else: + elif self.interval == "epoch": + self.lr_scheduler.step() + self.current_epoch = epoch + + def on_train_batch_end(self, batch: int, logs: Optional[Dict] = None) -> None: + """Takes a step at the end of every training batch.""" + if self.current_epoch < self.swa_start and self.interval == "step": self.lr_scheduler.step() def on_fit_end(self) -> None: diff --git a/src/training/trainer/callbacks/progress_bar.py b/src/training/trainer/callbacks/progress_bar.py index 7829fa0..6c4305a 100644 --- a/src/training/trainer/callbacks/progress_bar.py +++ b/src/training/trainer/callbacks/progress_bar.py @@ -11,6 +11,7 @@ class ProgressBar(Callback): def __init__(self, epochs: int, log_batch_frequency: int = None) -> None: """Initializes the tqdm callback.""" self.epochs = epochs + print(epochs, type(epochs)) self.log_batch_frequency = log_batch_frequency self.progress_bar = None self.val_metrics = {} diff --git a/src/training/trainer/callbacks/wandb_callbacks.py b/src/training/trainer/callbacks/wandb_callbacks.py index 6643a44..d2df4d7 100644 --- a/src/training/trainer/callbacks/wandb_callbacks.py +++ b/src/training/trainer/callbacks/wandb_callbacks.py @@ -32,6 +32,7 @@ class WandbCallback(Callback): def on_train_batch_end(self, batch: int, logs: Optional[Dict] = None) -> None: """Logs training metrics.""" if logs is not None: + logs["lr"] = self.model.optimizer.param_groups[0]["lr"] self._on_batch_end(batch, logs) def on_validation_batch_end(self, batch: int, logs: Optional[Dict] = None) -> None: diff --git a/src/training/trainer/train.py b/src/training/trainer/train.py index b240157..bd6a491 100644 --- a/src/training/trainer/train.py +++ b/src/training/trainer/train.py @@ -9,7 +9,7 @@ import numpy as np import torch from torch import Tensor from torch.optim.swa_utils import update_bn -from training.trainer.callbacks import Callback, CallbackList +from training.trainer.callbacks import Callback, CallbackList, LRScheduler, SWA from training.trainer.util import log_val_metric, RunningAverage import wandb @@ -47,8 +47,14 @@ class Trainer: self.model = None def _configure_callbacks(self) -> None: + """Instantiate the CallbackList.""" if not self.callbacks_configured: - # Instantiate a CallbackList. + # If learning rate schedulers are present, they need to be added to the callbacks. + if self.model.swa_scheduler is not None: + self.callbacks.append(SWA()) + elif self.model.lr_scheduler is not None: + self.callbacks.append(LRScheduler()) + self.callbacks = CallbackList(self.model, self.callbacks) def compute_metrics( @@ -91,7 +97,7 @@ class Trainer: # Forward pass. # Get the network prediction. - output = self.model.network(data) + output = self.model.forward(data) # Compute the loss. loss = self.model.loss_fn(output, targets) @@ -130,7 +136,6 @@ class Trainer: batch: int, samples: Tuple[Tensor, Tensor], loss_avg: Type[RunningAverage], - use_swa: bool = False, ) -> Dict: """Performs the validation step.""" # Pass the tensor to the device for computation. @@ -143,10 +148,7 @@ class Trainer: # Forward pass. # Get the network prediction. # Use SWA if available and using test dataset. - if use_swa and self.model.swa_network is None: - output = self.model.swa_network(data) - else: - output = self.model.network(data) + output = self.model.forward(data) # Compute the loss. loss = self.model.loss_fn(output, targets) @@ -238,7 +240,7 @@ class Trainer: self.model.eval() # Check if SWA network is available. - use_swa = True if self.model.swa_network is not None else False + self.model.use_swa_model() # Running average for the loss. loss_avg = RunningAverage() @@ -247,7 +249,7 @@ class Trainer: summary = [] for batch, samples in enumerate(self.model.test_dataloader()): - metrics = self.validation_step(batch, samples, loss_avg, use_swa) + metrics = self.validation_step(batch, samples, loss_avg) summary.append(metrics) # Compute mean of all test metrics. |