summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--notebooks/04-efficientnet-transformer.ipynb219
-rw-r--r--notebooks/04-efficientnet.ipynb279
-rw-r--r--notebooks/04-vq-transformer.ipynb253
-rw-r--r--notebooks/04-vqvae.ipynb233
4 files changed, 984 insertions, 0 deletions
diff --git a/notebooks/04-efficientnet-transformer.ipynb b/notebooks/04-efficientnet-transformer.ipynb
new file mode 100644
index 0000000..427c98c
--- /dev/null
+++ b/notebooks/04-efficientnet-transformer.ipynb
@@ -0,0 +1,219 @@
+{
+ "cells": [
+ {
+ "cell_type": "code",
+ "execution_count": 2,
+ "id": "7c02ae76-b540-4b16-9492-e9210b3b9249",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import os\n",
+ "os.environ['CUDA_VISIBLE_DEVICE'] = ''\n",
+ "import random\n",
+ "\n",
+ "%matplotlib inline\n",
+ "import matplotlib.pyplot as plt\n",
+ "\n",
+ "import numpy as np\n",
+ "from omegaconf import OmegaConf\n",
+ "\n",
+ "%load_ext autoreload\n",
+ "%autoreload 2\n",
+ "\n",
+ "from importlib.util import find_spec\n",
+ "if find_spec(\"text_recognizer\") is None:\n",
+ " import sys\n",
+ " sys.path.append('..')"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 3,
+ "id": "ccdb6dde-47e5-429a-88f2-0764fb7e259a",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from hydra import compose, initialize\n",
+ "from omegaconf import OmegaConf\n",
+ "from hydra.utils import instantiate"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 4,
+ "id": "3cf50475-39f2-4642-a7d1-5bcbc0a036f7",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "path = \"../training/conf/experiment/cnn_htr_char_lines.yaml\""
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 5,
+ "id": "e52ecb01-c975-4e55-925d-1182c7aea473",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "with open(path, \"rb\") as f:\n",
+ " cfg = OmegaConf.load(f)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 6,
+ "id": "f939aa37-7b1d-45cc-885c-323c4540bda1",
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "{'defaults': [{'override /mapping': None}, {'override /criterion': None}, {'override /datamodule': None}, {'override /network': None}, {'override /model': None}, {'override /lr_schedulers': None}, {'override /optimizers': None}], 'criterion': {'_target_': 'torch.nn.CrossEntropyLoss', 'ignore_index': 3}, 'mapping': {'_target_': 'text_recognizer.data.emnist_mapping.EmnistMapping'}, 'optimizers': {'madgrad': {'_target_': 'madgrad.MADGRAD', 'lr': 0.0001, 'momentum': 0.9, 'weight_decay': 0, 'eps': 1e-06, 'parameters': 'network'}}, 'lr_schedulers': {'network': {'_target_': 'torch.optim.lr_scheduler.CosineAnnealingLR', 'T_max': 1024, 'eta_min': 4.5e-06, 'last_epoch': -1, 'interval': 'epoch', 'monitor': 'val/loss'}}, 'datamodule': {'_target_': 'text_recognizer.data.iam_lines.IAMLines', 'batch_size': 8, 'num_workers': 12, 'train_fraction': 0.8, 'augment': False, 'pin_memory': False}, 'network': {'_target_': 'text_recognizer.networks.conv_transformer.ConvTransformer', 'input_dims': [1, 56, 1024], 'hidden_dim': 128, 'encoder_dim': 1280, 'dropout_rate': 0.2, 'num_classes': 58, 'pad_index': 3, 'encoder': {'_target_': 'text_recognizer.networks.encoders.efficientnet.EfficientNet', 'arch': 'b0', 'out_channels': 1280, 'stochastic_dropout_rate': 0.2, 'bn_momentum': 0.99, 'bn_eps': 0.001}, 'decoder': {'_target_': 'text_recognizer.networks.transformer.Decoder', 'dim': 128, 'depth': 2, 'num_heads': 4, 'attn_fn': 'text_recognizer.networks.transformer.attention.Attention', 'attn_kwargs': {'dim_head': 32, 'dropout_rate': 0.2}, 'norm_fn': 'torch.nn.LayerNorm', 'ff_fn': 'text_recognizer.networks.transformer.mlp.FeedForward', 'ff_kwargs': {'dim_out': None, 'expansion_factor': 4, 'glu': True, 'dropout_rate': 0.2}, 'cross_attend': True, 'pre_norm': True, 'rotary_emb': None}}, 'model': {'_target_': 'text_recognizer.models.transformer.TransformerLitModel', 'max_output_len': 89, 'start_token': '<s>', 'end_token': '<e>', 'pad_token': '<p>'}, 'trainer': {'_target_': 'pytorch_lightning.Trainer', 'stochastic_weight_avg': False, 'auto_scale_batch_size': 'binsearch', 'auto_lr_find': False, 'gradient_clip_val': 0, 'fast_dev_run': False, 'gpus': 1, 'precision': 16, 'max_epochs': 1024, 'terminate_on_nan': True, 'weights_summary': 'top', 'limit_train_batches': 1.0, 'limit_val_batches': 1.0, 'limit_test_batches': 1.0, 'resume_from_checkpoint': None, 'accumulate_grad_batches': 4}}"
+ ]
+ },
+ "execution_count": 6,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "cfg"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 7,
+ "id": "aaeab329-aeb0-4a1b-aa35-5a2aab81b1d0",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "net = instantiate(cfg.network)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 9,
+ "id": "618b997c-e6a6-4487-b70c-9d260cb556d3",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from torchinfo import summary"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 10,
+ "id": "25759b7b-8deb-4163-b75d-a1357c9fe88f",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "torch.Size([4, 4, 89, 1024])\n",
+ "torch.Size([4, 4, 89, 1024])\n",
+ "torch.Size([4, 4, 89, 1024])\n",
+ "torch.Size([4, 4, 32, 1024])\n",
+ "torch.Size([4, 4, 89, 1024])\n",
+ "torch.Size([4, 4, 89, 1024])\n",
+ "torch.Size([4, 4, 89, 1024])\n",
+ "torch.Size([4, 4, 32, 1024])\n"
+ ]
+ },
+ {
+ "data": {
+ "text/plain": [
+ "====================================================================================================\n",
+ "Layer (type:depth-idx) Output Shape Param #\n",
+ "====================================================================================================\n",
+ "ConvTransformer -- --\n",
+ "├─EfficientNet: 1 -- --\n",
+ "│ └─ModuleList: 2-1 -- --\n",
+ "├─Decoder: 1 -- --\n",
+ "│ └─ModuleList: 2-2 -- --\n",
+ "│ │ └─ModuleList: 3-1 -- 2,097,536\n",
+ "│ │ └─ModuleList: 3-2 -- 2,097,536\n",
+ "│ │ └─ModuleList: 3-3 -- 198,016\n",
+ "│ │ └─ModuleList: 3-4 -- 2,097,536\n",
+ "│ │ └─ModuleList: 3-5 -- 2,097,536\n",
+ "│ │ └─ModuleList: 3-6 -- 198,016\n",
+ "├─EfficientNet: 1-1 [4, 1280, 1, 32] --\n",
+ "│ └─Sequential: 2-3 [4, 32, 28, 512] --\n",
+ "│ │ └─ZeroPad2d: 3-7 [4, 1, 57, 1025] --\n",
+ "│ │ └─Conv2d: 3-8 [4, 32, 28, 512] 288\n",
+ "│ │ └─BatchNorm2d: 3-9 [4, 32, 28, 512] 64\n",
+ "│ │ └─Mish: 3-10 [4, 32, 28, 512] --\n",
+ "│ └─ModuleList: 2-1 -- --\n",
+ "│ │ └─MBConvBlock: 3-11 [4, 16, 28, 512] 1,448\n",
+ "│ │ └─MBConvBlock: 3-12 [4, 24, 14, 256] 9,864\n",
+ "│ │ └─MBConvBlock: 3-13 [4, 24, 14, 256] 19,380\n",
+ "│ │ └─MBConvBlock: 3-14 [4, 40, 7, 128] 24,020\n",
+ "│ │ └─MBConvBlock: 3-15 [4, 40, 7, 128] 55,340\n",
+ "│ │ └─MBConvBlock: 3-16 [4, 80, 3, 64] 61,180\n",
+ "│ │ └─MBConvBlock: 3-17 [4, 80, 3, 64] 199,000\n",
+ "│ │ └─MBConvBlock: 3-18 [4, 80, 3, 64] 199,000\n",
+ "│ │ └─MBConvBlock: 3-19 [4, 112, 3, 64] 222,104\n",
+ "│ │ └─MBConvBlock: 3-20 [4, 112, 3, 64] 396,872\n",
+ "│ │ └─MBConvBlock: 3-21 [4, 112, 3, 64] 396,872\n",
+ "│ │ └─MBConvBlock: 3-22 [4, 192, 1, 32] 450,792\n",
+ "│ │ └─MBConvBlock: 3-23 [4, 192, 1, 32] 1,141,152\n",
+ "│ │ └─MBConvBlock: 3-24 [4, 192, 1, 32] 1,141,152\n",
+ "│ │ └─MBConvBlock: 3-25 [4, 192, 1, 32] 1,141,152\n",
+ "│ │ └─MBConvBlock: 3-26 [4, 320, 1, 32] 1,270,432\n",
+ "│ └─Sequential: 2-4 [4, 1280, 1, 32] --\n",
+ "│ │ └─Conv2d: 3-27 [4, 1280, 1, 32] 409,600\n",
+ "│ │ └─BatchNorm2d: 3-28 [4, 1280, 1, 32] 2,560\n",
+ "├─Sequential: 1-2 [4, 128, 32] --\n",
+ "│ └─Conv2d: 2-5 [4, 128, 1, 32] 163,968\n",
+ "│ └─PositionalEncoding2D: 2-6 [4, 128, 1, 32] --\n",
+ "│ └─Flatten: 2-7 [4, 128, 32] --\n",
+ "├─Embedding: 1-3 [4, 89, 128] 7,424\n",
+ "├─PositionalEncoding: 1-4 [4, 89, 128] --\n",
+ "│ └─Dropout: 2-8 [4, 89, 128] --\n",
+ "├─Decoder: 1-5 [4, 89, 128] --\n",
+ "├─Linear: 1-6 [4, 89, 58] 7,482\n",
+ "====================================================================================================\n",
+ "Total params: 16,107,322\n",
+ "Trainable params: 16,107,322\n",
+ "Non-trainable params: 0\n",
+ "Total mult-adds (G): 2.84\n",
+ "====================================================================================================\n",
+ "Input size (MB): 0.92\n",
+ "Forward/backward pass size (MB): 677.01\n",
+ "Params size (MB): 64.43\n",
+ "Estimated Total Size (MB): 742.36\n",
+ "===================================================================================================="
+ ]
+ },
+ "execution_count": 10,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "summary(net, ((4, 1, 56, 1024), (4, 89)), device=\"cpu\")"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3 (ipykernel)",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.9.7"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}
diff --git a/notebooks/04-efficientnet.ipynb b/notebooks/04-efficientnet.ipynb
new file mode 100644
index 0000000..4148e7d
--- /dev/null
+++ b/notebooks/04-efficientnet.ipynb
@@ -0,0 +1,279 @@
+{
+ "cells": [
+ {
+ "cell_type": "code",
+ "execution_count": 2,
+ "id": "7c02ae76-b540-4b16-9492-e9210b3b9249",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import os\n",
+ "os.environ['CUDA_VISIBLE_DEVICE'] = ''\n",
+ "import random\n",
+ "\n",
+ "%matplotlib inline\n",
+ "import matplotlib.pyplot as plt\n",
+ "\n",
+ "import numpy as np\n",
+ "from omegaconf import OmegaConf\n",
+ "\n",
+ "%load_ext autoreload\n",
+ "%autoreload 2\n",
+ "\n",
+ "from importlib.util import find_spec\n",
+ "if find_spec(\"text_recognizer\") is None:\n",
+ " import sys\n",
+ " sys.path.append('..')"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 3,
+ "id": "ccdb6dde-47e5-429a-88f2-0764fb7e259a",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from hydra import compose, initialize\n",
+ "from omegaconf import OmegaConf\n",
+ "from hydra.utils import instantiate"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 4,
+ "id": "3cf50475-39f2-4642-a7d1-5bcbc0a036f7",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "path = \"../training/conf/network/encoder/efficientnet.yaml\""
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 5,
+ "id": "e52ecb01-c975-4e55-925d-1182c7aea473",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "with open(path, \"rb\") as f:\n",
+ " cfg = OmegaConf.load(f)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 6,
+ "id": "f939aa37-7b1d-45cc-885c-323c4540bda1",
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "{'_target_': 'text_recognizer.networks.encoders.efficientnet.EfficientNet', 'arch': 'b0', 'out_channels': 1280, 'stochastic_dropout_rate': 0.2, 'bn_momentum': 0.99, 'bn_eps': 0.001}"
+ ]
+ },
+ "execution_count": 6,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "cfg"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 7,
+ "id": "aaeab329-aeb0-4a1b-aa35-5a2aab81b1d0",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "net = instantiate(cfg)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 8,
+ "id": "618b997c-e6a6-4487-b70c-9d260cb556d3",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from torchinfo import summary"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 9,
+ "id": "25759b7b-8deb-4163-b75d-a1357c9fe88f",
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "==========================================================================================\n",
+ "Layer (type:depth-idx) Output Shape Param #\n",
+ "==========================================================================================\n",
+ "EfficientNet -- --\n",
+ "├─ModuleList: 1-1 -- --\n",
+ "├─Sequential: 1-2 [2, 32, 288, 320] --\n",
+ "│ └─ZeroPad2d: 2-1 [2, 1, 577, 641] --\n",
+ "│ └─Conv2d: 2-2 [2, 32, 288, 320] 288\n",
+ "│ └─BatchNorm2d: 2-3 [2, 32, 288, 320] 64\n",
+ "│ └─Mish: 2-4 [2, 32, 288, 320] --\n",
+ "├─ModuleList: 1-1 -- --\n",
+ "│ └─MBConvBlock: 2-5 [2, 16, 288, 320] --\n",
+ "│ │ └─Sequential: 3-1 [2, 32, 288, 320] 352\n",
+ "│ │ └─Sequential: 3-2 [2, 32, 288, 320] 552\n",
+ "│ │ └─Sequential: 3-3 [2, 16, 288, 320] 544\n",
+ "│ └─MBConvBlock: 2-6 [2, 24, 144, 160] --\n",
+ "│ │ └─Sequential: 3-4 [2, 96, 288, 320] 1,728\n",
+ "│ │ └─Sequential: 3-5 [2, 96, 144, 160] 1,056\n",
+ "│ │ └─Sequential: 3-6 [2, 96, 144, 160] 4,728\n",
+ "│ │ └─Sequential: 3-7 [2, 24, 144, 160] 2,352\n",
+ "│ └─MBConvBlock: 2-7 [2, 24, 144, 160] --\n",
+ "│ │ └─Sequential: 3-8 [2, 144, 144, 160] 3,744\n",
+ "│ │ └─Sequential: 3-9 [2, 144, 144, 160] 1,584\n",
+ "│ │ └─Sequential: 3-10 [2, 144, 144, 160] 10,548\n",
+ "│ │ └─Sequential: 3-11 [2, 24, 144, 160] 3,504\n",
+ "│ └─MBConvBlock: 2-8 [2, 40, 72, 80] --\n",
+ "│ │ └─Sequential: 3-12 [2, 144, 144, 160] 3,744\n",
+ "│ │ └─Sequential: 3-13 [2, 144, 72, 80] 3,888\n",
+ "│ │ └─Sequential: 3-14 [2, 144, 72, 80] 10,548\n",
+ "│ │ └─Sequential: 3-15 [2, 40, 72, 80] 5,840\n",
+ "│ └─MBConvBlock: 2-9 [2, 40, 72, 80] --\n",
+ "│ │ └─Sequential: 3-16 [2, 240, 72, 80] 10,080\n",
+ "│ │ └─Sequential: 3-17 [2, 240, 72, 80] 6,480\n",
+ "│ │ └─Sequential: 3-18 [2, 240, 72, 80] 29,100\n",
+ "│ │ └─Sequential: 3-19 [2, 40, 72, 80] 9,680\n",
+ "│ └─MBConvBlock: 2-10 [2, 80, 36, 40] --\n",
+ "│ │ └─Sequential: 3-20 [2, 240, 72, 80] 10,080\n",
+ "│ │ └─Sequential: 3-21 [2, 240, 36, 40] 2,640\n",
+ "│ │ └─Sequential: 3-22 [2, 240, 36, 40] 29,100\n",
+ "│ │ └─Sequential: 3-23 [2, 80, 36, 40] 19,360\n",
+ "│ └─MBConvBlock: 2-11 [2, 80, 36, 40] --\n",
+ "│ │ └─Sequential: 3-24 [2, 480, 36, 40] 39,360\n",
+ "│ │ └─Sequential: 3-25 [2, 480, 36, 40] 5,280\n",
+ "│ │ └─Sequential: 3-26 [2, 480, 36, 40] 115,800\n",
+ "│ │ └─Sequential: 3-27 [2, 80, 36, 40] 38,560\n",
+ "│ └─MBConvBlock: 2-12 [2, 80, 36, 40] --\n",
+ "│ │ └─Sequential: 3-28 [2, 480, 36, 40] 39,360\n",
+ "│ │ └─Sequential: 3-29 [2, 480, 36, 40] 5,280\n",
+ "│ │ └─Sequential: 3-30 [2, 480, 36, 40] 115,800\n",
+ "│ │ └─Sequential: 3-31 [2, 80, 36, 40] 38,560\n",
+ "│ └─MBConvBlock: 2-13 [2, 112, 36, 40] --\n",
+ "│ │ └─Sequential: 3-32 [2, 480, 36, 40] 39,360\n",
+ "│ │ └─Sequential: 3-33 [2, 480, 36, 40] 12,960\n",
+ "│ │ └─Sequential: 3-34 [2, 480, 36, 40] 115,800\n",
+ "│ │ └─Sequential: 3-35 [2, 112, 36, 40] 53,984\n",
+ "│ └─MBConvBlock: 2-14 [2, 112, 36, 40] --\n",
+ "│ │ └─Sequential: 3-36 [2, 672, 36, 40] 76,608\n",
+ "│ │ └─Sequential: 3-37 [2, 672, 36, 40] 18,144\n",
+ "│ │ └─Sequential: 3-38 [2, 672, 36, 40] 226,632\n",
+ "│ │ └─Sequential: 3-39 [2, 112, 36, 40] 75,488\n",
+ "│ └─MBConvBlock: 2-15 [2, 112, 36, 40] --\n",
+ "│ │ └─Sequential: 3-40 [2, 672, 36, 40] 76,608\n",
+ "│ │ └─Sequential: 3-41 [2, 672, 36, 40] 18,144\n",
+ "│ │ └─Sequential: 3-42 [2, 672, 36, 40] 226,632\n",
+ "│ │ └─Sequential: 3-43 [2, 112, 36, 40] 75,488\n",
+ "│ └─MBConvBlock: 2-16 [2, 192, 18, 20] --\n",
+ "│ │ └─Sequential: 3-44 [2, 672, 36, 40] 76,608\n",
+ "│ │ └─Sequential: 3-45 [2, 672, 18, 20] 18,144\n",
+ "│ │ └─Sequential: 3-46 [2, 672, 18, 20] 226,632\n",
+ "│ │ └─Sequential: 3-47 [2, 192, 18, 20] 129,408\n",
+ "│ └─MBConvBlock: 2-17 [2, 192, 18, 20] --\n",
+ "│ │ └─Sequential: 3-48 [2, 1152, 18, 20] 223,488\n",
+ "│ │ └─Sequential: 3-49 [2, 1152, 18, 20] 31,104\n",
+ "│ │ └─Sequential: 3-50 [2, 1152, 18, 20] 664,992\n",
+ "│ │ └─Sequential: 3-51 [2, 192, 18, 20] 221,568\n",
+ "│ └─MBConvBlock: 2-18 [2, 192, 18, 20] --\n",
+ "│ │ └─Sequential: 3-52 [2, 1152, 18, 20] 223,488\n",
+ "│ │ └─Sequential: 3-53 [2, 1152, 18, 20] 31,104\n",
+ "│ │ └─Sequential: 3-54 [2, 1152, 18, 20] 664,992\n",
+ "│ │ └─Sequential: 3-55 [2, 192, 18, 20] 221,568\n",
+ "│ └─MBConvBlock: 2-19 [2, 192, 18, 20] --\n",
+ "│ │ └─Sequential: 3-56 [2, 1152, 18, 20] 223,488\n",
+ "│ │ └─Sequential: 3-57 [2, 1152, 18, 20] 31,104\n",
+ "│ │ └─Sequential: 3-58 [2, 1152, 18, 20] 664,992\n",
+ "│ │ └─Sequential: 3-59 [2, 192, 18, 20] 221,568\n",
+ "│ └─MBConvBlock: 2-20 [2, 320, 18, 20] --\n",
+ "│ │ └─Sequential: 3-60 [2, 1152, 18, 20] 223,488\n",
+ "│ │ └─Sequential: 3-61 [2, 1152, 18, 20] 12,672\n",
+ "│ │ └─Sequential: 3-62 [2, 1152, 18, 20] 664,992\n",
+ "│ │ └─Sequential: 3-63 [2, 320, 18, 20] 369,280\n",
+ "├─Sequential: 1-3 [2, 1280, 18, 20] --\n",
+ "│ └─Conv2d: 2-21 [2, 1280, 18, 20] 409,600\n",
+ "│ └─BatchNorm2d: 2-22 [2, 1280, 18, 20] 2,560\n",
+ "==========================================================================================\n",
+ "Total params: 7,142,272\n",
+ "Trainable params: 7,142,272\n",
+ "Non-trainable params: 0\n",
+ "Total mult-adds (G): 11.27\n",
+ "==========================================================================================\n",
+ "Input size (MB): 2.95\n",
+ "Forward/backward pass size (MB): 1922.96\n",
+ "Params size (MB): 28.57\n",
+ "Estimated Total Size (MB): 1954.48\n",
+ "=========================================================================================="
+ ]
+ },
+ "execution_count": 9,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "summary(net, (2, 1, 576, 640), device=\"cpu\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 11,
+ "id": "3ef95a63-7044-45bf-a085-faf5ea0c03ec",
+ "metadata": {},
+ "outputs": [
+ {
+ "ename": "TypeError",
+ "evalue": "'EfficientNet' object is not subscriptable",
+ "output_type": "error",
+ "traceback": [
+ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+ "\u001b[0;31mTypeError\u001b[0m Traceback (most recent call last)",
+ "\u001b[0;32m/tmp/ipykernel_2800/4064962505.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mnet\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
+ "\u001b[0;31mTypeError\u001b[0m: 'EfficientNet' object is not subscriptable"
+ ]
+ }
+ ],
+ "source": [
+ "net[:-2]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "62ca0d97-625c-474b-8d6c-d0caba79e198",
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3 (ipykernel)",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.9.7"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}
diff --git a/notebooks/04-vq-transformer.ipynb b/notebooks/04-vq-transformer.ipynb
new file mode 100644
index 0000000..69d2688
--- /dev/null
+++ b/notebooks/04-vq-transformer.ipynb
@@ -0,0 +1,253 @@
+{
+ "cells": [
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "7c02ae76-b540-4b16-9492-e9210b3b9249",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import os\n",
+ "os.environ['CUDA_VISIBLE_DEVICE'] = ''\n",
+ "import random\n",
+ "\n",
+ "%matplotlib inline\n",
+ "import matplotlib.pyplot as plt\n",
+ "\n",
+ "import numpy as np\n",
+ "from omegaconf import OmegaConf\n",
+ "\n",
+ "%load_ext autoreload\n",
+ "%autoreload 2\n",
+ "\n",
+ "from importlib.util import find_spec\n",
+ "if find_spec(\"text_recognizer\") is None:\n",
+ " import sys\n",
+ " sys.path.append('..')"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "ccdb6dde-47e5-429a-88f2-0764fb7e259a",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from hydra import compose, initialize\n",
+ "from omegaconf import OmegaConf\n",
+ "from hydra.utils import instantiate"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "3cf50475-39f2-4642-a7d1-5bcbc0a036f7",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "path = \"../training/conf/experiment/vqgan_htr_char_iam_lines.yaml\""
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "e52ecb01-c975-4e55-925d-1182c7aea473",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "with open(path, \"rb\") as f:\n",
+ " cfg = OmegaConf.load(f)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "f939aa37-7b1d-45cc-885c-323c4540bda1",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "cfg"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "aaeab329-aeb0-4a1b-aa35-5a2aab81b1d0",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "net = instantiate(cfg.network)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "a564ac7a-b67f-4bc1-af36-0fe0a58c1bc9",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import torch"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "aeddcc5c-e48d-4d90-8efa-963011ef40bc",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "x = torch.randn((16, 1, 16, 64))\n",
+ "y = torch.randint(0, 56, (16, 89))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "0f0d78bc-7e0a-4d06-8e38-49b29ad25933",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "y.shape"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "e9f4ee2a-c93f-4461-8d75-40c8c12d9d48",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "yy = net(x, y)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "7a7493a9-0e1d-46ef-8180-27605e18d082",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "yy[0].shape"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "75bc9695-2afd-455c-a4fb-2e182456ccbd",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "z = torch.randn((16, 8, 32))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "3df6f9a0-6e66-4f46-a5b7-c0bb71b16b9b",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "z, _ = net.encode(x)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "6d6e9dd1-c56e-4169-8216-bcc84ea980e3",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "z.shape"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "8f1539cb-b9b2-40b7-a843-d7479ddbddd7",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "yy = net.decode(z, y[:, :2])"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "5cdba0a9-da7d-4e33-b209-7f360d1a38e5",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "yy.shape"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "6da8065f-f93f-4aec-a60e-408712a28c3b",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "torch.argmax(yy,dim=-2).shape"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "beabbda7-6a1f-4294-8f01-f9d866ffe088",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "yy[0].shape"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "618b997c-e6a6-4487-b70c-9d260cb556d3",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from torchinfo import summary"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "25759b7b-8deb-4163-b75d-a1357c9fe88f",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "summary(net, (1, 1, 576, 640), device=\"cpu\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "62ca0d97-625c-474b-8d6c-d0caba79e198",
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3 (ipykernel)",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.9.7"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}
diff --git a/notebooks/04-vqvae.ipynb b/notebooks/04-vqvae.ipynb
new file mode 100644
index 0000000..1b31671
--- /dev/null
+++ b/notebooks/04-vqvae.ipynb
@@ -0,0 +1,233 @@
+{
+ "cells": [
+ {
+ "cell_type": "code",
+ "execution_count": 7,
+ "id": "136a80f5-10e1-40c4-973a-a7eb7939bb1f",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "The autoreload extension is already loaded. To reload it, use:\n",
+ " %reload_ext autoreload\n"
+ ]
+ }
+ ],
+ "source": [
+ "import os\n",
+ "os.environ['CUDA_VISIBLE_DEVICE'] = ''\n",
+ "import random\n",
+ "\n",
+ "%matplotlib inline\n",
+ "import matplotlib.pyplot as plt\n",
+ "\n",
+ "import numpy as np\n",
+ "from omegaconf import OmegaConf\n",
+ "from hydra import compose, initialize\n",
+ "from omegaconf import OmegaConf\n",
+ "from hydra.utils import instantiate\n",
+ "from torchinfo import summary\n",
+ "%load_ext autoreload\n",
+ "%autoreload 2\n",
+ "\n",
+ "from importlib.util import find_spec\n",
+ "if find_spec(\"text_recognizer\") is None:\n",
+ " import sys\n",
+ " sys.path.append('..')"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 2,
+ "id": "1a0fb9ca-1886-4fd4-839f-dc111a450cfd",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "path = \"../training/conf/network/vqvae.yaml\""
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 5,
+ "id": "0182a614-5781-44a6-b659-008e7c584fa7",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "encoder:\n",
+ " _target_: text_recognizer.networks.vqvae.encoder.Encoder\n",
+ " in_channels: 1\n",
+ " hidden_dim: 32\n",
+ " channels_multipliers:\n",
+ " - 1\n",
+ " - 2\n",
+ " - 4\n",
+ " dropout_rate: 0.0\n",
+ " activation: mish\n",
+ " use_norm: true\n",
+ " num_residuals: 4\n",
+ " residual_channels: 32\n",
+ "decoder:\n",
+ " _target_: text_recognizer.networks.vqvae.decoder.Decoder\n",
+ " out_channels: 1\n",
+ " hidden_dim: 32\n",
+ " channels_multipliers:\n",
+ " - 4\n",
+ " - 2\n",
+ " - 1\n",
+ " dropout_rate: 0.0\n",
+ " activation: mish\n",
+ " use_norm: true\n",
+ " num_residuals: 4\n",
+ " residual_channels: 32\n",
+ "_target_: text_recognizer.networks.vqvae.vqvae.VQVAE\n",
+ "hidden_dim: 128\n",
+ "embedding_dim: 32\n",
+ "num_embeddings: 8192\n",
+ "decay: 0.99\n",
+ "\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "/home/aktersnurra/.cache/pypoetry/virtualenvs/text-recognizer-ejNaVa9M-py3.9/lib/python3.9/site-packages/hydra/_internal/defaults_list.py:251: UserWarning: In 'vqvae': Defaults list is missing `_self_`. See https://hydra.cc/docs/upgrades/1.0_to_1.1/default_composition_order for more information\n",
+ " warnings.warn(msg, UserWarning)\n"
+ ]
+ }
+ ],
+ "source": [
+ "with initialize(config_path=\"../training/conf/network/\", job_name=\"test_app\"):\n",
+ " cfg = compose(config_name=\"vqvae\")\n",
+ " print(OmegaConf.to_yaml(cfg))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 6,
+ "id": "a500f94c-7dae-477e-a3fb-2a2d62ee7b72",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "net = instantiate(cfg)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 9,
+ "id": "7f3b3559-5e23-485e-bf57-9405568a1fbf",
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "====================================================================================================\n",
+ "Layer (type:depth-idx) Output Shape Param #\n",
+ "====================================================================================================\n",
+ "VQVAE -- --\n",
+ "├─Encoder: 1-1 [1, 128, 72, 80] --\n",
+ "│ └─Sequential: 2-1 [1, 128, 72, 80] --\n",
+ "│ │ └─Conv2d: 3-1 [1, 32, 576, 640] 320\n",
+ "│ │ └─Normalize: 3-2 [1, 32, 576, 640] 64\n",
+ "│ │ └─Mish: 3-3 [1, 32, 576, 640] --\n",
+ "│ │ └─Mish: 3-4 [1, 32, 576, 640] --\n",
+ "│ │ └─Mish: 3-5 [1, 32, 576, 640] --\n",
+ "│ │ └─Conv2d: 3-6 [1, 32, 288, 320] 16,416\n",
+ "│ │ └─Normalize: 3-7 [1, 32, 288, 320] 64\n",
+ "│ │ └─Mish: 3-8 [1, 32, 288, 320] --\n",
+ "│ │ └─Mish: 3-9 [1, 32, 288, 320] --\n",
+ "│ │ └─Mish: 3-10 [1, 32, 288, 320] --\n",
+ "│ │ └─Conv2d: 3-11 [1, 64, 144, 160] 32,832\n",
+ "│ │ └─Normalize: 3-12 [1, 64, 144, 160] 128\n",
+ "│ │ └─Mish: 3-13 [1, 64, 144, 160] --\n",
+ "│ │ └─Mish: 3-14 [1, 64, 144, 160] --\n",
+ "│ │ └─Mish: 3-15 [1, 64, 144, 160] --\n",
+ "│ │ └─Conv2d: 3-16 [1, 128, 72, 80] 131,200\n",
+ "│ │ └─Residual: 3-17 [1, 128, 72, 80] 41,280\n",
+ "│ │ └─Residual: 3-18 [1, 128, 72, 80] 41,280\n",
+ "│ │ └─Residual: 3-19 [1, 128, 72, 80] 41,280\n",
+ "│ │ └─Residual: 3-20 [1, 128, 72, 80] 41,280\n",
+ "├─Conv2d: 1-2 [1, 32, 72, 80] 4,128\n",
+ "├─VectorQuantizer: 1-3 [1, 32, 72, 80] --\n",
+ "├─Conv2d: 1-4 [1, 128, 72, 80] 4,224\n",
+ "├─Decoder: 1-5 [1, 1, 576, 640] --\n",
+ "│ └─Sequential: 2-2 [1, 1, 576, 640] --\n",
+ "│ │ └─Residual: 3-21 [1, 128, 72, 80] 41,280\n",
+ "│ │ └─Residual: 3-22 [1, 128, 72, 80] 41,280\n",
+ "│ │ └─Residual: 3-23 [1, 128, 72, 80] 41,280\n",
+ "│ │ └─Residual: 3-24 [1, 128, 72, 80] 41,280\n",
+ "│ │ └─Normalize: 3-25 [1, 128, 72, 80] 256\n",
+ "│ │ └─Mish: 3-26 [1, 128, 72, 80] --\n",
+ "│ │ └─Mish: 3-27 [1, 128, 72, 80] --\n",
+ "│ │ └─Mish: 3-28 [1, 128, 72, 80] --\n",
+ "│ │ └─ConvTranspose2d: 3-29 [1, 64, 144, 160] 131,136\n",
+ "│ │ └─Normalize: 3-30 [1, 64, 144, 160] 128\n",
+ "│ │ └─Mish: 3-31 [1, 64, 144, 160] --\n",
+ "│ │ └─Mish: 3-32 [1, 64, 144, 160] --\n",
+ "│ │ └─Mish: 3-33 [1, 64, 144, 160] --\n",
+ "│ │ └─ConvTranspose2d: 3-34 [1, 32, 288, 320] 32,800\n",
+ "│ │ └─Normalize: 3-35 [1, 32, 288, 320] 64\n",
+ "│ │ └─Mish: 3-36 [1, 32, 288, 320] --\n",
+ "│ │ └─Mish: 3-37 [1, 32, 288, 320] --\n",
+ "│ │ └─Mish: 3-38 [1, 32, 288, 320] --\n",
+ "│ │ └─ConvTranspose2d: 3-39 [1, 32, 576, 640] 16,416\n",
+ "│ │ └─Normalize: 3-40 [1, 32, 576, 640] 64\n",
+ "│ │ └─Conv2d: 3-41 [1, 1, 576, 640] 289\n",
+ "====================================================================================================\n",
+ "Total params: 700,769\n",
+ "Trainable params: 700,769\n",
+ "Non-trainable params: 0\n",
+ "Total mult-adds (G): 17.28\n",
+ "====================================================================================================\n",
+ "Input size (MB): 1.47\n",
+ "Forward/backward pass size (MB): 659.13\n",
+ "Params size (MB): 2.80\n",
+ "Estimated Total Size (MB): 663.41\n",
+ "===================================================================================================="
+ ]
+ },
+ "execution_count": 9,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "summary(net, (1, 1, 576, 640), device=\"cpu\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "9f880b03-d641-4640-acd3-aa5666ca5184",
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3 (ipykernel)",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.9.7"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}
generated by cgit v1.2.3-70-g09d2 (git 2.46.0) at 2024-12-25 18:16:16 +0000