summaryrefslogtreecommitdiff
path: root/notebooks/04-efficientnet-transformer.ipynb
diff options
context:
space:
mode:
Diffstat (limited to 'notebooks/04-efficientnet-transformer.ipynb')
-rw-r--r--notebooks/04-efficientnet-transformer.ipynb173
1 files changed, 106 insertions, 67 deletions
diff --git a/notebooks/04-efficientnet-transformer.ipynb b/notebooks/04-efficientnet-transformer.ipynb
index 427c98c..3ee1cb2 100644
--- a/notebooks/04-efficientnet-transformer.ipynb
+++ b/notebooks/04-efficientnet-transformer.ipynb
@@ -2,7 +2,7 @@
"cells": [
{
"cell_type": "code",
- "execution_count": 2,
+ "execution_count": 1,
"id": "7c02ae76-b540-4b16-9492-e9210b3b9249",
"metadata": {},
"outputs": [],
@@ -28,7 +28,7 @@
},
{
"cell_type": "code",
- "execution_count": 3,
+ "execution_count": 2,
"id": "ccdb6dde-47e5-429a-88f2-0764fb7e259a",
"metadata": {},
"outputs": [],
@@ -40,7 +40,7 @@
},
{
"cell_type": "code",
- "execution_count": 4,
+ "execution_count": 3,
"id": "3cf50475-39f2-4642-a7d1-5bcbc0a036f7",
"metadata": {},
"outputs": [],
@@ -50,7 +50,7 @@
},
{
"cell_type": "code",
- "execution_count": 5,
+ "execution_count": 7,
"id": "e52ecb01-c975-4e55-925d-1182c7aea473",
"metadata": {},
"outputs": [],
@@ -61,17 +61,17 @@
},
{
"cell_type": "code",
- "execution_count": 6,
+ "execution_count": 8,
"id": "f939aa37-7b1d-45cc-885c-323c4540bda1",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
- "{'defaults': [{'override /mapping': None}, {'override /criterion': None}, {'override /datamodule': None}, {'override /network': None}, {'override /model': None}, {'override /lr_schedulers': None}, {'override /optimizers': None}], 'criterion': {'_target_': 'torch.nn.CrossEntropyLoss', 'ignore_index': 3}, 'mapping': {'_target_': 'text_recognizer.data.emnist_mapping.EmnistMapping'}, 'optimizers': {'madgrad': {'_target_': 'madgrad.MADGRAD', 'lr': 0.0001, 'momentum': 0.9, 'weight_decay': 0, 'eps': 1e-06, 'parameters': 'network'}}, 'lr_schedulers': {'network': {'_target_': 'torch.optim.lr_scheduler.CosineAnnealingLR', 'T_max': 1024, 'eta_min': 4.5e-06, 'last_epoch': -1, 'interval': 'epoch', 'monitor': 'val/loss'}}, 'datamodule': {'_target_': 'text_recognizer.data.iam_lines.IAMLines', 'batch_size': 8, 'num_workers': 12, 'train_fraction': 0.8, 'augment': False, 'pin_memory': False}, 'network': {'_target_': 'text_recognizer.networks.conv_transformer.ConvTransformer', 'input_dims': [1, 56, 1024], 'hidden_dim': 128, 'encoder_dim': 1280, 'dropout_rate': 0.2, 'num_classes': 58, 'pad_index': 3, 'encoder': {'_target_': 'text_recognizer.networks.encoders.efficientnet.EfficientNet', 'arch': 'b0', 'out_channels': 1280, 'stochastic_dropout_rate': 0.2, 'bn_momentum': 0.99, 'bn_eps': 0.001}, 'decoder': {'_target_': 'text_recognizer.networks.transformer.Decoder', 'dim': 128, 'depth': 2, 'num_heads': 4, 'attn_fn': 'text_recognizer.networks.transformer.attention.Attention', 'attn_kwargs': {'dim_head': 32, 'dropout_rate': 0.2}, 'norm_fn': 'torch.nn.LayerNorm', 'ff_fn': 'text_recognizer.networks.transformer.mlp.FeedForward', 'ff_kwargs': {'dim_out': None, 'expansion_factor': 4, 'glu': True, 'dropout_rate': 0.2}, 'cross_attend': True, 'pre_norm': True, 'rotary_emb': None}}, 'model': {'_target_': 'text_recognizer.models.transformer.TransformerLitModel', 'max_output_len': 89, 'start_token': '<s>', 'end_token': '<e>', 'pad_token': '<p>'}, 'trainer': {'_target_': 'pytorch_lightning.Trainer', 'stochastic_weight_avg': False, 'auto_scale_batch_size': 'binsearch', 'auto_lr_find': False, 'gradient_clip_val': 0, 'fast_dev_run': False, 'gpus': 1, 'precision': 16, 'max_epochs': 1024, 'terminate_on_nan': True, 'weights_summary': 'top', 'limit_train_batches': 1.0, 'limit_val_batches': 1.0, 'limit_test_batches': 1.0, 'resume_from_checkpoint': None, 'accumulate_grad_batches': 4}}"
+ "{'defaults': [{'override /mapping': None}, {'override /criterion': None}, {'override /datamodule': None}, {'override /network': None}, {'override /model': None}, {'override /lr_schedulers': None}, {'override /optimizers': None}], 'criterion': {'_target_': 'torch.nn.CrossEntropyLoss', 'ignore_index': 3}, 'mapping': {'_target_': 'text_recognizer.data.emnist_mapping.EmnistMapping'}, 'callbacks': {'stochastic_weight_averaging': {'_target_': 'pytorch_lightning.callbacks.StochasticWeightAveraging', 'swa_epoch_start': 0.8, 'swa_lrs': 0.05, 'annealing_epochs': 10, 'annealing_strategy': 'cos', 'device': None}}, 'optimizers': {'madgrad': {'_target_': 'madgrad.MADGRAD', 'lr': 0.0001, 'momentum': 0.9, 'weight_decay': 0, 'eps': 1e-06, 'parameters': 'network'}}, 'lr_schedulers': {'network': {'_target_': 'torch.optim.lr_scheduler.CosineAnnealingLR', 'T_max': 1024, 'eta_min': 4.5e-06, 'last_epoch': -1, 'interval': 'epoch', 'monitor': 'val/loss'}}, 'datamodule': {'_target_': 'text_recognizer.data.iam_lines.IAMLines', 'batch_size': 24, 'num_workers': 12, 'train_fraction': 0.8, 'augment': True, 'pin_memory': False}, 'network': {'_target_': 'text_recognizer.networks.conv_transformer.ConvTransformer', 'input_dims': [1, 56, 1024], 'hidden_dim': 128, 'encoder_dim': 1280, 'dropout_rate': 0.2, 'num_classes': 58, 'pad_index': 3, 'encoder': {'_target_': 'text_recognizer.networks.encoders.efficientnet.EfficientNet', 'arch': 'b0', 'out_channels': 1280, 'stochastic_dropout_rate': 0.2, 'bn_momentum': 0.99, 'bn_eps': 0.001}, 'decoder': {'_target_': 'text_recognizer.networks.transformer.Decoder', 'dim': 128, 'depth': 3, 'num_heads': 4, 'attn_fn': 'text_recognizer.networks.transformer.attention.Attention', 'attn_kwargs': {'dim_head': 32, 'dropout_rate': 0.2}, 'norm_fn': 'torch.nn.LayerNorm', 'ff_fn': 'text_recognizer.networks.transformer.mlp.FeedForward', 'ff_kwargs': {'dim_out': None, 'expansion_factor': 4, 'glu': True, 'dropout_rate': 0.2}, 'cross_attend': True, 'pre_norm': True, 'rotary_emb': None}}, 'model': {'_target_': 'text_recognizer.models.transformer.TransformerLitModel', 'max_output_len': 89, 'start_token': '<s>', 'end_token': '<e>', 'pad_token': '<p>'}, 'trainer': {'_target_': 'pytorch_lightning.Trainer', 'stochastic_weight_avg': False, 'auto_scale_batch_size': 'binsearch', 'auto_lr_find': False, 'gradient_clip_val': 0, 'fast_dev_run': False, 'gpus': 1, 'precision': 16, 'max_epochs': 1024, 'terminate_on_nan': True, 'weights_summary': 'top', 'limit_train_batches': 1.0, 'limit_val_batches': 1.0, 'limit_test_batches': 1.0, 'resume_from_checkpoint': None, 'accumulate_grad_batches': 4, 'overfit_batches': 0.0}, 'summary': [[1, 1, 56, 1024], [1, 89]]}"
]
},
- "execution_count": 6,
+ "execution_count": 8,
"metadata": {},
"output_type": "execute_result"
}
@@ -82,7 +82,49 @@
},
{
"cell_type": "code",
- "execution_count": 7,
+ "execution_count": 23,
+ "id": "2fdd47e6-aac0-47e9-b224-f46c1f015f17",
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "[[1, 1, 56, 1024], [1, 89]]"
+ ]
+ },
+ "execution_count": 23,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "list(map(lambda x: list(x), cfg.summary))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 17,
+ "id": "92405411-f944-4519-9a6d-13b0761c2de6",
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "list"
+ ]
+ },
+ "execution_count": 17,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "type(list(cfg.summary))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 11,
"id": "aaeab329-aeb0-4a1b-aa35-5a2aab81b1d0",
"metadata": {},
"outputs": [],
@@ -92,7 +134,7 @@
},
{
"cell_type": "code",
- "execution_count": 9,
+ "execution_count": 12,
"id": "618b997c-e6a6-4487-b70c-9d260cb556d3",
"metadata": {},
"outputs": [],
@@ -102,25 +144,11 @@
},
{
"cell_type": "code",
- "execution_count": 10,
+ "execution_count": 24,
"id": "25759b7b-8deb-4163-b75d-a1357c9fe88f",
"metadata": {},
"outputs": [
{
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "torch.Size([4, 4, 89, 1024])\n",
- "torch.Size([4, 4, 89, 1024])\n",
- "torch.Size([4, 4, 89, 1024])\n",
- "torch.Size([4, 4, 32, 1024])\n",
- "torch.Size([4, 4, 89, 1024])\n",
- "torch.Size([4, 4, 89, 1024])\n",
- "torch.Size([4, 4, 89, 1024])\n",
- "torch.Size([4, 4, 32, 1024])\n"
- ]
- },
- {
"data": {
"text/plain": [
"====================================================================================================\n",
@@ -137,62 +165,73 @@
"│ │ └─ModuleList: 3-4 -- 2,097,536\n",
"│ │ └─ModuleList: 3-5 -- 2,097,536\n",
"│ │ └─ModuleList: 3-6 -- 198,016\n",
- "├─EfficientNet: 1-1 [4, 1280, 1, 32] --\n",
- "│ └─Sequential: 2-3 [4, 32, 28, 512] --\n",
- "│ │ └─ZeroPad2d: 3-7 [4, 1, 57, 1025] --\n",
- "│ │ └─Conv2d: 3-8 [4, 32, 28, 512] 288\n",
- "│ │ └─BatchNorm2d: 3-9 [4, 32, 28, 512] 64\n",
- "│ │ └─Mish: 3-10 [4, 32, 28, 512] --\n",
+ "│ │ └─ModuleList: 3-7 -- 2,097,536\n",
+ "│ │ └─ModuleList: 3-8 -- 2,097,536\n",
+ "│ │ └─ModuleList: 3-9 -- 198,016\n",
+ "├─EfficientNet: 1-1 [1, 1280, 1, 32] --\n",
+ "│ └─Sequential: 2-3 [1, 32, 28, 512] --\n",
+ "│ │ └─ZeroPad2d: 3-10 [1, 1, 57, 1025] --\n",
+ "│ │ └─Conv2d: 3-11 [1, 32, 28, 512] 288\n",
+ "│ │ └─BatchNorm2d: 3-12 [1, 32, 28, 512] 64\n",
+ "│ │ └─Mish: 3-13 [1, 32, 28, 512] --\n",
"│ └─ModuleList: 2-1 -- --\n",
- "│ │ └─MBConvBlock: 3-11 [4, 16, 28, 512] 1,448\n",
- "│ │ └─MBConvBlock: 3-12 [4, 24, 14, 256] 9,864\n",
- "│ │ └─MBConvBlock: 3-13 [4, 24, 14, 256] 19,380\n",
- "│ │ └─MBConvBlock: 3-14 [4, 40, 7, 128] 24,020\n",
- "│ │ └─MBConvBlock: 3-15 [4, 40, 7, 128] 55,340\n",
- "│ │ └─MBConvBlock: 3-16 [4, 80, 3, 64] 61,180\n",
- "│ │ └─MBConvBlock: 3-17 [4, 80, 3, 64] 199,000\n",
- "│ │ └─MBConvBlock: 3-18 [4, 80, 3, 64] 199,000\n",
- "│ │ └─MBConvBlock: 3-19 [4, 112, 3, 64] 222,104\n",
- "│ │ └─MBConvBlock: 3-20 [4, 112, 3, 64] 396,872\n",
- "│ │ └─MBConvBlock: 3-21 [4, 112, 3, 64] 396,872\n",
- "│ │ └─MBConvBlock: 3-22 [4, 192, 1, 32] 450,792\n",
- "│ │ └─MBConvBlock: 3-23 [4, 192, 1, 32] 1,141,152\n",
- "│ │ └─MBConvBlock: 3-24 [4, 192, 1, 32] 1,141,152\n",
- "│ │ └─MBConvBlock: 3-25 [4, 192, 1, 32] 1,141,152\n",
- "│ │ └─MBConvBlock: 3-26 [4, 320, 1, 32] 1,270,432\n",
- "│ └─Sequential: 2-4 [4, 1280, 1, 32] --\n",
- "│ │ └─Conv2d: 3-27 [4, 1280, 1, 32] 409,600\n",
- "│ │ └─BatchNorm2d: 3-28 [4, 1280, 1, 32] 2,560\n",
- "├─Sequential: 1-2 [4, 128, 32] --\n",
- "│ └─Conv2d: 2-5 [4, 128, 1, 32] 163,968\n",
- "│ └─PositionalEncoding2D: 2-6 [4, 128, 1, 32] --\n",
- "│ └─Flatten: 2-7 [4, 128, 32] --\n",
- "├─Embedding: 1-3 [4, 89, 128] 7,424\n",
- "├─PositionalEncoding: 1-4 [4, 89, 128] --\n",
- "│ └─Dropout: 2-8 [4, 89, 128] --\n",
- "├─Decoder: 1-5 [4, 89, 128] --\n",
- "├─Linear: 1-6 [4, 89, 58] 7,482\n",
+ "│ │ └─MBConvBlock: 3-14 [1, 16, 28, 512] 1,448\n",
+ "│ │ └─MBConvBlock: 3-15 [1, 24, 14, 256] 9,864\n",
+ "│ │ └─MBConvBlock: 3-16 [1, 24, 14, 256] 19,380\n",
+ "│ │ └─MBConvBlock: 3-17 [1, 40, 7, 128] 24,020\n",
+ "│ │ └─MBConvBlock: 3-18 [1, 40, 7, 128] 55,340\n",
+ "│ │ └─MBConvBlock: 3-19 [1, 80, 3, 64] 61,180\n",
+ "│ │ └─MBConvBlock: 3-20 [1, 80, 3, 64] 199,000\n",
+ "│ │ └─MBConvBlock: 3-21 [1, 80, 3, 64] 199,000\n",
+ "│ │ └─MBConvBlock: 3-22 [1, 112, 3, 64] 222,104\n",
+ "│ │ └─MBConvBlock: 3-23 [1, 112, 3, 64] 396,872\n",
+ "│ │ └─MBConvBlock: 3-24 [1, 112, 3, 64] 396,872\n",
+ "│ │ └─MBConvBlock: 3-25 [1, 192, 1, 32] 450,792\n",
+ "│ │ └─MBConvBlock: 3-26 [1, 192, 1, 32] 1,141,152\n",
+ "│ │ └─MBConvBlock: 3-27 [1, 192, 1, 32] 1,141,152\n",
+ "│ │ └─MBConvBlock: 3-28 [1, 192, 1, 32] 1,141,152\n",
+ "│ │ └─MBConvBlock: 3-29 [1, 320, 1, 32] 1,270,432\n",
+ "│ └─Sequential: 2-4 [1, 1280, 1, 32] --\n",
+ "│ │ └─Conv2d: 3-30 [1, 1280, 1, 32] 409,600\n",
+ "│ │ └─BatchNorm2d: 3-31 [1, 1280, 1, 32] 2,560\n",
+ "├─Sequential: 1-2 [1, 128, 32] --\n",
+ "│ └─Conv2d: 2-5 [1, 128, 1, 32] 163,968\n",
+ "│ └─PositionalEncoding2D: 2-6 [1, 128, 1, 32] --\n",
+ "│ └─Flatten: 2-7 [1, 128, 32] --\n",
+ "├─Embedding: 1-3 [1, 89, 128] 7,424\n",
+ "├─PositionalEncoding: 1-4 [1, 89, 128] --\n",
+ "│ └─Dropout: 2-8 [1, 89, 128] --\n",
+ "├─Decoder: 1-5 [1, 89, 128] --\n",
+ "├─Linear: 1-6 [1, 89, 58] 7,482\n",
"====================================================================================================\n",
- "Total params: 16,107,322\n",
- "Trainable params: 16,107,322\n",
+ "Total params: 20,500,410\n",
+ "Trainable params: 20,500,410\n",
"Non-trainable params: 0\n",
- "Total mult-adds (G): 2.84\n",
+ "Total mult-adds (M): 714.87\n",
"====================================================================================================\n",
- "Input size (MB): 0.92\n",
- "Forward/backward pass size (MB): 677.01\n",
- "Params size (MB): 64.43\n",
- "Estimated Total Size (MB): 742.36\n",
+ "Input size (MB): 0.23\n",
+ "Forward/backward pass size (MB): 184.29\n",
+ "Params size (MB): 82.00\n",
+ "Estimated Total Size (MB): 266.52\n",
"===================================================================================================="
]
},
- "execution_count": 10,
+ "execution_count": 24,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
- "summary(net, ((4, 1, 56, 1024), (4, 89)), device=\"cpu\")"
+ "summary(net, list(map(lambda x: list(x), cfg.summary)), device=\"cpu\")"
]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "bf9d8d67-d7d2-4cf7-b166-377a79d5fd70",
+ "metadata": {},
+ "outputs": [],
+ "source": []
}
],
"metadata": {