summaryrefslogtreecommitdiff
path: root/notebooks/04-efficientnet-transformer.ipynb
blob: 3ee1cb24461de6764b64e6da75fd2b5ff7854233 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "7c02ae76-b540-4b16-9492-e9210b3b9249",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ['CUDA_VISIBLE_DEVICE'] = ''\n",
    "import random\n",
    "\n",
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import numpy as np\n",
    "from omegaconf import OmegaConf\n",
    "\n",
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "from importlib.util import find_spec\n",
    "if find_spec(\"text_recognizer\") is None:\n",
    "    import sys\n",
    "    sys.path.append('..')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "ccdb6dde-47e5-429a-88f2-0764fb7e259a",
   "metadata": {},
   "outputs": [],
   "source": [
    "from hydra import compose, initialize\n",
    "from omegaconf import OmegaConf\n",
    "from hydra.utils import instantiate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "3cf50475-39f2-4642-a7d1-5bcbc0a036f7",
   "metadata": {},
   "outputs": [],
   "source": [
    "path = \"../training/conf/experiment/cnn_htr_char_lines.yaml\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "e52ecb01-c975-4e55-925d-1182c7aea473",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(path, \"rb\") as f:\n",
    "    cfg = OmegaConf.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "f939aa37-7b1d-45cc-885c-323c4540bda1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'defaults': [{'override /mapping': None}, {'override /criterion': None}, {'override /datamodule': None}, {'override /network': None}, {'override /model': None}, {'override /lr_schedulers': None}, {'override /optimizers': None}], 'criterion': {'_target_': 'torch.nn.CrossEntropyLoss', 'ignore_index': 3}, 'mapping': {'_target_': 'text_recognizer.data.emnist_mapping.EmnistMapping'}, 'callbacks': {'stochastic_weight_averaging': {'_target_': 'pytorch_lightning.callbacks.StochasticWeightAveraging', 'swa_epoch_start': 0.8, 'swa_lrs': 0.05, 'annealing_epochs': 10, 'annealing_strategy': 'cos', 'device': None}}, 'optimizers': {'madgrad': {'_target_': 'madgrad.MADGRAD', 'lr': 0.0001, 'momentum': 0.9, 'weight_decay': 0, 'eps': 1e-06, 'parameters': 'network'}}, 'lr_schedulers': {'network': {'_target_': 'torch.optim.lr_scheduler.CosineAnnealingLR', 'T_max': 1024, 'eta_min': 4.5e-06, 'last_epoch': -1, 'interval': 'epoch', 'monitor': 'val/loss'}}, 'datamodule': {'_target_': 'text_recognizer.data.iam_lines.IAMLines', 'batch_size': 24, 'num_workers': 12, 'train_fraction': 0.8, 'augment': True, 'pin_memory': False}, 'network': {'_target_': 'text_recognizer.networks.conv_transformer.ConvTransformer', 'input_dims': [1, 56, 1024], 'hidden_dim': 128, 'encoder_dim': 1280, 'dropout_rate': 0.2, 'num_classes': 58, 'pad_index': 3, 'encoder': {'_target_': 'text_recognizer.networks.encoders.efficientnet.EfficientNet', 'arch': 'b0', 'out_channels': 1280, 'stochastic_dropout_rate': 0.2, 'bn_momentum': 0.99, 'bn_eps': 0.001}, 'decoder': {'_target_': 'text_recognizer.networks.transformer.Decoder', 'dim': 128, 'depth': 3, 'num_heads': 4, 'attn_fn': 'text_recognizer.networks.transformer.attention.Attention', 'attn_kwargs': {'dim_head': 32, 'dropout_rate': 0.2}, 'norm_fn': 'torch.nn.LayerNorm', 'ff_fn': 'text_recognizer.networks.transformer.mlp.FeedForward', 'ff_kwargs': {'dim_out': None, 'expansion_factor': 4, 'glu': True, 'dropout_rate': 0.2}, 'cross_attend': True, 'pre_norm': True, 'rotary_emb': None}}, 'model': {'_target_': 'text_recognizer.models.transformer.TransformerLitModel', 'max_output_len': 89, 'start_token': '<s>', 'end_token': '<e>', 'pad_token': '<p>'}, 'trainer': {'_target_': 'pytorch_lightning.Trainer', 'stochastic_weight_avg': False, 'auto_scale_batch_size': 'binsearch', 'auto_lr_find': False, 'gradient_clip_val': 0, 'fast_dev_run': False, 'gpus': 1, 'precision': 16, 'max_epochs': 1024, 'terminate_on_nan': True, 'weights_summary': 'top', 'limit_train_batches': 1.0, 'limit_val_batches': 1.0, 'limit_test_batches': 1.0, 'resume_from_checkpoint': None, 'accumulate_grad_batches': 4, 'overfit_batches': 0.0}, 'summary': [[1, 1, 56, 1024], [1, 89]]}"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cfg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "2fdd47e6-aac0-47e9-b224-f46c1f015f17",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[1, 1, 56, 1024], [1, 89]]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(map(lambda x: list(x), cfg.summary))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "92405411-f944-4519-9a6d-13b0761c2de6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "list"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(list(cfg.summary))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "aaeab329-aeb0-4a1b-aa35-5a2aab81b1d0",
   "metadata": {},
   "outputs": [],
   "source": [
    "net = instantiate(cfg.network)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "618b997c-e6a6-4487-b70c-9d260cb556d3",
   "metadata": {},
   "outputs": [],
   "source": [
    "from torchinfo import summary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "25759b7b-8deb-4163-b75d-a1357c9fe88f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "====================================================================================================\n",
       "Layer (type:depth-idx)                             Output Shape              Param #\n",
       "====================================================================================================\n",
       "ConvTransformer                                    --                        --\n",
       "├─EfficientNet: 1                                  --                        --\n",
       "│    └─ModuleList: 2-1                             --                        --\n",
       "├─Decoder: 1                                       --                        --\n",
       "│    └─ModuleList: 2-2                             --                        --\n",
       "│    │    └─ModuleList: 3-1                        --                        2,097,536\n",
       "│    │    └─ModuleList: 3-2                        --                        2,097,536\n",
       "│    │    └─ModuleList: 3-3                        --                        198,016\n",
       "│    │    └─ModuleList: 3-4                        --                        2,097,536\n",
       "│    │    └─ModuleList: 3-5                        --                        2,097,536\n",
       "│    │    └─ModuleList: 3-6                        --                        198,016\n",
       "│    │    └─ModuleList: 3-7                        --                        2,097,536\n",
       "│    │    └─ModuleList: 3-8                        --                        2,097,536\n",
       "│    │    └─ModuleList: 3-9                        --                        198,016\n",
       "├─EfficientNet: 1-1                                [1, 1280, 1, 32]          --\n",
       "│    └─Sequential: 2-3                             [1, 32, 28, 512]          --\n",
       "│    │    └─ZeroPad2d: 3-10                        [1, 1, 57, 1025]          --\n",
       "│    │    └─Conv2d: 3-11                           [1, 32, 28, 512]          288\n",
       "│    │    └─BatchNorm2d: 3-12                      [1, 32, 28, 512]          64\n",
       "│    │    └─Mish: 3-13                             [1, 32, 28, 512]          --\n",
       "│    └─ModuleList: 2-1                             --                        --\n",
       "│    │    └─MBConvBlock: 3-14                      [1, 16, 28, 512]          1,448\n",
       "│    │    └─MBConvBlock: 3-15                      [1, 24, 14, 256]          9,864\n",
       "│    │    └─MBConvBlock: 3-16                      [1, 24, 14, 256]          19,380\n",
       "│    │    └─MBConvBlock: 3-17                      [1, 40, 7, 128]           24,020\n",
       "│    │    └─MBConvBlock: 3-18                      [1, 40, 7, 128]           55,340\n",
       "│    │    └─MBConvBlock: 3-19                      [1, 80, 3, 64]            61,180\n",
       "│    │    └─MBConvBlock: 3-20                      [1, 80, 3, 64]            199,000\n",
       "│    │    └─MBConvBlock: 3-21                      [1, 80, 3, 64]            199,000\n",
       "│    │    └─MBConvBlock: 3-22                      [1, 112, 3, 64]           222,104\n",
       "│    │    └─MBConvBlock: 3-23                      [1, 112, 3, 64]           396,872\n",
       "│    │    └─MBConvBlock: 3-24                      [1, 112, 3, 64]           396,872\n",
       "│    │    └─MBConvBlock: 3-25                      [1, 192, 1, 32]           450,792\n",
       "│    │    └─MBConvBlock: 3-26                      [1, 192, 1, 32]           1,141,152\n",
       "│    │    └─MBConvBlock: 3-27                      [1, 192, 1, 32]           1,141,152\n",
       "│    │    └─MBConvBlock: 3-28                      [1, 192, 1, 32]           1,141,152\n",
       "│    │    └─MBConvBlock: 3-29                      [1, 320, 1, 32]           1,270,432\n",
       "│    └─Sequential: 2-4                             [1, 1280, 1, 32]          --\n",
       "│    │    └─Conv2d: 3-30                           [1, 1280, 1, 32]          409,600\n",
       "│    │    └─BatchNorm2d: 3-31                      [1, 1280, 1, 32]          2,560\n",
       "├─Sequential: 1-2                                  [1, 128, 32]              --\n",
       "│    └─Conv2d: 2-5                                 [1, 128, 1, 32]           163,968\n",
       "│    └─PositionalEncoding2D: 2-6                   [1, 128, 1, 32]           --\n",
       "│    └─Flatten: 2-7                                [1, 128, 32]              --\n",
       "├─Embedding: 1-3                                   [1, 89, 128]              7,424\n",
       "├─PositionalEncoding: 1-4                          [1, 89, 128]              --\n",
       "│    └─Dropout: 2-8                                [1, 89, 128]              --\n",
       "├─Decoder: 1-5                                     [1, 89, 128]              --\n",
       "├─Linear: 1-6                                      [1, 89, 58]               7,482\n",
       "====================================================================================================\n",
       "Total params: 20,500,410\n",
       "Trainable params: 20,500,410\n",
       "Non-trainable params: 0\n",
       "Total mult-adds (M): 714.87\n",
       "====================================================================================================\n",
       "Input size (MB): 0.23\n",
       "Forward/backward pass size (MB): 184.29\n",
       "Params size (MB): 82.00\n",
       "Estimated Total Size (MB): 266.52\n",
       "===================================================================================================="
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "summary(net, list(map(lambda x: list(x), cfg.summary)), device=\"cpu\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bf9d8d67-d7d2-4cf7-b166-377a79d5fd70",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}