summaryrefslogtreecommitdiff
path: root/notebooks/04-efficientnet-transformer.ipynb
blob: 4a6fd6499fb0cb5c3f9ee6fb890eeca6bb79dc33 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "7c02ae76-b540-4b16-9492-e9210b3b9249",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ['CUDA_VISIBLE_DEVICE'] = ''\n",
    "import random\n",
    "\n",
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import torch\n",
    "import numpy as np\n",
    "from omegaconf import OmegaConf\n",
    "\n",
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "from importlib.util import find_spec\n",
    "if find_spec(\"text_recognizer\") is None:\n",
    "    import sys\n",
    "    sys.path.append('..')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "ccdb6dde-47e5-429a-88f2-0764fb7e259a",
   "metadata": {},
   "outputs": [],
   "source": [
    "from hydra import compose, initialize\n",
    "from omegaconf import OmegaConf\n",
    "from hydra.utils import instantiate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "3cf50475-39f2-4642-a7d1-5bcbc0a036f7",
   "metadata": {},
   "outputs": [],
   "source": [
    "path = \"../training/conf/experiment/conv_transformer_paragraphs.yaml\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "id": "e52ecb01-c975-4e55-925d-1182c7aea473",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(path, \"rb\") as f:\n",
    "    cfg = OmegaConf.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "id": "f939aa37-7b1d-45cc-885c-323c4540bda1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'dim': 192, 'num_heads': 4, 'dim_head': 64, 'dropout_rate': 0.05, '_target_': 'text_recognizer.networks.transformer.attention.Attention', 'causal': False}"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cfg.network.decoder.cross_attn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "id": "aaeab329-aeb0-4a1b-aa35-5a2aab81b1d0",
   "metadata": {},
   "outputs": [],
   "source": [
    "net = instantiate(cfg.network)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "id": "618b997c-e6a6-4487-b70c-9d260cb556d3",
   "metadata": {},
   "outputs": [],
   "source": [
    "from torchinfo import summary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "id": "66118c10-2e59-469f-99d6-ddea4bfd0d73",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "====================================================================================================\n",
       "Layer (type:depth-idx)                             Output Shape              Param #\n",
       "====================================================================================================\n",
       "ConvTransformer                                    --                        --\n",
       "├─EfficientNet: 1                                  --                        --\n",
       "│    └─ModuleList: 2-1                             --                        --\n",
       "├─Decoder: 1                                       --                        --\n",
       "│    └─ModuleList: 2-2                             --                        --\n",
       "│    │    └─ModuleList: 3-1                        --                        --\n",
       "│    │    └─ModuleList: 3-2                        --                        --\n",
       "│    │    └─ModuleList: 3-3                        --                        --\n",
       "│    │    └─ModuleList: 3-4                        --                        --\n",
       "│    │    └─ModuleList: 3-5                        --                        --\n",
       "│    │    └─ModuleList: 3-6                        --                        --\n",
       "│    │    └─ModuleList: 3-7                        --                        --\n",
       "│    │    └─ModuleList: 3-8                        --                        --\n",
       "│    │    └─ModuleList: 3-9                        --                        --\n",
       "├─EfficientNet: 1-1                                [2, 1280, 18, 20]         --\n",
       "│    └─Sequential: 2-3                             [2, 32, 288, 320]         --\n",
       "│    │    └─ZeroPad2d: 3-10                        [2, 1, 577, 641]          --\n",
       "│    │    └─Conv2d: 3-11                           [2, 32, 288, 320]         288\n",
       "│    │    └─BatchNorm2d: 3-12                      [2, 32, 288, 320]         64\n",
       "│    │    └─Mish: 3-13                             [2, 32, 288, 320]         --\n",
       "│    └─ModuleList: 2-1                             --                        --\n",
       "│    │    └─MBConvBlock: 3-14                      [2, 16, 288, 320]         --\n",
       "│    │    │    └─Sequential: 4-1                   [2, 32, 288, 320]         352\n",
       "│    │    │    └─Sequential: 4-2                   [2, 32, 288, 320]         552\n",
       "│    │    │    └─Sequential: 4-3                   [2, 16, 288, 320]         544\n",
       "│    │    └─MBConvBlock: 3-15                      [2, 24, 144, 160]         --\n",
       "│    │    │    └─Sequential: 4-4                   [2, 96, 288, 320]         1,728\n",
       "│    │    │    └─Sequential: 4-5                   [2, 96, 144, 160]         1,056\n",
       "│    │    │    └─Sequential: 4-6                   [2, 96, 144, 160]         4,728\n",
       "│    │    │    └─Sequential: 4-7                   [2, 24, 144, 160]         2,352\n",
       "│    │    └─MBConvBlock: 3-16                      [2, 24, 144, 160]         --\n",
       "│    │    │    └─Sequential: 4-8                   [2, 144, 144, 160]        3,744\n",
       "│    │    │    └─Sequential: 4-9                   [2, 144, 144, 160]        1,584\n",
       "│    │    │    └─Sequential: 4-10                  [2, 144, 144, 160]        10,548\n",
       "│    │    │    └─Sequential: 4-11                  [2, 24, 144, 160]         3,504\n",
       "│    │    └─MBConvBlock: 3-17                      [2, 40, 72, 80]           --\n",
       "│    │    │    └─Sequential: 4-12                  [2, 144, 144, 160]        3,744\n",
       "│    │    │    └─Sequential: 4-13                  [2, 144, 72, 80]          3,888\n",
       "│    │    │    └─Sequential: 4-14                  [2, 144, 72, 80]          10,548\n",
       "│    │    │    └─Sequential: 4-15                  [2, 40, 72, 80]           5,840\n",
       "│    │    └─MBConvBlock: 3-18                      [2, 40, 72, 80]           --\n",
       "│    │    │    └─Sequential: 4-16                  [2, 240, 72, 80]          10,080\n",
       "│    │    │    └─Sequential: 4-17                  [2, 240, 72, 80]          6,480\n",
       "│    │    │    └─Sequential: 4-18                  [2, 240, 72, 80]          29,100\n",
       "│    │    │    └─Sequential: 4-19                  [2, 40, 72, 80]           9,680\n",
       "│    │    └─MBConvBlock: 3-19                      [2, 80, 36, 40]           --\n",
       "│    │    │    └─Sequential: 4-20                  [2, 240, 72, 80]          10,080\n",
       "│    │    │    └─Sequential: 4-21                  [2, 240, 36, 40]          2,640\n",
       "│    │    │    └─Sequential: 4-22                  [2, 240, 36, 40]          29,100\n",
       "│    │    │    └─Sequential: 4-23                  [2, 80, 36, 40]           19,360\n",
       "│    │    └─MBConvBlock: 3-20                      [2, 80, 36, 40]           --\n",
       "│    │    │    └─Sequential: 4-24                  [2, 480, 36, 40]          39,360\n",
       "│    │    │    └─Sequential: 4-25                  [2, 480, 36, 40]          5,280\n",
       "│    │    │    └─Sequential: 4-26                  [2, 480, 36, 40]          115,800\n",
       "│    │    │    └─Sequential: 4-27                  [2, 80, 36, 40]           38,560\n",
       "│    │    └─MBConvBlock: 3-21                      [2, 80, 36, 40]           --\n",
       "│    │    │    └─Sequential: 4-28                  [2, 480, 36, 40]          39,360\n",
       "│    │    │    └─Sequential: 4-29                  [2, 480, 36, 40]          5,280\n",
       "│    │    │    └─Sequential: 4-30                  [2, 480, 36, 40]          115,800\n",
       "│    │    │    └─Sequential: 4-31                  [2, 80, 36, 40]           38,560\n",
       "│    │    └─MBConvBlock: 3-22                      [2, 112, 36, 40]          --\n",
       "│    │    │    └─Sequential: 4-32                  [2, 480, 36, 40]          39,360\n",
       "│    │    │    └─Sequential: 4-33                  [2, 480, 36, 40]          12,960\n",
       "│    │    │    └─Sequential: 4-34                  [2, 480, 36, 40]          115,800\n",
       "│    │    │    └─Sequential: 4-35                  [2, 112, 36, 40]          53,984\n",
       "│    │    └─MBConvBlock: 3-23                      [2, 112, 36, 40]          --\n",
       "│    │    │    └─Sequential: 4-36                  [2, 672, 36, 40]          76,608\n",
       "│    │    │    └─Sequential: 4-37                  [2, 672, 36, 40]          18,144\n",
       "│    │    │    └─Sequential: 4-38                  [2, 672, 36, 40]          226,632\n",
       "│    │    │    └─Sequential: 4-39                  [2, 112, 36, 40]          75,488\n",
       "│    │    └─MBConvBlock: 3-24                      [2, 112, 36, 40]          --\n",
       "│    │    │    └─Sequential: 4-40                  [2, 672, 36, 40]          76,608\n",
       "│    │    │    └─Sequential: 4-41                  [2, 672, 36, 40]          18,144\n",
       "│    │    │    └─Sequential: 4-42                  [2, 672, 36, 40]          226,632\n",
       "│    │    │    └─Sequential: 4-43                  [2, 112, 36, 40]          75,488\n",
       "│    │    └─MBConvBlock: 3-25                      [2, 192, 18, 20]          --\n",
       "│    │    │    └─Sequential: 4-44                  [2, 672, 36, 40]          76,608\n",
       "│    │    │    └─Sequential: 4-45                  [2, 672, 18, 20]          18,144\n",
       "│    │    │    └─Sequential: 4-46                  [2, 672, 18, 20]          226,632\n",
       "│    │    │    └─Sequential: 4-47                  [2, 192, 18, 20]          129,408\n",
       "│    │    └─MBConvBlock: 3-26                      [2, 192, 18, 20]          --\n",
       "│    │    │    └─Sequential: 4-48                  [2, 1152, 18, 20]         223,488\n",
       "│    │    │    └─Sequential: 4-49                  [2, 1152, 18, 20]         31,104\n",
       "│    │    │    └─Sequential: 4-50                  [2, 1152, 18, 20]         664,992\n",
       "│    │    │    └─Sequential: 4-51                  [2, 192, 18, 20]          221,568\n",
       "│    │    └─MBConvBlock: 3-27                      [2, 192, 18, 20]          --\n",
       "│    │    │    └─Sequential: 4-52                  [2, 1152, 18, 20]         223,488\n",
       "│    │    │    └─Sequential: 4-53                  [2, 1152, 18, 20]         31,104\n",
       "│    │    │    └─Sequential: 4-54                  [2, 1152, 18, 20]         664,992\n",
       "│    │    │    └─Sequential: 4-55                  [2, 192, 18, 20]          221,568\n",
       "│    │    └─MBConvBlock: 3-28                      [2, 192, 18, 20]          --\n",
       "│    │    │    └─Sequential: 4-56                  [2, 1152, 18, 20]         223,488\n",
       "│    │    │    └─Sequential: 4-57                  [2, 1152, 18, 20]         31,104\n",
       "│    │    │    └─Sequential: 4-58                  [2, 1152, 18, 20]         664,992\n",
       "│    │    │    └─Sequential: 4-59                  [2, 192, 18, 20]          221,568\n",
       "│    │    └─MBConvBlock: 3-29                      [2, 320, 18, 20]          --\n",
       "│    │    │    └─Sequential: 4-60                  [2, 1152, 18, 20]         223,488\n",
       "│    │    │    └─Sequential: 4-61                  [2, 1152, 18, 20]         12,672\n",
       "│    │    │    └─Sequential: 4-62                  [2, 1152, 18, 20]         664,992\n",
       "│    │    │    └─Sequential: 4-63                  [2, 320, 18, 20]          369,280\n",
       "│    └─Sequential: 2-4                             [2, 1280, 18, 20]         --\n",
       "│    │    └─Conv2d: 3-30                           [2, 1280, 18, 20]         409,600\n",
       "│    │    └─BatchNorm2d: 3-31                      [2, 1280, 18, 20]         2,560\n",
       "├─Sequential: 1-2                                  [2, 192, 360]             --\n",
       "│    └─Conv2d: 2-5                                 [2, 192, 18, 20]          245,952\n",
       "│    └─AxialPositionalEmbedding: 2-6               [2, 192, 18, 20]          7,296\n",
       "│    └─Flatten: 2-7                                [2, 192, 360]             --\n",
       "├─Embedding: 1-3                                   [1, 682, 192]             11,136\n",
       "├─Decoder: 1-4                                     [2, 682, 192]             --\n",
       "│    └─ModuleList: 2-2                             --                        --\n",
       "│    │    └─ModuleList: 3-1                        --                        --\n",
       "│    │    │    └─ScaleNorm: 4-64                   [1, 682, 192]             1\n",
       "│    │    │    └─LocalAttention: 4-65              [1, 682, 192]             196,800\n",
       "│    │    │    └─Residual: 4-66                    [1, 682, 192]             --\n",
       "│    │    └─ModuleList: 3-2                        --                        --\n",
       "│    │    │    └─ScaleNorm: 4-67                   [1, 682, 192]             1\n",
       "│    │    │    └─Attention: 4-68                   [2, 682, 192]             196,800\n",
       "│    │    │    └─Residual: 4-69                    [2, 682, 192]             --\n",
       "│    │    └─ModuleList: 3-3                        --                        --\n",
       "│    │    │    └─ScaleNorm: 4-70                   [2, 682, 192]             1\n",
       "│    │    │    └─FeedForward: 4-71                 [2, 682, 192]             444,096\n",
       "│    │    │    └─Residual: 4-72                    [2, 682, 192]             --\n",
       "│    │    └─ModuleList: 3-4                        --                        --\n",
       "│    │    │    └─ScaleNorm: 4-73                   [2, 682, 192]             1\n",
       "│    │    │    └─LocalAttention: 4-74              [2, 682, 192]             196,800\n",
       "│    │    │    └─Residual: 4-75                    [2, 682, 192]             --\n",
       "│    │    └─ModuleList: 3-5                        --                        --\n",
       "│    │    │    └─ScaleNorm: 4-76                   [2, 682, 192]             1\n",
       "│    │    │    └─Attention: 4-77                   [2, 682, 192]             196,800\n",
       "│    │    │    └─Residual: 4-78                    [2, 682, 192]             --\n",
       "│    │    └─ModuleList: 3-6                        --                        --\n",
       "│    │    │    └─ScaleNorm: 4-79                   [2, 682, 192]             1\n",
       "│    │    │    └─FeedForward: 4-80                 [2, 682, 192]             444,096\n",
       "│    │    │    └─Residual: 4-81                    [2, 682, 192]             --\n",
       "│    │    └─ModuleList: 3-7                        --                        --\n",
       "│    │    │    └─ScaleNorm: 4-82                   [2, 682, 192]             1\n",
       "│    │    │    └─Attention: 4-83                   [2, 682, 192]             196,800\n",
       "│    │    │    └─Residual: 4-84                    [2, 682, 192]             --\n",
       "│    │    └─ModuleList: 3-8                        --                        --\n",
       "│    │    │    └─ScaleNorm: 4-85                   [2, 682, 192]             1\n",
       "│    │    │    └─Attention: 4-86                   [2, 682, 192]             196,800\n",
       "│    │    │    └─Residual: 4-87                    [2, 682, 192]             --\n",
       "│    │    └─ModuleList: 3-9                        --                        --\n",
       "│    │    │    └─ScaleNorm: 4-88                   [2, 682, 192]             1\n",
       "│    │    │    └─FeedForward: 4-89                 [2, 682, 192]             444,096\n",
       "│    │    │    └─Residual: 4-90                    [2, 682, 192]             --\n",
       "├─Linear: 1-5                                      [2, 682, 58]              11,194\n",
       "====================================================================================================\n",
       "Total params: 9,930,947\n",
       "Trainable params: 9,930,947\n",
       "Non-trainable params: 0\n",
       "Total mult-adds (G): 11.45\n",
       "====================================================================================================\n",
       "Input size (MB): 2.95\n",
       "Forward/backward pass size (MB): 2048.49\n",
       "Params size (MB): 39.72\n",
       "Estimated Total Size (MB): 2091.17\n",
       "===================================================================================================="
      ]
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "summary(net, ((2, 1, 576, 640), (1, 682)), device=\"cpu\", depth=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b13ac47c-322d-47d4-bcee-43e5341f74a7",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}