summaryrefslogtreecommitdiff
path: root/text_recognizer/networks/transformer/embeddings/absolute.py
diff options
context:
space:
mode:
Diffstat (limited to 'text_recognizer/networks/transformer/embeddings/absolute.py')
-rw-r--r--text_recognizer/networks/transformer/embeddings/absolute.py17
1 files changed, 17 insertions, 0 deletions
diff --git a/text_recognizer/networks/transformer/embeddings/absolute.py b/text_recognizer/networks/transformer/embeddings/absolute.py
new file mode 100644
index 0000000..7140537
--- /dev/null
+++ b/text_recognizer/networks/transformer/embeddings/absolute.py
@@ -0,0 +1,17 @@
+"""Absolute positional embedding."""
+import torch
+from torch import nn, Tensor
+
+
+class AbsolutePositionalEmbedding(nn.Module):
+ def __init__(self, dim: int, max_seq_len: int) -> None:
+ super().__init__()
+ self.emb = nn.Embedding(max_seq_len, dim)
+ self._weight_init()
+
+ def _weight_init(self) -> None:
+ nn.init.normal_(self.emb.weight, std=0.02)
+
+ def forward(self, x: Tensor) -> Tensor:
+ n = torch.arange(x.shape[1], device=x.device)
+ return self.emb(n)[None, :, :]