text_recognizer/network/transformer/embedding/token.py


1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18

from torch import nn, Tensor

from text_recognizer.network.transformer.embedding.l2_norm import l2_norm


class TokenEmbedding(nn.Module):
    def __init__(self, num_tokens: int, dim: int, use_l2: bool = True) -> None:
        super().__init__()
        self.use_l2 = use_l2
        self.to_embedding = nn.Embedding(num_tokens, dim)
        if self.use_l2:
            nn.init.normal_(self.to_embedding.weight, std=1e-5)
        else:
            nn.init.kaiming_normal_(self.to_embedding.weight)

    def forward(self, x: Tensor) -> Tensor:
        embedding = self.to_embedding(x)
        return l2_norm(embedding) if self.use_l2 else embedding