diff options
Diffstat (limited to 'text_recognizer/network/transformer/ff.py')
-rw-r--r-- | text_recognizer/network/transformer/ff.py | 22 |
1 files changed, 22 insertions, 0 deletions
diff --git a/text_recognizer/network/transformer/ff.py b/text_recognizer/network/transformer/ff.py new file mode 100644 index 0000000..9181323 --- /dev/null +++ b/text_recognizer/network/transformer/ff.py @@ -0,0 +1,22 @@ +"""Feedforward layer in transformer.""" +from torch import Tensor, nn + + +class FeedForward(nn.Module): + def __init__( + self, + dim: int, + inner_dim: int, + dropout_rate: float = 0.0, + ) -> None: + super().__init__() + self.ff = nn.Sequential( + nn.LayerNorm(dim), + nn.Linear(dim, inner_dim), + nn.GELU(), + nn.Dropout(dropout_rate), + nn.Linear(inner_dim, dim), + ) + + def forward(self, x: Tensor) -> Tensor: + return self.ff(x) |