diff options
| author | Gustaf Rydholm <gustaf.rydholm@gmail.com> | 2022-06-01 23:11:05 +0200 |
|---|---|---|
| committer | Gustaf Rydholm <gustaf.rydholm@gmail.com> | 2022-06-01 23:11:05 +0200 |
| commit | 729867b14754bffd98a3406b5d3a2eb9d09bc3bc (patch) | |
| tree | d22fbe9a9aacd2b7e457f8af4b667c5c8d927d73 /text_recognizer/networks/conformer/glu.py | |
| parent | db86cef2d308f58325278061c6aa177a535e7e03 (diff) | |
WIP conformer
Diffstat (limited to 'text_recognizer/networks/conformer/glu.py')
| -rw-r--r-- | text_recognizer/networks/conformer/glu.py | 12 |
1 files changed, 12 insertions, 0 deletions
diff --git a/text_recognizer/networks/conformer/glu.py b/text_recognizer/networks/conformer/glu.py new file mode 100644 index 0000000..016b684 --- /dev/null +++ b/text_recognizer/networks/conformer/glu.py @@ -0,0 +1,12 @@ +"""GLU layer.""" +from torch import nn, Tensor + + +class GLU(nn.Module): + def __init__(self, dim: int) -> None: + super().__init__() + self.dim = dim + + def forward(self, x: Tensor) -> Tensor: + out, gate = x.chunk(2, dim=self.dim) + return out * gate.sigmoid() |