1 files changed, 12 insertions, 0 deletions
diff --git a/text_recognizer/networks/conformer/glu.py b/text_recognizer/networks/conformer/glu.py
new file mode 100644
index 0000000..016b684
--- /dev/null
+++ b/text_recognizer/networks/conformer/glu.py
@@ -0,0 +1,12 @@
+"""GLU layer."""
+from torch import nn, Tensor
+
+
+class GLU(nn.Module):
+    def __init__(self, dim: int) -> None:
+        super().__init__()
+        self.dim = dim
+        
+    def forward(self, x: Tensor) -> Tensor:
+        out, gate = x.chunk(2, dim=self.dim)
+        return out * gate.sigmoid()