summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorGustaf Rydholm <gustaf.rydholm@gmail.com>2023-09-11 22:12:51 +0200
committerGustaf Rydholm <gustaf.rydholm@gmail.com>2023-09-11 22:12:51 +0200
commit6cf78f9cc4c3e75a8161002f87d3b81fd9605ec4 (patch)
treebcb1743b96c24cb07b3833a5b657293b2f440ced
parentfe693b4799cc48476d720fca5bb9fbf00fe58b23 (diff)
Add swiglu
-rw-r--r--text_recognizer/network/transformer/swiglu.py8
1 files changed, 8 insertions, 0 deletions
diff --git a/text_recognizer/network/transformer/swiglu.py b/text_recognizer/network/transformer/swiglu.py
new file mode 100644
index 0000000..e61662a
--- /dev/null
+++ b/text_recognizer/network/transformer/swiglu.py
@@ -0,0 +1,8 @@
+from torch import nn
+import torch.nn.functional as F
+
+
+class SwiGLU(nn.Module):
+ def forward(self, x):
+ x, gate = x.chunk(2, dim=-1)
+ return F.silu(gate) * x