From 6cf78f9cc4c3e75a8161002f87d3b81fd9605ec4 Mon Sep 17 00:00:00 2001
From: Gustaf Rydholm <gustaf.rydholm@gmail.com>
Date: Mon, 11 Sep 2023 22:12:51 +0200
Subject: Add swiglu

---
 text_recognizer/network/transformer/swiglu.py | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 text_recognizer/network/transformer/swiglu.py

(limited to 'text_recognizer/network/transformer/swiglu.py')

diff --git a/text_recognizer/network/transformer/swiglu.py b/text_recognizer/network/transformer/swiglu.py
new file mode 100644
index 0000000..e61662a
--- /dev/null
+++ b/text_recognizer/network/transformer/swiglu.py
@@ -0,0 +1,8 @@
+from torch import nn
+import torch.nn.functional as F
+
+
+class SwiGLU(nn.Module):
+    def forward(self, x):
+        x, gate = x.chunk(2, dim=-1)
+        return F.silu(gate) * x
-- 
cgit v1.2.3-70-g09d2