From 6cf78f9cc4c3e75a8161002f87d3b81fd9605ec4 Mon Sep 17 00:00:00 2001 From: Gustaf Rydholm Date: Mon, 11 Sep 2023 22:12:51 +0200 Subject: Add swiglu --- text_recognizer/network/transformer/swiglu.py | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 text_recognizer/network/transformer/swiglu.py (limited to 'text_recognizer') diff --git a/text_recognizer/network/transformer/swiglu.py b/text_recognizer/network/transformer/swiglu.py new file mode 100644 index 0000000..e61662a --- /dev/null +++ b/text_recognizer/network/transformer/swiglu.py @@ -0,0 +1,8 @@ +from torch import nn +import torch.nn.functional as F + + +class SwiGLU(nn.Module): + def forward(self, x): + x, gate = x.chunk(2, dim=-1) + return F.silu(gate) * x -- cgit v1.2.3-70-g09d2