diff options
| author | Gustaf Rydholm <gustaf.rydholm@gmail.com> | 2023-09-11 22:12:51 +0200 |
|---|---|---|
| committer | Gustaf Rydholm <gustaf.rydholm@gmail.com> | 2023-09-11 22:12:51 +0200 |
| commit | 6cf78f9cc4c3e75a8161002f87d3b81fd9605ec4 (patch) | |
| tree | bcb1743b96c24cb07b3833a5b657293b2f440ced /text_recognizer/network/transformer | |
| parent | fe693b4799cc48476d720fca5bb9fbf00fe58b23 (diff) | |
Add swiglu
Diffstat (limited to 'text_recognizer/network/transformer')
| -rw-r--r-- | text_recognizer/network/transformer/swiglu.py | 8 |
1 files changed, 8 insertions, 0 deletions
diff --git a/text_recognizer/network/transformer/swiglu.py b/text_recognizer/network/transformer/swiglu.py new file mode 100644 index 0000000..e61662a --- /dev/null +++ b/text_recognizer/network/transformer/swiglu.py @@ -0,0 +1,8 @@ +from torch import nn +import torch.nn.functional as F + + +class SwiGLU(nn.Module): + def forward(self, x): + x, gate = x.chunk(2, dim=-1) + return F.silu(gate) * x |