From 49ca6ade1a19f7f9c702171537fe4be0dfcda66d Mon Sep 17 00:00:00 2001
From: Gustaf Rydholm <gustaf.rydholm@gmail.com>
Date: Fri, 25 Aug 2023 23:19:14 +0200
Subject: Rename and add flash atten

---
 text_recognizer/network/transformer/ff.py | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)
 create mode 100644 text_recognizer/network/transformer/ff.py

(limited to 'text_recognizer/network/transformer/ff.py')

diff --git a/text_recognizer/network/transformer/ff.py b/text_recognizer/network/transformer/ff.py
new file mode 100644
index 0000000..9181323
--- /dev/null
+++ b/text_recognizer/network/transformer/ff.py
@@ -0,0 +1,22 @@
+"""Feedforward layer in transformer."""
+from torch import Tensor, nn
+
+
+class FeedForward(nn.Module):
+    def __init__(
+        self,
+        dim: int,
+        inner_dim: int,
+        dropout_rate: float = 0.0,
+    ) -> None:
+        super().__init__()
+        self.ff = nn.Sequential(
+            nn.LayerNorm(dim),
+            nn.Linear(dim, inner_dim),
+            nn.GELU(),
+            nn.Dropout(dropout_rate),
+            nn.Linear(inner_dim, dim),
+        )
+
+    def forward(self, x: Tensor) -> Tensor:
+        return self.ff(x)
-- 
cgit v1.2.3-70-g09d2