Rename and add flash atten

1 files changed, 22 insertions, 0 deletions

diff --git a/text_recognizer/network/transformer/norm.py b/text_recognizer/network/transformer/norm.py
new file mode 100644
index 0000000..2737754
--- /dev/null
+++ b/text_recognizer/network/transformer/norm.py
@@ -0,0 +1,22 @@
+"""Normalization layers for transformers.
+
+Copied from lucidrains:
+    https://github.com/lucidrains/x-transformers/blob/main/x_transformers/x_transformers.py
+
+"""
+import torch
+from torch import Tensor, nn
+import torch.nn.functional as F
+
+
+class RMSNorm(nn.Module):
+    """Root mean square layer normalization."""
+
+    def __init__(self, heads: int, dim: int) -> None:
+        super().__init__()
+        self.scale = dim**-0.5
+        self.gamma = nn.Parameter(torch.ones(heads, 1, dim))
+
+    def forward(self, x: Tensor) -> Tensor:
+        """Applies normalization."""
+        return F.normalize(x, dim=-1) * self.scale * self.gamma