From 49ca6ade1a19f7f9c702171537fe4be0dfcda66d Mon Sep 17 00:00:00 2001 From: Gustaf Rydholm Date: Fri, 25 Aug 2023 23:19:14 +0200 Subject: Rename and add flash atten --- text_recognizer/network/convnext/norm.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 text_recognizer/network/convnext/norm.py (limited to 'text_recognizer/network/convnext/norm.py') diff --git a/text_recognizer/network/convnext/norm.py b/text_recognizer/network/convnext/norm.py new file mode 100644 index 0000000..3355de9 --- /dev/null +++ b/text_recognizer/network/convnext/norm.py @@ -0,0 +1,18 @@ +"""Layer norm for conv layers.""" +import torch +from torch import Tensor, nn + + +class LayerNorm(nn.Module): + """Layer norm for convolutions.""" + + def __init__(self, dim: int) -> None: + super().__init__() + self.gamma = nn.Parameter(torch.ones(1, dim, 1, 1)) + + def forward(self, x: Tensor) -> Tensor: + """Applies layer norm.""" + eps = 1e-5 if x.dtype == torch.float32 else 1e-3 + var = torch.var(x, dim=1, unbiased=False, keepdim=True) + mean = torch.mean(x, dim=1, keepdim=True) + return (x - mean) / (var + eps).sqrt() * self.gamma -- cgit v1.2.3-70-g09d2