From 548f52b35062e258622ea638ed1b132d6759a07a Mon Sep 17 00:00:00 2001 From: Gustaf Rydholm Date: Sun, 9 May 2021 00:36:55 +0200 Subject: Attention layer soon done --- text_recognizer/networks/transformer/norm.py | 11 ----------- 1 file changed, 11 deletions(-) (limited to 'text_recognizer/networks/transformer/norm.py') diff --git a/text_recognizer/networks/transformer/norm.py b/text_recognizer/networks/transformer/norm.py index 58c8770..8bc3221 100644 --- a/text_recognizer/networks/transformer/norm.py +++ b/text_recognizer/networks/transformer/norm.py @@ -11,17 +11,6 @@ from torch import nn from torch import Tensor -class Rezero(nn.Module): - def __init__(self, fn: Callable) -> None: - super().__init__() - self.fn = fn - self.g = nn.Parameter(torch.zeros(1)) - - def forward(self, x: Tensor, **kwargs: Dict) -> Tensor: - x, *rest = self.fn(x, **kwargs) - return (x * self.g, *rest) - - class ScaleNorm(nn.Module): def __init__(self, dim: int, eps: float = 1.0e-5) -> None: super().__init__() -- cgit v1.2.3-70-g09d2