Attention layer soon done

1 files changed, 8 insertions, 0 deletions

diff --git a/text_recognizer/networks/transformer/residual.py b/text_recognizer/networks/transformer/residual.py
new file mode 100644
index 0000000..1547df6
--- /dev/null
+++ b/text_recognizer/networks/transformer/residual.py
@@ -0,0 +1,8 @@
+"""Residual function."""
+from torch import nn, Tensor
+
+
+class Residual(nn.Module):
+    def forward(self, x: Tensor, residual: Tensor) -> Tensor:
+        """Applies the residual function."""
+        return x + residual