Implementing CoaT transformer, continue tomorrow...

1 files changed, 9 insertions, 0 deletions

diff --git a/text_recognizer/networks/coat/factor_attention.py b/text_recognizer/networks/coat/factor_attention.py
new file mode 100644
index 0000000..f91c5ef
--- /dev/null
+++ b/text_recognizer/networks/coat/factor_attention.py
@@ -0,0 +1,9 @@
+"""Factorized attention with convolutional relative positional encodings."""
+from torch import nn
+
+
+class FactorAttention(nn.Module):
+    """Factorized attention with relative positional encodings."""
+    def __init__(self, dim: int, num_heads: int) -> None:
+        pass
+