From 8bb76745e43c6b4967c8e91ebaf4c4295d0b8d0b Mon Sep 17 00:00:00 2001 From: Gustaf Rydholm Date: Sun, 12 Jun 2022 23:16:20 +0200 Subject: Remove conformer --- text_recognizer/networks/conformer/attention.py | 49 ------------------------- 1 file changed, 49 deletions(-) delete mode 100644 text_recognizer/networks/conformer/attention.py (limited to 'text_recognizer/networks/conformer/attention.py') diff --git a/text_recognizer/networks/conformer/attention.py b/text_recognizer/networks/conformer/attention.py deleted file mode 100644 index e56e572..0000000 --- a/text_recognizer/networks/conformer/attention.py +++ /dev/null @@ -1,49 +0,0 @@ -"""Efficient self attention.""" -from einops import rearrange -import torch -import torch.nn.functional as F -from torch import einsum, nn, Tensor - - -class LayerNorm(nn.Module): - def __init__(self, dim: int) -> None: - super().__init__() - self.gamma = nn.Parameter(torch.ones(dim)) - self.register_buffer("beta", torch.zeros(dim)) - - def forward(self, x: Tensor) -> Tensor: - return F.layer_norm(x, x.shape[-1:], self.gamma, self.beta) - - -class SwiGLU(nn.Module): - def forward(self, x: Tensor) -> Tensor: - x, gate = x.chunk(2, dim=-1) - return F.silu(gate) * x - - -class Attention(nn.Module): - def __init__( - self, dim: int, dim_head: int = 64, heads: int = 8, mult: int = 4 - ) -> None: - super().__init__() - self.norm = LayerNorm(dim) - attn_inner_dim = heads * dim_head - ff_inner_dim = mult * dim - self.heads = heads - self.scale = dim_head ** -0.5 - self.fused_dims = (attn_inner_dim, dim_head, dim_head, (2 * ff_inner_dim)) - self.fused_attn_ff_proj = nn.Linear(dim, sum(self.fused_dims), bias=False) - self.attn_out = nn.Linear(attn_inner_dim, dim, bias=False) - self.ff_out = nn.Sequential(SwiGLU(), nn.Linear(ff_inner_dim, dim, bias=False)) - - def forward(self, x: Tensor) -> Tensor: - h = self.heads - x = self.norm(x) - q, k, v, ff = self.fused_attn_ff_proj(x).split(self.fused_dims, dim=-1) - q = rearrange(q, "b n (h d) -> b h n d", h=h) - q = q * self.scale - sim = einsum("b h i d, b j d -> b h i j", q, k) - attn = sim.softmax(dim=-1) - out = einsum("b h i j, b j d -> b h i d", attn, v) - out = rearrange(out, "b h n d -> b n (h d)") - return self.attn_out(out) + self.ff_out(ff) -- cgit v1.2.3-70-g09d2