summaryrefslogtreecommitdiff
path: root/text_recognizer/models
diff options
context:
space:
mode:
Diffstat (limited to 'text_recognizer/models')
-rw-r--r--text_recognizer/models/base.py9
-rw-r--r--text_recognizer/models/transformer.py66
2 files changed, 37 insertions, 38 deletions
diff --git a/text_recognizer/models/base.py b/text_recognizer/models/base.py
index bb4e695..f8f4b40 100644
--- a/text_recognizer/models/base.py
+++ b/text_recognizer/models/base.py
@@ -9,7 +9,7 @@ from pytorch_lightning import LightningModule
from torch import nn, Tensor
from torchmetrics import Accuracy
-from text_recognizer.data.mappings import EmnistMapping
+from text_recognizer.data.tokenizer import Tokenizer
class LitBase(LightningModule):
@@ -21,8 +21,7 @@ class LitBase(LightningModule):
loss_fn: Type[nn.Module],
optimizer_config: DictConfig,
lr_scheduler_config: Optional[DictConfig],
- mapping: EmnistMapping,
- ignore_index: Optional[int] = None,
+ tokenizer: Tokenizer,
) -> None:
super().__init__()
@@ -30,8 +29,8 @@ class LitBase(LightningModule):
self.loss_fn = loss_fn
self.optimizer_config = optimizer_config
self.lr_scheduler_config = lr_scheduler_config
- self.mapping = mapping
-
+ self.tokenizer = tokenizer
+ ignore_index = int(self.tokenizer.get_value("<p>"))
# Placeholders
self.train_acc = Accuracy(mdmc_reduce="samplewise", ignore_index=ignore_index)
self.val_acc = Accuracy(mdmc_reduce="samplewise", ignore_index=ignore_index)
diff --git a/text_recognizer/models/transformer.py b/text_recognizer/models/transformer.py
index 2c74b7e..752f3eb 100644
--- a/text_recognizer/models/transformer.py
+++ b/text_recognizer/models/transformer.py
@@ -1,11 +1,12 @@
"""Lightning model for base Transformers."""
+from collections.abc import Sequence
from typing import Optional, Tuple, Type
import torch
from omegaconf import DictConfig
from torch import nn, Tensor
-from text_recognizer.data.mappings import EmnistMapping
+from text_recognizer.data.tokenizer import Tokenizer
from text_recognizer.models.base import LitBase
from text_recognizer.models.metrics.cer import CharacterErrorRate
from text_recognizer.models.metrics.wer import WordErrorRate
@@ -19,33 +20,23 @@ class LitTransformer(LitBase):
network: Type[nn.Module],
loss_fn: Type[nn.Module],
optimizer_config: DictConfig,
- mapping: EmnistMapping,
+ tokenizer: Tokenizer,
lr_scheduler_config: Optional[DictConfig] = None,
max_output_len: int = 682,
- start_token: str = "<s>",
- end_token: str = "<e>",
- pad_token: str = "<p>",
) -> None:
- self.max_output_len = max_output_len
- self.start_token = start_token
- self.end_token = end_token
- self.pad_token = pad_token
- self.start_index = int(self.mapping.get_index(self.start_token))
- self.end_index = int(self.mapping.get_index(self.end_token))
- self.pad_index = int(self.mapping.get_index(self.pad_token))
- self.ignore_indices = set([self.start_index, self.end_index, self.pad_index])
- self.val_cer = CharacterErrorRate(self.ignore_indices)
- self.test_cer = CharacterErrorRate(self.ignore_indices)
- self.val_wer = WordErrorRate(self.ignore_indices)
- self.test_wer = WordErrorRate(self.ignore_indices)
super().__init__(
network,
loss_fn,
optimizer_config,
lr_scheduler_config,
- mapping,
- self.pad_index,
+ tokenizer,
)
+ self.max_output_len = max_output_len
+ self.ignore_indices = set([self.start_index, self.end_index, self.pad_index])
+ self.val_cer = CharacterErrorRate(self.ignore_indices)
+ self.test_cer = CharacterErrorRate(self.ignore_indices)
+ self.val_wer = WordErrorRate(self.ignore_indices)
+ self.test_wer = WordErrorRate(self.ignore_indices)
def forward(self, data: Tensor) -> Tensor:
"""Forward pass with the transformer network."""
@@ -63,11 +54,12 @@ class LitTransformer(LitBase):
"""Validation step."""
data, targets = batch
preds = self.predict(data)
- self.val_acc(preds, targets)
+ pred_text, target_text = self.get_text(preds, targets)
+ self.val_acc(pred_text, target_text)
self.log("val/acc", self.val_acc, on_step=False, on_epoch=True)
- self.val_cer(preds, targets)
+ self.val_cer(pred_text, target_text)
self.log("val/cer", self.val_cer, on_step=False, on_epoch=True, prog_bar=True)
- self.val_wer(preds, targets)
+ self.val_wer(pred_text, target_text)
self.log("val/wer", self.val_wer, on_step=False, on_epoch=True, prog_bar=True)
def test_step(self, batch: Tuple[Tensor, Tensor], batch_idx: int) -> None:
@@ -75,14 +67,22 @@ class LitTransformer(LitBase):
data, targets = batch
# Compute the text prediction.
- pred = self(data)
- self.test_acc(pred, targets)
+ preds = self(data)
+ pred_text, target_text = self.get_text(preds, targets)
+ self.test_acc(pred_text, target_text)
self.log("test/acc", self.test_acc, on_step=False, on_epoch=True)
- self.test_cer(pred, targets)
+ self.test_cer(pred_text, target_text)
self.log("test/cer", self.test_cer, on_step=False, on_epoch=True, prog_bar=True)
- self.test_wer(pred, targets)
+ self.test_wer(pred_text, target_text)
self.log("test/wer", self.test_wer, on_step=False, on_epoch=True, prog_bar=True)
+ def get_text(
+ self, preds: Tensor, targets: Tensor
+ ) -> Tuple[Sequence[str], Sequence[str]]:
+ pred_text = [self.tokenizer.decode(p) for p in preds]
+ target_text = [self.tokenizer.decode(t) for t in targets]
+ return pred_text, target_text
+
@torch.no_grad()
def predict(self, x: Tensor) -> Tensor:
"""Predicts text in image.
@@ -97,6 +97,9 @@ class LitTransformer(LitBase):
Returns:
Tensor: A tensor of token indices of the predictions from the model.
"""
+ start_index = self.tokenizer.start_index
+ end_index = self.tokenizer.start_index
+ pad_index = self.tokenizer.start_index
bsz = x.shape[0]
# Encode image(s) to latent vectors.
@@ -104,7 +107,7 @@ class LitTransformer(LitBase):
# Create a placeholder matrix for storing outputs from the network
output = torch.ones((bsz, self.max_output_len), dtype=torch.long).to(x.device)
- output[:, 0] = self.start_index
+ output[:, 0] = start_index
for Sy in range(1, self.max_output_len):
context = output[:, :Sy] # (B, Sy)
@@ -114,16 +117,13 @@ class LitTransformer(LitBase):
# Early stopping of prediction loop if token is end or padding token.
if (
- (output[:, Sy - 1] == self.end_index)
- | (output[:, Sy - 1] == self.pad_index)
+ (output[:, Sy - 1] == end_index) | (output[:, Sy - 1] == pad_index)
).all():
break
# Set all tokens after end token to pad token.
for Sy in range(1, self.max_output_len):
- idx = (output[:, Sy - 1] == self.end_index) | (
- output[:, Sy - 1] == self.pad_index
- )
- output[idx, Sy] = self.pad_index
+ idx = (output[:, Sy - 1] == end_index) | (output[:, Sy - 1] == pad_index)
+ output[idx, Sy] = pad_index
return output