diff options
Diffstat (limited to 'text_recognizer/models')
| -rw-r--r-- | text_recognizer/models/base.py | 9 | ||||
| -rw-r--r-- | text_recognizer/models/transformer.py | 66 |
2 files changed, 37 insertions, 38 deletions
diff --git a/text_recognizer/models/base.py b/text_recognizer/models/base.py index bb4e695..f8f4b40 100644 --- a/text_recognizer/models/base.py +++ b/text_recognizer/models/base.py @@ -9,7 +9,7 @@ from pytorch_lightning import LightningModule from torch import nn, Tensor from torchmetrics import Accuracy -from text_recognizer.data.mappings import EmnistMapping +from text_recognizer.data.tokenizer import Tokenizer class LitBase(LightningModule): @@ -21,8 +21,7 @@ class LitBase(LightningModule): loss_fn: Type[nn.Module], optimizer_config: DictConfig, lr_scheduler_config: Optional[DictConfig], - mapping: EmnistMapping, - ignore_index: Optional[int] = None, + tokenizer: Tokenizer, ) -> None: super().__init__() @@ -30,8 +29,8 @@ class LitBase(LightningModule): self.loss_fn = loss_fn self.optimizer_config = optimizer_config self.lr_scheduler_config = lr_scheduler_config - self.mapping = mapping - + self.tokenizer = tokenizer + ignore_index = int(self.tokenizer.get_value("<p>")) # Placeholders self.train_acc = Accuracy(mdmc_reduce="samplewise", ignore_index=ignore_index) self.val_acc = Accuracy(mdmc_reduce="samplewise", ignore_index=ignore_index) diff --git a/text_recognizer/models/transformer.py b/text_recognizer/models/transformer.py index 2c74b7e..752f3eb 100644 --- a/text_recognizer/models/transformer.py +++ b/text_recognizer/models/transformer.py @@ -1,11 +1,12 @@ """Lightning model for base Transformers.""" +from collections.abc import Sequence from typing import Optional, Tuple, Type import torch from omegaconf import DictConfig from torch import nn, Tensor -from text_recognizer.data.mappings import EmnistMapping +from text_recognizer.data.tokenizer import Tokenizer from text_recognizer.models.base import LitBase from text_recognizer.models.metrics.cer import CharacterErrorRate from text_recognizer.models.metrics.wer import WordErrorRate @@ -19,33 +20,23 @@ class LitTransformer(LitBase): network: Type[nn.Module], loss_fn: Type[nn.Module], optimizer_config: DictConfig, - mapping: EmnistMapping, + tokenizer: Tokenizer, lr_scheduler_config: Optional[DictConfig] = None, max_output_len: int = 682, - start_token: str = "<s>", - end_token: str = "<e>", - pad_token: str = "<p>", ) -> None: - self.max_output_len = max_output_len - self.start_token = start_token - self.end_token = end_token - self.pad_token = pad_token - self.start_index = int(self.mapping.get_index(self.start_token)) - self.end_index = int(self.mapping.get_index(self.end_token)) - self.pad_index = int(self.mapping.get_index(self.pad_token)) - self.ignore_indices = set([self.start_index, self.end_index, self.pad_index]) - self.val_cer = CharacterErrorRate(self.ignore_indices) - self.test_cer = CharacterErrorRate(self.ignore_indices) - self.val_wer = WordErrorRate(self.ignore_indices) - self.test_wer = WordErrorRate(self.ignore_indices) super().__init__( network, loss_fn, optimizer_config, lr_scheduler_config, - mapping, - self.pad_index, + tokenizer, ) + self.max_output_len = max_output_len + self.ignore_indices = set([self.start_index, self.end_index, self.pad_index]) + self.val_cer = CharacterErrorRate(self.ignore_indices) + self.test_cer = CharacterErrorRate(self.ignore_indices) + self.val_wer = WordErrorRate(self.ignore_indices) + self.test_wer = WordErrorRate(self.ignore_indices) def forward(self, data: Tensor) -> Tensor: """Forward pass with the transformer network.""" @@ -63,11 +54,12 @@ class LitTransformer(LitBase): """Validation step.""" data, targets = batch preds = self.predict(data) - self.val_acc(preds, targets) + pred_text, target_text = self.get_text(preds, targets) + self.val_acc(pred_text, target_text) self.log("val/acc", self.val_acc, on_step=False, on_epoch=True) - self.val_cer(preds, targets) + self.val_cer(pred_text, target_text) self.log("val/cer", self.val_cer, on_step=False, on_epoch=True, prog_bar=True) - self.val_wer(preds, targets) + self.val_wer(pred_text, target_text) self.log("val/wer", self.val_wer, on_step=False, on_epoch=True, prog_bar=True) def test_step(self, batch: Tuple[Tensor, Tensor], batch_idx: int) -> None: @@ -75,14 +67,22 @@ class LitTransformer(LitBase): data, targets = batch # Compute the text prediction. - pred = self(data) - self.test_acc(pred, targets) + preds = self(data) + pred_text, target_text = self.get_text(preds, targets) + self.test_acc(pred_text, target_text) self.log("test/acc", self.test_acc, on_step=False, on_epoch=True) - self.test_cer(pred, targets) + self.test_cer(pred_text, target_text) self.log("test/cer", self.test_cer, on_step=False, on_epoch=True, prog_bar=True) - self.test_wer(pred, targets) + self.test_wer(pred_text, target_text) self.log("test/wer", self.test_wer, on_step=False, on_epoch=True, prog_bar=True) + def get_text( + self, preds: Tensor, targets: Tensor + ) -> Tuple[Sequence[str], Sequence[str]]: + pred_text = [self.tokenizer.decode(p) for p in preds] + target_text = [self.tokenizer.decode(t) for t in targets] + return pred_text, target_text + @torch.no_grad() def predict(self, x: Tensor) -> Tensor: """Predicts text in image. @@ -97,6 +97,9 @@ class LitTransformer(LitBase): Returns: Tensor: A tensor of token indices of the predictions from the model. """ + start_index = self.tokenizer.start_index + end_index = self.tokenizer.start_index + pad_index = self.tokenizer.start_index bsz = x.shape[0] # Encode image(s) to latent vectors. @@ -104,7 +107,7 @@ class LitTransformer(LitBase): # Create a placeholder matrix for storing outputs from the network output = torch.ones((bsz, self.max_output_len), dtype=torch.long).to(x.device) - output[:, 0] = self.start_index + output[:, 0] = start_index for Sy in range(1, self.max_output_len): context = output[:, :Sy] # (B, Sy) @@ -114,16 +117,13 @@ class LitTransformer(LitBase): # Early stopping of prediction loop if token is end or padding token. if ( - (output[:, Sy - 1] == self.end_index) - | (output[:, Sy - 1] == self.pad_index) + (output[:, Sy - 1] == end_index) | (output[:, Sy - 1] == pad_index) ).all(): break # Set all tokens after end token to pad token. for Sy in range(1, self.max_output_len): - idx = (output[:, Sy - 1] == self.end_index) | ( - output[:, Sy - 1] == self.pad_index - ) - output[idx, Sy] = self.pad_index + idx = (output[:, Sy - 1] == end_index) | (output[:, Sy - 1] == pad_index) + output[idx, Sy] = pad_index return output |