1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
|
"""Defines the CNN-Transformer class."""
from typing import Callable, Dict, List, Optional, Tuple, Type, Union
import numpy as np
import torch
from torch import nn
from torch import Tensor
from torch.utils.data import Dataset
from text_recognizer.datasets import EmnistMapper
import text_recognizer.datasets.transforms as transforms
from text_recognizer.models.base import Model
from text_recognizer.networks import greedy_decoder
class TransformerModel(Model):
"""Model for predicting a sequence of characters from an image of a text line with a cnn-transformer."""
def __init__(
self,
network_fn: str,
dataset: str,
network_args: Optional[Dict] = None,
dataset_args: Optional[Dict] = None,
metrics: Optional[Dict] = None,
criterion: Optional[Callable] = None,
criterion_args: Optional[Dict] = None,
optimizer: Optional[Callable] = None,
optimizer_args: Optional[Dict] = None,
lr_scheduler: Optional[Callable] = None,
lr_scheduler_args: Optional[Dict] = None,
swa_args: Optional[Dict] = None,
device: Optional[str] = None,
) -> None:
super().__init__(
network_fn,
dataset,
network_args,
dataset_args,
metrics,
criterion,
criterion_args,
optimizer,
optimizer_args,
lr_scheduler,
lr_scheduler_args,
swa_args,
device,
)
self.init_token = dataset_args["args"]["init_token"]
self.pad_token = dataset_args["args"]["pad_token"]
self.eos_token = dataset_args["args"]["eos_token"]
self.lower = dataset_args["args"]["lower"]
self.max_len = 100
if self._mapper is None:
self._mapper = EmnistMapper(
init_token=self.init_token,
pad_token=self.pad_token,
eos_token=self.eos_token,
lower=self.lower,
)
self.tensor_transform = transforms.Compose(
[transforms.ToTensor(), transforms.Normalize(mean=[0.912], std=[0.168])]
)
self.softmax = nn.Softmax(dim=2)
@torch.no_grad()
def _generate_sentence(self, image: Tensor) -> Tuple[List, float]:
src = self.network.extract_image_features(image)
# Added for vqvae transformer.
if isinstance(src, Tuple):
src = src[0]
memory = self.network.encoder(src)
confidence_of_predictions = []
trg_indices = [self.mapper(self.init_token)]
for _ in range(self.max_len - 1):
trg = torch.tensor(trg_indices, device=self.device)[None, :].long()
trg = self.network.target_embedding(trg)
logits = self.network.decoder(trg=trg, memory=memory, trg_mask=None)
# Convert logits to probabilities.
probs = self.softmax(logits)
pred_token = probs.argmax(2)[:, -1].item()
confidence = probs.max(2).values[:, -1].item()
trg_indices.append(pred_token)
confidence_of_predictions.append(confidence)
if pred_token == self.mapper(self.eos_token):
break
confidence = np.min(confidence_of_predictions)
predicted_characters = "".join([self.mapper(x) for x in trg_indices[1:]])
return predicted_characters, confidence
@torch.no_grad()
def predict_on_image(self, image: Union[np.ndarray, Tensor]) -> Tuple[str, float]:
"""Predict on a single input."""
self.eval()
if image.dtype == np.uint8:
# Converts an image with range [0, 255] with to PyTorch Tensor with range [0, 1].
image = self.tensor_transform(image)
# Rescale image between 0 and 1.
if image.dtype == torch.uint8:
# If the image is an unscaled tensor.
image = image.type("torch.FloatTensor") / 255
# Put the image tensor on the device the model weights are on.
image = image.to(self.device)
(predicted_characters, confidence_of_prediction,) = self._generate_sentence(
image
)
return predicted_characters, confidence_of_prediction
|
