diff options
Diffstat (limited to 'training/conf/experiment/cnn_htr_char_lines.yaml')
| -rw-r--r-- | training/conf/experiment/cnn_htr_char_lines.yaml | 90 |
1 files changed, 54 insertions, 36 deletions
diff --git a/training/conf/experiment/cnn_htr_char_lines.yaml b/training/conf/experiment/cnn_htr_char_lines.yaml index 0f28ff9..0d62a73 100644 --- a/training/conf/experiment/cnn_htr_char_lines.yaml +++ b/training/conf/experiment/cnn_htr_char_lines.yaml @@ -10,28 +10,26 @@ defaults: - override /optimizers: null +epochs: &epochs 200 +ignore_index: &ignore_index 3 +num_classes: &num_classes 58 +max_output_len: &max_output_len 89 + criterion: _target_: text_recognizer.criterions.label_smoothing.LabelSmoothingLoss smoothing: 0.1 - ignore_index: 1000 + ignore_index: *ignore_index + # _target_: torch.nn.CrossEntropyLoss + # ignore_index: *ignore_index mapping: - _target_: text_recognizer.data.word_piece_mapping.WordPieceMapping - num_features: 1000 - tokens: iamdb_1kwp_tokens_1000.txt - lexicon: iamdb_1kwp_lex_1000.txt - data_dir: null - use_words: false - prepend_wordsep: false - special_tokens: [ <s>, <e>, <p> ] - # _target_: text_recognizer.data.emnist_mapping.EmnistMapping - # extra_symbols: [ "\n" ] + _target_: text_recognizer.data.emnist_mapping.EmnistMapping callbacks: stochastic_weight_averaging: _target_: pytorch_lightning.callbacks.StochasticWeightAveraging - swa_epoch_start: 0.8 - swa_lrs: 0.05 + swa_epoch_start: 0.75 + swa_lrs: 5.0e-5 annealing_epochs: 10 annealing_strategy: cos device: null @@ -39,7 +37,7 @@ callbacks: optimizers: madgrad: _target_: madgrad.MADGRAD - lr: 1.0e-4 + lr: 3.0e-4 momentum: 0.9 weight_decay: 0 eps: 1.0e-6 @@ -48,34 +46,42 @@ optimizers: lr_schedulers: network: - _target_: torch.optim.lr_scheduler.ReduceLROnPlateau - mode: min - factor: 0.1 - patience: 10 - threshold: 1.0e-4 - threshold_mode: rel - cooldown: 0 - min_lr: 1.0e-7 - eps: 1.0e-8 - interval: epoch + _target_: torch.optim.lr_scheduler.OneCycleLR + max_lr: 3.0e-4 + total_steps: null + epochs: *epochs + steps_per_epoch: 90 + pct_start: 0.1 + anneal_strategy: cos + cycle_momentum: true + base_momentum: 0.85 + max_momentum: 0.95 + div_factor: 25 + final_div_factor: 1.0e4 + three_phase: false + last_epoch: -1 + verbose: false + # Non-class arguments + interval: step monitor: val/loss datamodule: _target_: text_recognizer.data.iam_lines.IAMLines - batch_size: 16 + batch_size: 32 num_workers: 12 train_fraction: 0.8 augment: true - pin_memory: false + pin_memory: true + word_pieces: false network: _target_: text_recognizer.networks.conv_transformer.ConvTransformer input_dims: [1, 56, 1024] - hidden_dim: 128 + hidden_dim: &hidden_dim 128 encoder_dim: 1280 dropout_rate: 0.2 - num_classes: 1006 - pad_index: 1000 + num_classes: *num_classes + pad_index: *ignore_index encoder: _target_: text_recognizer.networks.encoders.efficientnet.EfficientNet arch: b0 @@ -85,14 +91,14 @@ network: bn_eps: 1.0e-3 decoder: _target_: text_recognizer.networks.transformer.Decoder - dim: 128 + dim: *hidden_dim depth: 3 num_heads: 4 attn_fn: text_recognizer.networks.transformer.attention.Attention attn_kwargs: dim_head: 32 dropout_rate: 0.2 - norm_fn: torch.nn.LayerNorm + norm_fn: text_recognizer.networks.transformer.norm.ScaleNorm ff_fn: text_recognizer.networks.transformer.mlp.FeedForward ff_kwargs: dim_out: null @@ -101,11 +107,23 @@ network: dropout_rate: 0.2 cross_attend: true pre_norm: true - rotary_emb: null + rotary_emb: + _target_: text_recognizer.networks.transformer.positional_encodings.rotary_embedding.RotaryEmbedding + dim: 32 + pixel_pos_embedding: + _target_: text_recognizer.networks.transformer.positional_encodings.PositionalEncoding2D + hidden_dim: *hidden_dim + max_h: 1 + max_w: 32 + token_pos_embedding: + _target_: text_recognizer.networks.transformer.positional_encodings.PositionalEncoding + hidden_dim: *hidden_dim + dropout_rate: 0.2 + max_len: *max_output_len model: _target_: text_recognizer.models.transformer.TransformerLitModel - max_output_len: 89 + max_output_len: *max_output_len start_token: <s> end_token: <e> pad_token: <p> @@ -115,11 +133,11 @@ trainer: stochastic_weight_avg: true auto_scale_batch_size: binsearch auto_lr_find: false - gradient_clip_val: 0 + gradient_clip_val: 0.5 fast_dev_run: false gpus: 1 precision: 16 - max_epochs: 1024 + max_epochs: *epochs terminate_on_nan: true weights_summary: null limit_train_batches: 1.0 @@ -127,6 +145,6 @@ trainer: limit_test_batches: 1.0 resume_from_checkpoint: null accumulate_grad_batches: 4 - overfit_batches: 0.0 + overfit_batches: 0 summary: [[1, 1, 56, 1024], [1, 89]] |