# @package _global_

defaults:
  - override /criterion: cross_entropy
  - override /callbacks: htr
  - override /datamodule: iam_extended_paragraphs
  - override /network: null
  - override /model: lit_transformer
  - override /lr_scheduler: null
  - override /optimizer: null

tags: [paragraphs]
epochs: &epochs 600
num_classes: &num_classes 58
ignore_index: &ignore_index 3
max_output_len: &max_output_len 682
# summary: [[1, 1, 576, 640], [1, 682]]

logger:
  wandb:
    tags: ${tags}
    id: 8je5lxmx

criterion:
  ignore_index: *ignore_index
  # label_smoothing: 0.05

callbacks:
  stochastic_weight_averaging:
    _target_: pytorch_lightning.callbacks.StochasticWeightAveraging
    swa_epoch_start: 0.75
    swa_lrs: 1.0e-5
    annealing_epochs: 10
    annealing_strategy: cos
    device: null

optimizer:
  _target_: adan_pytorch.Adan
  lr: 3.0e-4
  betas: [0.02, 0.08, 0.01]
  weight_decay: 0.02

lr_scheduler:
  _target_: torch.optim.lr_scheduler.ReduceLROnPlateau
  mode: min
  factor: 0.8
  patience: 10
  threshold: 1.0e-4
  threshold_mode: rel
  cooldown: 0
  min_lr: 1.0e-5
  eps: 1.0e-8
  verbose: false
  interval: epoch
  monitor: val/cer

datamodule:
  batch_size: 2
  train_fraction: 0.95

network:
  _target_: text_recognizer.networks.ConvTransformer
  input_dims: [1, 1, 576, 640]
  hidden_dim: &hidden_dim 128
  num_classes: *num_classes
  pad_index: 3
  encoder:
    _target_: text_recognizer.networks.convnext.ConvNext
    dim: 16
    dim_mults: [2, 4, 8, 8]
    depths: [3, 3, 6, 6]
    downsampling_factors: [[2, 2], [2, 2], [2, 2], [2, 1]]
    attn:
      _target_: text_recognizer.networks.convnext.TransformerBlock
      attn:
        _target_: text_recognizer.networks.convnext.Attention
        dim: 128
        heads: 4
        dim_head: 64
        scale: 8
      ff:
        _target_: text_recognizer.networks.convnext.FeedForward
        dim: 128
        mult: 4
  decoder:
    _target_: text_recognizer.networks.transformer.Decoder
    depth: 6
    block:
      _target_: text_recognizer.networks.transformer.DecoderBlock
      self_attn:
        _target_: text_recognizer.networks.transformer.Attention
        dim: *hidden_dim
        num_heads: 12
        dim_head: 64
        dropout_rate: &dropout_rate 0.2
        causal: true
        rotary_embedding:
          _target_: text_recognizer.networks.transformer.RotaryEmbedding
          dim: 64
      cross_attn:
        _target_: text_recognizer.networks.transformer.Attention
        dim: *hidden_dim
        num_heads: 12
        dim_head: 64
        dropout_rate: *dropout_rate
        causal: false
      norm:
        _target_: text_recognizer.networks.transformer.RMSNorm
        dim: *hidden_dim
      ff:
        _target_: text_recognizer.networks.transformer.FeedForward
        dim: *hidden_dim
        dim_out: null
        expansion_factor: 2
        glu: true
        dropout_rate: *dropout_rate
  pixel_embedding:
    _target_: "text_recognizer.networks.transformer.embeddings.axial.\
      AxialPositionalEmbeddingImage"
    dim: *hidden_dim
    axial_shape: [36, 80]
    axial_dims: [64, 64]
  token_pos_embedding:
    _target_: "text_recognizer.networks.transformer.embeddings.fourier.\
      PositionalEncoding"
    dim: *hidden_dim
    dropout_rate: 0.1
    max_len: 89

trainer:
  gradient_clip_val: 1.0
  max_epochs: *epochs
  accumulate_grad_batches: 6