From 8689e8da65ed02a13975956d70216d7f33db1688 Mon Sep 17 00:00:00 2001 From: Gustaf Rydholm Date: Sun, 19 Jun 2022 20:58:36 +0200 Subject: Fix lr and optim conf --- training/conf/config.yaml | 4 ++-- training/conf/experiment/conv_transformer_lines.yaml | 8 ++++---- .../conf/experiment/conv_transformer_paragraphs.yaml | 8 ++++---- training/conf/lr_scheduler/cosine_annealing.yaml | 7 +++++++ training/conf/lr_scheduler/one_cycle.yaml | 17 +++++++++++++++++ training/conf/lr_scheduler/reduce_on_plateau.yaml | 12 ++++++++++++ training/conf/lr_schedulers/cosine_annealing.yaml | 7 ------- training/conf/lr_schedulers/one_cycle.yaml | 17 ----------------- training/conf/lr_schedulers/reduce_on_plateau.yaml | 12 ------------ training/conf/optimizer/radam.yaml | 6 ++++++ training/conf/optimizers/radam.yaml | 6 ------ 11 files changed, 52 insertions(+), 52 deletions(-) create mode 100644 training/conf/lr_scheduler/cosine_annealing.yaml create mode 100644 training/conf/lr_scheduler/one_cycle.yaml create mode 100644 training/conf/lr_scheduler/reduce_on_plateau.yaml delete mode 100644 training/conf/lr_schedulers/cosine_annealing.yaml delete mode 100644 training/conf/lr_schedulers/one_cycle.yaml delete mode 100644 training/conf/lr_schedulers/reduce_on_plateau.yaml create mode 100644 training/conf/optimizer/radam.yaml delete mode 100644 training/conf/optimizers/radam.yaml (limited to 'training/conf') diff --git a/training/conf/config.yaml b/training/conf/config.yaml index fdeb5d2..46a1e43 100644 --- a/training/conf/config.yaml +++ b/training/conf/config.yaml @@ -7,10 +7,10 @@ defaults: - datamodule: iam_extended_paragraphs - hydra: default - logger: wandb - - lr_schedulers: cosine_annealing + - lr_scheduler: cosine_annealing - model: lit_transformer - network: conv_transformer - - optimizers: radam + - optimizer: radam - trainer: default seed: 4711 diff --git a/training/conf/experiment/conv_transformer_lines.yaml b/training/conf/experiment/conv_transformer_lines.yaml index 48df78d..eb9bc9e 100644 --- a/training/conf/experiment/conv_transformer_lines.yaml +++ b/training/conf/experiment/conv_transformer_lines.yaml @@ -6,8 +6,8 @@ defaults: - override /datamodule: iam_lines - override /network: conv_transformer - override /model: lit_transformer - - override /lr_schedulers: null - - override /optimizers: null + - override /lr_scheduler: null + - override /optimizer: null epochs: &epochs 512 ignore_index: &ignore_index 3 @@ -28,7 +28,7 @@ callbacks: annealing_strategy: cos device: null -optimizers: +optimizer: _target_: torch.optim.RAdam lr: 3.0e-4 betas: [0.9, 0.999] @@ -36,7 +36,7 @@ optimizers: eps: 1.0e-8 parameters: network -lr_schedulers: +lr_scheduler: _target_: torch.optim.lr_scheduler.OneCycleLR max_lr: 3.0e-4 total_steps: null diff --git a/training/conf/experiment/conv_transformer_paragraphs.yaml b/training/conf/experiment/conv_transformer_paragraphs.yaml index d0d0314..7c6e231 100644 --- a/training/conf/experiment/conv_transformer_paragraphs.yaml +++ b/training/conf/experiment/conv_transformer_paragraphs.yaml @@ -6,8 +6,8 @@ defaults: - override /datamodule: iam_extended_paragraphs - override /network: conv_transformer - override /model: lit_transformer - - override /lr_schedulers: null - - override /optimizers: null + - override /lr_scheduler: null + - override /optimizer: null epochs: &epochs 600 num_classes: &num_classes 58 @@ -28,7 +28,7 @@ callbacks: annealing_strategy: cos device: null -optimizers: +optimizer: _target_: torch.optim.RAdam lr: 3.0e-4 betas: [0.9, 0.999] @@ -36,7 +36,7 @@ optimizers: eps: 1.0e-8 parameters: network -lr_schedulers: +lr_scheduler: _target_: torch.optim.lr_scheduler.OneCycleLR max_lr: 3.0e-4 total_steps: null diff --git a/training/conf/lr_scheduler/cosine_annealing.yaml b/training/conf/lr_scheduler/cosine_annealing.yaml new file mode 100644 index 0000000..e8364f0 --- /dev/null +++ b/training/conf/lr_scheduler/cosine_annealing.yaml @@ -0,0 +1,7 @@ +_target_: torch.optim.lr_scheduler.CosineAnnealingLR +T_max: 256 +eta_min: 0.0 +last_epoch: -1 + +interval: epoch +monitor: val/loss diff --git a/training/conf/lr_scheduler/one_cycle.yaml b/training/conf/lr_scheduler/one_cycle.yaml new file mode 100644 index 0000000..20eab9f --- /dev/null +++ b/training/conf/lr_scheduler/one_cycle.yaml @@ -0,0 +1,17 @@ +_target_: torch.optim.lr_scheduler.OneCycleLR +max_lr: 1.0e-3 +total_steps: null +epochs: 512 +steps_per_epoch: 4992 +pct_start: 0.3 +anneal_strategy: cos +cycle_momentum: true +base_momentum: 0.85 +max_momentum: 0.95 +div_factor: 25.0 +final_div_factor: 10000.0 +three_phase: true +last_epoch: -1 +verbose: false +interval: step +monitor: val/loss diff --git a/training/conf/lr_scheduler/reduce_on_plateau.yaml b/training/conf/lr_scheduler/reduce_on_plateau.yaml new file mode 100644 index 0000000..9aa9de4 --- /dev/null +++ b/training/conf/lr_scheduler/reduce_on_plateau.yaml @@ -0,0 +1,12 @@ +_target_: torch.optim.lr_scheduler.ReduceLROnPlateau +mode: min +factor: 0.1 +patience: 10 +threshold: 1.0e-4 +threshold_mode: rel +cooldown: 0 +min_lr: 1.0e-5 +eps: 1.0e-8 +verbose: false +interval: epoch +monitor: val/loss diff --git a/training/conf/lr_schedulers/cosine_annealing.yaml b/training/conf/lr_schedulers/cosine_annealing.yaml deleted file mode 100644 index e8364f0..0000000 --- a/training/conf/lr_schedulers/cosine_annealing.yaml +++ /dev/null @@ -1,7 +0,0 @@ -_target_: torch.optim.lr_scheduler.CosineAnnealingLR -T_max: 256 -eta_min: 0.0 -last_epoch: -1 - -interval: epoch -monitor: val/loss diff --git a/training/conf/lr_schedulers/one_cycle.yaml b/training/conf/lr_schedulers/one_cycle.yaml deleted file mode 100644 index 20eab9f..0000000 --- a/training/conf/lr_schedulers/one_cycle.yaml +++ /dev/null @@ -1,17 +0,0 @@ -_target_: torch.optim.lr_scheduler.OneCycleLR -max_lr: 1.0e-3 -total_steps: null -epochs: 512 -steps_per_epoch: 4992 -pct_start: 0.3 -anneal_strategy: cos -cycle_momentum: true -base_momentum: 0.85 -max_momentum: 0.95 -div_factor: 25.0 -final_div_factor: 10000.0 -three_phase: true -last_epoch: -1 -verbose: false -interval: step -monitor: val/loss diff --git a/training/conf/lr_schedulers/reduce_on_plateau.yaml b/training/conf/lr_schedulers/reduce_on_plateau.yaml deleted file mode 100644 index 9aa9de4..0000000 --- a/training/conf/lr_schedulers/reduce_on_plateau.yaml +++ /dev/null @@ -1,12 +0,0 @@ -_target_: torch.optim.lr_scheduler.ReduceLROnPlateau -mode: min -factor: 0.1 -patience: 10 -threshold: 1.0e-4 -threshold_mode: rel -cooldown: 0 -min_lr: 1.0e-5 -eps: 1.0e-8 -verbose: false -interval: epoch -monitor: val/loss diff --git a/training/conf/optimizer/radam.yaml b/training/conf/optimizer/radam.yaml new file mode 100644 index 0000000..d11fcb5 --- /dev/null +++ b/training/conf/optimizer/radam.yaml @@ -0,0 +1,6 @@ +_target_: torch.optim.RAdam +lr: 1.5e-4 +betas: [0.9, 0.999] +weight_decay: 1.0e-4 +eps: 1.0e-8 +parameters: network diff --git a/training/conf/optimizers/radam.yaml b/training/conf/optimizers/radam.yaml deleted file mode 100644 index d11fcb5..0000000 --- a/training/conf/optimizers/radam.yaml +++ /dev/null @@ -1,6 +0,0 @@ -_target_: torch.optim.RAdam -lr: 1.5e-4 -betas: [0.9, 0.999] -weight_decay: 1.0e-4 -eps: 1.0e-8 -parameters: network -- cgit v1.2.3-70-g09d2