diff options
| author | Gustaf Rydholm <gustaf.rydholm@gmail.com> | 2022-01-26 23:22:13 +0100 |
|---|---|---|
| committer | Gustaf Rydholm <gustaf.rydholm@gmail.com> | 2022-01-26 23:22:13 +0100 |
| commit | 69e93847bb0adf6e819f48e70e6a88da4cf0b7f6 (patch) | |
| tree | b7e533674d51f8e34a9318b7a4f7344aae56f5de /training | |
| parent | 68640a70a769568fd571e50322d6e3fa40c78271 (diff) | |
fix: nuke madgrad, add radam
Diffstat (limited to 'training')
| -rw-r--r-- | training/conf/config.yaml | 6 | ||||
| -rw-r--r-- | training/conf/optimizers/madgrad.yaml | 8 | ||||
| -rw-r--r-- | training/conf/optimizers/radam.yaml | 7 |
3 files changed, 10 insertions, 11 deletions
diff --git a/training/conf/config.yaml b/training/conf/config.yaml index e783efd..11bb551 100644 --- a/training/conf/config.yaml +++ b/training/conf/config.yaml @@ -7,13 +7,13 @@ defaults: - datamodule: iam_extended_paragraphs - hydra: default - logger: wandb - - lr_schedulers: + - lr_schedulers: - cosine_annealing - mapping: characters # word_piece - model: lit_transformer - network: conv_transformer - - optimizers: - - madgrad + - optimizers: + - radam - trainer: default seed: 4711 diff --git a/training/conf/optimizers/madgrad.yaml b/training/conf/optimizers/madgrad.yaml deleted file mode 100644 index b6507b9..0000000 --- a/training/conf/optimizers/madgrad.yaml +++ /dev/null @@ -1,8 +0,0 @@ -madgrad: - _target_: madgrad.MADGRAD - lr: 1.0e-4 - momentum: 0.9 - weight_decay: 0 - eps: 1.0e-6 - - parameters: network diff --git a/training/conf/optimizers/radam.yaml b/training/conf/optimizers/radam.yaml new file mode 100644 index 0000000..7ee1234 --- /dev/null +++ b/training/conf/optimizers/radam.yaml @@ -0,0 +1,7 @@ +radam: + _target_: torch.optim.RAdam + lr: 1.5e-4 + betas: [0.9, 0.999] + weight_decay: 1.0e-4 + eps: 1.0e-8 + parameters: network |