From 69e93847bb0adf6e819f48e70e6a88da4cf0b7f6 Mon Sep 17 00:00:00 2001
From: Gustaf Rydholm <gustaf.rydholm@gmail.com>
Date: Wed, 26 Jan 2022 23:22:13 +0100
Subject: fix: nuke madgrad, add radam

---
 training/conf/config.yaml             | 6 +++---
 training/conf/optimizers/madgrad.yaml | 8 --------
 training/conf/optimizers/radam.yaml   | 7 +++++++
 3 files changed, 10 insertions(+), 11 deletions(-)
 delete mode 100644 training/conf/optimizers/madgrad.yaml
 create mode 100644 training/conf/optimizers/radam.yaml

diff --git a/training/conf/config.yaml b/training/conf/config.yaml
index e783efd..11bb551 100644
--- a/training/conf/config.yaml
+++ b/training/conf/config.yaml
@@ -7,13 +7,13 @@ defaults:
   - datamodule: iam_extended_paragraphs
   - hydra: default
   - logger: wandb
-  - lr_schedulers: 
+  - lr_schedulers:
       - cosine_annealing
   - mapping: characters # word_piece
   - model: lit_transformer
   - network: conv_transformer
-  - optimizers: 
-      - madgrad
+  - optimizers:
+      - radam
   - trainer: default
 
 seed: 4711
diff --git a/training/conf/optimizers/madgrad.yaml b/training/conf/optimizers/madgrad.yaml
deleted file mode 100644
index b6507b9..0000000
--- a/training/conf/optimizers/madgrad.yaml
+++ /dev/null
@@ -1,8 +0,0 @@
-madgrad:
-  _target_: madgrad.MADGRAD
-  lr: 1.0e-4
-  momentum: 0.9
-  weight_decay: 0
-  eps: 1.0e-6
-
-  parameters: network
diff --git a/training/conf/optimizers/radam.yaml b/training/conf/optimizers/radam.yaml
new file mode 100644
index 0000000..7ee1234
--- /dev/null
+++ b/training/conf/optimizers/radam.yaml
@@ -0,0 +1,7 @@
+radam:
+  _target_: torch.optim.RAdam
+  lr: 1.5e-4
+  betas: [0.9, 0.999]
+  weight_decay: 1.0e-4
+  eps: 1.0e-8
+  parameters: network
-- 
cgit v1.2.3-70-g09d2