From 56dc112cfb649217cd624b4ff305e2db83a383b7 Mon Sep 17 00:00:00 2001 From: Gustaf Rydholm Date: Mon, 11 Sep 2023 22:15:26 +0200 Subject: Update configs --- training/conf/optimizer/adamw.yaml | 5 +++++ training/conf/optimizer/adan.yaml | 4 ++++ training/conf/optimizer/lion.yaml | 5 +++++ 3 files changed, 14 insertions(+) create mode 100644 training/conf/optimizer/adamw.yaml create mode 100644 training/conf/optimizer/adan.yaml create mode 100644 training/conf/optimizer/lion.yaml (limited to 'training/conf/optimizer') diff --git a/training/conf/optimizer/adamw.yaml b/training/conf/optimizer/adamw.yaml new file mode 100644 index 0000000..568c67b --- /dev/null +++ b/training/conf/optimizer/adamw.yaml @@ -0,0 +1,5 @@ +_target_: torch.optim.AdamW +lr: 1.5e-4 +betas: [0.95, 0.98] +weight_decay: 1.0e-2 +eps: 1.0e-6 diff --git a/training/conf/optimizer/adan.yaml b/training/conf/optimizer/adan.yaml new file mode 100644 index 0000000..6950a86 --- /dev/null +++ b/training/conf/optimizer/adan.yaml @@ -0,0 +1,4 @@ +_target_: adan_pytorch.Adan +lr: 2.0e-4 +betas: [0.02, 0.08, 0.01] +weight_decay: 0.02 diff --git a/training/conf/optimizer/lion.yaml b/training/conf/optimizer/lion.yaml new file mode 100644 index 0000000..cbf386a --- /dev/null +++ b/training/conf/optimizer/lion.yaml @@ -0,0 +1,5 @@ +_target_: lion_pytorch.Lion +lr: 5e-5 +betas: [0.95, 0.99] +weight_decay: 0.1 +use_triton: true -- cgit v1.2.3-70-g09d2