From 07dd14116fe1d8148fb614b160245287533620fc Mon Sep 17 00:00:00 2001
From: aktersnurra <gustaf.rydholm@gmail.com>
Date: Mon, 3 Aug 2020 23:33:34 +0200
Subject: Working Emnist lines dataset.

---
 src/training/callbacks/__init__.py                 |  20 +-
 src/training/callbacks/base.py                     | 231 ++++++++++++++++----
 src/training/callbacks/early_stopping.py           | 106 +++++++++
 src/training/callbacks/lr_schedulers.py            |  97 +++++++++
 src/training/callbacks/wandb_callbacks.py          |  93 ++++++++
 .../0721_231455/config.yml                         |  48 -----
 .../0721_231455/model/best.pt                      | Bin 14483400 -> 0 bytes
 .../0721_231455/model/last.pt                      | Bin 14483400 -> 0 bytes
 .../0722_190746/config.yml                         |  48 -----
 .../0722_190746/model/best.pt                      | Bin 14483400 -> 0 bytes
 .../0722_190746/model/last.pt                      | Bin 14483400 -> 0 bytes
 .../0721_124928/config.yml                         |  43 ----
 .../0721_141139/config.yml                         |  47 ----
 .../0721_141213/config.yml                         |  47 ----
 .../0721_141213/model/best.pt                      | Bin 1901268 -> 0 bytes
 .../0721_141213/model/last.pt                      | Bin 1901268 -> 0 bytes
 .../0721_141433/config.yml                         |  47 ----
 .../0721_141433/model/best.pt                      | Bin 1901268 -> 0 bytes
 .../0721_141433/model/last.pt                      | Bin 1901268 -> 0 bytes
 .../0721_141702/config.yml                         |  47 ----
 .../0721_141702/model/best.pt                      | Bin 1901268 -> 0 bytes
 .../0721_141702/model/last.pt                      | Bin 1901268 -> 0 bytes
 .../0721_145028/config.yml                         |  47 ----
 .../0721_150212/config.yml                         |  47 ----
 .../0721_150301/config.yml                         |  47 ----
 .../0721_150317/config.yml                         |  47 ----
 .../0721_151135/config.yml                         |  47 ----
 .../0721_151135/model/best.pt                      | Bin 1901268 -> 0 bytes
 .../0721_151135/model/last.pt                      | Bin 1901268 -> 0 bytes
 .../0721_151408/config.yml                         |  47 ----
 .../0721_153144/config.yml                         |  47 ----
 .../0721_153207/config.yml                         |  47 ----
 .../0721_153310/config.yml                         |  47 ----
 .../0721_153310/model/best.pt                      | Bin 1702142 -> 0 bytes
 .../0721_153310/model/last.pt                      | Bin 1702142 -> 0 bytes
 .../0721_175150/config.yml                         |  47 ----
 .../0721_175150/model/best.pt                      | Bin 1702142 -> 0 bytes
 .../0721_175150/model/last.pt                      | Bin 1702142 -> 0 bytes
 .../0721_180741/config.yml                         |  47 ----
 .../0721_180741/model/best.pt                      | Bin 1702142 -> 0 bytes
 .../0721_180741/model/last.pt                      | Bin 1702142 -> 0 bytes
 .../0721_181933/config.yml                         |  46 ----
 .../0721_181933/model/best.pt                      | Bin 1702114 -> 0 bytes
 .../0721_181933/model/last.pt                      | Bin 1702114 -> 0 bytes
 .../0721_183347/config.yml                         |  46 ----
 .../0721_183347/model/best.pt                      | Bin 1702114 -> 0 bytes
 .../0721_183347/model/last.pt                      | Bin 1702114 -> 0 bytes
 .../0721_190044/config.yml                         |  46 ----
 .../0721_190044/model/best.pt                      | Bin 1702135 -> 0 bytes
 .../0721_190044/model/last.pt                      | Bin 1702135 -> 0 bytes
 .../0721_190633/config.yml                         |  46 ----
 .../0721_190633/model/best.pt                      | Bin 1702135 -> 0 bytes
 .../0721_190633/model/last.pt                      | Bin 1702135 -> 0 bytes
 .../0721_190738/config.yml                         |  46 ----
 .../0721_190738/model/best.pt                      | Bin 1702135 -> 0 bytes
 .../0721_190738/model/last.pt                      | Bin 1702135 -> 0 bytes
 .../0721_191111/config.yml                         |  46 ----
 .../0721_191310/config.yml                         |  46 ----
 .../0721_191310/model/best.pt                      | Bin 1702135 -> 0 bytes
 .../0721_191310/model/last.pt                      | Bin 1702135 -> 0 bytes
 .../0721_191412/config.yml                         |  42 ----
 .../0721_191412/model/best.pt                      | Bin 1135058 -> 0 bytes
 .../0721_191412/model/last.pt                      | Bin 1135058 -> 0 bytes
 .../0721_191504/config.yml                         |  42 ----
 .../0721_191504/model/best.pt                      | Bin 1135058 -> 0 bytes
 .../0721_191504/model/last.pt                      | Bin 1135058 -> 0 bytes
 .../0721_191826/config.yml                         |  47 ----
 .../0721_191826/model/best.pt                      | Bin 1702135 -> 0 bytes
 .../0721_191826/model/last.pt                      | Bin 1702135 -> 0 bytes
 .../0722_191559/config.yml                         |  49 -----
 .../0722_191559/model/best.pt                      | Bin 1702233 -> 0 bytes
 .../0722_191559/model/last.pt                      | Bin 1702249 -> 0 bytes
 .../0722_213125/config.yml                         |  49 -----
 .../0722_213413/config.yml                         |  49 -----
 .../0722_213413/model/best.pt                      | Bin 1702233 -> 0 bytes
 .../0722_213413/model/last.pt                      | Bin 1702233 -> 0 bytes
 .../0722_213549/config.yml                         |  49 -----
 .../0722_213549/model/best.pt                      | Bin 1702233 -> 0 bytes
 .../0722_213549/model/last.pt                      | Bin 1702233 -> 0 bytes
 src/training/experiments/sample.yml                |  43 ----
 src/training/experiments/sample_experiment.yml     |  34 ++-
 src/training/prepare_experiments.py                |  18 +-
 src/training/run_experiment.py                     |  45 +++-
 src/training/train.py                              | 237 ++++++++++++---------
 84 files changed, 705 insertions(+), 1712 deletions(-)
 create mode 100644 src/training/callbacks/lr_schedulers.py
 create mode 100644 src/training/callbacks/wandb_callbacks.py
 delete mode 100644 src/training/experiments/CharacterModel_Emnist_LeNet/0721_231455/config.yml
 delete mode 100644 src/training/experiments/CharacterModel_Emnist_LeNet/0721_231455/model/best.pt
 delete mode 100644 src/training/experiments/CharacterModel_Emnist_LeNet/0721_231455/model/last.pt
 delete mode 100644 src/training/experiments/CharacterModel_Emnist_LeNet/0722_190746/config.yml
 delete mode 100644 src/training/experiments/CharacterModel_Emnist_LeNet/0722_190746/model/best.pt
 delete mode 100644 src/training/experiments/CharacterModel_Emnist_LeNet/0722_190746/model/last.pt
 delete mode 100644 src/training/experiments/CharacterModel_Emnist_MLP/0721_124928/config.yml
 delete mode 100644 src/training/experiments/CharacterModel_Emnist_MLP/0721_141139/config.yml
 delete mode 100644 src/training/experiments/CharacterModel_Emnist_MLP/0721_141213/config.yml
 delete mode 100644 src/training/experiments/CharacterModel_Emnist_MLP/0721_141213/model/best.pt
 delete mode 100644 src/training/experiments/CharacterModel_Emnist_MLP/0721_141213/model/last.pt
 delete mode 100644 src/training/experiments/CharacterModel_Emnist_MLP/0721_141433/config.yml
 delete mode 100644 src/training/experiments/CharacterModel_Emnist_MLP/0721_141433/model/best.pt
 delete mode 100644 src/training/experiments/CharacterModel_Emnist_MLP/0721_141433/model/last.pt
 delete mode 100644 src/training/experiments/CharacterModel_Emnist_MLP/0721_141702/config.yml
 delete mode 100644 src/training/experiments/CharacterModel_Emnist_MLP/0721_141702/model/best.pt
 delete mode 100644 src/training/experiments/CharacterModel_Emnist_MLP/0721_141702/model/last.pt
 delete mode 100644 src/training/experiments/CharacterModel_Emnist_MLP/0721_145028/config.yml
 delete mode 100644 src/training/experiments/CharacterModel_Emnist_MLP/0721_150212/config.yml
 delete mode 100644 src/training/experiments/CharacterModel_Emnist_MLP/0721_150301/config.yml
 delete mode 100644 src/training/experiments/CharacterModel_Emnist_MLP/0721_150317/config.yml
 delete mode 100644 src/training/experiments/CharacterModel_Emnist_MLP/0721_151135/config.yml
 delete mode 100644 src/training/experiments/CharacterModel_Emnist_MLP/0721_151135/model/best.pt
 delete mode 100644 src/training/experiments/CharacterModel_Emnist_MLP/0721_151135/model/last.pt
 delete mode 100644 src/training/experiments/CharacterModel_Emnist_MLP/0721_151408/config.yml
 delete mode 100644 src/training/experiments/CharacterModel_Emnist_MLP/0721_153144/config.yml
 delete mode 100644 src/training/experiments/CharacterModel_Emnist_MLP/0721_153207/config.yml
 delete mode 100644 src/training/experiments/CharacterModel_Emnist_MLP/0721_153310/config.yml
 delete mode 100644 src/training/experiments/CharacterModel_Emnist_MLP/0721_153310/model/best.pt
 delete mode 100644 src/training/experiments/CharacterModel_Emnist_MLP/0721_153310/model/last.pt
 delete mode 100644 src/training/experiments/CharacterModel_Emnist_MLP/0721_175150/config.yml
 delete mode 100644 src/training/experiments/CharacterModel_Emnist_MLP/0721_175150/model/best.pt
 delete mode 100644 src/training/experiments/CharacterModel_Emnist_MLP/0721_175150/model/last.pt
 delete mode 100644 src/training/experiments/CharacterModel_Emnist_MLP/0721_180741/config.yml
 delete mode 100644 src/training/experiments/CharacterModel_Emnist_MLP/0721_180741/model/best.pt
 delete mode 100644 src/training/experiments/CharacterModel_Emnist_MLP/0721_180741/model/last.pt
 delete mode 100644 src/training/experiments/CharacterModel_Emnist_MLP/0721_181933/config.yml
 delete mode 100644 src/training/experiments/CharacterModel_Emnist_MLP/0721_181933/model/best.pt
 delete mode 100644 src/training/experiments/CharacterModel_Emnist_MLP/0721_181933/model/last.pt
 delete mode 100644 src/training/experiments/CharacterModel_Emnist_MLP/0721_183347/config.yml
 delete mode 100644 src/training/experiments/CharacterModel_Emnist_MLP/0721_183347/model/best.pt
 delete mode 100644 src/training/experiments/CharacterModel_Emnist_MLP/0721_183347/model/last.pt
 delete mode 100644 src/training/experiments/CharacterModel_Emnist_MLP/0721_190044/config.yml
 delete mode 100644 src/training/experiments/CharacterModel_Emnist_MLP/0721_190044/model/best.pt
 delete mode 100644 src/training/experiments/CharacterModel_Emnist_MLP/0721_190044/model/last.pt
 delete mode 100644 src/training/experiments/CharacterModel_Emnist_MLP/0721_190633/config.yml
 delete mode 100644 src/training/experiments/CharacterModel_Emnist_MLP/0721_190633/model/best.pt
 delete mode 100644 src/training/experiments/CharacterModel_Emnist_MLP/0721_190633/model/last.pt
 delete mode 100644 src/training/experiments/CharacterModel_Emnist_MLP/0721_190738/config.yml
 delete mode 100644 src/training/experiments/CharacterModel_Emnist_MLP/0721_190738/model/best.pt
 delete mode 100644 src/training/experiments/CharacterModel_Emnist_MLP/0721_190738/model/last.pt
 delete mode 100644 src/training/experiments/CharacterModel_Emnist_MLP/0721_191111/config.yml
 delete mode 100644 src/training/experiments/CharacterModel_Emnist_MLP/0721_191310/config.yml
 delete mode 100644 src/training/experiments/CharacterModel_Emnist_MLP/0721_191310/model/best.pt
 delete mode 100644 src/training/experiments/CharacterModel_Emnist_MLP/0721_191310/model/last.pt
 delete mode 100644 src/training/experiments/CharacterModel_Emnist_MLP/0721_191412/config.yml
 delete mode 100644 src/training/experiments/CharacterModel_Emnist_MLP/0721_191412/model/best.pt
 delete mode 100644 src/training/experiments/CharacterModel_Emnist_MLP/0721_191412/model/last.pt
 delete mode 100644 src/training/experiments/CharacterModel_Emnist_MLP/0721_191504/config.yml
 delete mode 100644 src/training/experiments/CharacterModel_Emnist_MLP/0721_191504/model/best.pt
 delete mode 100644 src/training/experiments/CharacterModel_Emnist_MLP/0721_191504/model/last.pt
 delete mode 100644 src/training/experiments/CharacterModel_Emnist_MLP/0721_191826/config.yml
 delete mode 100644 src/training/experiments/CharacterModel_Emnist_MLP/0721_191826/model/best.pt
 delete mode 100644 src/training/experiments/CharacterModel_Emnist_MLP/0721_191826/model/last.pt
 delete mode 100644 src/training/experiments/CharacterModel_Emnist_MLP/0722_191559/config.yml
 delete mode 100644 src/training/experiments/CharacterModel_Emnist_MLP/0722_191559/model/best.pt
 delete mode 100644 src/training/experiments/CharacterModel_Emnist_MLP/0722_191559/model/last.pt
 delete mode 100644 src/training/experiments/CharacterModel_Emnist_MLP/0722_213125/config.yml
 delete mode 100644 src/training/experiments/CharacterModel_Emnist_MLP/0722_213413/config.yml
 delete mode 100644 src/training/experiments/CharacterModel_Emnist_MLP/0722_213413/model/best.pt
 delete mode 100644 src/training/experiments/CharacterModel_Emnist_MLP/0722_213413/model/last.pt
 delete mode 100644 src/training/experiments/CharacterModel_Emnist_MLP/0722_213549/config.yml
 delete mode 100644 src/training/experiments/CharacterModel_Emnist_MLP/0722_213549/model/best.pt
 delete mode 100644 src/training/experiments/CharacterModel_Emnist_MLP/0722_213549/model/last.pt
 delete mode 100644 src/training/experiments/sample.yml

(limited to 'src/training')

diff --git a/src/training/callbacks/__init__.py b/src/training/callbacks/__init__.py
index 868d739..fbcc285 100644
--- a/src/training/callbacks/__init__.py
+++ b/src/training/callbacks/__init__.py
@@ -1 +1,19 @@
-"""TBC."""
+"""The callback modules used in the training script."""
+from .base import Callback, CallbackList, Checkpoint
+from .early_stopping import EarlyStopping
+from .lr_schedulers import CyclicLR, MultiStepLR, OneCycleLR, ReduceLROnPlateau, StepLR
+from .wandb_callbacks import WandbCallback, WandbImageLogger
+
+__all__ = [
+    "Callback",
+    "CallbackList",
+    "Checkpoint",
+    "EarlyStopping",
+    "WandbCallback",
+    "WandbImageLogger",
+    "CyclicLR",
+    "MultiStepLR",
+    "OneCycleLR",
+    "ReduceLROnPlateau",
+    "StepLR",
+]
diff --git a/src/training/callbacks/base.py b/src/training/callbacks/base.py
index d80a1e5..e0d91e6 100644
--- a/src/training/callbacks/base.py
+++ b/src/training/callbacks/base.py
@@ -1,12 +1,33 @@
 """Metaclass for callback functions."""
 
-from abc import ABC
-from typing import Callable, List, Type
+from enum import Enum
+from typing import Callable, Dict, List, Type, Union
 
+from loguru import logger
+import numpy as np
+import torch
 
-class Callback(ABC):
+from text_recognizer.models import Model
+
+
+class ModeKeys:
+    """Mode keys for CallbackList."""
+
+    TRAIN = "train"
+    VALIDATION = "validation"
+
+
+class Callback:
     """Metaclass for callbacks used in training."""
 
+    def __init__(self) -> None:
+        """Initializes the Callback instance."""
+        self.model = None
+
+    def set_model(self, model: Type[Model]) -> None:
+        """Set the model."""
+        self.model = model
+
     def on_fit_begin(self) -> None:
         """Called when fit begins."""
         pass
@@ -15,35 +36,27 @@ class Callback(ABC):
         """Called when fit ends."""
         pass
 
-    def on_train_epoch_begin(self) -> None:
-        """Called at the beginning of an epoch."""
-        pass
-
-    def on_train_epoch_end(self) -> None:
-        """Called at the end of an epoch."""
+    def on_epoch_begin(self, epoch: int, logs: Dict = {}) -> None:
+        """Called at the beginning of an epoch. Only used in training mode."""
         pass
 
-    def on_val_epoch_begin(self) -> None:
-        """Called at the beginning of an epoch."""
+    def on_epoch_end(self, epoch: int, logs: Dict = {}) -> None:
+        """Called at the end of an epoch. Only used in training mode."""
         pass
 
-    def on_val_epoch_end(self) -> None:
-        """Called at the end of an epoch."""
-        pass
-
-    def on_train_batch_begin(self) -> None:
+    def on_train_batch_begin(self, batch: int, logs: Dict = {}) -> None:
         """Called at the beginning of an epoch."""
         pass
 
-    def on_train_batch_end(self) -> None:
+    def on_train_batch_end(self, batch: int, logs: Dict = {}) -> None:
         """Called at the end of an epoch."""
         pass
 
-    def on_val_batch_begin(self) -> None:
+    def on_validation_batch_begin(self, batch: int, logs: Dict = {}) -> None:
         """Called at the beginning of an epoch."""
         pass
 
-    def on_val_batch_end(self) -> None:
+    def on_validation_batch_end(self, batch: int, logs: Dict = {}) -> None:
         """Called at the end of an epoch."""
         pass
 
@@ -51,9 +64,29 @@ class Callback(ABC):
 class CallbackList:
     """Container for abstracting away callback calls."""
 
-    def __init__(self, callbacks: List[Callable] = None) -> None:
-        """TBC."""
-        self._callbacks = callbacks if callbacks is not None else []
+    mode_keys = ModeKeys()
+
+    def __init__(self, model: Type[Model], callbacks: List[Callback] = None) -> None:
+        """Container for `Callback` instances.
+
+        This object wraps a list of `Callback` instances and allows them all to be
+        called via a single end point.
+
+        Args:
+            model (Type[Model]): A `Model` instance.
+            callbacks (List[Callback]): List of `Callback` instances. Defaults to None.
+
+        """
+
+        self._callbacks = callbacks or []
+        if model:
+            self.set_model(model)
+
+    def set_model(self, model: Type[Model]) -> None:
+        """Set the model for all callbacks."""
+        self.model = model
+        for callback in self._callbacks:
+            callback.set_model(model=self.model)
 
     def append(self, callback: Type[Callback]) -> None:
         """Append new callback to callback list."""
@@ -61,41 +94,147 @@ class CallbackList:
 
     def on_fit_begin(self) -> None:
         """Called when fit begins."""
-        for _ in self._callbacks:
-            pass
+        for callback in self._callbacks:
+            callback.on_fit_begin()
 
     def on_fit_end(self) -> None:
         """Called when fit ends."""
-        pass
+        for callback in self._callbacks:
+            callback.on_fit_end()
 
-    def on_train_epoch_begin(self) -> None:
+    def on_epoch_begin(self, epoch: int, logs: Dict = {}) -> None:
         """Called at the beginning of an epoch."""
-        pass
+        for callback in self._callbacks:
+            callback.on_epoch_begin(epoch, logs)
 
-    def on_train_epoch_end(self) -> None:
+    def on_epoch_end(self, epoch: int, logs: Dict = {}) -> None:
         """Called at the end of an epoch."""
-        pass
-
-    def on_val_epoch_begin(self) -> None:
+        for callback in self._callbacks:
+            callback.on_epoch_end(epoch, logs)
+
+    def _call_batch_hook(
+        self, mode: str, hook: str, batch: int, logs: Dict = {}
+    ) -> None:
+        """Helper function for all batch_{begin | end} methods."""
+        if hook == "begin":
+            self._call_batch_begin_hook(mode, batch, logs)
+        elif hook == "end":
+            self._call_batch_end_hook(mode, batch, logs)
+        else:
+            raise ValueError(f"Unrecognized hook {hook}.")
+
+    def _call_batch_begin_hook(self, mode: str, batch: int, logs: Dict = {}) -> None:
+        """Helper function for all `on_*_batch_begin` methods."""
+        hook_name = f"on_{mode}_batch_begin"
+        self._call_batch_hook_helper(hook_name, batch, logs)
+
+    def _call_batch_end_hook(self, mode: str, batch: int, logs: Dict = {}) -> None:
+        """Helper function for all `on_*_batch_end` methods."""
+        hook_name = f"on_{mode}_batch_end"
+        self._call_batch_hook_helper(hook_name, batch, logs)
+
+    def _call_batch_hook_helper(
+        self, hook_name: str, batch: int, logs: Dict = {}
+    ) -> None:
+        """Helper function for `on_*_batch_begin` methods."""
+        for callback in self._callbacks:
+            hook = getattr(callback, hook_name)
+            hook(batch, logs)
+
+    def on_train_batch_begin(self, batch: int, logs: Dict = {}) -> None:
         """Called at the beginning of an epoch."""
-        pass
+        self._call_batch_hook(self.mode_keys.TRAIN, "begin", batch)
 
-    def on_val_epoch_end(self) -> None:
+    def on_train_batch_end(self, batch: int, logs: Dict = {}) -> None:
         """Called at the end of an epoch."""
-        pass
+        self._call_batch_hook(self.mode_keys.TRAIN, "end", batch)
 
-    def on_train_batch_begin(self) -> None:
+    def on_validation_batch_begin(self, batch: int, logs: Dict = {}) -> None:
         """Called at the beginning of an epoch."""
-        pass
-
-    def on_train_batch_end(self) -> None:
-        """Called at the end of an epoch."""
-        pass
+        self._call_batch_hook(self.mode_keys.VALIDATION, "begin", batch)
 
-    def on_val_batch_begin(self) -> None:
-        """Called at the beginning of an epoch."""
-        pass
-
-    def on_val_batch_end(self) -> None:
+    def on_validation_batch_end(self, batch: int, logs: Dict = {}) -> None:
         """Called at the end of an epoch."""
-        pass
+        self._call_batch_hook(self.mode_keys.VALIDATION, "end", batch)
+
+    def __iter__(self) -> iter:
+        """Iter function for callback list."""
+        return iter(self._callbacks)
+
+
+class Checkpoint(Callback):
+    """Saving model parameters at the end of each epoch."""
+
+    mode_dict = {
+        "min": torch.lt,
+        "max": torch.gt,
+    }
+
+    def __init__(
+        self, monitor: str = "accuracy", mode: str = "auto", min_delta: float = 0.0
+    ) -> None:
+        """Monitors a quantity that will allow us to determine the best model weights.
+
+        Args:
+            monitor (str): Name of the quantity to monitor. Defaults to "accuracy".
+            mode (str): Description of parameter `mode`. Defaults to "auto".
+            min_delta (float): Description of parameter `min_delta`. Defaults to 0.0.
+
+        """
+        super().__init__()
+        self.monitor = monitor
+        self.mode = mode
+        self.min_delta = torch.tensor(min_delta)
+
+        if mode not in ["auto", "min", "max"]:
+            logger.warning(f"Checkpoint mode {mode} is unkown, fallback to auto mode.")
+
+            self.mode = "auto"
+
+        if self.mode == "auto":
+            if "accuracy" in self.monitor:
+                self.mode = "max"
+            else:
+                self.mode = "min"
+            logger.debug(
+                f"Checkpoint mode set to {self.mode} for monitoring {self.monitor}."
+            )
+
+        torch_inf = torch.tensor(np.inf)
+        self.min_delta *= 1 if self.monitor_op == torch.gt else -1
+        self.best_score = torch_inf if self.monitor_op == torch.lt else -torch_inf
+
+    @property
+    def monitor_op(self) -> float:
+        """Returns the comparison method."""
+        return self.mode_dict[self.mode]
+
+    def on_epoch_end(self, epoch: int, logs: Dict) -> None:
+        """Saves a checkpoint for the network parameters.
+
+        Args:
+            epoch (int): The current epoch.
+            logs (Dict): The log containing the monitored metrics.
+
+        """
+        current = self.get_monitor_value(logs)
+        if current is None:
+            return
+        if self.monitor_op(current - self.min_delta, self.best_score):
+            self.best_score = current
+            is_best = True
+        else:
+            is_best = False
+
+        self.model.save_checkpoint(is_best, epoch, self.monitor)
+
+    def get_monitor_value(self, logs: Dict) -> Union[float, None]:
+        """Extracts the monitored value."""
+        monitor_value = logs.get(self.monitor)
+        if monitor_value is None:
+            logger.warning(
+                f"Checkpoint is conditioned on metric {self.monitor} which is not available. Available"
+                + f"metrics are: {','.join(list(logs.keys()))}"
+            )
+            return None
+        return monitor_value
diff --git a/src/training/callbacks/early_stopping.py b/src/training/callbacks/early_stopping.py
index 4da0e85..c9b7907 100644
--- a/src/training/callbacks/early_stopping.py
+++ b/src/training/callbacks/early_stopping.py
@@ -1 +1,107 @@
 """Implements Early stopping for PyTorch model."""
+from typing import Dict, Union
+
+from loguru import logger
+import numpy as np
+import torch
+from training.callbacks import Callback
+
+
+class EarlyStopping(Callback):
+    """Stops training when a monitored metric stops improving."""
+
+    mode_dict = {
+        "min": torch.lt,
+        "max": torch.gt,
+    }
+
+    def __init__(
+        self,
+        monitor: str = "val_loss",
+        min_delta: float = 0.0,
+        patience: int = 3,
+        mode: str = "auto",
+    ) -> None:
+        """Initializes the EarlyStopping callback.
+
+        Args:
+            monitor (str): Description of parameter `monitor`. Defaults to "val_loss".
+            min_delta (float): Description of parameter `min_delta`. Defaults to 0.0.
+            patience (int): Description of parameter `patience`. Defaults to 3.
+            mode (str): Description of parameter `mode`. Defaults to "auto".
+
+        """
+        super().__init__()
+        self.monitor = monitor
+        self.patience = patience
+        self.min_delta = torch.tensor(min_delta)
+        self.mode = mode
+        self.wait_count = 0
+        self.stopped_epoch = 0
+
+        if mode not in ["auto", "min", "max"]:
+            logger.warning(
+                f"EarlyStopping mode {mode} is unkown, fallback to auto mode."
+            )
+
+            self.mode = "auto"
+
+        if self.mode == "auto":
+            if "accuracy" in self.monitor:
+                self.mode = "max"
+            else:
+                self.mode = "min"
+            logger.debug(
+                f"EarlyStopping mode set to {self.mode} for monitoring {self.monitor}."
+            )
+
+        self.torch_inf = torch.tensor(np.inf)
+        self.min_delta *= 1 if self.monitor_op == torch.gt else -1
+        self.best_score = (
+            self.torch_inf if self.monitor_op == torch.lt else -self.torch_inf
+        )
+
+    @property
+    def monitor_op(self) -> float:
+        """Returns the comparison method."""
+        return self.mode_dict[self.mode]
+
+    def on_fit_begin(self) -> Union[torch.lt, torch.gt]:
+        """Reset the early stopping variables for reuse."""
+        self.wait_count = 0
+        self.stopped_epoch = 0
+        self.best_score = (
+            self.torch_inf if self.monitor_op == torch.lt else -self.torch_inf
+        )
+
+    def on_epoch_end(self, epoch: int, logs: Dict) -> None:
+        """Computes the early stop criterion."""
+        current = self.get_monitor_value(logs)
+        if current is None:
+            return
+        if self.monitor_op(current - self.min_delta, self.best_score):
+            self.best_score = current
+            self.wait_count = 0
+        else:
+            self.wait_count += 1
+            if self.wait_count >= self.patience:
+                self.stopped_epoch = epoch
+                self.model.stop_training = True
+
+    def on_fit_end(self) -> None:
+        """Logs if early stopping was used."""
+        if self.stopped_epoch > 0:
+            logger.info(
+                f"Stopped training at epoch {self.stopped_epoch + 1} with early stopping."
+            )
+
+    def get_monitor_value(self, logs: Dict) -> Union[torch.Tensor, None]:
+        """Extracts the monitor value."""
+        monitor_value = logs.get(self.monitor)
+        if monitor_value is None:
+            logger.warning(
+                f"Early stopping is conditioned on metric {self.monitor} which is not available. Available"
+                + f"metrics are: {','.join(list(logs.keys()))}"
+            )
+            return None
+        return torch.tensor(monitor_value)
diff --git a/src/training/callbacks/lr_schedulers.py b/src/training/callbacks/lr_schedulers.py
new file mode 100644
index 0000000..00c7e9b
--- /dev/null
+++ b/src/training/callbacks/lr_schedulers.py
@@ -0,0 +1,97 @@
+"""Callbacks for learning rate schedulers."""
+from typing import Callable, Dict, List, Optional, Type
+
+from training.callbacks import Callback
+
+from text_recognizer.models import Model
+
+
+class StepLR(Callback):
+    """Callback for StepLR."""
+
+    def __init__(self) -> None:
+        """Initializes the callback."""
+        super().__init__()
+        self.lr_scheduler = None
+
+    def set_model(self, model: Type[Model]) -> None:
+        """Sets the model and lr scheduler."""
+        self.model = model
+        self.lr_scheduler = self.model.lr_scheduler
+
+    def on_epoch_end(self, epoch: int, logs: Dict = {}) -> None:
+        """Takes a step at the end of every epoch."""
+        self.lr_scheduler.step()
+
+
+class MultiStepLR(Callback):
+    """Callback for MultiStepLR."""
+
+    def __init__(self) -> None:
+        """Initializes the callback."""
+        super().__init__()
+        self.lr_scheduler = None
+
+    def set_model(self, model: Type[Model]) -> None:
+        """Sets the model and lr scheduler."""
+        self.model = model
+        self.lr_scheduler = self.model.lr_scheduler
+
+    def on_epoch_end(self, epoch: int, logs: Dict = {}) -> None:
+        """Takes a step at the end of every epoch."""
+        self.lr_scheduler.step()
+
+
+class ReduceLROnPlateau(Callback):
+    """Callback for ReduceLROnPlateau."""
+
+    def __init__(self) -> None:
+        """Initializes the callback."""
+        super().__init__()
+        self.lr_scheduler = None
+
+    def set_model(self, model: Type[Model]) -> None:
+        """Sets the model and lr scheduler."""
+        self.model = model
+        self.lr_scheduler = self.model.lr_scheduler
+
+    def on_epoch_end(self, epoch: int, logs: Dict = {}) -> None:
+        """Takes a step at the end of every epoch."""
+        val_loss = logs["val_loss"]
+        self.lr_scheduler.step(val_loss)
+
+
+class CyclicLR(Callback):
+    """Callback for CyclicLR."""
+
+    def __init__(self) -> None:
+        """Initializes the callback."""
+        super().__init__()
+        self.lr_scheduler = None
+
+    def set_model(self, model: Type[Model]) -> None:
+        """Sets the model and lr scheduler."""
+        self.model = model
+        self.lr_scheduler = self.model.lr_scheduler
+
+    def on_train_batch_end(self, batch: int, logs: Dict = {}) -> None:
+        """Takes a step at the end of every training batch."""
+        self.lr_scheduler.step()
+
+
+class OneCycleLR(Callback):
+    """Callback for OneCycleLR."""
+
+    def __init__(self) -> None:
+        """Initializes the callback."""
+        super().__init__()
+        self.lr_scheduler = None
+
+    def set_model(self, model: Type[Model]) -> None:
+        """Sets the model and lr scheduler."""
+        self.model = model
+        self.lr_scheduler = self.model.lr_scheduler
+
+    def on_train_batch_end(self, batch: int, logs: Dict = {}) -> None:
+        """Takes a step at the end of every training batch."""
+        self.lr_scheduler.step()
diff --git a/src/training/callbacks/wandb_callbacks.py b/src/training/callbacks/wandb_callbacks.py
new file mode 100644
index 0000000..f64cbe1
--- /dev/null
+++ b/src/training/callbacks/wandb_callbacks.py
@@ -0,0 +1,93 @@
+"""Callbacks using wandb."""
+from typing import Callable, Dict, List, Optional, Type
+
+import numpy as np
+from torchvision.transforms import Compose, ToTensor
+from training.callbacks import Callback
+import wandb
+
+from text_recognizer.datasets import Transpose
+from text_recognizer.models.base import Model
+
+
+class WandbCallback(Callback):
+    """A custom W&B metric logger for the trainer."""
+
+    def __init__(self, log_batch_frequency: int = None) -> None:
+        """Short summary.
+
+        Args:
+            log_batch_frequency (int): If None, metrics will be logged every epoch.
+                If set to an integer, callback will log every metrics every log_batch_frequency.
+
+        """
+        super().__init__()
+        self.log_batch_frequency = log_batch_frequency
+
+    def _on_batch_end(self, batch: int, logs: Dict) -> None:
+        if self.log_batch_frequency and batch % self.log_batch_frequency == 0:
+            wandb.log(logs, commit=True)
+
+    def on_train_batch_end(self, batch: int, logs: Dict = {}) -> None:
+        """Logs training metrics."""
+        if logs is not None:
+            self._on_batch_end(batch, logs)
+
+    def on_validation_batch_end(self, batch: int, logs: Dict = {}) -> None:
+        """Logs validation metrics."""
+        if logs is not None:
+            self._on_batch_end(batch, logs)
+
+    def on_epoch_end(self, epoch: int, logs: Dict) -> None:
+        """Logs at epoch end."""
+        wandb.log(logs, commit=True)
+
+
+class WandbImageLogger(Callback):
+    """Custom W&B callback for image logging."""
+
+    def __init__(
+        self,
+        example_indices: Optional[List] = None,
+        num_examples: int = 4,
+        transfroms: Optional[Callable] = None,
+    ) -> None:
+        """Initializes the WandbImageLogger with the model to train.
+
+        Args:
+            example_indices (Optional[List]): Indices for validation images. Defaults to None.
+            num_examples (int): Number of random samples to take if example_indices are not specified. Defaults to 4.
+            transfroms (Optional[Callable]): Transforms to use on the validation images, e.g. transpose. Defaults to
+                None.
+
+        """
+
+        super().__init__()
+        self.example_indices = example_indices
+        self.num_examples = num_examples
+        self.transfroms = transfroms
+        if self.transfroms is None:
+            self.transforms = Compose([Transpose()])
+
+    def set_model(self, model: Type[Model]) -> None:
+        """Sets the model and extracts validation images from the dataset."""
+        self.model = model
+        data_loader = self.model.data_loaders("val")
+        if self.example_indices is None:
+            self.example_indices = np.random.randint(
+                0, len(data_loader.dataset.data), self.num_examples
+            )
+        self.val_images = data_loader.dataset.data[self.example_indices]
+        self.val_targets = data_loader.dataset.targets[self.example_indices].numpy()
+
+    def on_epoch_end(self, epoch: int, logs: Dict) -> None:
+        """Get network predictions on validation images."""
+        images = []
+        for i, image in enumerate(self.val_images):
+            image = self.transforms(image)
+            pred, conf = self.model.predict_on_image(image)
+            ground_truth = self.model._mapping[self.val_targets[i]]
+            caption = f"Prediction: {pred} Confidence: {conf:.3f} Ground Truth: {ground_truth}"
+            images.append(wandb.Image(image, caption=caption))
+
+        wandb.log({"examples": images}, commit=False)
diff --git a/src/training/experiments/CharacterModel_Emnist_LeNet/0721_231455/config.yml b/src/training/experiments/CharacterModel_Emnist_LeNet/0721_231455/config.yml
deleted file mode 100644
index 2595325..0000000
--- a/src/training/experiments/CharacterModel_Emnist_LeNet/0721_231455/config.yml
+++ /dev/null
@@ -1,48 +0,0 @@
-criterion: CrossEntropyLoss
-criterion_args:
-  ignore_index: -100
-  reduction: mean
-  weight: null
-data_loader_args:
-  batch_size: 256
-  cuda: true
-  num_workers: 8
-  sample_to_balance: true
-  seed: 4711
-  shuffle: true
-  splits:
-  - train
-  - val
-  subsample_fraction: null
-  target_transform: null
-  transform: null
-dataloader: EmnistDataLoader
-device: cuda:0
-experiment_group: Sample Experiments
-lr_scheduler: OneCycleLR
-lr_scheduler_args:
-  epochs: 16
-  max_lr: 0.001
-  steps_per_epoch: 1314
-metrics:
-- accuracy
-model: CharacterModel
-network: LeNet
-network_args:
-  input_size:
-  - 28
-  - 28
-  output_size: 62
-optimizer: AdamW
-optimizer_args:
-  amsgrad: false
-  betas:
-  - 0.9
-  - 0.999
-  eps: 1.0e-08
-  lr: 0.01
-  weight_decay: 0
-train_args:
-  batch_size: 256
-  epochs: 16
-  val_metric: accuracy
diff --git a/src/training/experiments/CharacterModel_Emnist_LeNet/0721_231455/model/best.pt b/src/training/experiments/CharacterModel_Emnist_LeNet/0721_231455/model/best.pt
deleted file mode 100644
index 6d78bad..0000000
Binary files a/src/training/experiments/CharacterModel_Emnist_LeNet/0721_231455/model/best.pt and /dev/null differ
diff --git a/src/training/experiments/CharacterModel_Emnist_LeNet/0721_231455/model/last.pt b/src/training/experiments/CharacterModel_Emnist_LeNet/0721_231455/model/last.pt
deleted file mode 100644
index 6d78bad..0000000
Binary files a/src/training/experiments/CharacterModel_Emnist_LeNet/0721_231455/model/last.pt and /dev/null differ
diff --git a/src/training/experiments/CharacterModel_Emnist_LeNet/0722_190746/config.yml b/src/training/experiments/CharacterModel_Emnist_LeNet/0722_190746/config.yml
deleted file mode 100644
index 2595325..0000000
--- a/src/training/experiments/CharacterModel_Emnist_LeNet/0722_190746/config.yml
+++ /dev/null
@@ -1,48 +0,0 @@
-criterion: CrossEntropyLoss
-criterion_args:
-  ignore_index: -100
-  reduction: mean
-  weight: null
-data_loader_args:
-  batch_size: 256
-  cuda: true
-  num_workers: 8
-  sample_to_balance: true
-  seed: 4711
-  shuffle: true
-  splits:
-  - train
-  - val
-  subsample_fraction: null
-  target_transform: null
-  transform: null
-dataloader: EmnistDataLoader
-device: cuda:0
-experiment_group: Sample Experiments
-lr_scheduler: OneCycleLR
-lr_scheduler_args:
-  epochs: 16
-  max_lr: 0.001
-  steps_per_epoch: 1314
-metrics:
-- accuracy
-model: CharacterModel
-network: LeNet
-network_args:
-  input_size:
-  - 28
-  - 28
-  output_size: 62
-optimizer: AdamW
-optimizer_args:
-  amsgrad: false
-  betas:
-  - 0.9
-  - 0.999
-  eps: 1.0e-08
-  lr: 0.01
-  weight_decay: 0
-train_args:
-  batch_size: 256
-  epochs: 16
-  val_metric: accuracy
diff --git a/src/training/experiments/CharacterModel_Emnist_LeNet/0722_190746/model/best.pt b/src/training/experiments/CharacterModel_Emnist_LeNet/0722_190746/model/best.pt
deleted file mode 100644
index 43a3891..0000000
Binary files a/src/training/experiments/CharacterModel_Emnist_LeNet/0722_190746/model/best.pt and /dev/null differ
diff --git a/src/training/experiments/CharacterModel_Emnist_LeNet/0722_190746/model/last.pt b/src/training/experiments/CharacterModel_Emnist_LeNet/0722_190746/model/last.pt
deleted file mode 100644
index 61c03f0..0000000
Binary files a/src/training/experiments/CharacterModel_Emnist_LeNet/0722_190746/model/last.pt and /dev/null differ
diff --git a/src/training/experiments/CharacterModel_Emnist_MLP/0721_124928/config.yml b/src/training/experiments/CharacterModel_Emnist_MLP/0721_124928/config.yml
deleted file mode 100644
index 2aa52cd..0000000
--- a/src/training/experiments/CharacterModel_Emnist_MLP/0721_124928/config.yml
+++ /dev/null
@@ -1,43 +0,0 @@
-criterion: CrossEntropyLoss
-criterion_args:
-  ignore_index: -100
-  reduction: mean
-  weight: null
-data_loader_args:
-  batch_size: 256
-  cuda: true
-  num_workers: 0
-  sample_to_balance: true
-  seed: 4711
-  shuffle: true
-  splits:
-  - train
-  - val
-  subsample_fraction: null
-  target_transform: null
-  transform: null
-dataloader: EmnistDataLoader
-device: cuda:0
-experiment_group: Sample Experiments
-lr_scheduler: null
-metrics:
-- accuracy
-model: CharacterModel
-network: MLP
-network_args:
-  input_size: 784
-  num_layers: 3
-  output_size: 62
-optimizer: AdamW
-optimizer_args:
-  amsgrad: false
-  betas:
-  - 0.9
-  - 0.999
-  eps: 1.0e-08
-  lr: 0.001
-  weight_decay: 0
-train_args:
-  batch_size: 256
-  epochs: 16
-  val_metric: accuracy
diff --git a/src/training/experiments/CharacterModel_Emnist_MLP/0721_141139/config.yml b/src/training/experiments/CharacterModel_Emnist_MLP/0721_141139/config.yml
deleted file mode 100644
index 829297d..0000000
--- a/src/training/experiments/CharacterModel_Emnist_MLP/0721_141139/config.yml
+++ /dev/null
@@ -1,47 +0,0 @@
-criterion: CrossEntropyLoss
-criterion_args:
-  ignore_index: -100
-  reduction: mean
-  weight: null
-data_loader_args:
-  batch_size: 256
-  cuda: true
-  num_workers: 0
-  sample_to_balance: true
-  seed: 4711
-  shuffle: true
-  splits:
-  - train
-  - val
-  subsample_fraction: null
-  target_transform: null
-  transform: null
-dataloader: EmnistDataLoader
-device: cuda:0
-experiment_group: Sample Experiments
-lr_scheduler: OneCycleLR
-lr_scheduler_args:
-  epochs: 16
-  max_lr: 0.0003
-  steps_per_epoch: 1314
-metrics:
-- accuracy
-model: CharacterModel
-network: MLP
-network_args:
-  input_size: 784
-  num_layers: 3
-  output_size: 62
-optimizer: AdamW
-optimizer_args:
-  amsgrad: false
-  betas:
-  - 0.9
-  - 0.999
-  eps: 1.0e-08
-  lr: 0.0006
-  weight_decay: 5.0e-05
-train_args:
-  batch_size: 256
-  epochs: 16
-  val_metric: accuracy
diff --git a/src/training/experiments/CharacterModel_Emnist_MLP/0721_141213/config.yml b/src/training/experiments/CharacterModel_Emnist_MLP/0721_141213/config.yml
deleted file mode 100644
index 829297d..0000000
--- a/src/training/experiments/CharacterModel_Emnist_MLP/0721_141213/config.yml
+++ /dev/null
@@ -1,47 +0,0 @@
-criterion: CrossEntropyLoss
-criterion_args:
-  ignore_index: -100
-  reduction: mean
-  weight: null
-data_loader_args:
-  batch_size: 256
-  cuda: true
-  num_workers: 0
-  sample_to_balance: true
-  seed: 4711
-  shuffle: true
-  splits:
-  - train
-  - val
-  subsample_fraction: null
-  target_transform: null
-  transform: null
-dataloader: EmnistDataLoader
-device: cuda:0
-experiment_group: Sample Experiments
-lr_scheduler: OneCycleLR
-lr_scheduler_args:
-  epochs: 16
-  max_lr: 0.0003
-  steps_per_epoch: 1314
-metrics:
-- accuracy
-model: CharacterModel
-network: MLP
-network_args:
-  input_size: 784
-  num_layers: 3
-  output_size: 62
-optimizer: AdamW
-optimizer_args:
-  amsgrad: false
-  betas:
-  - 0.9
-  - 0.999
-  eps: 1.0e-08
-  lr: 0.0006
-  weight_decay: 5.0e-05
-train_args:
-  batch_size: 256
-  epochs: 16
-  val_metric: accuracy
diff --git a/src/training/experiments/CharacterModel_Emnist_MLP/0721_141213/model/best.pt b/src/training/experiments/CharacterModel_Emnist_MLP/0721_141213/model/best.pt
deleted file mode 100644
index d0db78b..0000000
Binary files a/src/training/experiments/CharacterModel_Emnist_MLP/0721_141213/model/best.pt and /dev/null differ
diff --git a/src/training/experiments/CharacterModel_Emnist_MLP/0721_141213/model/last.pt b/src/training/experiments/CharacterModel_Emnist_MLP/0721_141213/model/last.pt
deleted file mode 100644
index d0db78b..0000000
Binary files a/src/training/experiments/CharacterModel_Emnist_MLP/0721_141213/model/last.pt and /dev/null differ
diff --git a/src/training/experiments/CharacterModel_Emnist_MLP/0721_141433/config.yml b/src/training/experiments/CharacterModel_Emnist_MLP/0721_141433/config.yml
deleted file mode 100644
index 3df32bb..0000000
--- a/src/training/experiments/CharacterModel_Emnist_MLP/0721_141433/config.yml
+++ /dev/null
@@ -1,47 +0,0 @@
-criterion: CrossEntropyLoss
-criterion_args:
-  ignore_index: -100
-  reduction: mean
-  weight: null
-data_loader_args:
-  batch_size: 256
-  cuda: true
-  num_workers: 0
-  sample_to_balance: true
-  seed: 4711
-  shuffle: true
-  splits:
-  - train
-  - val
-  subsample_fraction: null
-  target_transform: null
-  transform: null
-dataloader: EmnistDataLoader
-device: cuda:0
-experiment_group: Sample Experiments
-lr_scheduler: OneCycleLR
-lr_scheduler_args:
-  epochs: 16
-  max_lr: 0.01
-  steps_per_epoch: 1314
-metrics:
-- accuracy
-model: CharacterModel
-network: MLP
-network_args:
-  input_size: 784
-  num_layers: 3
-  output_size: 62
-optimizer: AdamW
-optimizer_args:
-  amsgrad: false
-  betas:
-  - 0.9
-  - 0.999
-  eps: 1.0e-08
-  lr: 0.1
-  weight_decay: 5.0e-05
-train_args:
-  batch_size: 256
-  epochs: 16
-  val_metric: accuracy
diff --git a/src/training/experiments/CharacterModel_Emnist_MLP/0721_141433/model/best.pt b/src/training/experiments/CharacterModel_Emnist_MLP/0721_141433/model/best.pt
deleted file mode 100644
index 5914c8f..0000000
Binary files a/src/training/experiments/CharacterModel_Emnist_MLP/0721_141433/model/best.pt and /dev/null differ
diff --git a/src/training/experiments/CharacterModel_Emnist_MLP/0721_141433/model/last.pt b/src/training/experiments/CharacterModel_Emnist_MLP/0721_141433/model/last.pt
deleted file mode 100644
index 5ba44bb..0000000
Binary files a/src/training/experiments/CharacterModel_Emnist_MLP/0721_141433/model/last.pt and /dev/null differ
diff --git a/src/training/experiments/CharacterModel_Emnist_MLP/0721_141702/config.yml b/src/training/experiments/CharacterModel_Emnist_MLP/0721_141702/config.yml
deleted file mode 100644
index fb75736..0000000
--- a/src/training/experiments/CharacterModel_Emnist_MLP/0721_141702/config.yml
+++ /dev/null
@@ -1,47 +0,0 @@
-criterion: CrossEntropyLoss
-criterion_args:
-  ignore_index: -100
-  reduction: mean
-  weight: null
-data_loader_args:
-  batch_size: 256
-  cuda: true
-  num_workers: 0
-  sample_to_balance: true
-  seed: 4711
-  shuffle: true
-  splits:
-  - train
-  - val
-  subsample_fraction: null
-  target_transform: null
-  transform: null
-dataloader: EmnistDataLoader
-device: cuda:0
-experiment_group: Sample Experiments
-lr_scheduler: OneCycleLR
-lr_scheduler_args:
-  epochs: 16
-  max_lr: 0.001
-  steps_per_epoch: 1314
-metrics:
-- accuracy
-model: CharacterModel
-network: MLP
-network_args:
-  input_size: 784
-  num_layers: 3
-  output_size: 62
-optimizer: AdamW
-optimizer_args:
-  amsgrad: false
-  betas:
-  - 0.9
-  - 0.999
-  eps: 1.0e-08
-  lr: 0.01
-  weight_decay: 5.0e-05
-train_args:
-  batch_size: 256
-  epochs: 16
-  val_metric: accuracy
diff --git a/src/training/experiments/CharacterModel_Emnist_MLP/0721_141702/model/best.pt b/src/training/experiments/CharacterModel_Emnist_MLP/0721_141702/model/best.pt
deleted file mode 100644
index 96c21c1..0000000
Binary files a/src/training/experiments/CharacterModel_Emnist_MLP/0721_141702/model/best.pt and /dev/null differ
diff --git a/src/training/experiments/CharacterModel_Emnist_MLP/0721_141702/model/last.pt b/src/training/experiments/CharacterModel_Emnist_MLP/0721_141702/model/last.pt
deleted file mode 100644
index f024c0d..0000000
Binary files a/src/training/experiments/CharacterModel_Emnist_MLP/0721_141702/model/last.pt and /dev/null differ
diff --git a/src/training/experiments/CharacterModel_Emnist_MLP/0721_145028/config.yml b/src/training/experiments/CharacterModel_Emnist_MLP/0721_145028/config.yml
deleted file mode 100644
index fb75736..0000000
--- a/src/training/experiments/CharacterModel_Emnist_MLP/0721_145028/config.yml
+++ /dev/null
@@ -1,47 +0,0 @@
-criterion: CrossEntropyLoss
-criterion_args:
-  ignore_index: -100
-  reduction: mean
-  weight: null
-data_loader_args:
-  batch_size: 256
-  cuda: true
-  num_workers: 0
-  sample_to_balance: true
-  seed: 4711
-  shuffle: true
-  splits:
-  - train
-  - val
-  subsample_fraction: null
-  target_transform: null
-  transform: null
-dataloader: EmnistDataLoader
-device: cuda:0
-experiment_group: Sample Experiments
-lr_scheduler: OneCycleLR
-lr_scheduler_args:
-  epochs: 16
-  max_lr: 0.001
-  steps_per_epoch: 1314
-metrics:
-- accuracy
-model: CharacterModel
-network: MLP
-network_args:
-  input_size: 784
-  num_layers: 3
-  output_size: 62
-optimizer: AdamW
-optimizer_args:
-  amsgrad: false
-  betas:
-  - 0.9
-  - 0.999
-  eps: 1.0e-08
-  lr: 0.01
-  weight_decay: 5.0e-05
-train_args:
-  batch_size: 256
-  epochs: 16
-  val_metric: accuracy
diff --git a/src/training/experiments/CharacterModel_Emnist_MLP/0721_150212/config.yml b/src/training/experiments/CharacterModel_Emnist_MLP/0721_150212/config.yml
deleted file mode 100644
index fb75736..0000000
--- a/src/training/experiments/CharacterModel_Emnist_MLP/0721_150212/config.yml
+++ /dev/null
@@ -1,47 +0,0 @@
-criterion: CrossEntropyLoss
-criterion_args:
-  ignore_index: -100
-  reduction: mean
-  weight: null
-data_loader_args:
-  batch_size: 256
-  cuda: true
-  num_workers: 0
-  sample_to_balance: true
-  seed: 4711
-  shuffle: true
-  splits:
-  - train
-  - val
-  subsample_fraction: null
-  target_transform: null
-  transform: null
-dataloader: EmnistDataLoader
-device: cuda:0
-experiment_group: Sample Experiments
-lr_scheduler: OneCycleLR
-lr_scheduler_args:
-  epochs: 16
-  max_lr: 0.001
-  steps_per_epoch: 1314
-metrics:
-- accuracy
-model: CharacterModel
-network: MLP
-network_args:
-  input_size: 784
-  num_layers: 3
-  output_size: 62
-optimizer: AdamW
-optimizer_args:
-  amsgrad: false
-  betas:
-  - 0.9
-  - 0.999
-  eps: 1.0e-08
-  lr: 0.01
-  weight_decay: 5.0e-05
-train_args:
-  batch_size: 256
-  epochs: 16
-  val_metric: accuracy
diff --git a/src/training/experiments/CharacterModel_Emnist_MLP/0721_150301/config.yml b/src/training/experiments/CharacterModel_Emnist_MLP/0721_150301/config.yml
deleted file mode 100644
index fb75736..0000000
--- a/src/training/experiments/CharacterModel_Emnist_MLP/0721_150301/config.yml
+++ /dev/null
@@ -1,47 +0,0 @@
-criterion: CrossEntropyLoss
-criterion_args:
-  ignore_index: -100
-  reduction: mean
-  weight: null
-data_loader_args:
-  batch_size: 256
-  cuda: true
-  num_workers: 0
-  sample_to_balance: true
-  seed: 4711
-  shuffle: true
-  splits:
-  - train
-  - val
-  subsample_fraction: null
-  target_transform: null
-  transform: null
-dataloader: EmnistDataLoader
-device: cuda:0
-experiment_group: Sample Experiments
-lr_scheduler: OneCycleLR
-lr_scheduler_args:
-  epochs: 16
-  max_lr: 0.001
-  steps_per_epoch: 1314
-metrics:
-- accuracy
-model: CharacterModel
-network: MLP
-network_args:
-  input_size: 784
-  num_layers: 3
-  output_size: 62
-optimizer: AdamW
-optimizer_args:
-  amsgrad: false
-  betas:
-  - 0.9
-  - 0.999
-  eps: 1.0e-08
-  lr: 0.01
-  weight_decay: 5.0e-05
-train_args:
-  batch_size: 256
-  epochs: 16
-  val_metric: accuracy
diff --git a/src/training/experiments/CharacterModel_Emnist_MLP/0721_150317/config.yml b/src/training/experiments/CharacterModel_Emnist_MLP/0721_150317/config.yml
deleted file mode 100644
index fb75736..0000000
--- a/src/training/experiments/CharacterModel_Emnist_MLP/0721_150317/config.yml
+++ /dev/null
@@ -1,47 +0,0 @@
-criterion: CrossEntropyLoss
-criterion_args:
-  ignore_index: -100
-  reduction: mean
-  weight: null
-data_loader_args:
-  batch_size: 256
-  cuda: true
-  num_workers: 0
-  sample_to_balance: true
-  seed: 4711
-  shuffle: true
-  splits:
-  - train
-  - val
-  subsample_fraction: null
-  target_transform: null
-  transform: null
-dataloader: EmnistDataLoader
-device: cuda:0
-experiment_group: Sample Experiments
-lr_scheduler: OneCycleLR
-lr_scheduler_args:
-  epochs: 16
-  max_lr: 0.001
-  steps_per_epoch: 1314
-metrics:
-- accuracy
-model: CharacterModel
-network: MLP
-network_args:
-  input_size: 784
-  num_layers: 3
-  output_size: 62
-optimizer: AdamW
-optimizer_args:
-  amsgrad: false
-  betas:
-  - 0.9
-  - 0.999
-  eps: 1.0e-08
-  lr: 0.01
-  weight_decay: 5.0e-05
-train_args:
-  batch_size: 256
-  epochs: 16
-  val_metric: accuracy
diff --git a/src/training/experiments/CharacterModel_Emnist_MLP/0721_151135/config.yml b/src/training/experiments/CharacterModel_Emnist_MLP/0721_151135/config.yml
deleted file mode 100644
index fb75736..0000000
--- a/src/training/experiments/CharacterModel_Emnist_MLP/0721_151135/config.yml
+++ /dev/null
@@ -1,47 +0,0 @@
-criterion: CrossEntropyLoss
-criterion_args:
-  ignore_index: -100
-  reduction: mean
-  weight: null
-data_loader_args:
-  batch_size: 256
-  cuda: true
-  num_workers: 0
-  sample_to_balance: true
-  seed: 4711
-  shuffle: true
-  splits:
-  - train
-  - val
-  subsample_fraction: null
-  target_transform: null
-  transform: null
-dataloader: EmnistDataLoader
-device: cuda:0
-experiment_group: Sample Experiments
-lr_scheduler: OneCycleLR
-lr_scheduler_args:
-  epochs: 16
-  max_lr: 0.001
-  steps_per_epoch: 1314
-metrics:
-- accuracy
-model: CharacterModel
-network: MLP
-network_args:
-  input_size: 784
-  num_layers: 3
-  output_size: 62
-optimizer: AdamW
-optimizer_args:
-  amsgrad: false
-  betas:
-  - 0.9
-  - 0.999
-  eps: 1.0e-08
-  lr: 0.01
-  weight_decay: 5.0e-05
-train_args:
-  batch_size: 256
-  epochs: 16
-  val_metric: accuracy
diff --git a/src/training/experiments/CharacterModel_Emnist_MLP/0721_151135/model/best.pt b/src/training/experiments/CharacterModel_Emnist_MLP/0721_151135/model/best.pt
deleted file mode 100644
index f833a89..0000000
Binary files a/src/training/experiments/CharacterModel_Emnist_MLP/0721_151135/model/best.pt and /dev/null differ
diff --git a/src/training/experiments/CharacterModel_Emnist_MLP/0721_151135/model/last.pt b/src/training/experiments/CharacterModel_Emnist_MLP/0721_151135/model/last.pt
deleted file mode 100644
index f833a89..0000000
Binary files a/src/training/experiments/CharacterModel_Emnist_MLP/0721_151135/model/last.pt and /dev/null differ
diff --git a/src/training/experiments/CharacterModel_Emnist_MLP/0721_151408/config.yml b/src/training/experiments/CharacterModel_Emnist_MLP/0721_151408/config.yml
deleted file mode 100644
index fb75736..0000000
--- a/src/training/experiments/CharacterModel_Emnist_MLP/0721_151408/config.yml
+++ /dev/null
@@ -1,47 +0,0 @@
-criterion: CrossEntropyLoss
-criterion_args:
-  ignore_index: -100
-  reduction: mean
-  weight: null
-data_loader_args:
-  batch_size: 256
-  cuda: true
-  num_workers: 0
-  sample_to_balance: true
-  seed: 4711
-  shuffle: true
-  splits:
-  - train
-  - val
-  subsample_fraction: null
-  target_transform: null
-  transform: null
-dataloader: EmnistDataLoader
-device: cuda:0
-experiment_group: Sample Experiments
-lr_scheduler: OneCycleLR
-lr_scheduler_args:
-  epochs: 16
-  max_lr: 0.001
-  steps_per_epoch: 1314
-metrics:
-- accuracy
-model: CharacterModel
-network: MLP
-network_args:
-  input_size: 784
-  num_layers: 3
-  output_size: 62
-optimizer: AdamW
-optimizer_args:
-  amsgrad: false
-  betas:
-  - 0.9
-  - 0.999
-  eps: 1.0e-08
-  lr: 0.01
-  weight_decay: 5.0e-05
-train_args:
-  batch_size: 256
-  epochs: 16
-  val_metric: accuracy
diff --git a/src/training/experiments/CharacterModel_Emnist_MLP/0721_153144/config.yml b/src/training/experiments/CharacterModel_Emnist_MLP/0721_153144/config.yml
deleted file mode 100644
index 829297d..0000000
--- a/src/training/experiments/CharacterModel_Emnist_MLP/0721_153144/config.yml
+++ /dev/null
@@ -1,47 +0,0 @@
-criterion: CrossEntropyLoss
-criterion_args:
-  ignore_index: -100
-  reduction: mean
-  weight: null
-data_loader_args:
-  batch_size: 256
-  cuda: true
-  num_workers: 0
-  sample_to_balance: true
-  seed: 4711
-  shuffle: true
-  splits:
-  - train
-  - val
-  subsample_fraction: null
-  target_transform: null
-  transform: null
-dataloader: EmnistDataLoader
-device: cuda:0
-experiment_group: Sample Experiments
-lr_scheduler: OneCycleLR
-lr_scheduler_args:
-  epochs: 16
-  max_lr: 0.0003
-  steps_per_epoch: 1314
-metrics:
-- accuracy
-model: CharacterModel
-network: MLP
-network_args:
-  input_size: 784
-  num_layers: 3
-  output_size: 62
-optimizer: AdamW
-optimizer_args:
-  amsgrad: false
-  betas:
-  - 0.9
-  - 0.999
-  eps: 1.0e-08
-  lr: 0.0006
-  weight_decay: 5.0e-05
-train_args:
-  batch_size: 256
-  epochs: 16
-  val_metric: accuracy
diff --git a/src/training/experiments/CharacterModel_Emnist_MLP/0721_153207/config.yml b/src/training/experiments/CharacterModel_Emnist_MLP/0721_153207/config.yml
deleted file mode 100644
index fb75736..0000000
--- a/src/training/experiments/CharacterModel_Emnist_MLP/0721_153207/config.yml
+++ /dev/null
@@ -1,47 +0,0 @@
-criterion: CrossEntropyLoss
-criterion_args:
-  ignore_index: -100
-  reduction: mean
-  weight: null
-data_loader_args:
-  batch_size: 256
-  cuda: true
-  num_workers: 0
-  sample_to_balance: true
-  seed: 4711
-  shuffle: true
-  splits:
-  - train
-  - val
-  subsample_fraction: null
-  target_transform: null
-  transform: null
-dataloader: EmnistDataLoader
-device: cuda:0
-experiment_group: Sample Experiments
-lr_scheduler: OneCycleLR
-lr_scheduler_args:
-  epochs: 16
-  max_lr: 0.001
-  steps_per_epoch: 1314
-metrics:
-- accuracy
-model: CharacterModel
-network: MLP
-network_args:
-  input_size: 784
-  num_layers: 3
-  output_size: 62
-optimizer: AdamW
-optimizer_args:
-  amsgrad: false
-  betas:
-  - 0.9
-  - 0.999
-  eps: 1.0e-08
-  lr: 0.01
-  weight_decay: 5.0e-05
-train_args:
-  batch_size: 256
-  epochs: 16
-  val_metric: accuracy
diff --git a/src/training/experiments/CharacterModel_Emnist_MLP/0721_153310/config.yml b/src/training/experiments/CharacterModel_Emnist_MLP/0721_153310/config.yml
deleted file mode 100644
index fb75736..0000000
--- a/src/training/experiments/CharacterModel_Emnist_MLP/0721_153310/config.yml
+++ /dev/null
@@ -1,47 +0,0 @@
-criterion: CrossEntropyLoss
-criterion_args:
-  ignore_index: -100
-  reduction: mean
-  weight: null
-data_loader_args:
-  batch_size: 256
-  cuda: true
-  num_workers: 0
-  sample_to_balance: true
-  seed: 4711
-  shuffle: true
-  splits:
-  - train
-  - val
-  subsample_fraction: null
-  target_transform: null
-  transform: null
-dataloader: EmnistDataLoader
-device: cuda:0
-experiment_group: Sample Experiments
-lr_scheduler: OneCycleLR
-lr_scheduler_args:
-  epochs: 16
-  max_lr: 0.001
-  steps_per_epoch: 1314
-metrics:
-- accuracy
-model: CharacterModel
-network: MLP
-network_args:
-  input_size: 784
-  num_layers: 3
-  output_size: 62
-optimizer: AdamW
-optimizer_args:
-  amsgrad: false
-  betas:
-  - 0.9
-  - 0.999
-  eps: 1.0e-08
-  lr: 0.01
-  weight_decay: 5.0e-05
-train_args:
-  batch_size: 256
-  epochs: 16
-  val_metric: accuracy
diff --git a/src/training/experiments/CharacterModel_Emnist_MLP/0721_153310/model/best.pt b/src/training/experiments/CharacterModel_Emnist_MLP/0721_153310/model/best.pt
deleted file mode 100644
index cbbc5e1..0000000
Binary files a/src/training/experiments/CharacterModel_Emnist_MLP/0721_153310/model/best.pt and /dev/null differ
diff --git a/src/training/experiments/CharacterModel_Emnist_MLP/0721_153310/model/last.pt b/src/training/experiments/CharacterModel_Emnist_MLP/0721_153310/model/last.pt
deleted file mode 100644
index cbbc5e1..0000000
Binary files a/src/training/experiments/CharacterModel_Emnist_MLP/0721_153310/model/last.pt and /dev/null differ
diff --git a/src/training/experiments/CharacterModel_Emnist_MLP/0721_175150/config.yml b/src/training/experiments/CharacterModel_Emnist_MLP/0721_175150/config.yml
deleted file mode 100644
index fb75736..0000000
--- a/src/training/experiments/CharacterModel_Emnist_MLP/0721_175150/config.yml
+++ /dev/null
@@ -1,47 +0,0 @@
-criterion: CrossEntropyLoss
-criterion_args:
-  ignore_index: -100
-  reduction: mean
-  weight: null
-data_loader_args:
-  batch_size: 256
-  cuda: true
-  num_workers: 0
-  sample_to_balance: true
-  seed: 4711
-  shuffle: true
-  splits:
-  - train
-  - val
-  subsample_fraction: null
-  target_transform: null
-  transform: null
-dataloader: EmnistDataLoader
-device: cuda:0
-experiment_group: Sample Experiments
-lr_scheduler: OneCycleLR
-lr_scheduler_args:
-  epochs: 16
-  max_lr: 0.001
-  steps_per_epoch: 1314
-metrics:
-- accuracy
-model: CharacterModel
-network: MLP
-network_args:
-  input_size: 784
-  num_layers: 3
-  output_size: 62
-optimizer: AdamW
-optimizer_args:
-  amsgrad: false
-  betas:
-  - 0.9
-  - 0.999
-  eps: 1.0e-08
-  lr: 0.01
-  weight_decay: 5.0e-05
-train_args:
-  batch_size: 256
-  epochs: 16
-  val_metric: accuracy
diff --git a/src/training/experiments/CharacterModel_Emnist_MLP/0721_175150/model/best.pt b/src/training/experiments/CharacterModel_Emnist_MLP/0721_175150/model/best.pt
deleted file mode 100644
index c93e3c6..0000000
Binary files a/src/training/experiments/CharacterModel_Emnist_MLP/0721_175150/model/best.pt and /dev/null differ
diff --git a/src/training/experiments/CharacterModel_Emnist_MLP/0721_175150/model/last.pt b/src/training/experiments/CharacterModel_Emnist_MLP/0721_175150/model/last.pt
deleted file mode 100644
index c93e3c6..0000000
Binary files a/src/training/experiments/CharacterModel_Emnist_MLP/0721_175150/model/last.pt and /dev/null differ
diff --git a/src/training/experiments/CharacterModel_Emnist_MLP/0721_180741/config.yml b/src/training/experiments/CharacterModel_Emnist_MLP/0721_180741/config.yml
deleted file mode 100644
index 1be5113..0000000
--- a/src/training/experiments/CharacterModel_Emnist_MLP/0721_180741/config.yml
+++ /dev/null
@@ -1,47 +0,0 @@
-criterion: CrossEntropyLoss
-criterion_args:
-  ignore_index: -100
-  reduction: mean
-  weight: null
-data_loader_args:
-  batch_size: 256
-  cuda: true
-  num_workers: 0
-  sample_to_balance: true
-  seed: 4711
-  shuffle: true
-  splits:
-  - train
-  - val
-  subsample_fraction: null
-  target_transform: null
-  transform: null
-dataloader: EmnistDataLoader
-device: cuda:0
-experiment_group: Sample Experiments
-lr_scheduler: OneCycleLR
-lr_scheduler_args:
-  epochs: 16
-  max_lr: 0.001
-  steps_per_epoch: 1314
-metrics:
-- accuracy
-model: CharacterModel
-network: MLP
-network_args:
-  input_size: 784
-  num_layers: 3
-  output_size: 62
-optimizer: Adam
-optimizer_args:
-  amsgrad: false
-  betas:
-  - 0.9
-  - 0.999
-  eps: 1.0e-08
-  lr: 0.01
-  weight_decay: 5.0e-05
-train_args:
-  batch_size: 256
-  epochs: 16
-  val_metric: accuracy
diff --git a/src/training/experiments/CharacterModel_Emnist_MLP/0721_180741/model/best.pt b/src/training/experiments/CharacterModel_Emnist_MLP/0721_180741/model/best.pt
deleted file mode 100644
index 580bad2..0000000
Binary files a/src/training/experiments/CharacterModel_Emnist_MLP/0721_180741/model/best.pt and /dev/null differ
diff --git a/src/training/experiments/CharacterModel_Emnist_MLP/0721_180741/model/last.pt b/src/training/experiments/CharacterModel_Emnist_MLP/0721_180741/model/last.pt
deleted file mode 100644
index 97e245c..0000000
Binary files a/src/training/experiments/CharacterModel_Emnist_MLP/0721_180741/model/last.pt and /dev/null differ
diff --git a/src/training/experiments/CharacterModel_Emnist_MLP/0721_181933/config.yml b/src/training/experiments/CharacterModel_Emnist_MLP/0721_181933/config.yml
deleted file mode 100644
index d2f98a2..0000000
--- a/src/training/experiments/CharacterModel_Emnist_MLP/0721_181933/config.yml
+++ /dev/null
@@ -1,46 +0,0 @@
-criterion: CrossEntropyLoss
-criterion_args:
-  ignore_index: -100
-  reduction: mean
-  weight: null
-data_loader_args:
-  batch_size: 256
-  cuda: true
-  num_workers: 0
-  sample_to_balance: true
-  seed: 4711
-  shuffle: true
-  splits:
-  - train
-  - val
-  subsample_fraction: null
-  target_transform: null
-  transform: null
-dataloader: EmnistDataLoader
-device: cuda:0
-experiment_group: Sample Experiments
-lr_scheduler: OneCycleLR
-lr_scheduler_args:
-  epochs: 16
-  max_lr: 0.001
-  steps_per_epoch: 1314
-metrics:
-- accuracy
-model: CharacterModel
-network: MLP
-network_args:
-  input_size: 784
-  num_layers: 3
-  output_size: 62
-optimizer: Adamax
-optimizer_args:
-  betas:
-  - 0.9
-  - 0.999
-  eps: 1.0e-08
-  lr: 0.01
-  weight_decay: 0
-train_args:
-  batch_size: 256
-  epochs: 16
-  val_metric: accuracy
diff --git a/src/training/experiments/CharacterModel_Emnist_MLP/0721_181933/model/best.pt b/src/training/experiments/CharacterModel_Emnist_MLP/0721_181933/model/best.pt
deleted file mode 100644
index 5a3df56..0000000
Binary files a/src/training/experiments/CharacterModel_Emnist_MLP/0721_181933/model/best.pt and /dev/null differ
diff --git a/src/training/experiments/CharacterModel_Emnist_MLP/0721_181933/model/last.pt b/src/training/experiments/CharacterModel_Emnist_MLP/0721_181933/model/last.pt
deleted file mode 100644
index 7f28dc3..0000000
Binary files a/src/training/experiments/CharacterModel_Emnist_MLP/0721_181933/model/last.pt and /dev/null differ
diff --git a/src/training/experiments/CharacterModel_Emnist_MLP/0721_183347/config.yml b/src/training/experiments/CharacterModel_Emnist_MLP/0721_183347/config.yml
deleted file mode 100644
index d2f98a2..0000000
--- a/src/training/experiments/CharacterModel_Emnist_MLP/0721_183347/config.yml
+++ /dev/null
@@ -1,46 +0,0 @@
-criterion: CrossEntropyLoss
-criterion_args:
-  ignore_index: -100
-  reduction: mean
-  weight: null
-data_loader_args:
-  batch_size: 256
-  cuda: true
-  num_workers: 0
-  sample_to_balance: true
-  seed: 4711
-  shuffle: true
-  splits:
-  - train
-  - val
-  subsample_fraction: null
-  target_transform: null
-  transform: null
-dataloader: EmnistDataLoader
-device: cuda:0
-experiment_group: Sample Experiments
-lr_scheduler: OneCycleLR
-lr_scheduler_args:
-  epochs: 16
-  max_lr: 0.001
-  steps_per_epoch: 1314
-metrics:
-- accuracy
-model: CharacterModel
-network: MLP
-network_args:
-  input_size: 784
-  num_layers: 3
-  output_size: 62
-optimizer: Adamax
-optimizer_args:
-  betas:
-  - 0.9
-  - 0.999
-  eps: 1.0e-08
-  lr: 0.01
-  weight_decay: 0
-train_args:
-  batch_size: 256
-  epochs: 16
-  val_metric: accuracy
diff --git a/src/training/experiments/CharacterModel_Emnist_MLP/0721_183347/model/best.pt b/src/training/experiments/CharacterModel_Emnist_MLP/0721_183347/model/best.pt
deleted file mode 100644
index 6f09780..0000000
Binary files a/src/training/experiments/CharacterModel_Emnist_MLP/0721_183347/model/best.pt and /dev/null differ
diff --git a/src/training/experiments/CharacterModel_Emnist_MLP/0721_183347/model/last.pt b/src/training/experiments/CharacterModel_Emnist_MLP/0721_183347/model/last.pt
deleted file mode 100644
index 3bb103e..0000000
Binary files a/src/training/experiments/CharacterModel_Emnist_MLP/0721_183347/model/last.pt and /dev/null differ
diff --git a/src/training/experiments/CharacterModel_Emnist_MLP/0721_190044/config.yml b/src/training/experiments/CharacterModel_Emnist_MLP/0721_190044/config.yml
deleted file mode 100644
index a7c66c5..0000000
--- a/src/training/experiments/CharacterModel_Emnist_MLP/0721_190044/config.yml
+++ /dev/null
@@ -1,46 +0,0 @@
-criterion: CrossEntropyLoss
-criterion_args:
-  ignore_index: -100
-  reduction: mean
-  weight: null
-data_loader_args:
-  batch_size: 256
-  cuda: true
-  num_workers: 0
-  sample_to_balance: true
-  seed: 4711
-  shuffle: true
-  splits:
-  - train
-  - val
-  subsample_fraction: null
-  target_transform: null
-  transform: null
-dataloader: EmnistDataLoader
-device: cuda:0
-experiment_group: Sample Experiments
-lr_scheduler: OneCycleLR
-lr_scheduler_args:
-  epochs: 16
-  max_lr: 0.001
-  steps_per_epoch: 1314
-metrics:
-- accuracy
-model: CharacterModel
-network: MLP
-network_args:
-  input_size: 784
-  num_layers: 3
-  output_size: 62
-optimizer: AdamW
-optimizer_args:
-  betas:
-  - 0.9
-  - 0.999
-  eps: 1.0e-08
-  lr: 0.01
-  weight_decay: 0
-train_args:
-  batch_size: 256
-  epochs: 16
-  val_metric: accuracy
diff --git a/src/training/experiments/CharacterModel_Emnist_MLP/0721_190044/model/best.pt b/src/training/experiments/CharacterModel_Emnist_MLP/0721_190044/model/best.pt
deleted file mode 100644
index c3e3618..0000000
Binary files a/src/training/experiments/CharacterModel_Emnist_MLP/0721_190044/model/best.pt and /dev/null differ
diff --git a/src/training/experiments/CharacterModel_Emnist_MLP/0721_190044/model/last.pt b/src/training/experiments/CharacterModel_Emnist_MLP/0721_190044/model/last.pt
deleted file mode 100644
index c3e3618..0000000
Binary files a/src/training/experiments/CharacterModel_Emnist_MLP/0721_190044/model/last.pt and /dev/null differ
diff --git a/src/training/experiments/CharacterModel_Emnist_MLP/0721_190633/config.yml b/src/training/experiments/CharacterModel_Emnist_MLP/0721_190633/config.yml
deleted file mode 100644
index a7c66c5..0000000
--- a/src/training/experiments/CharacterModel_Emnist_MLP/0721_190633/config.yml
+++ /dev/null
@@ -1,46 +0,0 @@
-criterion: CrossEntropyLoss
-criterion_args:
-  ignore_index: -100
-  reduction: mean
-  weight: null
-data_loader_args:
-  batch_size: 256
-  cuda: true
-  num_workers: 0
-  sample_to_balance: true
-  seed: 4711
-  shuffle: true
-  splits:
-  - train
-  - val
-  subsample_fraction: null
-  target_transform: null
-  transform: null
-dataloader: EmnistDataLoader
-device: cuda:0
-experiment_group: Sample Experiments
-lr_scheduler: OneCycleLR
-lr_scheduler_args:
-  epochs: 16
-  max_lr: 0.001
-  steps_per_epoch: 1314
-metrics:
-- accuracy
-model: CharacterModel
-network: MLP
-network_args:
-  input_size: 784
-  num_layers: 3
-  output_size: 62
-optimizer: AdamW
-optimizer_args:
-  betas:
-  - 0.9
-  - 0.999
-  eps: 1.0e-08
-  lr: 0.01
-  weight_decay: 0
-train_args:
-  batch_size: 256
-  epochs: 16
-  val_metric: accuracy
diff --git a/src/training/experiments/CharacterModel_Emnist_MLP/0721_190633/model/best.pt b/src/training/experiments/CharacterModel_Emnist_MLP/0721_190633/model/best.pt
deleted file mode 100644
index 44d9b9b..0000000
Binary files a/src/training/experiments/CharacterModel_Emnist_MLP/0721_190633/model/best.pt and /dev/null differ
diff --git a/src/training/experiments/CharacterModel_Emnist_MLP/0721_190633/model/last.pt b/src/training/experiments/CharacterModel_Emnist_MLP/0721_190633/model/last.pt
deleted file mode 100644
index 44d9b9b..0000000
Binary files a/src/training/experiments/CharacterModel_Emnist_MLP/0721_190633/model/last.pt and /dev/null differ
diff --git a/src/training/experiments/CharacterModel_Emnist_MLP/0721_190738/config.yml b/src/training/experiments/CharacterModel_Emnist_MLP/0721_190738/config.yml
deleted file mode 100644
index a7c66c5..0000000
--- a/src/training/experiments/CharacterModel_Emnist_MLP/0721_190738/config.yml
+++ /dev/null
@@ -1,46 +0,0 @@
-criterion: CrossEntropyLoss
-criterion_args:
-  ignore_index: -100
-  reduction: mean
-  weight: null
-data_loader_args:
-  batch_size: 256
-  cuda: true
-  num_workers: 0
-  sample_to_balance: true
-  seed: 4711
-  shuffle: true
-  splits:
-  - train
-  - val
-  subsample_fraction: null
-  target_transform: null
-  transform: null
-dataloader: EmnistDataLoader
-device: cuda:0
-experiment_group: Sample Experiments
-lr_scheduler: OneCycleLR
-lr_scheduler_args:
-  epochs: 16
-  max_lr: 0.001
-  steps_per_epoch: 1314
-metrics:
-- accuracy
-model: CharacterModel
-network: MLP
-network_args:
-  input_size: 784
-  num_layers: 3
-  output_size: 62
-optimizer: AdamW
-optimizer_args:
-  betas:
-  - 0.9
-  - 0.999
-  eps: 1.0e-08
-  lr: 0.01
-  weight_decay: 0
-train_args:
-  batch_size: 256
-  epochs: 16
-  val_metric: accuracy
diff --git a/src/training/experiments/CharacterModel_Emnist_MLP/0721_190738/model/best.pt b/src/training/experiments/CharacterModel_Emnist_MLP/0721_190738/model/best.pt
deleted file mode 100644
index 4a0333c..0000000
Binary files a/src/training/experiments/CharacterModel_Emnist_MLP/0721_190738/model/best.pt and /dev/null differ
diff --git a/src/training/experiments/CharacterModel_Emnist_MLP/0721_190738/model/last.pt b/src/training/experiments/CharacterModel_Emnist_MLP/0721_190738/model/last.pt
deleted file mode 100644
index 4a0333c..0000000
Binary files a/src/training/experiments/CharacterModel_Emnist_MLP/0721_190738/model/last.pt and /dev/null differ
diff --git a/src/training/experiments/CharacterModel_Emnist_MLP/0721_191111/config.yml b/src/training/experiments/CharacterModel_Emnist_MLP/0721_191111/config.yml
deleted file mode 100644
index a7c66c5..0000000
--- a/src/training/experiments/CharacterModel_Emnist_MLP/0721_191111/config.yml
+++ /dev/null
@@ -1,46 +0,0 @@
-criterion: CrossEntropyLoss
-criterion_args:
-  ignore_index: -100
-  reduction: mean
-  weight: null
-data_loader_args:
-  batch_size: 256
-  cuda: true
-  num_workers: 0
-  sample_to_balance: true
-  seed: 4711
-  shuffle: true
-  splits:
-  - train
-  - val
-  subsample_fraction: null
-  target_transform: null
-  transform: null
-dataloader: EmnistDataLoader
-device: cuda:0
-experiment_group: Sample Experiments
-lr_scheduler: OneCycleLR
-lr_scheduler_args:
-  epochs: 16
-  max_lr: 0.001
-  steps_per_epoch: 1314
-metrics:
-- accuracy
-model: CharacterModel
-network: MLP
-network_args:
-  input_size: 784
-  num_layers: 3
-  output_size: 62
-optimizer: AdamW
-optimizer_args:
-  betas:
-  - 0.9
-  - 0.999
-  eps: 1.0e-08
-  lr: 0.01
-  weight_decay: 0
-train_args:
-  batch_size: 256
-  epochs: 16
-  val_metric: accuracy
diff --git a/src/training/experiments/CharacterModel_Emnist_MLP/0721_191310/config.yml b/src/training/experiments/CharacterModel_Emnist_MLP/0721_191310/config.yml
deleted file mode 100644
index 08c344c..0000000
--- a/src/training/experiments/CharacterModel_Emnist_MLP/0721_191310/config.yml
+++ /dev/null
@@ -1,46 +0,0 @@
-criterion: CrossEntropyLoss
-criterion_args:
-  ignore_index: -100
-  reduction: mean
-  weight: null
-data_loader_args:
-  batch_size: 256
-  cuda: true
-  num_workers: 1
-  sample_to_balance: true
-  seed: 4711
-  shuffle: true
-  splits:
-  - train
-  - val
-  subsample_fraction: null
-  target_transform: null
-  transform: null
-dataloader: EmnistDataLoader
-device: cuda:0
-experiment_group: Sample Experiments
-lr_scheduler: OneCycleLR
-lr_scheduler_args:
-  epochs: 16
-  max_lr: 0.001
-  steps_per_epoch: 1314
-metrics:
-- accuracy
-model: CharacterModel
-network: MLP
-network_args:
-  input_size: 784
-  num_layers: 3
-  output_size: 62
-optimizer: AdamW
-optimizer_args:
-  betas:
-  - 0.9
-  - 0.999
-  eps: 1.0e-08
-  lr: 0.01
-  weight_decay: 0
-train_args:
-  batch_size: 256
-  epochs: 16
-  val_metric: accuracy
diff --git a/src/training/experiments/CharacterModel_Emnist_MLP/0721_191310/model/best.pt b/src/training/experiments/CharacterModel_Emnist_MLP/0721_191310/model/best.pt
deleted file mode 100644
index 076aae1..0000000
Binary files a/src/training/experiments/CharacterModel_Emnist_MLP/0721_191310/model/best.pt and /dev/null differ
diff --git a/src/training/experiments/CharacterModel_Emnist_MLP/0721_191310/model/last.pt b/src/training/experiments/CharacterModel_Emnist_MLP/0721_191310/model/last.pt
deleted file mode 100644
index 076aae1..0000000
Binary files a/src/training/experiments/CharacterModel_Emnist_MLP/0721_191310/model/last.pt and /dev/null differ
diff --git a/src/training/experiments/CharacterModel_Emnist_MLP/0721_191412/config.yml b/src/training/experiments/CharacterModel_Emnist_MLP/0721_191412/config.yml
deleted file mode 100644
index 0b9b10e..0000000
--- a/src/training/experiments/CharacterModel_Emnist_MLP/0721_191412/config.yml
+++ /dev/null
@@ -1,42 +0,0 @@
-criterion: CrossEntropyLoss
-criterion_args:
-  ignore_index: -100
-  reduction: mean
-  weight: null
-data_loader_args:
-  batch_size: 256
-  cuda: true
-  num_workers: 1
-  sample_to_balance: true
-  seed: 4711
-  shuffle: true
-  splits:
-  - train
-  - val
-  subsample_fraction: null
-  target_transform: null
-  transform: null
-dataloader: EmnistDataLoader
-device: cuda:0
-experiment_group: Sample Experiments
-lr_scheduler: null
-metrics:
-- accuracy
-model: CharacterModel
-network: MLP
-network_args:
-  input_size: 784
-  num_layers: 3
-  output_size: 62
-optimizer: RMSprop
-optimizer_args:
-  alpha: 0.9
-  centered: false
-  eps: 1.0e-07
-  lr: 0.001
-  momentum: 0
-  weight_decay: 0
-train_args:
-  batch_size: 256
-  epochs: 16
-  val_metric: accuracy
diff --git a/src/training/experiments/CharacterModel_Emnist_MLP/0721_191412/model/best.pt b/src/training/experiments/CharacterModel_Emnist_MLP/0721_191412/model/best.pt
deleted file mode 100644
index 2fb0195..0000000
Binary files a/src/training/experiments/CharacterModel_Emnist_MLP/0721_191412/model/best.pt and /dev/null differ
diff --git a/src/training/experiments/CharacterModel_Emnist_MLP/0721_191412/model/last.pt b/src/training/experiments/CharacterModel_Emnist_MLP/0721_191412/model/last.pt
deleted file mode 100644
index 2fb0195..0000000
Binary files a/src/training/experiments/CharacterModel_Emnist_MLP/0721_191412/model/last.pt and /dev/null differ
diff --git a/src/training/experiments/CharacterModel_Emnist_MLP/0721_191504/config.yml b/src/training/experiments/CharacterModel_Emnist_MLP/0721_191504/config.yml
deleted file mode 100644
index 93c2854..0000000
--- a/src/training/experiments/CharacterModel_Emnist_MLP/0721_191504/config.yml
+++ /dev/null
@@ -1,42 +0,0 @@
-criterion: CrossEntropyLoss
-criterion_args:
-  ignore_index: -100
-  reduction: mean
-  weight: null
-data_loader_args:
-  batch_size: 256
-  cuda: true
-  num_workers: 4
-  sample_to_balance: true
-  seed: 4711
-  shuffle: true
-  splits:
-  - train
-  - val
-  subsample_fraction: null
-  target_transform: null
-  transform: null
-dataloader: EmnistDataLoader
-device: cuda:0
-experiment_group: Sample Experiments
-lr_scheduler: null
-metrics:
-- accuracy
-model: CharacterModel
-network: MLP
-network_args:
-  input_size: 784
-  num_layers: 3
-  output_size: 62
-optimizer: RMSprop
-optimizer_args:
-  alpha: 0.9
-  centered: false
-  eps: 1.0e-07
-  lr: 0.001
-  momentum: 0
-  weight_decay: 0
-train_args:
-  batch_size: 256
-  epochs: 16
-  val_metric: accuracy
diff --git a/src/training/experiments/CharacterModel_Emnist_MLP/0721_191504/model/best.pt b/src/training/experiments/CharacterModel_Emnist_MLP/0721_191504/model/best.pt
deleted file mode 100644
index 9acc5b1..0000000
Binary files a/src/training/experiments/CharacterModel_Emnist_MLP/0721_191504/model/best.pt and /dev/null differ
diff --git a/src/training/experiments/CharacterModel_Emnist_MLP/0721_191504/model/last.pt b/src/training/experiments/CharacterModel_Emnist_MLP/0721_191504/model/last.pt
deleted file mode 100644
index b8cc01c..0000000
Binary files a/src/training/experiments/CharacterModel_Emnist_MLP/0721_191504/model/last.pt and /dev/null differ
diff --git a/src/training/experiments/CharacterModel_Emnist_MLP/0721_191826/config.yml b/src/training/experiments/CharacterModel_Emnist_MLP/0721_191826/config.yml
deleted file mode 100644
index 7340941..0000000
--- a/src/training/experiments/CharacterModel_Emnist_MLP/0721_191826/config.yml
+++ /dev/null
@@ -1,47 +0,0 @@
-criterion: CrossEntropyLoss
-criterion_args:
-  ignore_index: -100
-  reduction: mean
-  weight: null
-data_loader_args:
-  batch_size: 256
-  cuda: true
-  num_workers: 8
-  sample_to_balance: true
-  seed: 4711
-  shuffle: true
-  splits:
-  - train
-  - val
-  subsample_fraction: null
-  target_transform: null
-  transform: null
-dataloader: EmnistDataLoader
-device: cuda:0
-experiment_group: Sample Experiments
-lr_scheduler: OneCycleLR
-lr_scheduler_args:
-  epochs: 16
-  max_lr: 0.001
-  steps_per_epoch: 1314
-metrics:
-- accuracy
-model: CharacterModel
-network: MLP
-network_args:
-  input_size: 784
-  num_layers: 3
-  output_size: 62
-optimizer: AdamW
-optimizer_args:
-  amsgrad: false
-  betas:
-  - 0.9
-  - 0.999
-  eps: 1.0e-08
-  lr: 0.01
-  weight_decay: 0
-train_args:
-  batch_size: 256
-  epochs: 16
-  val_metric: accuracy
diff --git a/src/training/experiments/CharacterModel_Emnist_MLP/0721_191826/model/best.pt b/src/training/experiments/CharacterModel_Emnist_MLP/0721_191826/model/best.pt
deleted file mode 100644
index 26bfb07..0000000
Binary files a/src/training/experiments/CharacterModel_Emnist_MLP/0721_191826/model/best.pt and /dev/null differ
diff --git a/src/training/experiments/CharacterModel_Emnist_MLP/0721_191826/model/last.pt b/src/training/experiments/CharacterModel_Emnist_MLP/0721_191826/model/last.pt
deleted file mode 100644
index 26bfb07..0000000
Binary files a/src/training/experiments/CharacterModel_Emnist_MLP/0721_191826/model/last.pt and /dev/null differ
diff --git a/src/training/experiments/CharacterModel_Emnist_MLP/0722_191559/config.yml b/src/training/experiments/CharacterModel_Emnist_MLP/0722_191559/config.yml
deleted file mode 100644
index 90f0e13..0000000
--- a/src/training/experiments/CharacterModel_Emnist_MLP/0722_191559/config.yml
+++ /dev/null
@@ -1,49 +0,0 @@
-criterion: CrossEntropyLoss
-criterion_args:
-  ignore_index: -100
-  reduction: mean
-  weight: null
-data_loader_args:
-  batch_size: 256
-  cuda: true
-  num_workers: 8
-  sample_to_balance: true
-  seed: 4711
-  shuffle: true
-  splits:
-  - train
-  - val
-  subsample_fraction: null
-  target_transform: null
-  transform: null
-dataloader: EmnistDataLoader
-device: cuda:0
-experiment_group: Sample Experiments
-lr_scheduler: OneCycleLR
-lr_scheduler_args:
-  epochs: 33
-  max_lr: 0.001
-  steps_per_epoch: 1314
-metrics:
-- accuracy
-model: CharacterModel
-network: MLP
-network_args:
-  input_size: 784
-  num_layers: 3
-  output_size: 62
-optimizer: AdamW
-optimizer_args:
-  amsgrad: false
-  betas:
-  - 0.9
-  - 0.999
-  eps: 1.0e-08
-  lr: 0.01
-  weight_decay: 0
-resume_experiment: last
-train_args:
-  batch_size: 256
-  epochs: 33
-  val_metric: accuracy
-verbosity: 1
diff --git a/src/training/experiments/CharacterModel_Emnist_MLP/0722_191559/model/best.pt b/src/training/experiments/CharacterModel_Emnist_MLP/0722_191559/model/best.pt
deleted file mode 100644
index f0f297b..0000000
Binary files a/src/training/experiments/CharacterModel_Emnist_MLP/0722_191559/model/best.pt and /dev/null differ
diff --git a/src/training/experiments/CharacterModel_Emnist_MLP/0722_191559/model/last.pt b/src/training/experiments/CharacterModel_Emnist_MLP/0722_191559/model/last.pt
deleted file mode 100644
index c1adda5..0000000
Binary files a/src/training/experiments/CharacterModel_Emnist_MLP/0722_191559/model/last.pt and /dev/null differ
diff --git a/src/training/experiments/CharacterModel_Emnist_MLP/0722_213125/config.yml b/src/training/experiments/CharacterModel_Emnist_MLP/0722_213125/config.yml
deleted file mode 100644
index 8d77de5..0000000
--- a/src/training/experiments/CharacterModel_Emnist_MLP/0722_213125/config.yml
+++ /dev/null
@@ -1,49 +0,0 @@
-criterion: CrossEntropyLoss
-criterion_args:
-  ignore_index: -100
-  reduction: mean
-  weight: null
-data_loader_args:
-  batch_size: 256
-  cuda: true
-  num_workers: 8
-  sample_to_balance: true
-  seed: 4711
-  shuffle: true
-  splits:
-  - train
-  - val
-  subsample_fraction: null
-  target_transform: null
-  transform: null
-dataloader: EmnistDataLoader
-device: cuda:0
-experiment_group: Sample Experiments
-lr_scheduler: OneCycleLR
-lr_scheduler_args:
-  epochs: 16
-  max_lr: 0.001
-  steps_per_epoch: 1314
-metrics:
-- accuracy
-model: CharacterModel
-network: MLP
-network_args:
-  input_size: 784
-  num_layers: 3
-  output_size: 62
-optimizer: AdamW
-optimizer_args:
-  amsgrad: false
-  betas:
-  - 0.9
-  - 0.999
-  eps: 1.0e-08
-  lr: 0.01
-  weight_decay: 0
-resume_experiment: null
-train_args:
-  batch_size: 256
-  epochs: 16
-  val_metric: accuracy
-verbosity: 2
diff --git a/src/training/experiments/CharacterModel_Emnist_MLP/0722_213413/config.yml b/src/training/experiments/CharacterModel_Emnist_MLP/0722_213413/config.yml
deleted file mode 100644
index 8d77de5..0000000
--- a/src/training/experiments/CharacterModel_Emnist_MLP/0722_213413/config.yml
+++ /dev/null
@@ -1,49 +0,0 @@
-criterion: CrossEntropyLoss
-criterion_args:
-  ignore_index: -100
-  reduction: mean
-  weight: null
-data_loader_args:
-  batch_size: 256
-  cuda: true
-  num_workers: 8
-  sample_to_balance: true
-  seed: 4711
-  shuffle: true
-  splits:
-  - train
-  - val
-  subsample_fraction: null
-  target_transform: null
-  transform: null
-dataloader: EmnistDataLoader
-device: cuda:0
-experiment_group: Sample Experiments
-lr_scheduler: OneCycleLR
-lr_scheduler_args:
-  epochs: 16
-  max_lr: 0.001
-  steps_per_epoch: 1314
-metrics:
-- accuracy
-model: CharacterModel
-network: MLP
-network_args:
-  input_size: 784
-  num_layers: 3
-  output_size: 62
-optimizer: AdamW
-optimizer_args:
-  amsgrad: false
-  betas:
-  - 0.9
-  - 0.999
-  eps: 1.0e-08
-  lr: 0.01
-  weight_decay: 0
-resume_experiment: null
-train_args:
-  batch_size: 256
-  epochs: 16
-  val_metric: accuracy
-verbosity: 2
diff --git a/src/training/experiments/CharacterModel_Emnist_MLP/0722_213413/model/best.pt b/src/training/experiments/CharacterModel_Emnist_MLP/0722_213413/model/best.pt
deleted file mode 100644
index e985997..0000000
Binary files a/src/training/experiments/CharacterModel_Emnist_MLP/0722_213413/model/best.pt and /dev/null differ
diff --git a/src/training/experiments/CharacterModel_Emnist_MLP/0722_213413/model/last.pt b/src/training/experiments/CharacterModel_Emnist_MLP/0722_213413/model/last.pt
deleted file mode 100644
index e985997..0000000
Binary files a/src/training/experiments/CharacterModel_Emnist_MLP/0722_213413/model/last.pt and /dev/null differ
diff --git a/src/training/experiments/CharacterModel_Emnist_MLP/0722_213549/config.yml b/src/training/experiments/CharacterModel_Emnist_MLP/0722_213549/config.yml
deleted file mode 100644
index 8d77de5..0000000
--- a/src/training/experiments/CharacterModel_Emnist_MLP/0722_213549/config.yml
+++ /dev/null
@@ -1,49 +0,0 @@
-criterion: CrossEntropyLoss
-criterion_args:
-  ignore_index: -100
-  reduction: mean
-  weight: null
-data_loader_args:
-  batch_size: 256
-  cuda: true
-  num_workers: 8
-  sample_to_balance: true
-  seed: 4711
-  shuffle: true
-  splits:
-  - train
-  - val
-  subsample_fraction: null
-  target_transform: null
-  transform: null
-dataloader: EmnistDataLoader
-device: cuda:0
-experiment_group: Sample Experiments
-lr_scheduler: OneCycleLR
-lr_scheduler_args:
-  epochs: 16
-  max_lr: 0.001
-  steps_per_epoch: 1314
-metrics:
-- accuracy
-model: CharacterModel
-network: MLP
-network_args:
-  input_size: 784
-  num_layers: 3
-  output_size: 62
-optimizer: AdamW
-optimizer_args:
-  amsgrad: false
-  betas:
-  - 0.9
-  - 0.999
-  eps: 1.0e-08
-  lr: 0.01
-  weight_decay: 0
-resume_experiment: null
-train_args:
-  batch_size: 256
-  epochs: 16
-  val_metric: accuracy
-verbosity: 2
diff --git a/src/training/experiments/CharacterModel_Emnist_MLP/0722_213549/model/best.pt b/src/training/experiments/CharacterModel_Emnist_MLP/0722_213549/model/best.pt
deleted file mode 100644
index 0dde787..0000000
Binary files a/src/training/experiments/CharacterModel_Emnist_MLP/0722_213549/model/best.pt and /dev/null differ
diff --git a/src/training/experiments/CharacterModel_Emnist_MLP/0722_213549/model/last.pt b/src/training/experiments/CharacterModel_Emnist_MLP/0722_213549/model/last.pt
deleted file mode 100644
index e02738b..0000000
Binary files a/src/training/experiments/CharacterModel_Emnist_MLP/0722_213549/model/last.pt and /dev/null differ
diff --git a/src/training/experiments/sample.yml b/src/training/experiments/sample.yml
deleted file mode 100644
index 0ed560d..0000000
--- a/src/training/experiments/sample.yml
+++ /dev/null
@@ -1,43 +0,0 @@
-experiment_group: Sample Experiments
-experiments:
-    - dataloader: EmnistDataLoader
-      model: CharacterModel
-      metrics: [accuracy]
-      network: MLP
-      network_args:
-        input_shape: 784
-        num_layers: 2
-      train_args:
-        batch_size: 256
-        epochs: 16
-      criterion: CrossEntropyLoss
-      criterion_args:
-        weight: null
-        ignore_index: -100
-        reduction: mean
-      optimizer: AdamW
-      optimizer_args:
-        lr: 3.e-4
-        betas: [0.9, 0.999]
-        eps: 1.e-08
-        weight_decay: 0
-        amsgrad: false
-      lr_scheduler: OneCycleLR
-      lr_scheduler_args:
-        max_lr: 3.e-5
-        epochs: 16
-    # - dataloader: EmnistDataLoader
-    #   model: CharacterModel
-    #   network: MLP
-    #   network_args:
-    #     input_shape: 784
-    #     num_layers: 4
-    #   train_args:
-    #     batch_size: 256
-    # - dataloader: EmnistDataLoader
-    #   model: CharacterModel
-    #   network: LeNet
-    #   network_args:
-    #     input_shape: [28, 28]
-    #   train_args:
-    #     batch_size: 256
diff --git a/src/training/experiments/sample_experiment.yml b/src/training/experiments/sample_experiment.yml
index e8d5023..70edb63 100644
--- a/src/training/experiments/sample_experiment.yml
+++ b/src/training/experiments/sample_experiment.yml
@@ -1,6 +1,6 @@
 experiment_group: Sample Experiments
 experiments:
-    - dataloader: EmnistDataLoader
+    - dataloader: EmnistDataLoaders
       data_loader_args:
         splits: [train, val]
         sample_to_balance: true
@@ -14,19 +14,18 @@ experiments:
         seed: 4711
       model: CharacterModel
       metrics: [accuracy]
-      network: MLP
-      network_args:
-        input_size: 784
-        output_size: 62
-        num_layers: 3
-      # network: LeNet
+      # network: MLP
       # network_args:
-      #   input_size: [28, 28]
+      #   input_size: 784
       #   output_size: 62
+      #   num_layers: 3
+      network: LeNet
+      network_args:
+        input_size: [28, 28]
+        output_size: 62
       train_args:
         batch_size: 256
         epochs: 16
-        val_metric: accuracy
       criterion: CrossEntropyLoss
       criterion_args:
         weight: null
@@ -52,5 +51,20 @@ experiments:
       lr_scheduler_args:
         max_lr: 1.e-3
         epochs: 16
-      verbosity: 2 # 0, 1, 2
+      callbacks: [Checkpoint, EarlyStopping, WandbCallback, WandbImageLogger, OneCycleLR]
+      callback_args:
+        Checkpoint:
+          monitor: val_accuracy
+        EarlyStopping:
+          monitor: val_loss
+          min_delta: 0.0
+          patience: 3
+          mode: min
+        WandbCallback:
+          log_batch_frequency: 10
+        WandbImageLogger:
+          num_examples: 4
+        OneCycleLR:
+          null
+      verbosity: 1 # 0, 1, 2
       resume_experiment: null
diff --git a/src/training/prepare_experiments.py b/src/training/prepare_experiments.py
index eb872d7..5a665b3 100644
--- a/src/training/prepare_experiments.py
+++ b/src/training/prepare_experiments.py
@@ -1,12 +1,13 @@
 """Run a experiment from a config file."""
 import json
-from subprocess import check_call
+from subprocess import run
 
 import click
 from loguru import logger
 import yaml
 
 
+# flake8: noqa: S404,S607,S603
 def run_experiments(experiments_filename: str) -> None:
     """Run experiment from file."""
     with open(experiments_filename) as f:
@@ -15,10 +16,19 @@ def run_experiments(experiments_filename: str) -> None:
     for index in range(num_experiments):
         experiment_config = experiments_config["experiments"][index]
         experiment_config["experiment_group"] = experiments_config["experiment_group"]
-        # cmd = f"python training/run_experiment.py --gpu=-1 '{json.dumps(experiment_config)}'"
-        cmd = f"poetry run run-experiment --gpu=-1 --save --experiment_config '{json.dumps(experiment_config)}'"
+        cmd = f"poetry run run-experiment --gpu=-1 --save --experiment_config={json.dumps(experiment_config)}"
         print(cmd)
-        check_call(cmd, shell=True)
+        run(
+            [
+                "poetry",
+                "run",
+                "run-experiment",
+                "--gpu=-1",
+                "--save",
+                f"--experiment_config={json.dumps(experiment_config)}",
+            ],
+            check=True,
+        )
 
 
 @click.command()
diff --git a/src/training/run_experiment.py b/src/training/run_experiment.py
index 0b29ce9..c133ce5 100644
--- a/src/training/run_experiment.py
+++ b/src/training/run_experiment.py
@@ -12,8 +12,10 @@ import click
 from loguru import logger
 import torch
 from tqdm import tqdm
+from training.callbacks import CallbackList
 from training.gpu_manager import GPUManager
 from training.train import Trainer
+import wandb
 import yaml
 
 
@@ -48,9 +50,8 @@ def create_experiment_dir(model: Callable, experiment_config: Dict) -> Path:
             logger.debug(f"Resuming the latest experiment {experiment}")
         else:
             experiment = experiment_config["resume_experiment"]
-            assert (
-                str(experiment_dir / experiment) in available_experiments
-            ), "Experiment does not exist."
+            if not str(experiment_dir / experiment) in available_experiments:
+                raise FileNotFoundError("Experiment does not exist.")
             logger.debug(f"Resuming the experiment {experiment}")
 
     experiment_dir = experiment_dir / experiment
@@ -87,6 +88,13 @@ def load_modules_and_arguments(experiment_config: Dict) -> Tuple[Callable, Dict]
     optimizer_ = getattr(torch.optim, experiment_config["optimizer"])
     optimizer_args = experiment_config.get("optimizer_args", {})
 
+    # Callbacks
+    callback_modules = importlib.import_module("training.callbacks")
+    callbacks = []
+    for callback in experiment_config["callbacks"]:
+        args = experiment_config["callback_args"][callback] or {}
+        callbacks.append(getattr(callback_modules, callback)(**args))
+
     # Learning rate scheduler
     if experiment_config["lr_scheduler"] is not None:
         lr_scheduler_ = getattr(
@@ -111,7 +119,7 @@ def load_modules_and_arguments(experiment_config: Dict) -> Tuple[Callable, Dict]
         "lr_scheduler_args": lr_scheduler_args,
     }
 
-    return model_class_, model_args
+    return model_class_, model_args, callbacks
 
 
 def run_experiment(
@@ -120,11 +128,14 @@ def run_experiment(
     """Runs an experiment."""
 
     # Load the modules and model arguments.
-    model_class_, model_args = load_modules_and_arguments(experiment_config)
+    model_class_, model_args, callbacks = load_modules_and_arguments(experiment_config)
 
     # Initializes the model with experiment config.
     model = model_class_(**model_args, device=device)
 
+    # Instantiate a CallbackList.
+    callbacks = CallbackList(model, callbacks)
+
     # Create new experiment.
     experiment_dir = create_experiment_dir(model, experiment_config)
 
@@ -132,6 +143,9 @@ def run_experiment(
     log_dir = experiment_dir / "log"
     model_dir = experiment_dir / "model"
 
+    # Set the model dir to be able to save checkpoints.
+    model.model_dir = model_dir
+
     # Get checkpoint path.
     checkpoint_path = model_dir / "last.pt"
     if not checkpoint_path.exists():
@@ -162,6 +176,13 @@ def run_experiment(
 
     logger.info(f"The class mapping is {model.mapping}")
 
+    # Initializes Weights & Biases
+    if use_wandb:
+        wandb.init(project="text-recognizer", config=experiment_config)
+
+        # Lets W&B save the model and track the gradients and optional parameters.
+        wandb.watch(model.network)
+
     # Pŕints a summary of the network in terminal.
     model.summary()
 
@@ -181,21 +202,26 @@ def run_experiment(
     with open(str(config_path), "w") as f:
         yaml.dump(experiment_config, f)
 
-    # TODO: wandb
     trainer = Trainer(
         model=model,
         model_dir=model_dir,
-        epochs=experiment_config["train_args"]["epochs"],
-        val_metric=experiment_config["train_args"]["val_metric"],
+        train_args=experiment_config["train_args"],
+        callbacks=callbacks,
         checkpoint_path=checkpoint_path,
     )
 
     trainer.fit()
 
+    logger.info("Loading checkpoint with the best weights.")
+    model.load_checkpoint(model_dir / "best.pt")
+
     score = trainer.validate()
 
     logger.info(f"Validation set evaluation: {score}")
 
+    if use_wandb:
+        wandb.log({"validation_metric": score["val_accuracy"]})
+
     if save_weights:
         model.save_weights(model_dir)
 
@@ -220,12 +246,11 @@ def main(experiment_config: str, gpu: int, save: bool, nowandb: bool) -> None:
     if gpu < 0:
         gpu_manager = GPUManager(True)
         gpu = gpu_manager.get_free_gpu()
-
     device = "cuda:" + str(gpu)
 
     experiment_config = json.loads(experiment_config)
     os.environ["CUDA_VISIBLE_DEVICES"] = f"{gpu}"
-    run_experiment(experiment_config, save, device, nowandb)
+    run_experiment(experiment_config, save, device, use_wandb=not nowandb)
 
 
 if __name__ == "__main__":
diff --git a/src/training/train.py b/src/training/train.py
index 8cd5110..3334c2e 100644
--- a/src/training/train.py
+++ b/src/training/train.py
@@ -2,17 +2,19 @@
 
 from pathlib import Path
 import time
-from typing import Dict, Optional, Type
+from typing import Dict, List, Optional, Tuple, Type
 
 from loguru import logger
 import numpy as np
 import torch
 from tqdm import tqdm, trange
+from training.callbacks import Callback, CallbackList
 from training.util import RunningAverage
 import wandb
 
 from text_recognizer.models import Model
 
+
 torch.backends.cudnn.benchmark = True
 np.random.seed(4711)
 torch.manual_seed(4711)
@@ -22,51 +24,82 @@ torch.cuda.manual_seed(4711)
 class Trainer:
     """Trainer for training PyTorch models."""
 
-    # TODO implement wandb.
-    # TODO implement Bayesian parameter search.
-
     def __init__(
         self,
         model: Type[Model],
         model_dir: Path,
-        epochs: int,
-        val_metric: str = "accuracy",
+        train_args: Dict,
+        callbacks: CallbackList,
         checkpoint_path: Optional[Path] = None,
-        use_wandb: Optional[bool] = False,
     ) -> None:
         """Initialization of the Trainer.
 
         Args:
             model (Type[Model]): A model object.
             model_dir (Path): Path to the model directory.
-            epochs (int): Number of epochs to train.
-            val_metric (str): The validation metric to evaluate the model on. Defaults to "accuracy".
+            train_args (Dict): The training arguments.
+            callbacks (CallbackList): List of callbacks to be called.
             checkpoint_path (Optional[Path]): The path to a previously trained model. Defaults to None.
-            use_wandb (Optional[bool]): Sync training to wandb.
 
         """
         self.model = model
         self.model_dir = model_dir
-        self.epochs = epochs
         self.checkpoint_path = checkpoint_path
-        self.start_epoch = 0
+        self.start_epoch = 1
+        self.epochs = train_args["epochs"] + self.start_epoch
+        self.callbacks = callbacks
 
         if self.checkpoint_path is not None:
-            self.start_epoch = self.model.load_checkpoint(self.checkpoint_path)
-
-        if use_wandb:
-            # TODO implement wandb logging.
-            pass
-
-        self.val_metric = val_metric
-        self.best_val_metric = 0.0
+            self.start_epoch = self.model.load_checkpoint(self.checkpoint_path) + 1
 
         # Parse the name of the experiment.
         experiment_dir = str(self.model_dir.parents[1]).split("/")
         self.experiment_name = experiment_dir[-2] + "/" + experiment_dir[-1]
 
+    def training_step(
+        self,
+        batch: int,
+        samples: Tuple[torch.Tensor, torch.Tensor],
+        loss_avg: Type[RunningAverage],
+    ) -> Dict:
+        """Performs the training step."""
+        # Pass the tensor to the device for computation.
+        data, targets = samples
+        data, targets = (
+            data.to(self.model.device),
+            targets.to(self.model.device),
+        )
+
+        # Forward pass.
+        # Get the network prediction.
+        output = self.model.network(data)
+
+        # Compute the loss.
+        loss = self.model.criterion(output, targets)
+
+        # Backward pass.
+        # Clear the previous gradients.
+        self.model.optimizer.zero_grad()
+
+        # Compute the gradients.
+        loss.backward()
+
+        # Perform updates using calculated gradients.
+        self.model.optimizer.step()
+
+        # Compute metrics.
+        loss_avg.update(loss.item())
+        output = output.data.cpu()
+        targets = targets.data.cpu()
+        metrics = {
+            metric: self.model.metrics[metric](output, targets)
+            for metric in self.model.metrics
+        }
+        metrics["loss"] = loss_avg()
+        return metrics
+
     def train(self) -> None:
-        """Training loop."""
+        """Runs the training loop for one epoch."""
         # Set model to traning mode.
         self.model.train()
 
@@ -79,57 +112,54 @@ class Trainer:
             total=len(data_loader),
             leave=False,
             unit="step",
-            bar_format="{n_fmt}/{total_fmt} |{bar:20}| {remaining} {rate_inv_fmt}{postfix}",
+            bar_format="{n_fmt}/{total_fmt} |{bar:30}| {remaining} {rate_inv_fmt}{postfix}",
         ) as t:
-            for data, targets in data_loader:
+            for batch, samples in enumerate(data_loader):
+                self.callbacks.on_train_batch_begin(batch)
 
-                data, targets = (
-                    data.to(self.model.device),
-                    targets.to(self.model.device),
-                )
+                metrics = self.training_step(batch, samples, loss_avg)
 
-                # Forward pass.
-                # Get the network prediction.
-                output = self.model.network(data)
-
-                # Compute the loss.
-                loss = self.model.criterion(output, targets)
-
-                # Backward pass.
-                # Clear the previous gradients.
-                self.model.optimizer.zero_grad()
-
-                # Compute the gradients.
-                loss.backward()
-
-                # Perform updates using calculated gradients.
-                self.model.optimizer.step()
-
-                # Compute metrics.
-                loss_avg.update(loss.item())
-                output = output.data.cpu()
-                targets = targets.data.cpu()
-                metrics = {
-                    metric: self.model.metrics[metric](output, targets)
-                    for metric in self.model.metrics
-                }
-                metrics["loss"] = loss_avg()
+                self.callbacks.on_train_batch_end(batch, logs=metrics)
 
                 # Update Tqdm progress bar.
                 t.set_postfix(**metrics)
                 t.update()
 
-                # If the model has a learning rate scheduler, compute a step.
-                if self.model.lr_scheduler is not None:
-                    self.model.lr_scheduler.step()
-
-    def validate(self) -> Dict:
-        """Evaluation loop.
+    def validation_step(
+        self,
+        batch: int,
+        samples: Tuple[torch.Tensor, torch.Tensor],
+        loss_avg: Type[RunningAverage],
+    ) -> Dict:
+        """Performs the validation step."""
+        # Pass the tensor to the device for computation.
+        data, targets = samples
+        data, targets = (
+            data.to(self.model.device),
+            targets.to(self.model.device),
+        )
+
+        # Forward pass.
+        # Get the network prediction.
+        output = self.model.network(data)
+
+        # Compute the loss.
+        loss = self.model.criterion(output, targets)
+
+        # Compute metrics.
+        loss_avg.update(loss.item())
+        output = output.data.cpu()
+        targets = targets.data.cpu()
+        metrics = {
+            metric: self.model.metrics[metric](output, targets)
+            for metric in self.model.metrics
+        }
+        metrics["loss"] = loss.item()
 
-        Returns:
-            Dict: A dictionary of evaluation metrics.
+        return metrics
 
-        """
+    def validate(self, epoch: Optional[int] = None) -> Dict:
+        """Runs the validation loop for one epoch."""
         # Set model to eval mode.
         self.model.eval()
 
@@ -146,44 +176,37 @@ class Trainer:
             total=len(data_loader),
             leave=False,
             unit="step",
-            bar_format="{n_fmt}/{total_fmt} |{bar:20}| {remaining} {rate_inv_fmt}{postfix}",
+            bar_format="{n_fmt}/{total_fmt} |{bar:30}| {remaining} {rate_inv_fmt}{postfix}",
         ) as t:
-            for data, targets in data_loader:
-                data, targets = (
-                    data.to(self.model.device),
-                    targets.to(self.model.device),
-                )
-
-                with torch.no_grad():
-                    # Forward pass.
-                    # Get the network prediction.
-                    output = self.model.network(data)
-
-                    # Compute the loss.
-                    loss = self.model.criterion(output, targets)
-
-                # Compute metrics.
-                loss_avg.update(loss.item())
-                output = output.data.cpu()
-                targets = targets.data.cpu()
-                metrics = {
-                    metric: self.model.metrics[metric](output, targets)
-                    for metric in self.model.metrics
-                }
-                metrics["loss"] = loss.item()
-
-                summary.append(metrics)
+            with torch.no_grad():
+                for batch, samples in enumerate(data_loader):
+                    self.callbacks.on_validation_batch_begin(batch)
 
-                # Update Tqdm progress bar.
-                t.set_postfix(**metrics)
-                t.update()
+                    metrics = self.validation_step(batch, samples, loss_avg)
+
+                    self.callbacks.on_validation_batch_end(batch, logs=metrics)
+
+                    summary.append(metrics)
+
+                    # Update Tqdm progress bar.
+                    t.set_postfix(**metrics)
+                    t.update()
 
         # Compute mean of all metrics.
         metrics_mean = {
-            metric: np.mean([x[metric] for x in summary]) for metric in summary[0]
+            "val_" + metric: np.mean([x[metric] for x in summary])
+            for metric in summary[0]
         }
-        metrics_str = " - ".join(f"{k}: {v}" for k, v in metrics_mean.items())
-        logger.debug(metrics_str)
+        if epoch:
+            logger.debug(
+                f"Validation metrics at epoch {epoch} - "
+                + " - ".join(f"{k}: {v:.4f}" for k, v in metrics_mean.items())
+            )
+        else:
+            logger.debug(
+                "Validation metrics - "
+                + " - ".join(f"{k}: {v:.4f}" for k, v in metrics_mean.items())
+            )
 
         return metrics_mean
 
@@ -192,31 +215,35 @@ class Trainer:
 
         logger.debug(f"Running an experiment called {self.experiment_name}.")
         t_start = time.time()
+
+        self.callbacks.on_fit_begin()
+
+        # TODO: fix progress bar as callback.
         # Run the training loop.
         for epoch in trange(
+            self.start_epoch,
             self.epochs,
-            initial=self.start_epoch,
             leave=False,
-            bar_format="{desc}: {n_fmt}/{total_fmt} |{bar:10}| {remaining}{postfix}",
+            bar_format="{desc}: {n_fmt}/{total_fmt} |{bar:30}| {remaining}{postfix}",
             desc="Epoch",
         ):
+            self.callbacks.on_epoch_begin(epoch)
+
             # Perform one training pass over the training set.
             self.train()
 
             # Evaluate the model on the validation set.
-            val_metrics = self.validate()
+            val_metrics = self.validate(epoch)
 
-            # The validation metric to evaluate the model on, e.g. accuracy.
-            val_metric = val_metrics[self.val_metric]
-            is_best = val_metric >= self.best_val_metric
-            self.best_val_metric = val_metric if is_best else self.best_val_metric
-            # Save checkpoint.
-            self.model.save_checkpoint(self.model_dir, is_best, epoch, self.val_metric)
+            self.callbacks.on_epoch_end(epoch, logs=val_metrics)
 
-            if self.start_epoch > 0 and epoch + self.start_epoch == self.epochs:
-                logger.debug(f"Trained the model for {self.epochs} number of epochs.")
+            if self.model.stop_training:
                 break
 
+        # Calculate the total training time.
         t_end = time.time()
         t_training = t_end - t_start
+
+        self.callbacks.on_fit_end()
+
         logger.info(f"Training took {t_training:.2f} s.")
-- 
cgit v1.2.3-70-g09d2