optimizer act_fn

Author: bartzbeielstein

src/spotPython/data/light_hyper_dict.json

@@ -20,14 +20,14 @@
                     "lower": 1,
                     "upper": 4},
         "act_fn": {
-                "levels": ["ReLU"],
+                "levels": ["Sigmoid", "Tanh", "ReLU", "LeakyReLU", "ELU", "Swish"],
                 "type": "factor",
                 "default": "ReLU",
                 "transform": "None",
-                "class_name": "torch.nn",
+                "class_name": "spotPython.torch.activation", 
                 "core_model_parameter_type": "instance()",
                 "lower": 0,
-                "upper": 0},
+                "upper": 2},
         "optimizer": {
             "levels": ["Adadelta", "Adagrad", "Adam", "AdamW", "SparseAdam", "Adamax", "ASGD", "NAdam", "RAdam", "RMSprop", "Rprop", "SGD"],
             "type": "factor",
@@ -36,13 +36,19 @@
             "class_name": "torch.optim",
             "core_model_parameter_type": "str",
             "lower": 0,
-            "upper": 12},
+            "upper": 11},
         "dropout_prob": {
             "type": "float",
             "default": 0.01,
             "transform": "None",
             "lower": 0.0,
-            "upper": 0.1}
+            "upper": 0.1},
+        "lr_mult": {
+            "type": "float",
+            "default": 1.0,
+            "transform": "None",
+            "lower": 0.1,
+            "upper": 10.0}
     },
     "LitModel":
     {

src/spotPython/light/netlightbase.py

@@ -4,12 +4,11 @@
 from torch import nn
 from torchmetrics.functional import accuracy
 from spotPython.torch.mapk import MAPK
+from spotPython.hyperparameters.optimizer import optimizer_handler
 
 
 class NetLightBase(L.LightningModule):
-    def __init__(
-        self, l1, epochs, batch_size, act_fn, optimizer, dropout_prob, learning_rate=2e-4, _L_in=64, _L_out=11
-    ):
+    def __init__(self, l1, epochs, batch_size, act_fn, optimizer, dropout_prob, lr_mult, _L_in=64, _L_out=11):
         super().__init__()
 
         # We take in input dimensions as parameters and use those to dynamically build model.
@@ -23,7 +22,7 @@ def __init__(
         self.act_fn = act_fn
         self.optimizer = optimizer
         self.dropout_prob = dropout_prob
-        self.learning_rate = learning_rate
+        self.lr_mult = lr_mult
         self.train_mapk = MAPK(k=3)
         self.valid_mapk = MAPK(k=3)
         self.test_mapk = MAPK(k=3)
@@ -79,5 +78,6 @@ def test_step(self, batch, batch_idx, prog_bar=False):
         return loss, acc
 
     def configure_optimizers(self):
-        optimizer = torch.optim.Adam(self.parameters(), lr=self.learning_rate)
+        # optimizer = torch.optim.Adam(self.parameters(), lr=self.learning_rate)
+        optimizer = optimizer_handler(optimizer_name=self.optimizer, params=self.parameters(), lr_mult=self.lr_mult)
         return optimizer