0.15.25

batch_norm as a new option

Author: bartzbeielstein

pyproject.toml

@@ -7,7 +7,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "spotpython"
-version = "0.15.24"
+version = "0.15.25"
 authors = [
   { name="T. Bartz-Beielstein", email="tbb@bartzundbartz.de" }
 ]

src/spotpython/hyperdict/light_hyper_dict.json

@@ -391,6 +391,18 @@
             "lower": 2,
             "upper": 6
         },
+        "batch_norm": {
+            "levels": [
+                0,
+                1
+            ],
+            "type": "factor",
+            "default": 0,
+            "transform": "None",
+            "core_model_parameter_type": "bool",
+            "lower": 0,
+            "upper": 1
+        },
         "initialization": {
             "levels": [
                 "Default",

src/spotpython/light/regression/nn_linear_regressor.py

@@ -29,6 +29,8 @@ class NNLinearRegressor(L.LightningModule):
             The learning rate multiplier for the optimizer.
         patience (int):
             The number of epochs to wait before early stopping.
+        batch_norm (bool):
+            Whether to use batch normalization or not.
         _L_in (int):
             The number of input features.
         _L_out (int):
@@ -104,6 +106,7 @@ def __init__(
         dropout_prob: float,
         lr_mult: float,
         patience: int,
+        batch_norm: bool,
         _L_in: int,
         _L_out: int,
         _torchmetric: str,
@@ -130,6 +133,8 @@ def __init__(
                 The learning rate multiplier for the optimizer.
             patience (int):
                 The number of epochs to wait before early stopping.
+            batch_norm (bool):
+                Whether to use batch normalization or not.
             _L_in (int):
                 The number of input features. Not a hyperparameter, but needed to create the network.
             _L_out (int):
@@ -173,12 +178,19 @@ def __init__(
         for i in range(len(layer_sizes) - 1):
             current_layer_size = layer_sizes[i]
             next_layer_size = layer_sizes[i + 1]
-            layers += [
-                nn.Linear(current_layer_size, next_layer_size),
-                nn.BatchNorm1d(next_layer_size),  # Add Batch Normalization here
-                self.hparams.act_fn,
-                nn.Dropout(self.hparams.dropout_prob),
-            ]
+            if self.hparams.initialization:
+                layers += [
+                    nn.Linear(current_layer_size, next_layer_size),
+                    nn.BatchNorm1d(next_layer_size),  # Add Batch Normalization here
+                    self.hparams.act_fn,
+                    nn.Dropout(self.hparams.dropout_prob),
+                ]
+            else:
+                layers += [
+                    nn.Linear(current_layer_size, next_layer_size),
+                    self.hparams.act_fn,
+                    nn.Dropout(self.hparams.dropout_prob),
+                ]
         layers += [nn.Linear(layer_sizes[-1], self._L_out)]
 
         # Wrap the layers into a sequential container

test/test_nn_linear_regressor.py

@@ -45,6 +45,7 @@
     "dropout_prob": 0.1,
     "lr_mult": 0.1,
     "patience": 5,
+    "batch_norm": 0,
     "_L_in": 10,
     "_L_out": 1,
     "_torchmetric": "mean_squared_error",