0.10.68

torch lightning regression models require a new argument: "_torchmetric"

Author: bartzbeielstein

pyproject.toml

@@ -7,7 +7,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "spotPython"
-version = "0.10.67"
+version = "0.10.68"
 authors = [
   { name="T. Bartz-Beielstein", email="tbb@bartzundbartz.de" }
 ]

src/spotPython/light/cvmodel.py

@@ -39,6 +39,7 @@ def cv_model(config: dict, fun_control: dict) -> float:
     """
     _L_in = fun_control["_L_in"]
     _L_out = fun_control["_L_out"]
+    _torchmetric = fun_control["_torchmetric"]
     if fun_control["enable_progress_bar"] is None:
         enable_progress_bar = False
     else:
@@ -52,7 +53,7 @@ def cv_model(config: dict, fun_control: dict) -> float:
     for k in range(num_folds):
         print("k:", k)
 
-        model = fun_control["core_model"](**config, _L_in=_L_in, _L_out=_L_out)
+        model = fun_control["core_model"](**config, _L_in=_L_in, _L_out=_L_out, _torchmetric=_torchmetric)
         initialization = config["initialization"]
         if initialization == "Xavier":
             xavier_init(model)

src/spotPython/light/predictmodel.py

@@ -54,6 +54,7 @@ def predict_model(config: dict, fun_control: dict) -> Tuple[float, float]:
     """
     _L_in = fun_control["_L_in"]
     _L_out = fun_control["_L_out"]
+    _torchmetric = fun_control["_torchmetric"]
     if fun_control["enable_progress_bar"] is None:
         enable_progress_bar = False
     else:
@@ -72,7 +73,7 @@ def predict_model(config: dict, fun_control: dict) -> Tuple[float, float]:
     )
     dm.setup(stage="train")
     # Init model from datamodule's attributes
-    model = fun_control["core_model"](**config, _L_in=_L_in, _L_out=_L_out)
+    model = fun_control["core_model"](**config, _L_in=_L_in, _L_out=_L_out, _torchmetric=_torchmetric)
     initialization = config["initialization"]
     if initialization == "Xavier":
         xavier_init(model)

src/spotPython/light/regression/netlightregression.py

@@ -3,6 +3,7 @@
 import torch.nn.functional as F
 from torch import nn
 from spotPython.hyperparameters.optimizer import optimizer_handler
+import torchmetrics.functional.regression
 
 
 class NetLightRegression(L.LightningModule):
@@ -32,6 +33,8 @@ class NetLightRegression(L.LightningModule):
             The number of input features.
         _L_out (int):
             The number of output classes.
+        _torchmetric (str):
+            The metric to use for the loss function, e.g., "mean_squared_error".
         layers (nn.Sequential):
             The neural network model.
 
@@ -104,6 +107,7 @@ def __init__(
         patience: int,
         _L_in: int,
         _L_out: int,
+        _torchmetric: str,
     ):
         """
         Initializes the NetLightRegression object.
@@ -120,6 +124,7 @@ def __init__(
             patience (int): The number of epochs to wait before early stopping.
             _L_in (int): The number of input features. Not a hyperparameter, but needed to create the network.
             _L_out (int): The number of output classes. Not a hyperparameter, but needed to create the network.
+            _torchmetric (str): The metric to use for the loss function, e.g., "mean_squared_error".
 
         Returns:
             (NoneType): None
@@ -136,8 +141,10 @@ def __init__(
         #
         self._L_in = _L_in
         self._L_out = _L_out
+        self._torchmetric = _torchmetric
         # _L_in and _L_out are not hyperparameters, but are needed to create the network
-        self.save_hyperparameters(ignore=["_L_in", "_L_out"])
+        # _torchmetric is not a hyperparameter, but is needed to calculate the loss
+        self.save_hyperparameters(ignore=["_L_in", "_L_out", "_torchmetric"])
         # set dummy input array for Tensorboard Graphs
         # set log_graph=True in Trainer to see the graph (in traintest.py)
         self.example_input_array = torch.zeros((batch_size, self._L_in))
@@ -189,7 +196,9 @@ def training_step(self, batch: tuple) -> torch.Tensor:
         x, y = batch
         y = y.view(len(y), 1)
         y_hat = self(x)
-        val_loss = F.mse_loss(y_hat, y)
+        # val_loss = F.mse_loss(y_hat, y)
+        metric = getattr(torchmetrics.functional.regression, self._torchmetric)
+        val_loss = metric(y_hat, y)
         # mae_loss = F.l1_loss(y_hat, y)
         # self.log("train_loss", val_loss, on_step=True, on_epoch=True, prog_bar=True)
         # self.log("train_mae_loss", mae_loss, on_step=True, on_epoch=True, prog_bar=True)

src/spotPython/light/regression/netlightregression2.py

@@ -1,9 +1,9 @@
 import lightning as L
 import torch
-import torch.nn.functional as F
 from torch import nn
 from spotPython.hyperparameters.optimizer import optimizer_handler
 from spotPython.utils.math import generate_div2_list
+import torchmetrics.functional.regression
 
 
 class NetLightRegression2(L.LightningModule):
@@ -33,6 +33,8 @@ class NetLightRegression2(L.LightningModule):
             The number of input features.
         _L_out (int):
             The number of output classes.
+        _torchmetric (str):
+            The metric to use for the loss function, e.g., "mean_squared_error".
         layers (nn.Sequential):
             The neural network model.
 
@@ -105,6 +107,7 @@ def __init__(
         patience: int,
         _L_in: int,
         _L_out: int,
+        _torchmetric: str,
     ):
         """
         Initializes the NetLightRegression2 object.
@@ -121,6 +124,7 @@ def __init__(
             patience (int): The number of epochs to wait before early stopping.
             _L_in (int): The number of input features. Not a hyperparameter, but needed to create the network.
             _L_out (int): The number of output classes. Not a hyperparameter, but needed to create the network.
+            _torchmetric (str): The metric to use for the loss function, e.g., "mean_squared_error".
 
         Returns:
             (NoneType): None
@@ -137,8 +141,10 @@ def __init__(
         #
         self._L_in = _L_in
         self._L_out = _L_out
+        self.metric = getattr(torchmetrics.functional.regression, _torchmetric)
         # _L_in and _L_out are not hyperparameters, but are needed to create the network
-        self.save_hyperparameters(ignore=["_L_in", "_L_out"])
+        # _torchmetric is not a hyperparameter, but is needed to calculate the loss
+        self.save_hyperparameters(ignore=["_L_in", "_L_out", "_torchmetric"])
         # set dummy input array for Tensorboard Graphs
         # set log_graph=True in Trainer to see the graph (in traintest.py)
         self.example_input_array = torch.zeros((batch_size, self._L_in))
@@ -196,7 +202,8 @@ def _calculate_loss(self, batch, mode="train"):
         x, y = batch
         y = y.view(len(y), 1)
         y_hat = self(x)
-        loss = F.mse_loss(y_hat, y)
+        # loss = F.mse_loss(y_hat, y)
+        loss = self.metric(y_hat, y)
         if mode == "val" or mode == "test":
             self.log(f"{mode}_loss", loss, prog_bar=True)
             self.log("hp_metric", loss, prog_bar=True)

src/spotPython/light/regression/rnnlightregression.py

@@ -3,6 +3,7 @@
 import torch.nn.functional as F
 from torch import nn
 from spotPython.hyperparameters.optimizer import optimizer_handler
+import torchmetrics.functional.regression
 
 
 class RNNLightRegression(L.LightningModule):
@@ -32,6 +33,8 @@ class RNNLightRegression(L.LightningModule):
             The number of input features.
         _L_out (int):
             The number of output classes.
+        _torchmetric (str):
+            The metric to use for the loss function, e.g., "mean_squared_error".
         layers (nn.Sequential):
             The neural network model.
 
@@ -104,6 +107,7 @@ def __init__(
         patience: int,
         _L_in: int,
         _L_out: int,
+        _torchmetric: str,
     ):
         """
         Initializes the NetLightRegression object.
@@ -120,6 +124,7 @@ def __init__(
             patience (int): The number of epochs to wait before early stopping.
             _L_in (int): The number of input features. Not a hyperparameter, but needed to create the network.
             _L_out (int): The number of output classes. Not a hyperparameter, but needed to create the network.
+            _torchmetric (str): The metric to use for the loss function, e.g., "mean_squared_error".
 
         Returns:
             (NoneType): None
@@ -133,8 +138,10 @@ def __init__(
         #
         self._L_in = _L_in
         self._L_out = _L_out
+        self._torchmetric = _torchmetric
         # _L_in and _L_out are not hyperparameters, but are needed to create the network
-        self.save_hyperparameters(ignore=["_L_in", "_L_out"])
+        # _torchmetric is not a hyperparameter, but is needed to calculate the loss
+        self.save_hyperparameters(ignore=["_L_in", "_L_out", "_torchmetric"])
         # set dummy input array for Tensorboard Graphs
         # set log_graph=True in Trainer to see the graph (in traintest.py)
         self.example_input_array = torch.zeros((batch_size, self._L_in))
@@ -224,7 +231,9 @@ def training_step(self, batch: tuple, prog_bar: bool = False) -> torch.Tensor:
         # Note: the number of rows in x is equal to the number of rows in y
         y_hat = self(x)
         # Note: the number of rows in y_hat is equal to the number of rows in y
-        train_loss = F.mse_loss(y_hat, y)
+        # train_loss = F.mse_loss(y_hat, y)
+        metric = getattr(torchmetrics.functional.regression, self._torchmetric)
+        train_loss = metric(y_hat, y)
         # mae_loss = F.l1_loss(y_hat, y)
         # self.log("train_loss", val_loss, on_step=True, on_epoch=True, prog_bar=True)
         # self.log("train_mae_loss", mae_loss, on_step=True, on_epoch=True, prog_bar=True)

src/spotPython/light/regression/transformerlightregression.py

@@ -6,6 +6,7 @@
 from spotPython.light.transformer.skiplinear import SkipLinear
 from spotPython.light.transformer.positionalEncoding import PositionalEncoding
 from spotPython.utils.math import generate_div2_list
+import torchmetrics.functional.regression
 
 
 class TransformerLightRegression(L.LightningModule):
@@ -35,6 +36,8 @@ class TransformerLightRegression(L.LightningModule):
             The number of input features.
         _L_out (int):
             The number of output classes.
+        _torchmetric (str):
+            The metric to use for the loss function, e.g., "mean_squared_error".
         layers (nn.Sequential):
             The neural network model.
 
@@ -110,6 +113,7 @@ def __init__(
         patience: int,
         _L_in: int,
         _L_out: int,
+        _torchmetric: str,
     ):
         """
         Initializes the TransformerLightRegression object.
@@ -126,6 +130,7 @@ def __init__(
             patience (int): The number of epochs to wait before early stopping.
             _L_in (int): The number of input features. Not a hyperparameter, but needed to create the network.
             _L_out (int): The number of output classes. Not a hyperparameter, but needed to create the network.
+            _torchmetric (str): The metric to use for the loss function, e.g., "mean_squared_error".
 
         Returns:
             (NoneType): None
@@ -142,9 +147,11 @@ def __init__(
         #
         self._L_in = _L_in
         self._L_out = _L_out
+        self.metric = getattr(torchmetrics.functional.regression, _torchmetric)
         self.d_mult = d_mult
         # _L_in and _L_out are not hyperparameters, but are needed to create the network
-        self.save_hyperparameters(ignore=["_L_in", "_L_out"])
+        # _torchmetric is not a hyperparameter, but is needed to calculate the loss
+        self.save_hyperparameters(ignore=["_L_in", "_L_out", "_torchmetric"])
         # set dummy input array for Tensorboard Graphs
         # set log_graph=True in Trainer to see the graph (in traintest.py)
         self.example_input_array = torch.zeros((batch_size, self._L_in))

src/spotPython/light/testmodel.py

@@ -39,7 +39,8 @@ def test_model(config: dict, fun_control: dict) -> Tuple[float, float]:
             import spotPython.light.testmodel as tm
             fun_control = fun_control_init(
                 _L_in=10,
-                _L_out=1,)
+                _L_out=1,
+                _torchmetric="mean_squared_error")
             dataset = Diabetes()
             set_control_key_value(control_dict=fun_control,
                                     key="data_set",
@@ -54,6 +55,7 @@ def test_model(config: dict, fun_control: dict) -> Tuple[float, float]:
     """
     _L_in = fun_control["_L_in"]
     _L_out = fun_control["_L_out"]
+    _torchmetric = fun_control["_torchmetric"]
     if fun_control["enable_progress_bar"] is None:
         enable_progress_bar = False
     else:
@@ -72,7 +74,7 @@ def test_model(config: dict, fun_control: dict) -> Tuple[float, float]:
     )
     dm.setup()
     # Init model from datamodule's attributes
-    model = fun_control["core_model"](**config, _L_in=_L_in, _L_out=_L_out)
+    model = fun_control["core_model"](**config, _L_in=_L_in, _L_out=_L_out, _torchmetric=_torchmetric)
     initialization = config["initialization"]
     if initialization == "Xavier":
         xavier_init(model)

src/spotPython/light/trainmodel.py

@@ -67,14 +67,15 @@ def train_model(config: dict, fun_control: dict) -> float:
     """
     _L_in = fun_control["_L_in"]
     _L_out = fun_control["_L_out"]
+    _torchmetric = fun_control["_torchmetric"]
     if fun_control["enable_progress_bar"] is None:
         enable_progress_bar = False
     else:
         enable_progress_bar = fun_control["enable_progress_bar"]
     # config id is unique. Since the model is not loaded from a checkpoint,
     # the config id is generated here with a timestamp.
     config_id = generate_config_id(config, timestamp=True)
-    model = fun_control["core_model"](**config, _L_in=_L_in, _L_out=_L_out)
+    model = fun_control["core_model"](**config, _L_in=_L_in, _L_out=_L_out, _torchmetric=_torchmetric)
     initialization = config["initialization"]
     if initialization == "Xavier":
         xavier_init(model)

src/spotPython/utils/init.py

@@ -11,6 +11,7 @@
 def fun_control_init(
     _L_in=None,
     _L_out=None,
+    _torchmetric=None,
     PREFIX=None,
     TENSORBOARD_CLEAN=False,
     SUMMARY_WRITER=True,
@@ -69,6 +70,10 @@ def fun_control_init(
             The number of input features.
         _L_out (int):
             The number of output features.
+        _torchmetric (str):
+            The metric to be used by the Lighting Trainer.
+            For example "mean_squared_error",
+            see https://lightning.ai/docs/torchmetrics/stable/regression/mean_squared_error.html
         accelerator (str):
             The accelerator to be used by the Lighting Trainer.
             It can be either "auto", "dp", "ddp", "ddp2", "ddp_spawn", "ddp_cpu", "gpu", "tpu".
@@ -294,6 +299,7 @@ def fun_control_init(
         "TENSORBOARD_PATH": TENSORBOARD_PATH,
         "_L_in": _L_in,
         "_L_out": _L_out,
+        "_torchmetric": _torchmetric,
         "accelerator": accelerator,
         "converters": converters,
         "core_model": core_model,