conversion to light

Author: bartzbeielstein

pyproject.toml

@@ -7,7 +7,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "spotPython"
-version = "0.2.51"
+version = "0.2.52"
 authors = [
   { name="T. Bartz-Beielstein", email="tbb@bartzundbartz.de" }
 ]

src/spotPython/light/crossvalidationdatamodule.py

@@ -13,13 +13,13 @@ def __init__(
         k: int = 1,
         split_seed: int = 42,
         num_splits: int = 10,
-        data_dir: str = "./data",
+        DATASET_PATH: str = "./data",
         num_workers: int = 0,
         pin_memory: bool = False,
     ):
         super().__init__()
         self.batch_size = batch_size
-        self.data_dir = data_dir
+        self.DATASET_PATH = DATASET_PATH
         self.num_workers = num_workers
         self.k = k
         self.split_seed = split_seed
@@ -35,6 +35,7 @@ def __init__(
         self.data_val: Optional[Dataset] = None
 
     def prepare_data(self):
+        # download
         pass
 
     def setup(self, stage=None):

src/spotPython/light/csvdatamodule.py

@@ -5,10 +5,9 @@
 
 
 class CSVDataModule(L.LightningDataModule):
-    def __init__(self, batch_size, data_dir: str = "./data", num_workers: int = 0):
+    def __init__(self, batch_size, DATASET_PATH: str = "./data", num_workers: int = 0):
         super().__init__()
         self.batch_size = batch_size
-        self.data_dir = data_dir
         self.num_workers = num_workers
 
     def prepare_data(self):

src/spotPython/light/netlightbase.py

@@ -18,38 +18,38 @@ def __init__(
         optimizer,
         dropout_prob,
         lr_mult,
-        patience=3,
-        _L_in=64,
-        _L_out=11,
+        patience,
+        _L_in,
+        _L_out,
     ):
         super().__init__()
-
-        # Attribute 'act_fn' is an instance of `nn.Module` and is already saved during checkpointing.
-        # It is recommended to ignore them using `self.save_hyperparameters(ignore=['act_fn'])`
-        self.save_hyperparameters(ignore=["act_fn"])
+        # Attribute 'act_fn' is an instance of `nn.Module` and is already saved during
+        # checkpointing. It is recommended to ignore them
+        # using `self.save_hyperparameters(ignore=['act_fn'])`
+        # self.save_hyperparameters(ignore=["act_fn"])
+        #
+        self._L_in = _L_in
         self._L_out = _L_out
-        if l1 < 4:
+        # _L_in and _L_out are not hyperparameters, but are needed to create the network
+        self.save_hyperparameters(ignore=["_L_in", "_L_out"])
+        if self.hparams.l1 < 4:
             raise ValueError("l1 must be at least 4")
-        self.l1 = l1
-        hidden_sizes = [l1, l1 // 2, l1 // 2, l1 // 4]
-        self.epochs = epochs
-        self.patience = patience
-        self.batch_size = batch_size
-        self.initialization = initialization
-        self.act_fn = act_fn
-        self.optimizer = optimizer
-        self.dropout_prob = dropout_prob
-        self.lr_mult = lr_mult
+
+        hidden_sizes = [self.hparams.l1, self.hparams.l1 // 2, self.hparams.l1 // 2, self.hparams.l1 // 4]
         self.train_mapk = MAPK(k=3)
         self.valid_mapk = MAPK(k=3)
         self.test_mapk = MAPK(k=3)
 
         # Create the network based on the specified hidden sizes
         layers = []
-        layer_sizes = [_L_in] + hidden_sizes
+        layer_sizes = [self._L_in] + hidden_sizes
         layer_size_last = layer_sizes[0]
         for layer_size in layer_sizes[1:]:
-            layers += [nn.Linear(layer_size_last, layer_size), act_fn, nn.Dropout(self.dropout_prob)]
+            layers += [
+                nn.Linear(layer_size_last, layer_size),
+                self.hparams.act_fn,
+                nn.Dropout(self.hparams.dropout_prob),
+            ]
             layer_size_last = layer_size
         layers += [nn.Linear(layer_sizes[-1], self._L_out)]
         # nn.Sequential summarizes a list of modules into a single module, applying them in sequence
@@ -98,5 +98,7 @@ def test_step(self, batch, batch_idx, prog_bar=False):
 
     def configure_optimizers(self):
         # optimizer = torch.optim.Adam(self.parameters(), lr=self.learning_rate)
-        optimizer = optimizer_handler(optimizer_name=self.optimizer, params=self.parameters(), lr_mult=self.lr_mult)
+        optimizer = optimizer_handler(
+            optimizer_name=self.hparams.optimizer, params=self.parameters(), lr_mult=self.hparams.lr_mult
+        )
         return optimizer

src/spotPython/light/traintest.py

@@ -3,23 +3,23 @@
 from spotPython.light.crossvalidationdatamodule import CrossValidationDataModule
 from spotPython.utils.eda import generate_config_id
 from pytorch_lightning.loggers import TensorBoardLogger
+from lightning.pytorch.callbacks import ModelCheckpoint
 from lightning.pytorch.callbacks.early_stopping import EarlyStopping
 from spotPython.torch.initialization import kaiming_init, xavier_init
+import os
 
 
 def train_model(config, fun_control):
+    _L_in = fun_control["_L_in"]
+    _L_out = fun_control["_L_out"]
+    print(f"_L_in: {_L_in}")
+    print(f"_L_out: {_L_out}")
     if fun_control["enable_progress_bar"] is None:
         enable_progress_bar = False
     else:
         enable_progress_bar = fun_control["enable_progress_bar"]
     config_id = generate_config_id(config)
-    # Init DataModule
-    dm = CSVDataModule(
-        batch_size=config["batch_size"], num_workers=fun_control["num_workers"], data_dir=fun_control["data_dir"]
-    )
-    # Init model from datamodule's attributes
-    # model = LitModel(*dm.dims, dm.num_classes)
-    model = fun_control["core_model"](**config, _L_in=64, _L_out=11)
+    model = fun_control["core_model"](**config, _L_in=_L_in, _L_out=_L_out)
     initialization = config["initialization"]
     if initialization == "Xavier":
         xavier_init(model)
@@ -28,12 +28,22 @@ def train_model(config, fun_control):
     else:
         pass
     print(f"model: {model}")
+
+    # Init DataModule
+    dm = CSVDataModule(
+        batch_size=config["batch_size"],
+        num_workers=fun_control["num_workers"],
+        DATASET_PATH=fun_control["DATASET_PATH"],
+    )
+
     # Init trainer
     trainer = L.Trainer(
-        max_epochs=model.epochs,
+        # Where to save models
+        default_root_dir=os.path.join(fun_control["CHECKPOINT_PATH"], config_id),
+        max_epochs=model.hparams.epochs,
         accelerator="auto",
         devices=1,
-        logger=TensorBoardLogger(save_dir=fun_control["tensorboard_path"], version=config_id, default_hp_metric=True),
+        logger=TensorBoardLogger(save_dir=fun_control["TENSORBOARD_PATH"], version=config_id, default_hp_metric=True),
         callbacks=[
             EarlyStopping(monitor="val_loss", patience=config["patience"], mode="min", strict=False, verbose=False)
         ],
@@ -51,18 +61,22 @@ def train_model(config, fun_control):
 
 
 def test_model(config, fun_control):
+    _L_in = fun_control["_L_in"]
+    _L_out = fun_control["_L_out"]
     if fun_control["enable_progress_bar"] is None:
         enable_progress_bar = False
     else:
         enable_progress_bar = fun_control["enable_progress_bar"]
-    config_id = generate_config_id(config)
+    # Add "TEST" postfix to config_id
+    config_id = generate_config_id(config) + "_TEST"
     # Init DataModule
     dm = CSVDataModule(
-        batch_size=config["batch_size"], num_workers=fun_control["num_workers"], data_dir=fun_control["data_dir"]
+        batch_size=config["batch_size"],
+        num_workers=fun_control["num_workers"],
+        DATASET_PATH=fun_control["DATASET_PATH"],
     )
     # Init model from datamodule's attributes
-    # model = LitModel(*dm.dims, dm.num_classes)
-    model = fun_control["core_model"](**config, _L_in=64, _L_out=11)
+    model = fun_control["core_model"](**config, _L_in=_L_in, _L_out=_L_out)
     initialization = config["initialization"]
     if initialization == "Xavier":
         xavier_init(model)
@@ -73,12 +87,15 @@ def test_model(config, fun_control):
     print(f"model: {model}")
     # Init trainer
     trainer = L.Trainer(
-        max_epochs=model.epochs,
+        # Where to save models
+        default_root_dir=os.path.join(fun_control["CHECKPOINT_PATH"], config_id),
+        max_epochs=model.hparams.epochs,
         accelerator="auto",
         devices=1,
-        logger=TensorBoardLogger(save_dir=fun_control["tensorboard_path"], version=config_id, default_hp_metric=True),
+        logger=TensorBoardLogger(save_dir=fun_control["TENSORBOARD_PATH"], version=config_id, default_hp_metric=True),
         callbacks=[
-            EarlyStopping(monitor="val_loss", patience=config["patience"], mode="min", strict=False, verbose=False)
+            EarlyStopping(monitor="val_loss", patience=config["patience"], mode="min", strict=False, verbose=False),
+            ModelCheckpoint(save_last=True),  # Save the last checkpoint
         ],
         enable_progress_bar=enable_progress_bar,
     )
@@ -91,15 +108,18 @@ def test_model(config, fun_control):
 
 
 def cv_model(config, fun_control):
-    config_id = generate_config_id(config)
+    _L_in = fun_control["_L_in"]
+    _L_out = fun_control["_L_out"]
     if fun_control["enable_progress_bar"] is None:
         enable_progress_bar = False
     else:
         enable_progress_bar = fun_control["enable_progress_bar"]
+    # Add "CV" postfix to config_id
+    config_id = generate_config_id(config) + "_CV"
     results = []
     num_folds = 10
     split_seed = 12345
-    model = fun_control["core_model"](**config, _L_in=64, _L_out=11)
+    model = fun_control["core_model"](**config, _L_in=_L_in, _L_out=_L_out)
     initialization = config["initialization"]
     if initialization == "Xavier":
         xavier_init(model)
@@ -116,7 +136,7 @@ def cv_model(config, fun_control):
             num_splits=num_folds,
             split_seed=split_seed,
             batch_size=config["batch_size"],
-            data_dir=fun_control["data_dir"],
+            DATASET_PATH=fun_control["DATASET_PATH"],
         )
         dm.prepare_data()
         dm.setup()
@@ -125,11 +145,13 @@ def cv_model(config, fun_control):
         print(f"model: {model}")
         # Init trainer
         trainer = L.Trainer(
-            max_epochs=model.epochs,
+            # Where to save models
+            default_root_dir=os.path.join(fun_control["CHECKPOINT_PATH"], config_id),
+            max_epochs=model.hparams.epochs,
             accelerator="auto",
             devices=1,
             logger=TensorBoardLogger(
-                save_dir=fun_control["tensorboard_path"], version=config_id, default_hp_metric=True
+                save_dir=fun_control["TENSORBOARD_PATH"], version=config_id, default_hp_metric=True
             ),
             callbacks=[
                 EarlyStopping(monitor="val_loss", patience=config["patience"], mode="min", strict=False, verbose=False)
@@ -150,3 +172,16 @@ def cv_model(config, fun_control):
     mapk_score = sum(results) / num_folds
     print(f"cv_model mapk result: {mapk_score}")
     return mapk_score
+
+
+def load_light_from_checkpoint(config, fun_control, postfix="_TEST"):
+    config_id = generate_config_id(config) + postfix
+    default_root_dir = fun_control["TENSORBOARD_PATH"] + "lightning_logs/" + config_id + "/checkpoints/last.ckpt"
+    # default_root_dir = os.path.join(fun_control["CHECKPOINT_PATH"], config_id)
+    print(f"Loading model from {default_root_dir}")
+    model = fun_control["core_model"].load_from_checkpoint(
+        default_root_dir, _L_in=fun_control["_L_in"], _L_out=fun_control["_L_out"]
+    )
+    # disable randomness, dropout, etc...
+    model.eval()
+    return model

src/spotPython/light/utils.py

@@ -0,0 +1,7 @@
+from spotPython.hyperparameters.values import get_one_config_from_X
+
+
+def get_tuned_architecture(spot_tuner, fun_control):
+    X = spot_tuner.to_all_dim(spot_tuner.min_X.reshape(1, -1))
+    config = get_one_config_from_X(X, fun_control)
+    return config

src/spotPython/utils/eda.py

@@ -167,7 +167,7 @@ def visualize_activations(net, device="cpu", color="C0"):
         sns.histplot(data=activations[key], bins=50, ax=key_ax, color=color, kde=True, stat="density")
         key_ax.set_title(f"Layer {key} - {net.layers[key].__class__.__name__}")
         fig_index += 1
-    fig.suptitle(f"Activation distribution for activation function {net.act_fn}", fontsize=14)
+    fig.suptitle(f"Activation distribution for activation function {net.hparams.act_fn}", fontsize=14)
     fig.subplots_adjust(hspace=0.4, wspace=0.4)
     plt.show()
     plt.close()