v0.5.8

hparams update, plot_progress fixed

Author: bartzbeielstein

pyproject.toml

@@ -7,7 +7,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "spotPython"
-version = "0.5.7"
+version = "0.5.8"
 authors = [
   { name="T. Bartz-Beielstein", email="tbb@bartzundbartz.de" }
 ]

src/spotPython/build/kriging.py

@@ -310,18 +310,18 @@ def update_log(self):
         if self.spot_writer is not None:
             writer = self.spot_writer
             negLnLike = self.negLnLike.copy()
-            writer.add_scalar("negLnLike", negLnLike, self.counter+self.log_length)
+            writer.add_scalar("spot_negLnLike", negLnLike, self.counter+self.log_length)
             # add the self.n_theta theta values to the writer with one key "theta",
             # i.e, the same key for all theta values
             theta = self.theta.copy()
-            writer.add_scalars("theta", {f"theta_{i}": theta[i] for i in range(self.n_theta)},
+            writer.add_scalars("spot_theta", {f"theta_{i}": theta[i] for i in range(self.n_theta)},
                                self.counter+self.log_length)
             if self.noise:
                 Lambda = self.Lambda.copy()
-                writer.add_scalar("Lambda", Lambda, self.counter+self.log_length)
+                writer.add_scalar("spot_Lambda", Lambda, self.counter+self.log_length)
             if self.optim_p:
                 p = self.p.copy()
-                writer.add_scalars("p", {f"p_{i}": p[i] for i in range(self.n_p)}, self.counter+self.log_length)
+                writer.add_scalars("spot_p", {f"p_{i}": p[i] for i in range(self.n_p)}, self.counter+self.log_length)
             writer.flush()
 
     def fit_old(self, nat_X, nat_y):

src/spotPython/light/traintest.py

@@ -12,8 +12,8 @@
 def train_model(config, fun_control):
     _L_in = fun_control["_L_in"]
     _L_out = fun_control["_L_out"]
-    print(f"_L_in: {_L_in}")
-    print(f"_L_out: {_L_out}")
+    # print(f"_L_in: {_L_in}")
+    # print(f"_L_out: {_L_out}")
     if fun_control["enable_progress_bar"] is None:
         enable_progress_bar = False
     else:
@@ -27,7 +27,7 @@ def train_model(config, fun_control):
         kaiming_init(model)
     else:
         pass
-    print(f"model: {model}")
+    # print(f"model: {model}")
 
     # Init DataModule
     dm = CSVDataModule(
@@ -56,7 +56,7 @@ def train_model(config, fun_control):
     result = trainer.validate(model=model, datamodule=dm)
     # unlist the result (from a list of one dict)
     result = result[0]
-    print(f"train_model result: {result}")
+    # print(f"train_model result: {result}")
     return result["val_loss"]
 
 
@@ -84,7 +84,7 @@ def test_model(config, fun_control):
         kaiming_init(model)
     else:
         pass
-    print(f"model: {model}")
+    # print(f"model: {model}")
     # Init trainer
     trainer = L.Trainer(
         # Where to save models
@@ -103,7 +103,7 @@ def test_model(config, fun_control):
     trainer.fit(model=model, datamodule=dm)
     test_result = trainer.test(datamodule=dm, ckpt_path="last")
     test_result = test_result[0]
-    print(f"test_model result: {test_result}")
+    # print(f"test_model result: {test_result}")
     return test_result["val_loss"], test_result["val_acc"]
 
 
@@ -131,7 +131,7 @@ def cv_model(config, fun_control):
             kaiming_init(model)
         else:
             pass
-        print(f"model: {model}")
+        # print(f"model: {model}")
 
         dm = CrossValidationDataModule(
             k=k,
@@ -170,7 +170,7 @@ def cv_model(config, fun_control):
         results.append(score["valid_mapk"])
 
     mapk_score = sum(results) / num_folds
-    print(f"cv_model mapk result: {mapk_score}")
+    # print(f"cv_model mapk result: {mapk_score}")
     return mapk_score
 
 

src/spotPython/spot/spot.py

@@ -326,6 +326,8 @@ def append_X_ocba(self, X_ocba, X0):
 
     def run(self, X_start=None):
         self.initialize_design(X_start)
+        # New: self.update_stats() moved here:
+        self.update_stats()
         # (S-5) Calling the spotLoop Function
         # and
         # (S-9) Termination Criteria, Conditions:
@@ -365,8 +367,22 @@ def initialize_design(self, X_start=None):
         self.y = self.fun(X=X_all, fun_control=self.fun_control)
         # TODO: Error if only nan values are returned
         logger.debug("New y value: %s", self.y)
+        #
+        self.counter = self.y.size
+        if self.spot_writer is not None:
+            writer = self.spot_writer
+            # range goes to init_size -1 because the last value is added by update_stats(),
+            # which always adds the last value.
+            for j in range(len(self.y) - 1):
+                X_j = self.X[j].copy()
+                y_j = self.y[j].copy()
+                config = {self.var_name[i]: X_j[i] for i in range(self.k)}
+                writer.add_hparams(config, {"spot_y": y_j})
+                writer.flush()
+        #
         self.X, self.y = remove_nan(self.X, self.y)
-        self.update_stats()
+        # self.update_stats() moved to run()!
+        # self.update_stats()
         # (S-4): Imputation:
         # Not implemented yet.
         # (S-11) Surrogate Fit:
@@ -422,13 +438,16 @@ def update_stats(self):
             y_min = self.min_y.copy()
             y_last = self.last_y.copy()
             X_min = self.min_X.copy()
+            # y_min: best y value so far
+            # y_last: last y value, can he worse than y_min
             writer.add_scalars("spot_y", {"min": y_min, "last": y_last}, self.counter)
+            # X_min: X value of the best y value so far
             writer.add_scalars("spot_X", {f"X_{i}": X_min[i] for i in range(self.k)}, self.counter)
             # get last value of self.X and convert to dict. take the values from self.var_name as keys:
             X_last = self.X[-1].copy()
             config = {self.var_name[i]: X_last[i] for i in range(self.k)}
-            y_val = self.y[-1].copy()
-            writer.add_hparams(config, {"fun_torch: loss": y_val})
+            # hyperparameters X and value y of the last configuration:
+            writer.add_hparams(config, {"spot_y": y_last})
             writer.flush()
         # Update aggregated x and y values (if noise):
         if self.noise:
@@ -440,11 +459,17 @@ def update_stats(self):
             self.min_mean_X = self.mean_X[argmin(self.mean_y)]
             if self.spot_writer is not None:
                 writer = self.spot_writer
+                # y_min_mean: best mean y value so far
                 y_min_mean = self.min_mean_y.copy()
+                # X_min_mean: X value of the best mean y value so far
                 X_min_mean = self.min_mean_X.copy()
+                # y_var: variance of the y values
                 y_var = self.var_y.copy()
-                writer.add_scalars("spot_y_noise", {"min_mean_y": y_min_mean, "last": y_last}, self.counter)
                 writer.add_scalar("spot_y_var", y_var, self.counter)
+                # y_min_mean: best mean y value so far (see above)
+                # y_last: last y value, can he worse than y_min_mean
+                writer.add_scalars("spot_y_noise", {"min_mean_y": y_min_mean, "last": y_last}, self.counter)
+                # X_min_mean: X value of the best mean y value so far (see above)
                 writer.add_scalars(
                     "spot_X_noise", {f"X_min_mean{i}": X_min_mean[i] for i in range(self.k)}, self.counter
                 )
@@ -574,10 +599,10 @@ def plot_progress(
             range(1, n_init + 1),
             s_y[:n_init],
             style[0],
-            range(1, n_init + 1),
-            [s_c[:n_init].min()] * n_init,
+            range(1, n_init + 2),
+            [s_c[:n_init].min()] * (n_init + 1),
             style[1],
-            range(n_init, len(s_c)),
+            range(n_init + 1, len(s_c) + 1),
             s_c[n_init:],
             style[2],
         )

src/spotPython/utils/file.py

@@ -4,6 +4,7 @@
 from datetime import datetime
 from dateutil.tz import tzlocal
 import pickle
+import os
 
 
 def load_data(data_dir="./data"):
@@ -74,3 +75,15 @@ def load_pickle(filename: str):
     with open(filename, "rb") as f:
         obj = pickle.load(f)
     return obj
+
+
+def get_spot_tensorboard_path(experiment_name):
+    """Get the path to the spot tensorboard files.
+    Args:
+        experiment_name (str): The name of the experiment.
+    Returns:
+        spot_tensorboard_path (str): The path to the folder where the spot tensorboard files are saved.
+    """
+    spot_tensorboard_path = os.environ.get("PATH_TENSORBOARD", "runs/spot_logs/")
+    spot_tensorboard_path = os.path.join(spot_tensorboard_path, experiment_name)
+    return spot_tensorboard_path

src/spotPython/utils/init.py

@@ -10,7 +10,7 @@ def fun_control_init(
     _L_in=None,
     _L_out=None,
     enable_progress_bar=False,
-    tensorboard_path=None,
+    spot_tensorboard_path="runs_spot/",
     num_workers=0,
     device=None,
 ):
@@ -20,7 +20,8 @@ def fun_control_init(
         _L_in (int): The number of input features.
         _L_out (int): The number of output features.
         enable_progress_bar (bool): Whether to enable the progress bar or not.
-        tensorboard_path (str): The path to the folder where the tensorboard files are saved.
+        spot_tensorboard_path (str): The path to the folder where the spot tensorboard files are saved.
+        If None, no spot tensorboard files are saved.
         num_workers (int): The number of workers to use for the data loading.
         device (str): The device to use for the training. It can be either "cpu", "mps", or "cuda".
     Returns:
@@ -66,15 +67,6 @@ def fun_control_init(
     # Setting the seed
     L.seed_everything(42)
 
-    if tensorboard_path is not None:
-        check_and_create_dir(tensorboard_path)
-        # Starting with v0.2.41, Summary Writer should be not initialized here but by Lightning
-        # it is only available for compatibility reasons.
-        # So, set this to None and let Lightning manage the logging.
-        spot_writer = SummaryWriter(tensorboard_path)
-    else:
-        spot_writer = None
-
     # Path to the folder where the pretrained models are saved
     CHECKPOINT_PATH = os.environ.get("PATH_CHECKPOINT", "saved_models/")
     os.makedirs(CHECKPOINT_PATH, exist_ok=True)
@@ -87,6 +79,12 @@ def fun_control_init(
     # Path to the folder where the tensorboard files are saved
     TENSORBOARD_PATH = os.environ.get("PATH_TENSORBOARD", "runs/")
     os.makedirs(TENSORBOARD_PATH, exist_ok=True)
+    if spot_tensorboard_path is not None:
+        os.makedirs(spot_tensorboard_path, exist_ok=True)
+        spot_writer = SummaryWriter(spot_tensorboard_path)
+
+    if not os.path.exists("./figures"):
+        os.makedirs("./figures")
 
     fun_control = {
         "CHECKPOINT_PATH": CHECKPOINT_PATH,
@@ -119,7 +117,7 @@ def fun_control_init(
         "train": None,
         "test": None,
         "task": task,
-        "tensorboard_path": tensorboard_path,
+        "spot_tensorboard_path": spot_tensorboard_path,
         "weights": 1.0,
         "spot_writer": spot_writer,
     }