0.32.0

New Kriging isotropic/anisotropic

Author: bartzbeielstein

notebooks/00_spotPython_tests.ipynb

@@ -7,7 +7,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "spotpython"
-version = "0.31.16"
+version = "0.32.0"
 authors = [
   { name="T. Bartz-Beielstein", email="tbb@bartzundbartz.de" }
 ]

pyproject.toml

@@ -1470,8 +1470,7 @@ def get_tuned_hyperparameters(spot_tuner, fun_control=None) -> dict:
                         ] )
             from spotpython.utils.init import design_control_init, surrogate_control_init
             design_control = design_control_init(init_size=INIT_SIZE)
-            surrogate_control = surrogate_control_init(noise=True,
-                                                        n_theta=2)
+            surrogate_control = surrogate_control_init(noise=True)
             from spotpython.fun.hyperlight import HyperLight
             fun = HyperLight(log_level=50).fun
             from spotpython.spot import spot

src/spotpython/hyperparameters/values.py

@@ -171,7 +171,6 @@ def objective_function(X, fun_control=None):
                         model_fun_evals=10000,
                         min_theta=-3,
                         max_theta=3,
-                        n_theta=2,
                         theta_init_zero=False,
                         n_p=1,
                         optim_p=False,
@@ -422,17 +421,6 @@ def _surrogate_control_setup(self) -> None:
         if self.surrogate_control["model_optimizer"] is None or self.optimizer is not None:
             self.surrogate_control.update({"model_optimizer": self.optimizer})
 
-        # if self.surrogate_control["n_theta"] is a string and == isotropic, use 1 theta value:
-        if isinstance(self.surrogate_control["n_theta"], str):
-            if self.surrogate_control["n_theta"] == "anisotropic":
-                self.surrogate_control.update({"n_theta": self.k})
-            else:
-                # case "isotropic":
-                self.surrogate_control.update({"n_theta": 1})
-        if isinstance(self.surrogate_control["n_theta"], int):
-            if self.surrogate_control["n_theta"] > 1:
-                self.surrogate_control.update({"n_theta": self.k})
-
     def surrogate_setup(self, surrogate) -> None:
         """Set up the surrogate model for the optimization process.
         This method initializes the surrogate model. If no surrogate model is provided,
@@ -455,7 +443,6 @@ def surrogate_setup(self, surrogate) -> None:
                 - model_optimizer: Optimizer for model parameter tuning
                 - model_fun_evals: Number of function evaluations for model optimization
                 - min/max_theta: Bounds for theta parameters
-                - n_theta: Number of theta parameters
                 - theta_init_zero: Whether to initialize theta at zero
                 - p_val: Power parameter p value
                 - n_p: Number of p parameters
@@ -479,15 +466,14 @@ def surrogate_setup(self, surrogate) -> None:
             ... )
             >>> surrogate_control = surrogate_control_init(
             ...     method="interpolation",
-            ...     n_theta="anisotropic"
             ... )
             >>> S = spot.Spot(
             ...     fun=Analytical().fun_sphere,
             ...     fun_control=fun_control,
             ...     surrogate_control=surrogate_control
             ... )
-            >>> print(S.surrogate.n_theta)
-            2
+            >>> print(S.surrogate.isotropic)
+            False
 
             >>> # Setup with custom surrogate
             >>> from sklearn.gaussian_process import GaussianProcessRegressor
@@ -514,7 +500,7 @@ def surrogate_setup(self, surrogate) -> None:
                 model_fun_evals=self.surrogate_control["model_fun_evals"],
                 min_theta=self.surrogate_control["min_theta"],
                 max_theta=self.surrogate_control["max_theta"],
-                n_theta=self.surrogate_control["n_theta"],
+                isotropic=self.surrogate_control["isotropic"],
                 theta_init_zero=self.surrogate_control["theta_init_zero"],
                 p_val=self.surrogate_control["p_val"],
                 n_p=self.surrogate_control["n_p"],
@@ -2072,7 +2058,7 @@ def plot_progress(
                     tolerance_x = np.sqrt(np.spacing(1))
                     )
                 design_control=design_control_init(init_size=ni)
-                surrogate_control=surrogate_control_init(n_theta=3)
+                surrogate_control=surrogate_control_init()
                 S = spot.Spot(fun=fun,
                             fun_control=fun_control
                             design_control=design_control,
@@ -2303,8 +2289,7 @@ def get_tuned_hyperparameters(self, fun_control=None) -> dict:
                             ] )
                 from spotpython.utils.init import design_control_init, surrogate_control_init
                 design_control = design_control_init(init_size=INIT_SIZE)
-                surrogate_control = surrogate_control_init(method="regression",
-                                                            n_theta=2)
+                surrogate_control = surrogate_control_init(method="regression")
                 from spotpython.fun.hyperlight import HyperLight
                 fun = HyperLight(log_level=50).fun
                 from spotpython.spot import spot
@@ -2662,7 +2647,7 @@ def plot_important_hyperparameter_contour(
                     tolerance_x = np.sqrt(np.spacing(1))
                     )
                 design_control=design_control_init(init_size=ni)
-                surrogate_control=surrogate_control_init(n_theta=3)
+                surrogate_control=surrogate_control_init)
                 S = spot.Spot(fun=fun,
                             fun_control=fun_control,
                             design_control=design_control,
@@ -2733,7 +2718,6 @@ def get_importance(self) -> list:
             >>> design_control = design_control_init(init_size=10)
             >>> # Setup surrogate model with multiple theta values
             >>> surrogate_control = surrogate_control_init(
-            ...     n_theta="anisotropic",
             ...     method="interpolation"
             ... )
             >>> # Initialize and run spot
@@ -2753,7 +2737,7 @@ def get_importance(self) -> list:
             x3: 76.15%
             >>> # Try with single theta (should return zeros)
             >>> surrogate_control = surrogate_control_init(
-            ...     n_theta=1,
+            ...     isotropie=True,
             ...     method="interpolation"
             ... )
             >>> S2 = spot.Spot(
@@ -2772,16 +2756,16 @@ def get_importance(self) -> list:
             print("No surrogate model available.")
             return []
 
-        # Check if surrogate has n_theta attribute
-        if not hasattr(self.surrogate, "n_theta"):
-            print("Surrogate model does not have n_theta attribute.")
-            return []
-
         # Check if surrogate has theta attribute for multi-theta models
-        if self.surrogate.n_theta > 1 and not hasattr(self.surrogate, "theta"):
+        if not hasattr(self.surrogate, "theta"):
             print("Surrogate model does not have theta attribute.")
             return []
 
+        # Check if surrogate is not isotropic
+        if self.surrogate.isotropic:
+            print("Surrogate model is isotropic.")
+            return []
+
         # Check if all required attributes exist for importance calculation
         if not hasattr(self, "all_var_name"):
             print("Variable names (all_var_name) not available.")
@@ -2838,7 +2822,7 @@ def print_importance(self, threshold=0.1, print_screen=True) -> list:
                             print(var_name[i] + ": ", imp[i])
                     output.append([var_name[i], imp[i]])
         else:
-            print("Importance requires more than one theta values (n_theta>1).")
+            print("Importance requires more than one theta value (n_theta>1).")
         return output
 
     def plot_importance(self, threshold=0.1, filename=None, dpi=300, show=True, tkagg=False, figsize=(9, 6)) -> None:
@@ -2917,7 +2901,7 @@ def parallel_plot(self, show=False) -> go.Figure:
                     tolerance_x = np.sqrt(np.spacing(1))
                     )
                 design_control=design_control_init(init_size=ni)
-                surrogate_control=surrogate_control_init(n_theta=3)
+                surrogate_control=surrogate_control_init()
                 S = spot.Spot(fun=fun,
                             fun_control=fun_control,
                             design_control=design_control,

src/spotpython/spot/spot.py

@@ -22,6 +22,23 @@ class Kriging(BaseEstimator, RegressorMixin):
         U_ (np.ndarray): The Cholesky factor of the correlation matrix after fit().
         X_ (np.ndarray): The training input data (n x d).
         y_ (np.ndarray): The training target values (n,).
+        negLnLike (float): The negative log-likelihood of the model.
+        Psi_ (np.ndarray): The correlation matrix after fit().
+        method (str): The fitting method used, can be "interpolation", "regression", or "reinterpolation".
+        isotropic (bool): Whether the model is isotropic or not.
+
+    Methods:
+        __init__: Initializes the Kriging model with hyperparameters.
+        _get_eps: Returns the square root of machine epsilon.
+        _set_variable_types: Sets variable types for the model.
+        get_model_params: Returns additional model parameters not included in get_params().
+        _update_log: Updates the log with current model parameters.
+        fit: Fits the Kriging model to training data X and y.
+        predict: Predicts the Kriging response at a set of points X.
+        build_Psi: Constructs a new correlation matrix Psi.
+        likelihood: Computes the negative concentrated log-likelihood and correlation matrix.
+        build_psi_vec: Builds the psi vector for predictive methods.
+        _pred: Computes a single-point Kriging prediction.
     """
 
     def __init__(
@@ -37,7 +54,6 @@ def __init__(
         model_fun_evals: Optional[int] = None,
         min_theta: float = -3.0,
         max_theta: float = 2.0,
-        n_theta: int = None,
         theta_init_zero: bool = False,
         p_val: float = 2.0,
         n_p: int = 1,
@@ -50,6 +66,7 @@ def __init__(
         spot_writer=None,
         counter=None,
         metric_factorial="canberra",
+        isotropic: bool = False,
         **kwargs,
     ):
         """
@@ -66,6 +83,9 @@ def __init__(
                 not positive-definite. Defaults to 1e4.
             method (str, optional):
                 The type how the model uis fitted. Can be "interpolation", "regression", or "reinterpolation". Defaults to "regression".
+            isotropic (bool, optional):
+                If True, the model is isotropic, meaning all variables are treated equally (only one theta value is used).
+                If False, the model can handle different theta values, one for each dimension. Defaults to False.
         """
         if eps is None:
             self.eps = self._get_eps()
@@ -90,7 +110,8 @@ def __init__(
         self.max_Lambda = max_Lambda
         self.min_p = min_p
         self.max_p = max_p
-        self.n_theta = n_theta
+        self.n_theta = None  # Will be set in fit()
+        self.isotropic = isotropic
         self.p_val = p_val
         self.n_p = n_p
         self.optim_p = optim_p
@@ -256,8 +277,13 @@ def fit(self, X: np.ndarray, y: np.ndarray, bounds: Optional[List[Tuple[float, f
         self.y_ = y
         self.n, self.k = self.X_.shape
         self._set_variable_types()
-        if self.n_theta is None:
+        if self.isotropic:
+            # If isotropic, set n_theta to 1
+            self.n_theta = 1
+            print(f"Isotropic model: n_theta set to {self.n_theta}")
+        else:
             self.n_theta = self.k
+            print(f"Anisotropic model: n_theta set to {self.n_theta}")
         # Calculate and store min and max of X
         self.min_X = np.min(self.X_, axis=0)
         self.max_X = np.max(self.X_, axis=0)

src/spotpython/surrogate/kriging.py

@@ -739,7 +739,7 @@ def surrogate_control_init(
     model_fun_evals=10000,
     min_theta=-3.0,  # log10
     max_theta=2.0,  # log10
-    n_theta="anisotropic",
+    isotropic=False,
     p_val=2.0,
     n_p=1,
     optim_p=False,
@@ -764,14 +764,15 @@ def surrogate_control_init(
              Default is -3.
         max_theta (float): The maximum value of theta. Note that the base10-logarithm is used.
             Default is 3.
+        isotropic (bool):
+            Whether to use isotropic or anisotropic theta values. If True, the theta values are
+            isotropic, i.e., the same value is used for all dimensions. If False,
+            the theta values are anisotropic, i.e., different values are used for each dimension.
+            Default is False.
         method (str):
             The method to be used for the surrogate model. Default is "regression".
             Can be one of ["regression", "interpolation", "reinterpolation"].
             Note: Will also be set in the Spot class, if None.
-        n_theta (int):
-            The number of theta values. If larger than 1 or set to the string "anisotropic",
-            then the k theta values are used, where k is the problem dimension.
-            This is handled in spot.py. Default is "anisotropic".
         p_val (float):
                 p value. Used as an initial value if optim_p = True. Otherwise as a constant. Defaults to 2.0.
         n_p (int):
@@ -807,11 +808,8 @@ def surrogate_control_init(
                of `var_type` in the Spot class fun_control dictionary and the dimension of the problem.
                If the Kriging model is used as a surrogate in the Spot class, the setting from
                 surrogate_control_init() is overwritten.
-            * `n_theta`: If self.surrogate_control["n_theta"] > 1,
-               use k theta values, where k is the problem dimension specified in the Spot class.
-               The problem dimension is set in the Spot class based on the
-               length of the lower bounds.
-        * This value `model_fun_evals` will used for the optimization of the surrogate model, e.g., theta values.
+            * `isotropic`: If the `isotropic` value is set to `True`, then the theta values are isotropic, i.e., the same value is used for all dimensions. If it is set to `False`, then the theta values are anisotropic, i.e., different values are used for each dimension.
+        * The value `model_fun_evals` will used for the optimization of the surrogate model, e.g., theta values.
           Differential evaluation uses `maxiter = 1000` and sets the number of function evaluations to
           (maxiter + 1) * popsize * N, which results in 1000 * 15 * k,
           because the default popsize is 15 and N is the number of parameters. This is already sufficient
@@ -826,7 +824,7 @@ def surrogate_control_init(
         "model_fun_evals": model_fun_evals,
         "min_theta": min_theta,
         "max_theta": max_theta,
-        "n_theta": n_theta,
+        "isotropic": isotropic,
         "p_val": p_val,
         "n_p": n_p,
         "optim_p": optim_p,

src/spotpython/utils/init.py

@@ -47,7 +47,7 @@ def test_build_Psi():
     S_spot.min_X = X[argmin(y)]
     # Kriging:
 
-    S = Kriging(name="kriging", seed=124, n_theta=2, noise=True, cod_type="norm")
+    S = Kriging(name="kriging", seed=124, noise=True, cod_type="norm")
     S.nat_X = copy.deepcopy(X)
     S.nat_y = copy.deepcopy(y)
     S.n = S.nat_X.shape[0]

test/test_build_Psi.py

@@ -41,7 +41,7 @@ def test_build_U():
     S_spot.min_y = min(y)
     S_spot.min_X = X[argmin(y)]
     # Kriging:
-    S = Kriging(name="kriging", seed=124, n_theta=2, noise=True, cod_type="norm")
+    S = Kriging(name="kriging", seed=124, noise=True, cod_type="norm")
     S.nat_X = copy.deepcopy(X)
     S.nat_y = copy.deepcopy(y)
     S.n = S.nat_X.shape[0]

test/test_build_U.py

@@ -8,7 +8,7 @@ def test_build_psi_vec():
     X_train = np.array([[1.0, 2.0], [2.0, 4.0], [3.0, 6.0]])
     y_train = np.array([1.0, 2.0, 3.0])
 
-    S = Kriging(name="kriging", seed=123, log_level=50, n_theta=1, noise=False, cod_type="norm")
+    S = Kriging(name="kriging", seed=123, log_level=50, isotropic=True, noise=False, cod_type="norm")
     S.fit(X_train, y_train)
 
     # force theta to simple values: