Skip to content

Commit b2f024b

Browse files
bartzbeielsteinbartzbeielstein
committed
Merge branch 'main' of github.com:sequential-parameter-optimization/spotPython
2 parents 1be6484 + a097cbb commit b2f024b

2 files changed

Lines changed: 521 additions & 0 deletions

File tree

‎src/spotpython/fun/xai_hyperlight.py‎

Lines changed: 184 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,184 @@
1+
import logging
2+
import numpy as np
3+
import pprint
4+
from numpy.random import default_rng
5+
from spotpython.light.trainmodel import train_model_xai
6+
from spotpython.hyperparameters.values import assign_values, generate_one_config_from_var_dict, get_var_name
7+
8+
logger = logging.getLogger(__name__)
9+
py_handler = logging.FileHandler(f"{__name__}.log", mode="w")
10+
py_formatter = logging.Formatter("%(name)s %(asctime)s %(levelname)s %(message)s")
11+
py_handler.setFormatter(py_formatter)
12+
logger.addHandler(py_handler)
13+
14+
15+
class XAI_HyperLight:
16+
"""
17+
Hyperparameter Tuning for Lightning considering XAI inconsistency.
18+
19+
Args:
20+
seed (int): seed for the random number generator. See Numpy Random Sampling.
21+
log_level (int): log level for the logger.
22+
23+
Attributes:
24+
seed (int): seed for the random number generator.
25+
rng (Generator): random number generator.
26+
fun_control (dict): dictionary containing control parameters for the hyperparameter tuning.
27+
log_level (int): log level for the logger.
28+
29+
Examples:
30+
>>> hyper_light = HyperLight(seed=126, log_level=50)
31+
>>> print(hyper_light.seed)
32+
126
33+
"""
34+
35+
def __init__(self, seed: int = 126, log_level: int = 50) -> None:
36+
self.seed = seed
37+
self.rng = default_rng(seed=self.seed)
38+
self.log_level = log_level
39+
logger.setLevel(log_level)
40+
logger.info(f"Starting the logger at level {log_level} for module {__name__}:")
41+
42+
def check_X_shape(self, X: np.ndarray, fun_control: dict) -> np.ndarray:
43+
"""
44+
Checks the shape of the input array X and raises an exception if it is not valid.
45+
46+
Args:
47+
X (np.ndarray):
48+
input array.
49+
fun_control (dict):
50+
dictionary containing control parameters for the hyperparameter tuning.
51+
52+
Returns:
53+
np.ndarray:
54+
input array with valid shape.
55+
56+
Raises:
57+
Exception:
58+
if the shape of the input array is not valid.
59+
60+
Examples:
61+
>>> import numpy as np
62+
from spotpython.utils.init import fun_control_init
63+
from spotpython.light.regression.netlightregression import NetLightRegression
64+
from spotpython.hyperdict.light_hyper_dict import LightHyperDict
65+
from spotpython.hyperparameters.values import add_core_model_to_fun_control
66+
from spotpython.fun.hyperlight import HyperLight
67+
from spotpython.hyperparameters.values import get_var_name
68+
fun_control = fun_control_init()
69+
add_core_model_to_fun_control(core_model=NetLightRegression,
70+
fun_control=fun_control,
71+
hyper_dict=LightHyperDict)
72+
hyper_light = HyperLight(seed=126, log_level=50)
73+
n_hyperparams = len(get_var_name(fun_control))
74+
# generate a random np.array X with shape (2, n_hyperparams)
75+
X = np.random.rand(2, n_hyperparams)
76+
X == hyper_light.check_X_shape(X, fun_control)
77+
array([[ True, True, True, True, True, True, True, True, True],
78+
[ True, True, True, True, True, True, True, True, True]])
79+
80+
"""
81+
try:
82+
X.shape[1]
83+
except ValueError:
84+
X = np.array([X])
85+
if X.shape[1] != len(get_var_name(fun_control)):
86+
raise Exception("Invalid shape of input array X.")
87+
return X
88+
89+
def fun(self, X: np.ndarray, fun_control: dict = None) -> np.ndarray:
90+
"""
91+
Evaluates the function for the given input array X and control parameters.
92+
Calls the train_model function from spotpython.light.trainmodel
93+
to train the model and evaluate the results.
94+
95+
Args:
96+
X (np.ndarray):
97+
input array.
98+
fun_control (dict):
99+
dictionary containing control parameters for the hyperparameter tuning.
100+
101+
Returns:
102+
(np.ndarray):
103+
array containing the evaluation results.
104+
105+
Examples:
106+
>>> from math import inf
107+
import numpy as np
108+
from spotpython.data.diabetes import Diabetes
109+
from spotpython.hyperdict.light_hyper_dict import LightHyperDict
110+
from spotpython.fun.hyperlight import HyperLight
111+
from spotpython.utils.init import fun_control_init
112+
from spotpython.utils.eda import print_exp_table
113+
from spotpython.spot import spot
114+
from spotpython.hyperparameters.values import get_default_hyperparameters_as_array
115+
PREFIX="000"
116+
data_set = Diabetes()
117+
fun_control = fun_control_init(
118+
PREFIX=PREFIX,
119+
save_experiment=True,
120+
fun_evals=inf,
121+
max_time=1,
122+
data_set = data_set,
123+
core_model_name="light.regression.NNLinearRegressor",
124+
hyperdict=LightHyperDict,
125+
_L_in=10,
126+
_L_out=1,
127+
TENSORBOARD_CLEAN=True,
128+
tensorboard_log=True,
129+
seed=42,)
130+
print_exp_table(fun_control)
131+
X = get_default_hyperparameters_as_array(fun_control)
132+
# set epochs to 2^8:
133+
X[0, 1] = 8
134+
# set patience to 2^10:
135+
X[0, 7] = 10
136+
print(f"X: {X}")
137+
# combine X and X to a np.array with shape (2, n_hyperparams)
138+
# so that two values are returned
139+
X = np.vstack((X, X))
140+
hyper_light = HyperLight(seed=125, log_level=50)
141+
hyper_light.fun(X, fun_control)
142+
"""
143+
z_res = np.array([], dtype=float)
144+
xai_res = np.array([], dtype=float)
145+
self.check_X_shape(X=X, fun_control=fun_control)
146+
var_dict = assign_values(X, get_var_name(fun_control))
147+
# type information and transformations are considered in generate_one_config_from_var_dict:
148+
for config in generate_one_config_from_var_dict(var_dict, fun_control):
149+
if fun_control["show_config"]:
150+
print("\nIn fun(): config:")
151+
pprint.pprint(config)
152+
logger.debug(f"\nconfig: {config}")
153+
# extract parameters like epochs, batch_size, lr, etc. from config
154+
# config_id = generate_config_id(config)
155+
try:
156+
logger.debug("fun: Calling train_model")
157+
df_eval, xai_attr = train_model_xai(config, fun_control)
158+
logger.debug("fun: train_model returned")
159+
except Exception as err:
160+
if fun_control["verbosity"] > 0:
161+
print("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!")
162+
if fun_control["verbosity"] > 1:
163+
pprint.pprint(fun_control)
164+
print(f"Error in fun(). Call to train_model failed. {err=}, {type(err)=}")
165+
print("Setting df_eval to np.nan\n")
166+
logger.error(f"Error in fun(). Call to train_model failed. {err=}, {type(err)=}")
167+
logger.error("Setting df_eval to np.nan")
168+
df_eval = np.nan
169+
# Multiply results by the weights. Positive weights mean that the result is to be minimized.
170+
# Negative weights mean that the result is to be maximized, e.g., accuracy.
171+
z_val = fun_control["weights"] * df_eval
172+
print("Attribution : ", xai_attr)
173+
xai_incons = fun_control["xai_weight"] * xai_attr
174+
175+
# Append, since several configurations can be evaluated at once.
176+
z_res = np.append(z_res, z_val)
177+
xai_res = np.append(xai_res, xai_incons)
178+
179+
print("Performance loss: ", z_val)
180+
print("XAI inconsistency: ", xai_incons)
181+
182+
res = z_res + xai_res
183+
184+
return res

0 commit comments

Comments
 (0)