v0.2.50

initialization

Author: bartzbeielstein

pyproject.toml

@@ -7,7 +7,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "spotPython"
-version = "0.2.49"
+version = "0.2.50"
 authors = [
   { name="T. Bartz-Beielstein", email="tbb@bartzundbartz.de" }
 ]

src/spotPython/data/light_hyper_dict.json

@@ -32,7 +32,7 @@
                 "class_name": "spotPython.torch.activation",
                 "core_model_parameter_type": "instance()",
                 "lower": 0,
-                "upper": 2},
+                "upper": 5},
         "optimizer": {
             "levels": ["Adadelta",
                        "Adagrad",
@@ -71,7 +71,15 @@
             "transform": "transform_power_2_int",
             "lower": 2,
             "upper": 6
-        }
+        },
+        "initialization": {
+            "levels": ["Default", "Kaiming", "Xavier"],
+            "type": "factor",
+            "default": "Default",
+            "transform": "None",
+            "core_model_parameter_type": "str",
+            "lower": 0,
+            "upper": 2}        
     },
     "LitModel":
     {

src/spotPython/light/netlightbase.py

@@ -9,10 +9,24 @@
 
 class NetLightBase(L.LightningModule):
     def __init__(
-        self, l1, epochs, batch_size, act_fn, optimizer, dropout_prob, lr_mult, patience=3, _L_in=64, _L_out=11
+        self,
+        l1,
+        epochs,
+        batch_size,
+        initialization,
+        act_fn,
+        optimizer,
+        dropout_prob,
+        lr_mult,
+        patience=3,
+        _L_in=64,
+        _L_out=11,
     ):
         super().__init__()
-        self.save_hyperparameters()
+
+        # Attribute 'act_fn' is an instance of `nn.Module` and is already saved during checkpointing.
+        # It is recommended to ignore them using `self.save_hyperparameters(ignore=['act_fn'])`
+        self.save_hyperparameters(ignore=["act_fn"])
         self._L_out = _L_out
         if l1 < 4:
             raise ValueError("l1 must be at least 4")
@@ -21,6 +35,7 @@ def __init__(
         self.epochs = epochs
         self.patience = patience
         self.batch_size = batch_size
+        self.initialization = initialization
         self.act_fn = act_fn
         self.optimizer = optimizer
         self.dropout_prob = dropout_prob
@@ -38,10 +53,10 @@ def __init__(
             layer_size_last = layer_size
         layers += [nn.Linear(layer_sizes[-1], self._L_out)]
         # nn.Sequential summarizes a list of modules into a single module, applying them in sequence
-        self.model = nn.Sequential(*layers)
+        self.layers = nn.Sequential(*layers)
 
     def forward(self, x):
-        x = self.model(x)
+        x = self.layers(x)
         return F.softmax(x, dim=1)
 
     def training_step(self, batch):

src/spotPython/light/traintest.py

@@ -4,6 +4,7 @@
 from spotPython.utils.eda import generate_config_id
 from pytorch_lightning.loggers import TensorBoardLogger
 from lightning.pytorch.callbacks.early_stopping import EarlyStopping
+from spotPython.torch.initialization import kaiming_init, xavier_init
 
 
 def train_model(config, fun_control):
@@ -19,6 +20,13 @@ def train_model(config, fun_control):
     # Init model from datamodule's attributes
     # model = LitModel(*dm.dims, dm.num_classes)
     model = fun_control["core_model"](**config, _L_in=64, _L_out=11)
+    initialization = config["initialization"]
+    if initialization == "Xavier":
+        xavier_init(model)
+    elif initialization == "Kaiming":
+        kaiming_init(model)
+    else:
+        pass
     print(f"model: {model}")
     # Init trainer
     trainer = L.Trainer(
@@ -55,6 +63,13 @@ def test_model(config, fun_control):
     # Init model from datamodule's attributes
     # model = LitModel(*dm.dims, dm.num_classes)
     model = fun_control["core_model"](**config, _L_in=64, _L_out=11)
+    initialization = config["initialization"]
+    if initialization == "Xavier":
+        xavier_init(model)
+    elif initialization == "Kaiming":
+        kaiming_init(model)
+    else:
+        pass
     print(f"model: {model}")
     # Init trainer
     trainer = L.Trainer(
@@ -85,6 +100,13 @@ def cv_model(config, fun_control):
     num_folds = 10
     split_seed = 12345
     model = fun_control["core_model"](**config, _L_in=64, _L_out=11)
+    initialization = config["initialization"]
+    if initialization == "Xavier":
+        xavier_init(model)
+    elif initialization == "Kaiming":
+        kaiming_init(model)
+    else:
+        pass
     print(f"model: {model}")
 
     for k in range(num_folds):

src/spotPython/torch/initialization.py

@@ -0,0 +1,20 @@
+import math
+
+
+def kaiming_init(model):
+    for name, param in model.named_parameters():
+        if name.endswith(".bias"):
+            param.data.fill_(0)
+        elif name.startswith("layers.0"):  # The first layer does not have ReLU applied on its input
+            param.data.normal_(0, 1 / math.sqrt(param.shape[1]))
+        else:
+            param.data.normal_(0, math.sqrt(2) / math.sqrt(param.shape[1]))
+
+
+def xavier_init(model):
+    for name, param in model.named_parameters():
+        if name.endswith(".bias"):
+            param.data.fill_(0)
+        else:
+            bound = math.sqrt(6) / math.sqrt(param.shape[0] + param.shape[1])
+            param.data.uniform_(-bound, bound)

src/spotPython/utils/eda.py

@@ -6,6 +6,12 @@
     get_var_type,
     get_transform,
 )
+import torch
+from spotPython.light.csvdataset import CSVDataset
+from torch.utils.data import DataLoader
+import matplotlib.pyplot as plt
+import math
+import seaborn as sns
 
 
 def get_stars(input_list) -> list:
@@ -110,3 +116,58 @@ def generate_config_id(config):
     for key in config:
         config_id += str(config[key]) + "_"
     return config_id[:-1]
+
+
+def visualize_activations(net, device="cpu", color="C0"):
+    """Visualizes the activations of a neural network.
+    Code is based on:
+    PyTorch Lightning TUTORIAL 2: ACTIVATION FUNCTIONS,
+    Author: Phillip Lippe,
+    License: CC BY-SA.
+
+    Args:
+        net (object): A neural network.
+        device (str, optional): The device to use. Defaults to "cpu".
+        color (str, optional): The color to use. Defaults to "C0".
+    Example:
+        >>> from spotPython.hyperparameters.values import get_one_config_from_X
+        >>> X = spot_tuner.to_all_dim(spot_tuner.min_X.reshape(1,-1))
+        >>> config = get_one_config_from_X(X, fun_control)
+        >>> model = fun_control["core_model"](**config, _L_in=64, _L_out=11)
+        >>> visualize_activations(model, device="cpu", color=f"C{0}")
+    """
+    activations = {}
+    net.eval()
+    # Create an instance of CSVDataset
+    dataset = CSVDataset(csv_file="./data/VBDP/train.csv", train=True)
+    # Set batch size for DataLoader
+    batch_size = 128
+    # Create DataLoader
+    dataloader = DataLoader(dataset, batch_size=batch_size, shuffle=True)
+    # for batch in dataloader:
+    #     inputs, targets = batch
+    # small_loader = data.DataLoader(train_set, batch_size=1024)
+    inputs, _ = next(iter(dataloader))
+    with torch.no_grad():
+        layer_index = 0
+        inputs = inputs.to(device)
+        inputs = inputs.view(inputs.size(0), -1)
+        # We need to manually loop through the layers to save all activations
+        for layer_index, layer in enumerate(net.layers[:-1]):
+            inputs = layer(inputs)
+            activations[layer_index] = inputs.view(-1).cpu().numpy()
+
+    # Plotting
+    columns = 4
+    rows = math.ceil(len(activations) / columns)
+    fig, ax = plt.subplots(rows, columns, figsize=(columns * 2.7, rows * 2.5))
+    fig_index = 0
+    for key in activations:
+        key_ax = ax[fig_index // columns][fig_index % columns]
+        sns.histplot(data=activations[key], bins=50, ax=key_ax, color=color, kde=True, stat="density")
+        key_ax.set_title(f"Layer {key} - {net.layers[key].__class__.__name__}")
+        fig_index += 1
+    fig.suptitle(f"Activation distribution for activation function {net.act_fn}", fontsize=14)
+    fig.subplots_adjust(hspace=0.4, wspace=0.4)
+    plt.show()
+    plt.close()