v0.1.1

Author: bartzbeielstein

notebooks/14_spot_ray_hpt_torch_cifar10.ipynb

@@ -7,7 +7,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "spotPython"
-version = "0.1.0"
+version = "0.1.1"
 authors = [
   { name="T. Bartz-Beielstein", email="tbb@bartzundbartz.de" }
 ]

pyproject.toml

@@ -43,7 +43,23 @@
             "transform": "None",
             "lower": 2,
             "upper": 10
-        }
+        },
+        "optimizer": {
+            "levels": ["Adadelta", "Adagrad", "Adam", "AdamW", "SparseAdam", "Adamax", "ASGD", "LBFGS", "NAdam", "RAdam", "RMSprop", "Rprop", "SGD"],
+            "type": "factor",
+            "default": "SGD",
+            "transform": "None",
+            "core_model_parameter_type": "str",
+            "lower": 0,
+            "upper": 12},
+        "criterion": {
+            "levels": ["L1Loss", "MSELoss", "CrossEntropyLoss", "CTCLoss", "NLLLoss", "PoissonNLLLoss", "GaussianNLLLoss", "KLDivLoss", "BCELoss", "BCEWithLogitsLoss", "MarginRankingLoss", "HingeEmbeddingLoss", "MultiLabelMarginLoss", "HuberLoss", "SmoothL1Loss", "SoftMarginLoss", "MultiLabelSoftMarginLoss", "CosineEmbeddingLoss", "MultiMarginLoss", "TripletMarginLoss", "TripletMarginWithDistanceLoss"],
+            "type": "factor",
+            "default": "CrossEntropyLoss",
+            "transform": "None",
+            "core_model_parameter_type": "str",
+            "lower": 0,
+            "upper": 20}
     },
     "Net_CIFAR10":
     {
@@ -89,7 +105,25 @@
             "transform": "None",
             "lower": 2,
             "upper": 10
-        }
+        },
+        "optimizer": {
+            "levels": ["Adadelta", "Adagrad", "Adam", "AdamW", "SparseAdam", "Adamax", "ASGD", "LBFGS", "NAdam", "RAdam", "RMSprop", "Rprop", "SGD"],
+            "type": "factor",
+            "default": "SGD",
+            "transform": "None",
+            "class_name": "torch.optim",
+            "core_model_parameter_type": "instance",
+            "lower": 0,
+            "upper": 12},
+        "criterion": {
+            "levels": ["L1Loss", "MSELoss", "CrossEntropyLoss", "CTCLoss", "NLLLoss", "PoissonNLLLoss", "GaussianNLLLoss", "KLDivLoss", "BCELoss", "BCEWithLogitsLoss", "MarginRankingLoss", "HingeEmbeddingLoss", "MultiLabelMarginLoss", "HuberLoss", "SmoothL1Loss", "SoftMarginLoss", "MultiLabelSoftMarginLoss", "CosineEmbeddingLoss", "MultiMarginLoss", "TripletMarginLoss", "TripletMarginWithDistanceLoss"],
+            "type": "factor",
+            "default": "CrossEntropyLoss",
+            "transform": "None",
+            "class_name": "torch.nn",
+            "core_model_parameter_type": "instance()",
+            "lower": 0,
+            "upper": 20}
     },
     "Template":
     {

src/spotPython/data/torch_hyper_dict.json

@@ -4,9 +4,15 @@
 
 
 class Net_CIFAR10(netcore.Net_Core):
-    def __init__(self, l1, l2, lr, batch_size, epochs, k_folds, patience):
+    def __init__(self, l1, l2, lr, batch_size, epochs, k_folds, patience, criterion, optimizer):
         super(Net_CIFAR10, self).__init__(
-            lr=lr, batch_size=batch_size, epochs=epochs, k_folds=k_folds, patience=patience
+            lr=lr,
+            batch_size=batch_size,
+            epochs=epochs,
+            k_folds=k_folds,
+            patience=patience,
+            criterion=criterion,
+            optimizer=optimizer,
         )
         self.conv1 = nn.Conv2d(3, 6, 5)
         self.pool = nn.MaxPool2d(2, 2)

src/spotPython/torch/netcifar10.py

@@ -2,10 +2,12 @@
 
 
 class Net_Core(nn.Module):
-    def __init__(self, lr, batch_size, epochs, k_folds, patience):
+    def __init__(self, lr, batch_size, epochs, k_folds, patience, criterion, optimizer):
         super(Net_Core, self).__init__()
         self.lr = lr
         self.batch_size = batch_size
         self.epochs = epochs
         self.k_folds = k_folds
         self.patience = patience
+        self.criterion = criterion
+        self.optimizer = optimizer

src/spotPython/torch/netcore.py

@@ -3,9 +3,15 @@
 
 
 class Net_fashionMNIST(netcore.Net_Core):
-    def __init__(self, l1, l2, lr, batch_size, epochs, k_folds, patience):
+    def __init__(self, l1, l2, lr, batch_size, epochs, k_folds, patience, criterion, optimizer):
         super(Net_fashionMNIST, self).__init__(
-            lr=lr, batch_size=batch_size, epochs=epochs, k_folds=k_folds, patience=patience
+            lr=lr,
+            batch_size=batch_size,
+            epochs=epochs,
+            k_folds=k_folds,
+            patience=patience,
+            criterion=criterion,
+            optimizer=optimizer,
         )
         self.flatten = nn.Flatten()
         self.linear_relu_stack = nn.Sequential(

src/spotPython/torch/netfashionMNIST.py

@@ -1,8 +1,7 @@
 import numpy as np
 from sklearn.model_selection import KFold
 import torch
-from torch import nn
-import torch.optim as optim
+from torch import nn as nn
 from spotPython.utils.device import getDevice
 from torch.utils.data import random_split
 from spotPython.utils.classes import get_additional_attributes
@@ -77,6 +76,8 @@ def evaluate_cv(
     epochs = net.epochs
     batch_size = net.batch_size
     k_folds = net.k_folds
+    criterion_instance = net.criterion
+    optimizer_instance = net.optimizer
     removed_attributes = get_additional_attributes(net)
     net = remove_attributes(net, removed_attributes)
     results = {}
@@ -88,8 +89,10 @@ def evaluate_cv(
                 print("We will use", torch.cuda.device_count(), "GPUs!")
                 net = nn.DataParallel(net)
         net.to(device)
-        criterion = nn.CrossEntropyLoss()
-        optimizer = optim.Adam(net.parameters(), lr=lr)
+        # criterion = nn.CrossEntropyLoss()
+        # optimizer = optim.Adam(net.parameters(), lr=lr)
+        optimizer = optimizer_instance(net.parameters(), lr=lr)
+        criterion = criterion_instance
         kfold = KFold(n_splits=k_folds, shuffle=shuffle)
         for fold, (train_ids, val_ids) in enumerate(kfold.split(dataset)):
             print(f"Fold: {fold + 1}")
@@ -116,6 +119,8 @@ def evaluate_cv(
     net.epochs = epochs
     net.batch_size = batch_size
     net.k_folds = k_folds
+    net.criterion = criterion_instance
+    net.optimizer = optimizer_instance
     return df_eval, df_preds
 
 
@@ -126,6 +131,8 @@ def evaluate_hold_out(
     epochs = net.epochs
     batch_size = net.batch_size
     patience = net.patience
+    criterion_instance = net.criterion
+    optimizer_instance = net.optimizer
     removed_attributes = get_additional_attributes(net)
     net = remove_attributes(net, removed_attributes)
     try:
@@ -136,9 +143,11 @@ def evaluate_hold_out(
                 print("We will use", torch.cuda.device_count(), "GPUs!")
                 net = nn.DataParallel(net)
         net.to(device)
-        criterion = nn.CrossEntropyLoss()
+        # criterion = nn.CrossEntropyLoss()
         # TODO: optimizer = optim.Adam(net.parameters(), lr=lr)
-        optimizer = optim.SGD(net.parameters(), lr=lr, momentum=0.9)
+        # optimizer = optim.SGD(net.parameters(), lr=lr, momentum=0.9)
+        optimizer = optimizer_instance(net.parameters(), lr=lr)
+        criterion = criterion_instance
         if test_dataset is None:
             trainloader, valloader = create_train_val_data_loaders(
                 dataset=train_dataset, batch_size=batch_size, shuffle=shuffle
@@ -189,6 +198,8 @@ def evaluate_hold_out(
     net.epochs = epochs
     net.batch_size = batch_size
     net.patience = patience
+    net.criterion = criterion_instance
+    net.optimizer = optimizer_instance
     print(f"Returned to Spot: Validation loss: {df_eval}")
     print("----------------------------------------------")
     return df_eval, df_preds
@@ -270,6 +281,8 @@ def train_save(net, train_dataset, shuffle, device=None, show_batch_interval=10_
     epochs = net.epochs
     batch_size = net.batch_size
     patience = net.patience
+    criterion_instance = net.criterion
+    optimizer_instance = net.optimizer
     removed_attributes = get_additional_attributes(net)
     net = remove_attributes(net, removed_attributes)
     try:
@@ -280,9 +293,11 @@ def train_save(net, train_dataset, shuffle, device=None, show_batch_interval=10_
                 print("We will use", torch.cuda.device_count(), "GPUs!")
                 net = nn.DataParallel(net)
         net.to(device)
-        criterion = nn.CrossEntropyLoss()
+        # criterion = nn.CrossEntropyLoss()
         # TODO: optimizer = optim.Adam(net.parameters(), lr=lr)
-        optimizer = optim.SGD(net.parameters(), lr=lr, momentum=0.9)
+        # optimizer = optim.SGD(net.parameters(), lr=lr, momentum=0.9)
+        optimizer = optimizer_instance(net.parameters(), lr=lr)
+        criterion = criterion_instance
         trainloader, valloader = create_train_val_data_loaders(
             dataset=train_dataset, batch_size=batch_size, shuffle=shuffle
         )
@@ -329,6 +344,8 @@ def train_save(net, train_dataset, shuffle, device=None, show_batch_interval=10_
     net.epochs = epochs
     net.batch_size = batch_size
     net.patience = patience
+    net.criterion = criterion_instance
+    net.optimizer = optimizer_instance
     print(f"Returned to Spot: Validation loss: {df_eval}")
     print("----------------------------------------------")
     return df_eval, df_preds
@@ -339,6 +356,8 @@ def test_saved(net, shuffle, test_dataset=None, device=None, show_batch_interval
     epochs = net.epochs
     batch_size = net.batch_size
     patience = net.patience
+    criterion_instance = net.criterion
+    optimizer_instance = net.optimizer
     removed_attributes = get_additional_attributes(net)
     net = remove_attributes(net, removed_attributes)
     try:
@@ -349,7 +368,8 @@ def test_saved(net, shuffle, test_dataset=None, device=None, show_batch_interval
                 print("We will use", torch.cuda.device_count(), "GPUs!")
                 net = nn.DataParallel(net)
         net.to(device)
-        criterion = nn.CrossEntropyLoss()
+        # criterion = nn.CrossEntropyLoss()
+        criterion = criterion_instance
         valloader = torch.utils.data.DataLoader(
             test_dataset, batch_size=int(batch_size), shuffle=shuffle, num_workers=0
         )
@@ -364,6 +384,8 @@ def test_saved(net, shuffle, test_dataset=None, device=None, show_batch_interval
     net.epochs = epochs
     net.batch_size = batch_size
     net.patience = patience
+    net.criterion = criterion_instance
+    net.optimizer = optimizer_instance
     print(f"Returned to Spot: Validation loss: {df_eval}")
     print("----------------------------------------------")
     return df_eval, df_preds

src/spotPython/torch/traintest.py

@@ -16,6 +16,7 @@ def fun_control_init():
                 'device': None}
     """
     fun_control = {
+        "criterion": None,
         "data": None,
         "train": None,
         "test": None,
@@ -24,6 +25,7 @@ def fun_control_init():
         "shuffle": None,
         "eval": None,
         "k_folds": None,
+        "optimizer": None,
         "device": None,
         "metric_params": {},
         "show_batch_interval": 1_000_000,