0.18.8

Author: bartzbeielstein

RELEASE_NOTES.txt

@@ -1,18 +1,23 @@
+spotpython 0.18.8:
+
+- lightdatamodule.py:
+    - New function: setup restructured
+- lightcrossvalidationdatamodule.py:
+    - accepts verbosity as an argument
+
 
 spotpython 0.18.7:
 
-Separate train and test data sets can be passed to Lightning DataModules
+- Separate train and test data sets can be passed to Lightning DataModules
 
 spotpython 0.18.6:
 
 - split.py:
     New function: compute_lengths_from_fractions()
-
 - lightdatamodule.py:
-    train, val, test set computaion updated
+    train, val, test set computation updated
 
 spotpython 0.18.4:
-
 - architecture.py:
     documentation updated
 

pyproject.toml

@@ -7,7 +7,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "spotpython"
-version = "0.18.8"
+version = "0.18.9"
 authors = [
   { name="T. Bartz-Beielstein", email="tbb@bartzundbartz.de" }
 ]

src/spotpython/data/lightcrossvalidationdatamodule.py

@@ -25,6 +25,7 @@ class LightCrossValidationDataModule(L.LightningDataModule):
         data_dir (str): The path to the dataset. Defaults to "./data".
         num_workers (int): The number of workers for data loading. Defaults to 0.
         pin_memory (bool): Whether to pin memory for data loading. Defaults to False.
+        verbosity (int): The verbosity level. Defaults to 0.
 
     Attributes:
         data_train (Optional[Dataset]): The training dataset.
@@ -55,6 +56,7 @@ def __init__(
         num_workers: int = 0,
         pin_memory: bool = False,
         scaler: Optional[object] = None,
+        verbosity: int = 0,
     ):
         super().__init__()
         self.batch_size = batch_size
@@ -70,6 +72,7 @@ def __init__(
         self.scaler = scaler
         self.save_hyperparameters(logger=False)
         assert 0 <= self.k < self.num_splits, "incorrect fold number"
+        self.verbosity = verbosity
 
         # no data transformations
         self.transforms = None
@@ -96,9 +99,10 @@ def setup(self, stage: Optional[str] = None) -> None:
             train_indexes, val_indexes = all_splits[self.hparams.k]
             train_indexes, val_indexes = train_indexes.tolist(), val_indexes.tolist()
             self.data_train = Subset(dataset_full, train_indexes)
-            print(f"Train Dataset Size: {len(self.data_train)}")
             self.data_val = Subset(dataset_full, val_indexes)
-            print(f"Val Dataset Size: {len(self.data_val)}")
+            if self.verbosity > 0:
+                print(f"Train Dataset Size: {len(self.data_train)}")
+                print(f"Val Dataset Size: {len(self.data_val)}")
 
         if self.scaler is not None:
             # Fit the scaler on training data and transform both train and val data

src/spotpython/data/lightdatamodule.py

@@ -126,6 +126,94 @@ def prepare_data(self) -> None:
         # download
         pass
 
+    def _setup_full_data_provided(self, stage) -> None:
+        full_size = len(self.data_full)
+        test_size = self.test_size
+
+        # consider the case when test_size is a float
+        if isinstance(self.test_size, float):
+            full_train_size = 1.0 - self.test_size
+            val_size = full_train_size * self.test_size
+            train_size = full_train_size - val_size
+        else:
+            # test_size is an int, training size calculation directly based on it
+            full_train_size = full_size - self.test_size
+            val_size = floor(full_train_size * self.test_size / full_size)
+            train_size = full_size - val_size - test_size
+
+        # Assign train/val datasets for use in dataloaders
+        if stage == "fit" or stage is None:
+            if self.verbosity > 0:
+                print(f"train_size: {train_size}, val_size: {val_size} used for train & val data.")
+            generator_fit = torch.Generator().manual_seed(self.test_seed)
+            self.data_train, self.data_val, _ = random_split(self.data_full, [train_size, val_size, test_size], generator=generator_fit)
+            # Handle scaling and transformation if scaler is provided
+            if self.scaler is not None:
+                self.handle_scaling_and_transform()
+
+        # Assign test dataset for use in dataloader(s)
+        if stage == "test" or stage is None:
+            if self.verbosity > 0:
+                print(f"test_size: {test_size} used for test dataset.")
+            generator_test = torch.Generator().manual_seed(self.test_seed)
+            self.data_test, _, _ = random_split(self.data_full, [test_size, train_size, val_size], generator=generator_test)
+            if self.scaler is not None:
+                # Transform the test data
+                self.data_test = self.transform_dataset(self.data_test)
+
+        # Assign pred dataset for use in dataloader(s)
+        if stage == "predict" or stage is None:
+            if self.verbosity > 0:
+                print(f"test_size: {test_size} used for predict dataset.")
+            generator_predict = torch.Generator().manual_seed(self.test_seed)
+            self.data_predict, _, _ = random_split(self.data_full, [test_size, train_size, val_size], generator=generator_predict)
+            if self.scaler is not None:
+                # Transform the predict data
+                self.data_predict = self.transform_dataset(self.data_predict)
+
+    def _setup_test_data_provided(self, stage) -> None:
+        # New functionality with separate full_train and test datasets. Use these datasets directly.
+        full_train_size = len(self.data_full_train)
+        test_size = self.test_size
+        # consider the case when test_size is a float
+        if isinstance(self.test_size, float):
+            val_size = self.test_size
+            train_size = 1 - self.test_size
+        else:
+            # test_size is an int, training size calculation directly based on it
+            full_size = len(self.data_full_train) + len(self.data_test)
+            full_train_size = len(self.data_full_train)
+            val_size = floor(full_train_size * self.test_size / full_size)
+            train_size = full_train_size - val_size
+
+        # Assign train/val datasets for use in dataloaders
+        if stage == "fit" or stage is None:
+            if self.verbosity > 0:
+                print(f"train_size: {train_size}, val_size: {val_size} used for train & val data.")
+            generator_fit = torch.Generator().manual_seed(self.test_seed)
+            self.data_train, self.data_val = random_split(self.data_full_train, [train_size, val_size], generator=generator_fit)
+            # Handle scaling and transformation if scaler is provided
+            if self.scaler is not None:
+                self.handle_scaling_and_transform()
+
+        # Assign test dataset for use in dataloader(s)
+        if stage == "test" or stage is None:
+            if self.verbosity > 0:
+                print(f"test_size: {test_size} used for test dataset.")
+            self.data_test = self.data_test
+            if self.scaler is not None:
+                # Transform the test data
+                self.data_test = self.transform_dataset(self.data_test)
+
+        # Assign pred dataset for use in dataloader(s)
+        if stage == "predict" or stage is None:
+            if self.verbosity > 0:
+                print(f"test_size: {test_size} used for predict dataset.")
+            self.data_predict = self.data_test
+            if self.scaler is not None:
+                # Transform the predict data
+                self.data_predict = self.transform_dataset(self.data_predict)
+
     def setup(self, stage: Optional[str] = None) -> None:
         """
         Splits the data for use in training, validation, and testing.
@@ -151,91 +239,9 @@ def setup(self, stage: Optional[str] = None) -> None:
 
         """
         if self.data_full is not None:
-            full_size = len(self.data_full)
-            test_size = self.test_size
-
-            # consider the case when test_size is a float
-            if isinstance(self.test_size, float):
-                full_train_size = 1.0 - self.test_size
-                val_size = full_train_size * self.test_size
-                train_size = full_train_size - val_size
-            else:
-                # test_size is an int, training size calculation directly based on it
-                full_train_size = full_size - self.test_size
-                val_size = floor(full_train_size * self.test_size / full_size)
-                train_size = full_size - val_size - test_size
-
-            # Assign train/val datasets for use in dataloaders
-            if stage == "fit" or stage is None:
-                if self.verbosity > 0:
-                    print(f"train_size: {train_size}, val_size: {val_size} used for train & val data.")
-                generator_fit = torch.Generator().manual_seed(self.test_seed)
-                self.data_train, self.data_val, _ = random_split(self.data_full, [train_size, val_size, test_size], generator=generator_fit)
-                # Handle scaling and transformation if scaler is provided
-                if self.scaler is not None:
-                    self.handle_scaling_and_transform()
-
-            # Assign test dataset for use in dataloader(s)
-            if stage == "test" or stage is None:
-                if self.verbosity > 0:
-                    print(f"test_size: {test_size} used for test dataset.")
-                generator_test = torch.Generator().manual_seed(self.test_seed)
-                self.data_test, _, _ = random_split(self.data_full, [test_size, train_size, val_size], generator=generator_test)
-                if self.scaler is not None:
-                    # Transform the test data
-                    self.data_test = self.transform_dataset(self.data_test)
-
-            # Assign pred dataset for use in dataloader(s)
-            if stage == "predict" or stage is None:
-                if self.verbosity > 0:
-                    print(f"test_size: {test_size} used for predict dataset.")
-                generator_predict = torch.Generator().manual_seed(self.test_seed)
-                self.data_predict, _, _ = random_split(self.data_full, [test_size, train_size, val_size], generator=generator_predict)
-                if self.scaler is not None:
-                    # Transform the predict data
-                    self.data_predict = self.transform_dataset(self.data_predict)
+            self._setup_full_data_provided(stage)
         else:
-            # New functionality with separate full_train and test datasets. Use these datasets directly.
-            full_train_size = len(self.data_full_train)
-            test_size = self.test_size
-            # consider the case when test_size is a float
-            if isinstance(self.test_size, float):
-                val_size = self.test_size
-                train_size = 1 - self.test_size
-            else:
-                # test_size is an int, training size calculation directly based on it
-                full_size = len(self.data_full_train) + len(self.data_test)
-                full_train_size = len(self.data_full_train)
-                val_size = floor(full_train_size * self.test_size / full_size)
-                train_size = full_train_size - val_size
-
-            # Assign train/val datasets for use in dataloaders
-            if stage == "fit" or stage is None:
-                if self.verbosity > 0:
-                    print(f"train_size: {train_size}, val_size: {val_size} used for train & val data.")
-                generator_fit = torch.Generator().manual_seed(self.test_seed)
-                self.data_train, self.data_val = random_split(self.data_full_train, [train_size, val_size], generator=generator_fit)
-                # Handle scaling and transformation if scaler is provided
-                if self.scaler is not None:
-                    self.handle_scaling_and_transform()
-
-            # Assign test dataset for use in dataloader(s)
-            if stage == "test" or stage is None:
-                if self.verbosity > 0:
-                    print(f"test_size: {test_size} used for test dataset.")
-                self.data_test = self.data_test
-                if self.scaler is not None:
-                    # Transform the test data
-                    self.data_test = self.transform_dataset(self.data_test)
-
-            # Assign pred dataset for use in dataloader(s)
-            if stage == "predict" or stage is None:
-                if self.verbosity > 0:
-                    print(f"test_size: {test_size} used for predict dataset.")
-                self.data_predict = self.data_test
-                if self.scaler is not None:
-                    # Transform the predict data
-                    self.data_predict = self.transform_dataset(self.data_predict)
+            self._setup_test_data_provided(stage)
 
     def train_dataloader(self) -> DataLoader:
         """