scaler test

Author: ahinterl94-th

notebooks/00_spotPython_tests.ipynb

@@ -4064,6 +4064,56 @@
         "print(f\"S.y: {S.y}\")"
       ]
     },
+    {
+      "cell_type": "code",
+      "execution_count": 25,
+      "metadata": {},
+      "outputs": [
+        {
+          "name": "stdout",
+          "output_type": "stream",
+          "text": [
+            "LightDataModule.setup(): stage: None\n",
+            "train_size: 0.25, val_size: 0.25 used for train & val data.\n",
+            "test_size: 0.5 used for test dataset.\n",
+            "test_size: 0.5 used for predict dataset.\n",
+            "LightDataModule.train_dataloader(). data_train size: 5160\n"
+          ]
+        }
+      ],
+      "source": [
+        "import torch\n",
+        "from torch.utils.data import DataLoader\n",
+        "from spotPython.data.lightdatamodule import LightDataModule\n",
+        "from spotPython.data.csvdataset import CSVDataset\n",
+        "from spotPython.utils.scaler import TorchStandardScaler\n",
+        "from spotPython.data.california_housing import CaliforniaHousing\n",
+        "\n",
+        "\n",
+        "dataset = CaliforniaHousing(feature_type=torch.float32, target_type=torch.float32)\n",
+        "scaler = TorchStandardScaler()\n",
+        "data_module = LightDataModule(dataset=dataset, batch_size=5, test_size=0.5, scaler=scaler)\n",
+        "data_module.setup()\n",
+        "\n",
+        "loader = data_module.train_dataloader\n",
+        "\n",
+        "total_sum = None\n",
+        "total_count = 0\n",
+        "\n",
+        "# Iterate over batches in the DataLoader\n",
+        "for batch in loader():\n",
+        "    inputs, targets = batch\n",
+        "    if total_sum is None:\n",
+        "        total_sum = inputs.sum(dim=0)\n",
+        "    else:\n",
+        "        total_sum += inputs.sum(dim=0)\n",
+        "    total_count += inputs.shape[0]\n",
+        "\n",
+        "# Calculate the mean over all inputs\n",
+        "mean_inputs = total_sum / total_count\n",
+        "assert mean_inputs.mean() < 0.00001"
+      ]
+    },
     {
       "cell_type": "code",
       "execution_count": null,

test/test_scaler.py

@@ -0,0 +1,33 @@
+import torch
+from spotPython.data.lightdatamodule import LightDataModule
+from spotPython.data.csvdataset import CSVDataset
+from spotPython.utils.scaler import TorchStandardScaler
+from spotPython.data.california_housing import CaliforniaHousing
+
+def test_scaler():
+    dataset = CaliforniaHousing(feature_type=torch.float32, target_type=torch.float32)
+    scaler = TorchStandardScaler()
+    data_module = LightDataModule(dataset=dataset, batch_size=5, test_size=0.5, scaler=scaler)
+    data_module.setup()
+
+    loader = data_module.train_dataloader
+
+    total_sum = None
+    total_count = 0
+
+    # Iterate over batches in the DataLoader
+    for batch in loader():
+        inputs, targets = batch
+        if total_sum is None:
+            total_sum = inputs.sum(dim=0)
+        else:
+            total_sum += inputs.sum(dim=0)
+        total_count += inputs.shape[0]
+
+    # Calculate the mean over all inputs
+    mean_inputs = total_sum / total_count
+    overall_mean = mean_inputs.mean()
+    #assert that overall mean goes against zero
+    assert overall_mean < 0.00001
+    
+