0.10.52

Skip Linear uses Normal Dist Weights init

Author: bartzbeielstein

notebooks/00_spotPython_tests.ipynb

@@ -2852,6 +2852,65 @@
         "        result = result.reshape(-1, self.n_out)\n",
         "        return result\n"
       ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "class SkipLinear(torch.nn.Module):\n",
+        "\n",
+        "    class Core(torch.nn.Module):\n",
+        "        \"\"\"A simple linear layer with n outputs.\"\"\"\n",
+        "\n",
+        "        def __init__(self, n):\n",
+        "            \"\"\"\n",
+        "            Initialize the layer.\n",
+        "\n",
+        "            Args:\n",
+        "                n (int): The number of output nodes.\n",
+        "            \"\"\"\n",
+        "            super().__init__()\n",
+        "            self.weights = torch.nn.Parameter(torch.zeros((n, 1), dtype=torch.float32))\n",
+        "            self.biases = torch.nn.Parameter(torch.zeros(n, dtype=torch.float32))\n",
+        "            lim = 0.01\n",
+        "            torch.nn.init.uniform_(self.weights, -lim, lim)\n",
+        "\n",
+        "        def forward(self, x) -> torch.Tensor:\n",
+        "            \"\"\"\n",
+        "            Forward pass through the layer.\n",
+        "\n",
+        "            Args:\n",
+        "                x (torch.Tensor): The input tensor.\n",
+        "\n",
+        "            Returns:\n",
+        "                torch.Tensor: The output of the layer.\n",
+        "            \"\"\"\n",
+        "            return x @ self.weights.t() + self.biases\n",
+        "\n",
+        "    def __init__(self, n_in, n_out):\n",
+        "        super().__init__()\n",
+        "        self.n_in = n_in\n",
+        "        self.n_out = n_out\n",
+        "        if n_out % n_in != 0:\n",
+        "            raise ValueError(\"n_out % n_in != 0\")\n",
+        "        n = n_out // n_in  # num nodes per input\n",
+        "\n",
+        "        self.lst_modules = torch.nn.ModuleList([SkipLinear.Core(n) for i in range(n_in)])\n",
+        "\n",
+        "    def forward(self, x):\n",
+        "        lst_nodes = []\n",
+        "        for i in range(self.n_in):\n",
+        "            xi = x[:, i].reshape(-1, 1)\n",
+        "            oupt = self.lst_modules[i](xi)\n",
+        "            lst_nodes.append(oupt)\n",
+        "        result = torch.cat((lst_nodes[0], lst_nodes[1]), 1)\n",
+        "        for i in range(2, self.n_in):\n",
+        "            result = torch.cat((result, lst_nodes[i]), 1)\n",
+        "        result = result.reshape(-1, self.n_out)\n",
+        "        return result"
+      ]
     }
   ],
   "metadata": {

pyproject.toml

@@ -7,7 +7,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "spotPython"
-version = "0.10.53"
+version = "0.10.54"
 authors = [
   { name="T. Bartz-Beielstein", email="tbb@bartzundbartz.de" }
 ]

src/spotPython/light/transformer/skiplinear.py

@@ -61,10 +61,10 @@ def __init__(self, n):
                 n (int): The number of output nodes.
             """
             super().__init__()
-            self.weights = torch.nn.Parameter(torch.zeros((n, 1), dtype=torch.float32))
-            self.biases = torch.nn.Parameter(torch.zeros(n, dtype=torch.float32))
-            lim = 0.01
-            torch.nn.init.uniform_(self.weights, -lim, lim)
+            # initialize with random weights using normal distribution
+            self.weights = torch.nn.Parameter(torch.randn(1, n))
+            # self.weights = torch.nn.Parameter(torch.rand(1, n) * 2 - 1)
+            self.linear = torch.nn.Linear(1, n)
 
         def forward(self, x) -> torch.Tensor:
             """
@@ -76,7 +76,7 @@ def forward(self, x) -> torch.Tensor:
             Returns:
                 torch.Tensor: The output of the layer.
             """
-            return x @ self.weights.t() + self.biases
+            return self.linear(x)
 
     def __init__(self, n_in, n_out):
         super().__init__()
@@ -86,16 +86,12 @@ def __init__(self, n_in, n_out):
             raise ValueError("n_out % n_in != 0")
         n = n_out // n_in  # num nodes per input
 
-        self.lst_modules = torch.nn.ModuleList([SkipLinear.Core(n) for i in range(n_in)])
+        self.lst_modules = torch.nn.ModuleList([SkipLinear.Core(n) for _ in range(n_in)])
 
     def forward(self, x):
-        lst_nodes = []
-        for i in range(self.n_in):
-            xi = x[:, i].reshape(-1, 1)
-            oupt = self.lst_modules[i](xi)
-            lst_nodes.append(oupt)
-        result = torch.cat((lst_nodes[0], lst_nodes[1]), 1)
-        for i in range(2, self.n_in):
-            result = torch.cat((result, lst_nodes[i]), 1)
-        result = result.reshape(-1, self.n_out)
-        return result
+        # We want to apply each module to a slice of the input tensor x and collect the outputs.
+        # This applies the i-th module to the i-th column of x, reshaped as a column vector.
+        # The result is a list of output tensors, which are then concatenated to form the final output.
+        lst_nodes = [self.lst_modules[i](x[:, i].unsqueeze(1)) for i in range(self.n_in)]
+        result = torch.cat(lst_nodes, dim=1)
+        return result.reshape(-1, self.n_out)