feat(containerlab): Add two-node gateway canary

.github/workflows/publish.yaml

@@ -32,6 +32,19 @@ jobs:
       platforms: linux/amd64,linux/arm64
     secrets: inherit
 
+  publish-galactic-gateway-image:
+    permissions:
+      id-token: write
+      contents: read
+      packages: write
+      attestations: write
+    uses: datum-cloud/actions/.github/workflows/publish-docker.yaml@v1.20.0
+    with:
+      image-name: galactic-gateway
+      dockerfile-path: containers/galactic-gateway/Dockerfile
+      platforms: linux/amd64,linux/arm64
+    secrets: inherit
+
   publish-fabric-router-image:
     permissions:
       id-token: write
@@ -46,7 +59,13 @@ jobs:
     secrets: inherit
 
   publish-kustomize-bundles:
-    needs: [publish-galactic-cni-image, publish-galactic-router-image, publish-fabric-router-image]
+    needs:
+      [
+        publish-galactic-cni-image,
+        publish-galactic-router-image,
+        publish-galactic-gateway-image,
+        publish-fabric-router-image,
+      ]
     permissions:
       id-token: write
       contents: read
@@ -58,5 +77,12 @@ jobs:
       images: |
         - { path: config/cni, name: ghcr.io/datum-cloud/galactic-cni, tag: "${{ needs.publish-galactic-cni-image.outputs.tag }}" }
         - { path: config/router/base, name: ghcr.io/datum-cloud/galactic-router, tag: "${{ needs.publish-galactic-router-image.outputs.tag }}" }
+        # config/gateway/base's DaemonSet has two containers (galactic-router
+        # + galactic-gateway) -- the config/router/base entry above only
+        # stamps images within that path's own tree, not
+        # config/gateway/base's, so galactic-router's image needs its own
+        # entry here too, in addition to galactic-gateway's.
+        - { path: config/gateway/base, name: ghcr.io/datum-cloud/galactic-router, tag: "${{ needs.publish-galactic-router-image.outputs.tag }}" }
+        - { path: config/gateway/base, name: ghcr.io/datum-cloud/galactic-gateway, tag: "${{ needs.publish-galactic-gateway-image.outputs.tag }}" }
         - { path: config/fabric, name: ghcr.io/datum-cloud/fabric-router, tag: "${{ needs.publish-fabric-router-image.outputs.tag }}" }
     secrets: inherit

AGENTS.md

@@ -35,7 +35,7 @@ task test:e2e       # Kind cluster lifecycle test
 task lint           # golangci-lint; lint-fix applies safe auto-fixes
 ```
 
-There is no production release image build in this repo (`task docker-build` and the release workflow were removed after the shared image was found to advertise `galactic-router` without ever building it — see [docs/agents/ARCHITECTURE.md](docs/agents/ARCHITECTURE.md#known-constraints)). `containers/galactic-cni/Dockerfile` exists solely for `task test:e2e`.
+Production release images are built by `.github/workflows/publish.yaml`, not by any `task` in this file — see [docs/agents/ARCHITECTURE.md](docs/agents/ARCHITECTURE.md#cicd) for the full pipeline (per-binary `containers/*/Dockerfile`s pushed to `ghcr.io/datum-cloud/*`, plus a `config/` Kustomize OCI bundle). This replaced an earlier single shared image (`containers/galactic/Dockerfile`, `.github/workflows/release.yaml`, both removed — see that section's History note) that was found to advertise `galactic-router` without ever building it. `containers/galactic-cni/Dockerfile` is also used directly by `task test:e2e` (not just by the publish pipeline).
 
 **Before every PR:** `task ci` (lint → build → test:unit → test:e2e).
 
@@ -54,13 +54,14 @@ Summary:
 
 - **`config/system/`** — Creates the `galactic-system` namespace both components deploy into. Apply with `kubectl apply -k config/system/`.
 - **`config/cni/`** — Production manifests for the CNI installer DaemonSet, ConfigMap, RBAC, and ServiceAccount. Apply with `kubectl apply -k config/cni/`.
-- **`config/router/`** — Shared RBAC/ServiceAccount plus two DaemonSet roles, both running `GALACTIC_ROUTER_ROUTER_MODE=tenant`:
-  - **`config/router/tenant/`** — the per-node role (`galactic-router`); runs on every node except Kubernetes control-plane nodes and nodes labeled for the route-reflector role below.
+- **`config/router/`** — Shared RBAC/ServiceAccount plus DaemonSet roles, all running `GALACTIC_ROUTER_ROUTER_MODE=tenant`:
+  - **`config/router/tenant/`** — the per-node role (`galactic-router`); runs on every node except Kubernetes control-plane nodes and nodes labeled for the route-reflector or gateway roles.
   - **`config/router/tenant-control/`** — the BGP route-reflector role (`galactic-router-control`, `GALACTIC_ROUTER_REFLECTOR=true`); opt-in only, requires nodes labeled `galactic.datumapis.com/node: control` (stays at zero replicas otherwise). `GALACTIC_ROUTER_BGP_LOCAL_ADDRESS` is auto-detected from the host's `lo` interface by default; see the comments in `daemonset-patch.yaml` for when to override it.
-  - **`config/router/base/`** — the DaemonSet spec shared by both roles; not applied directly.
-  - Apply the whole router component (both roles) with `kubectl apply -k config/router/`, or a single role with e.g. `kubectl apply -k config/router/tenant/`.
-- **`config/fabric/`** — the FRR underlay eBGP DaemonSet (`fabric-router`; `galactic-router` needs a working underlay before it can start). Unlike `config/router/`, this is a single flat DaemonSet with no `tenant`/`tenant-control`-style role split — its affinity matches nodes labeled `galactic.datumapis.com/node` `In` `[edge, control]` directly, since (unlike galactic-router's route-reflector role) there's no env/config difference between running on a regular node vs. the control node. That affinity can legitimately match more than one node per cluster, and BGP underlay config (hostname, router-id, interface addresses, remote-AS) inherently differs per physical node — so `frr-init` gets the pod's node name via a `NODE_NAME` downward-API env var and selects a per-node `frr.conf.<nodename>` key from the ConfigMap, rather than assuming one shared `frr.conf` for the whole DaemonSet. **Not** part of the root `config/kustomization.yaml` and not covered by `kubectl apply -k config/` — unlike every other component here, it has no generic default: the deployer must hand-author a `fabric-config` ConfigMap with one `frr.conf.<nodename>` key per matching node (`daemons`/`vtysh.conf` are baked into the `fabric-router` image and only need to be in the ConfigMap if overriding those defaults) before applying `kubectl apply -k config/fabric/`.
-- **`deploy/containerlab/`** — ContainerLab topology (`gvpc.clab.yaml`) for three Kind clusters (dfw, iad, sjc) wired over an IPv6 SRv6 transit mesh. FRR runs as a hostNetwork DaemonSet on each worker for eBGP underlay; `galactic-router` (tenant role) handles EVPN path distribution over iBGP, and the iad route reflector builds on `config/router/tenant-control/`. See `deploy/containerlab/README.md` and `deploy/containerlab/Taskfile.yaml` for bring-up commands.
+  - **`config/router/base/`** — the DaemonSet spec shared by both roles above; not applied directly.
+  - Apply both roles with `kubectl apply -k config/router/`, or a single role with e.g. `kubectl apply -k config/router/tenant/`. `galactic-router` no longer has a gateway role of its own — see `config/gateway/` below.
+- **`config/gateway/`** — the edge XDP NAT+LB gateway control plane, a separate `galactic-gateway` binary rather than a `galactic-router` role, so a crash on either side no longer takes the other down with it. `config/gateway/{serviceaccount.yaml,rbac.yaml}` (safe/idempotent to apply cluster-wide) are what `kubectl apply -k config/gateway/` applies; `config/gateway/base/` (the two-container `galactic-router` + `galactic-gateway` pod, requiring nodes labeled `galactic.datumapis.com/node: gateway`) is **not** included in that kustomization and is **not** applied as-is — the same exemption as `config/fabric/` below, for the same reason: `GALACTIC_GATEWAY_SRV6_ADDRESS` must be unique per gateway node and has no generic default (no in-cluster mechanism yet derives it automatically — see `internal/controller/networkgateway_controller.go`'s `publishSelfAddress` doc comment). It's designed to be instantiated once per gateway node by a further overlay that pins it to one node (`kubernetes.io/hostname`) and sets that node's own public-interface/SRv6-address values; see `deploy/containerlab/resources/galactic-router-gateway/` for a worked two-node example. Also **not** part of the root `config/kustomization.yaml`'s default resource list, matching `config/fabric/`'s exemption.
+- **`config/fabric/`** — the FRR underlay eBGP DaemonSet (`fabric-router`; `galactic-router` needs a working underlay before it can start). Unlike `config/router/`, this is a single flat DaemonSet with no `tenant`/`tenant-control`-style role split — its affinity matches nodes labeled `galactic.datumapis.com/node` `In` `[edge, control, gateway]` directly, since (unlike galactic-router's route-reflector role and the gateway role above) there's no env/config difference between running on a regular node vs. those roles — fabric-router itself is identical everywhere; only the per-node BGP underlay config it reads from the ConfigMap differs. That affinity can legitimately match more than one node per cluster, and BGP underlay config (hostname, router-id, interface addresses, remote-AS) inherently differs per physical node — so `frr-init` gets the pod's node name via a `NODE_NAME` downward-API env var and selects a per-node `frr.conf.<nodename>` key from the ConfigMap, rather than assuming one shared `frr.conf` for the whole DaemonSet. **Not** part of the root `config/kustomization.yaml` and not covered by `kubectl apply -k config/` — unlike every other component here, it has no generic default: the deployer must hand-author a `fabric-config` ConfigMap with one `frr.conf.<nodename>` key per matching node (`daemons`/`vtysh.conf` are baked into the `fabric-router` image and only need to be in the ConfigMap if overriding those defaults) before applying `kubectl apply -k config/fabric/`.
+- **`deploy/containerlab/`** — ContainerLab topology (`gvpc.clab.yaml`) for three Kind clusters (dfw, iad, sjc) wired over an IPv6 SRv6 transit mesh. FRR runs as a hostNetwork DaemonSet on each worker for eBGP underlay; `galactic-router` (tenant role) handles EVPN path distribution over iBGP, the iad route reflector builds on `config/router/tenant-control/`, and iad additionally has two dedicated gateway-role nodes (`iad-gateway1`/`iad-gateway2`, `config/gateway/base/`) as a canary for the edge XDP NAT+LB gateway. See `deploy/containerlab/README.md` and `deploy/containerlab/Taskfile.yaml` for bring-up commands.
 
 ## New Developer Entry Points
 

Taskfile.yaml

@@ -140,6 +140,7 @@ tasks:
       - go build -ldflags "{{.LDFLAGS}}" -o bin/galactic-bgp ./cmd/galactic-bgp
       - go build -ldflags "{{.LDFLAGS}}" -o bin/galactic-route ./cmd/galactic-route
       - go build -ldflags "{{.LDFLAGS}}" -o bin/galactic-router ./cmd/galactic-router
+      - go build -ldflags "{{.LDFLAGS}}" -o bin/galactic-gateway ./cmd/galactic-gateway
       - go build -ldflags "{{.LDFLAGS}}" -o bin/vmtap-cni ./cmd/vmtap-cni
       - GOBIN={{.LOCALBIN}} go install github.com/containernetworking/plugins/plugins/main/host-device@v1.9.1
 

cmd/galactic-gateway/gateway.go

@@ -0,0 +1,116 @@
+// Copyright 2026 Datum Cloud, Inc.
+//
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+package main
+
+import (
+	"fmt"
+	"net"
+	"net/netip"
+
+	"github.com/cilium/ebpf/link"
+	"github.com/prometheus/client_golang/prometheus"
+
+	"go.datum.net/galactic/internal/gateway"
+	"go.datum.net/galactic/internal/plumbing/ebpf/edgeattach"
+	"go.datum.net/galactic/internal/plumbing/ebpf/edgemetrics"
+	"go.datum.net/galactic/internal/plumbing/ebpf/edgeprog"
+)
+
+// gatewayDatapathKeepAlive holds the loaded *edgeprog.EdgenatObjects and
+// the attached link.Link for the life of this process, once
+// setupGatewayDatapath's attach path succeeds. Neither is Closed anywhere
+// in this file — see setupGatewayDatapath's doc comment for why — but a
+// value that isn't stored somewhere reachable is exactly as good as
+// Closed: cilium/ebpf's *ebpf.Program, *ebpf.Map, and link.Link types all
+// register a runtime finalizer that closes their underlying fd once the
+// garbage collector determines nothing reachable still points at them,
+// with no error surfaced anywhere when that happens. gateway.KernelDatapath
+// only keeps objs.RuleTable (via edgemap.KernelTable) alive on its own, so
+// without this package-level var, objs.EdgeNat (the program) and the
+// link.Link returned by Attach — the two things actually keeping this
+// node's XDP attachment live on the wire — would eventually get GC'd and
+// silently detached, with every control-plane signal (the DaemonSet pod
+// healthy, ApplyRule succeeding, rule_table metrics populated) still
+// looking completely normal. Confirmed live: this is exactly what happened
+// the first time this path was ever exercised against a real interface
+// (ingress traffic for a registered rule was never intercepted at all,
+// bouncing between this node and its transit-facing peer via ordinary
+// kernel routing instead) — no unit test exercises this path with a real
+// attach for a GC cycle to occur during, and Phase D's "manifests and the
+// live pod/eBPF path" validation predates any live underlay BGP peering
+// that would have delivered real traffic to notice the gap.
+//
+// This var, and the rest of this file, moved here unchanged from
+// cmd/galactic-router/gateway.go: the edge NAT+LB gateway datapath now
+// lives in its own process rather than sharing one with the tenant BGP
+// reconcilers.
+var gatewayDatapathKeepAlive struct {
+	objs *edgeprog.EdgenatObjects
+	link link.Link
+}
+
+// setupGatewayDatapath loads and attaches the edge NAT+LB eBPF datapath to
+// publicInterface and returns the gateway.Datapath this node's Engine
+// should use. Unlike cmd/galactic-router's identically-named predecessor
+// (now removed), publicInterface and
+// srv6Address are both required here, not a jointly-optional pair with a
+// gateway.NoopDatapath{} fallback: config.GatewayConfig.Validate already
+// rejects either being empty before runCmd ever calls this function, since
+// this binary only exists to run the gateway role.
+//
+// The loaded *edgeprog.EdgenatObjects and the returned link.Link are
+// stashed in gatewayDatapathKeepAlive (see that var's doc comment for why)
+// rather than Closed here: they, and the XDP attachment itself, must
+// survive for the life of this process — same convention as
+// internal/plumbing/ebpf/attach.Start's identical choice for the SRv6 uSID
+// datapath.
+//
+// metricsReg additionally gets an edgemetrics.Collector registered against
+// it once objs is loaded, reading rule_table/conn_table/drop_reasons live
+// at every scrape — see that package's doc comment for why this is a
+// pull-based Collector rather than incrementally-updated Gauges.
+func setupGatewayDatapath(
+	publicInterface, srv6Address string, metricsReg prometheus.Registerer,
+) (gateway.Datapath, error) {
+	gwAddr, err := netip.ParseAddr(srv6Address)
+	if err != nil {
+		return nil, fmt.Errorf("parse gateway SRv6 address %q: %w", srv6Address, err)
+	}
+
+	objs, err := edgeattach.Load(edgeattach.PinDir)
+	if err != nil {
+		return nil, fmt.Errorf("load edge gateway eBPF datapath: %w", err)
+	}
+
+	xdpLink, err := edgeattach.Attach(objs.EdgeNat, publicInterface)
+	if err != nil {
+		_ = objs.Close()
+		return nil, fmt.Errorf("attach edge gateway datapath to public interface %q: %w", publicInterface, err)
+	}
+
+	if _, err := net.InterfaceByName(publicInterface); err != nil {
+		_ = xdpLink.Close()
+		_ = objs.Close()
+		return nil, fmt.Errorf("resolve public interface %q: %w", publicInterface, err)
+	}
+
+	datapath, err := gateway.NewKernelDatapath(objs, gwAddr)
+	if err != nil {
+		_ = xdpLink.Close()
+		_ = objs.Close()
+		return nil, fmt.Errorf("construct kernel datapath: %w", err)
+	}
+
+	if err := metricsReg.Register(edgemetrics.NewCollectorFromObjects(objs)); err != nil {
+		_ = xdpLink.Close()
+		_ = objs.Close()
+		return nil, fmt.Errorf("register edge gateway metrics collector: %w", err)
+	}
+
+	gatewayDatapathKeepAlive.objs = objs
+	gatewayDatapathKeepAlive.link = xdpLink
+
+	return datapath, nil
+}

cmd/galactic-gateway/gateway_test.go

@@ -0,0 +1,24 @@
+// Copyright 2026 Datum Cloud, Inc.
+//
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+package main
+
+import (
+	"testing"
+
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+// TestSetupGatewayDatapath_InvalidAddressIsError covers the address-parse
+// failure path, which runs before any kernel/eBPF interaction -- no root
+// needed. Unlike cmd/galactic-router's removed identically-named test
+// file, there is no "empty interface is a no-op" case to cover here: this
+// binary's config.GatewayConfig.Validate rejects an empty
+// PublicInterface/SRv6Address before setupGatewayDatapath is ever called.
+func TestSetupGatewayDatapath_InvalidAddressIsError(t *testing.T) {
+	_, err := setupGatewayDatapath("eth0", "not-an-ip-address", prometheus.NewRegistry())
+	if err == nil {
+		t.Error("setupGatewayDatapath with an invalid SRv6 address: want an error, got nil")
+	}
+}