From 33d1c4f322bb3364e1bb22ff82a08d1233216d4b Mon Sep 17 00:00:00 2001 From: Peter Sprygada Date: Wed, 12 Aug 2026 13:47:14 -0400 Subject: [PATCH] feat(containerlab): add two-node gateway canary for the edge XDP NAT+LB gateway Extends the containerlab lab with iad-gateway1/iad-gateway2, a dedicated active-active pair of gateway-role nodes exercising config/gateway/ end to end: - gvpc.clab.yaml, node_files/iad/config.yaml: two new gateway nodes, tainted galactic.datumapis.com/node=gateway:NoSchedule, uplinked to tr3 (node_files/tr3/frr.conf). - resources/fabric-router/iad/frr.conf.iad-gateway{1,2}: per-node FRR underlay eBGP config for the new nodes; existing per-site frr.conf files renamed to frr.conf. (dfw-worker, iad-worker, sjc-worker) to make room for the per-node key scheme fabric-router's ConfigMap needs once a site has more than one matching node. - resources/galactic-control/iad/bgppeer-gateway{1,2}.yaml: route reflector BGPPeer objects for the two new gateway nodes' iBGP sessions. - resources/galactic-router-gateway/: per-node overlay instantiating config/gateway/base/ for iad-gateway1/iad-gateway2 -- BGPPeer, BGPRouter, NetworkGateway, and the node-pinning patch (kubernetes.io/hostname, public interface, SRv6 address) each needs. iad/networkrule-ns60.yaml is a canary NetworkRule pointing at ns60's nginx pod. - resources/tenants/ns60/: replaces ns50 (removed) as the gateway canary's backend tenant -- a plain namespace + pod instead of ns50's multi-site Deployment, since the gateway canary only needs one reachable backend address to route traffic to. - scripts/deploy-fabric.sh, deploy-galactic-router.sh, deploy-system.sh: wire the new per-node resources into the deploy pipeline; verify-ns50.sh removed with ns50. - README.md, docs/tenants.md, docs/verification.md, Taskfile.yaml: document the new topology and canary. - deploy-system.sh, Taskfile.yaml: NetworkGateway/NetworkRule CRDs now fetch from datum-cloud/network like every other BGP CRD, and the galactic-router/-gateway/-cni image builds no longer need the '--build-context network=...' pointing at a sibling checkout -- both were stopgaps for those types not yet being upstream, which is no longer the case as of this stack's third branch. Seventh branch in the edge-gateway stack; builds on feat/edge-gateway-06-config-manifests. Co-Authored-By: Claude Sonnet 5 --- deploy/containerlab/README.md | 64 +++++++++++---- deploy/containerlab/Taskfile.yaml | 63 ++++++++++++--- deploy/containerlab/docs/tenants.md | 59 ++++++-------- deploy/containerlab/docs/verification.md | 4 +- deploy/containerlab/gvpc.clab.yaml | 79 +++++++++++++++++++ .../containerlab/node_files/iad/config.yaml | 36 +++++++++ deploy/containerlab/node_files/tr3/frr.conf | 14 ++++ .../fabric-router/base/fabric-lab-patch.yaml | 17 +++- .../dfw/{frr.conf => frr.conf.dfw-worker} | 0 .../fabric-router/dfw/kustomization.yaml | 2 +- .../fabric-router/iad/frr.conf.iad-gateway1 | 42 ++++++++++ .../fabric-router/iad/frr.conf.iad-gateway2 | 31 ++++++++ .../iad/{frr.conf => frr.conf.iad-worker} | 0 .../fabric-router/iad/kustomization.yaml | 4 +- .../sjc/{frr.conf => frr.conf.sjc-worker} | 0 .../fabric-router/sjc/kustomization.yaml | 2 +- .../iad/bgppeer-gateway1.yaml | 13 +++ .../iad/bgppeer-gateway2.yaml | 13 +++ .../galactic-control/iad/kustomization.yaml | 2 + .../base/kustomization.yaml | 15 ++++ .../base/router-lab-patch.yaml | 14 ++++ .../iad-gateway1/bgppeer.yaml | 14 ++++ .../iad-gateway1/bgprouter.yaml | 18 +++++ .../iad-gateway1/kustomization.yaml | 29 +++++++ .../iad-gateway1/networkgateway.yaml | 9 +++ .../iad-gateway1/node-patch.yaml | 52 ++++++++++++ .../iad-gateway2/bgppeer.yaml | 14 ++++ .../iad-gateway2/bgprouter.yaml | 18 +++++ .../iad-gateway2/kustomization.yaml | 21 +++++ .../iad-gateway2/networkgateway.yaml | 9 +++ .../iad-gateway2/node-patch.yaml | 52 ++++++++++++ .../iad/kustomization.yaml | 9 +++ .../iad/networkrule-ns60.yaml | 30 +++++++ .../tenants/ns50/base/kustomization.yaml | 17 ---- .../resources/tenants/ns50/dfw/nad.yaml | 52 ------------ .../tenants/ns50/iad/kustomization.yaml | 3 - .../resources/tenants/ns50/iad/nad.yaml | 31 -------- .../tenants/ns50/sjc/kustomization.yaml | 3 - .../tenants/ns60/base/kustomization.yaml | 13 +++ .../tenants/ns60/base/namespace.yaml | 14 ++++ .../resources/tenants/ns60/base/pod.yaml | 35 ++++++++ .../{ns50/dfw => ns60/iad}/kustomization.yaml | 0 .../tenants/{ns50/sjc => ns60/iad}/nad.yaml | 14 ++-- deploy/containerlab/scripts/deploy-fabric.sh | 27 ++++--- .../scripts/deploy-galactic-router.sh | 52 +++++++++++- deploy/containerlab/scripts/deploy-system.sh | 7 +- deploy/containerlab/scripts/verify-ns50.sh | 28 ------- 47 files changed, 820 insertions(+), 225 deletions(-) rename deploy/containerlab/resources/fabric-router/dfw/{frr.conf => frr.conf.dfw-worker} (100%) create mode 100644 deploy/containerlab/resources/fabric-router/iad/frr.conf.iad-gateway1 create mode 100644 deploy/containerlab/resources/fabric-router/iad/frr.conf.iad-gateway2 rename deploy/containerlab/resources/fabric-router/iad/{frr.conf => frr.conf.iad-worker} (100%) rename deploy/containerlab/resources/fabric-router/sjc/{frr.conf => frr.conf.sjc-worker} (100%) create mode 100644 deploy/containerlab/resources/galactic-control/iad/bgppeer-gateway1.yaml create mode 100644 deploy/containerlab/resources/galactic-control/iad/bgppeer-gateway2.yaml create mode 100644 deploy/containerlab/resources/galactic-router-gateway/base/kustomization.yaml create mode 100644 deploy/containerlab/resources/galactic-router-gateway/base/router-lab-patch.yaml create mode 100644 deploy/containerlab/resources/galactic-router-gateway/iad-gateway1/bgppeer.yaml create mode 100644 deploy/containerlab/resources/galactic-router-gateway/iad-gateway1/bgprouter.yaml create mode 100644 deploy/containerlab/resources/galactic-router-gateway/iad-gateway1/kustomization.yaml create mode 100644 deploy/containerlab/resources/galactic-router-gateway/iad-gateway1/networkgateway.yaml create mode 100644 deploy/containerlab/resources/galactic-router-gateway/iad-gateway1/node-patch.yaml create mode 100644 deploy/containerlab/resources/galactic-router-gateway/iad-gateway2/bgppeer.yaml create mode 100644 deploy/containerlab/resources/galactic-router-gateway/iad-gateway2/bgprouter.yaml create mode 100644 deploy/containerlab/resources/galactic-router-gateway/iad-gateway2/kustomization.yaml create mode 100644 deploy/containerlab/resources/galactic-router-gateway/iad-gateway2/networkgateway.yaml create mode 100644 deploy/containerlab/resources/galactic-router-gateway/iad-gateway2/node-patch.yaml create mode 100644 deploy/containerlab/resources/galactic-router-gateway/iad/kustomization.yaml create mode 100644 deploy/containerlab/resources/galactic-router-gateway/iad/networkrule-ns60.yaml delete mode 100644 deploy/containerlab/resources/tenants/ns50/base/kustomization.yaml delete mode 100644 deploy/containerlab/resources/tenants/ns50/dfw/nad.yaml delete mode 100644 deploy/containerlab/resources/tenants/ns50/iad/kustomization.yaml delete mode 100644 deploy/containerlab/resources/tenants/ns50/iad/nad.yaml delete mode 100644 deploy/containerlab/resources/tenants/ns50/sjc/kustomization.yaml create mode 100644 deploy/containerlab/resources/tenants/ns60/base/kustomization.yaml create mode 100644 deploy/containerlab/resources/tenants/ns60/base/namespace.yaml create mode 100644 deploy/containerlab/resources/tenants/ns60/base/pod.yaml rename deploy/containerlab/resources/tenants/{ns50/dfw => ns60/iad}/kustomization.yaml (100%) rename deploy/containerlab/resources/tenants/{ns50/sjc => ns60/iad}/nad.yaml (68%) delete mode 100755 deploy/containerlab/scripts/verify-ns50.sh diff --git a/deploy/containerlab/README.md b/deploy/containerlab/README.md index 02079dbf..00ff84e6 100644 --- a/deploy/containerlab/README.md +++ b/deploy/containerlab/README.md @@ -15,6 +15,8 @@ over iBGP to the route reflector on iad-control. (mesh) (mesh) tr3 ──eth5── iad-worker tr3 ──eth4── iad-worker-control + tr3 ──eth6── iad-gateway1 + tr3 ──eth7── iad-gateway2 ``` ### Node roles @@ -26,10 +28,23 @@ over iBGP to the route reflector on iad-control. | `iad-control-plane` | ext-container | Kind control-plane; runs Cilium, Multus | | `iad-worker` | ext-container | Kind worker; runs FRR PE + galactic-router PE | | `iad-worker2` (renamed `iad-worker-control` post-deploy) | ext-container | Kind worker; runs FRR PE + galactic-router RR | +| `iad-worker3` (renamed `iad-gateway1` post-deploy) | ext-container | Kind worker; edge XDP NAT+LB gateway canary (Phase D) | +| `iad-worker4` (renamed `iad-gateway2` post-deploy) | ext-container | Kind worker; edge XDP NAT+LB gateway canary (Phase D) | | `sjc-control-plane` | ext-container | Kind control-plane; runs Cilium, Multus | | `sjc-worker` | ext-container | Kind worker; runs FRR PE + galactic-router PE | | `tr1`–`tr4` | linux (FRR) | iBGP full mesh, AS 65100 | +`iad-gateway1`/`iad-gateway2` are tainted (`galactic.datumapis.com/node=gateway:NoSchedule`) +dedicated nodes, same idea as `iad-worker-control`'s taint: no tenant pods land there, only +DaemonSets with a blanket toleration (`fabric-router`, `galactic-gateway1`/`-gateway2` — each a +two-container pod, `galactic-router` + `galactic-gateway`). +They never run `galactic-cni` (config/cni's affinity is edge-only) or a route-reflector. +**Underlay BGP peering on their `tr3` uplinks is now wired** (`node_files/tr3/frr.conf`, +plus two `BGPPeer` objects in `resources/galactic-control/iad/` for the route reflector side) +and the full fabric converges, but real end-to-end ingress traffic through the datapath still +doesn't reach a backend in this topology — it currently stops on a veth-specific `XDP_TX` +behavior in this lab environment, not a code bug; see `resources/galactic-router-gateway/`. + `dfw`, `iad`, and `sjc` are the three Kind cluster names — not separate ContainerLab topology nodes. Each cluster's `control-plane`/`worker` nodes above are its members. @@ -37,7 +52,7 @@ topology nodes. Each cluster's `control-plane`/`worker` nodes above are its memb ``` AS 65000 (dfw fabric-router / FRR) ──eBGP── tr1 (AS 65100) -AS 65000 (iad fabric-router / FRR) ──eBGP── tr3:eth5 (AS 65100) +AS 65000 (iad fabric-router / FRR) ──eBGP── tr3:eth5,eth6,eth7 (AS 65100) AS 65000 (iad fabric-control / FRR) ──eBGP── tr3:eth4 (AS 65100) AS 65000 (sjc fabric-router / FRR) ──eBGP── tr2 (AS 65100) @@ -82,6 +97,8 @@ AS 65000 (sjc-tenant / galactic-router) ──iBGP── iad-control-tenant | sjc-worker – tr2 | 2001:db8:1:20::/64 | 2001:db8:1:20::1 | 2001:db8:1:20::2 | | iad-worker – tr3 | 2001:db8:1:30::/64 | 2001:db8:1:30::1 | 2001:db8:1:30::2 | | iad-worker-control – tr3 | 2001:db8:1:31::/64 | 2001:db8:1:31::1 | 2001:db8:1:31::2 | +| iad-gateway1 – tr3 | 2001:db8:1:32::/64 | 2001:db8:1:32::1 | 2001:db8:1:32::2 | +| iad-gateway2 – tr3 | 2001:db8:1:33::/64 | 2001:db8:1:33::1 | 2001:db8:1:33::2 | ### Cluster SRv6 addressing @@ -97,10 +114,11 @@ the ones with a pod running today. The FRR fabric DaemonSet advertises the same Each site's tenant node advertises its own `/56` SRv6 locator block into the fabric — never the site's full `/48` uSID Block, which would create an anycast ambiguity the instant a second tenant node joins a site. The test VPC -`ns50` (see [docs/tenants.md](docs/tenants.md)) gets a host address within its node's -block: +`ns10` (see [docs/tenants.md](docs/tenants.md)) gets a host address within its node's +block (illustrative only — the exact hextet depends on allocation order; see +docs/tenants.md's [SRv6 USID Argument allocation](docs/tenants.md#srv6-usid-argument-allocation)): -| Cluster | FRR loopback | Node locator block | USID ns50 | galactic-router address | +| Cluster | FRR loopback | Node locator block | USID ns10 | galactic-router address | |-------------|----------------|-------------------------|--------------------------------|-------------------------| | dfw | fc00:0:2::1/128 | 2001:db8:ff01:100::/56 | 2001:db8:ff01:100:c800::/128 | fc00:0:2::1 | | sjc | fc00:0:3::1/128 | 2001:db8:ff02:100::/56 | 2001:db8:ff02:100:c800::/128 | fc00:0:3::1 | @@ -111,6 +129,23 @@ per-cluster Kustomize patches — `galactic-router` auto-detects it from `lo` at startup (see `docs/router/configuration.md`), since it always matches the FRR loopback address on the same host. +### Gateway node self-addressing (Phase D canary) + +`iad-gateway1`/`iad-gateway2` each get a uFMT 48+16 uSID over iad's shared +`2001:db8:ff03::/48` locator, at the reserved Argument 0 (never registered +into any tenant VRF — see `internal/plumbing/ebpf/uformat.go`'s +`ArgumentMin`). Unlike a tenant's per-VPC uSID, `srv6.ComputeSID` can't +derive this value (it rejects `argument==0` by design), so these were +computed directly via `internal/plumbing/ebpf/uformat.Encode` and are +supplied statically through `GALACTIC_GATEWAY_SRV6_ADDRESS` (originally +`GALACTIC_ROUTER_GATEWAY_SRV6_ADDRESS` before the binary split) — see +`resources/galactic-router-gateway/iad-gateway{1,2}/node-patch.yaml`. + +| Node | FRR loopback | nodeID | SRv6 self-address (Argument 0) | +|---------------|------------------|--------|----------------------------------| +| iad-gateway1 | fc00:0:9::1/128 | 2 | 2001:db8:ff03:2:e000:: | +| iad-gateway2 | fc00:0:a::1/128 | 3 | 2001:db8:ff03:3:e000:: | + ### Management network (fc00:10::/64) | Node | Address | @@ -122,6 +157,8 @@ FRR loopback address on the same host. | iad-control-plane | fc00:10::112 | | iad-worker | fc00:10::113 | | iad-worker2 (renamed `iad-worker-control`) | fc00:10::114 | +| iad-worker3 (renamed `iad-gateway1`) | fc00:10::115 | +| iad-worker4 (renamed `iad-gateway2`) | fc00:10::116 | ## Lab layout @@ -139,11 +176,11 @@ deploy/containerlab/ │ ├── galactic-control/iad/ # galactic-router RR + BGP CRs (iad-control) │ └── tenants/ # test VPCs — one shared base/ (Namespace + netshoot │ ├── base/ # Deployment), each tenant patching its namespace and -│ ├── ns50/ # default-network annotation; per-site dirs hold each -│ ├── ns10/ # site's NAD(s): ns50 (IPv4, 3-site), ns10 (IPv6-only, -│ ├── ns20/ # 3-site), ns20 (dual-stack, 3-site), ns30 (IPv6-only, -│ ├── ns30/ # dfw only, 2 attachments), ns40 (IPv4-only, iad only, -│ └── ns40/ # 2 attachments) — ns30/ns40's two attachments are each +│ ├── ns10/ # default-network annotation; per-site dirs hold each +│ ├── ns20/ # site's NAD(s): ns10 (IPv6-only, 3-site), ns20 +│ ├── ns30/ # (dual-stack, 3-site), ns30 (IPv6-only, dfw only, 2 +│ └── ns40/ # attachments), ns40 (IPv4-only, iad only, 2 +│ # attachments) — ns30/ns40's two attachments are each │ # their own NAD+Deployment (distinct vpcattachment, │ # same vpc), not one NAD scaled to replicas: 2 — see │ # docs/tenants.md for why. @@ -202,24 +239,23 @@ task deploy | `deploy` | Build images, apply host sysctls, and deploy the lab | | `deploy:topology` | Deploy the ContainerLab topology (transit routers) | | `deploy:clusters` | Create the three Kind clusters and export their kubeconfigs | -| `deploy:rename-control` | Rename the `iad-worker2` Docker container to `iad-worker-control` | +| `deploy:rename-control` | Rename `iad-worker2`→`iad-worker-control`, `iad-worker3/4`→`iad-gateway1/2` | | `deploy:images` | Load container images into Kind clusters | | `deploy:system` | Install BGP and VPC CRDs; apply the galactic-system namespace and shared RBAC | | `deploy:cni` | Install Cilium and Multus, then the galactic-cni DaemonSet | | `deploy:fabric` | Apply FRR DaemonSets to all clusters | | `deploy:galactic-router` | Apply galactic-router DaemonSets and BGP CRs | | `deploy:scenarios` | Deploy all VPC test scenarios | -| `deploy:ns50` | Deploy ns50 test VPC (IPv4, 3-site) | | `deploy:ns10` | Deploy ns10 test VPC (IPv6-only, fd20 ULA) | | `deploy:ns20` | Deploy ns20 test VPC (dual-stack, fd20 ULA + IPv4) | | `deploy:ns30` | Deploy ns30 test VPC (dfw only, 2 pods) | | `deploy:ns40` | Deploy ns40 test VPC (iad only, 2 pods) | | `verify:scenarios` | Verify ping across all VPC test scenarios | -| `verify:ns50` | Verify ns50 ping (IPv4, 3-site mesh) | | `verify:ns10` | Verify ns10 ping (IPv6-only, 3-site mesh) | | `verify:ns20` | Verify ns20 ping (dual-stack, 3-site mesh) | | `verify:ns30` | Verify ns30 ping (dfw only, 2 pods) | | `verify:ns40` | Verify ns40 ping (iad only, 2 pods) | +| `verify:gateway` | Verify iad's gateway canary CRDs exist (manifests only, no live traffic) | | `destroy` | Destroy the lab and remove all Kind clusters | | `restart` | Full rebuild — destroy then redeploy | | `rebuild` | Full rebuild — clean (destroy + delete images/artifacts) then redeploy | @@ -233,10 +269,10 @@ task deploy See [docs/verification.md](docs/verification.md) for transit fabric, FRR, and galactic-router health checks, and [docs/tenants.md](docs/tenants.md) for deploying and verifying -the `ns50`/`ns10`/`ns20`/`ns30`/`ns40` test VPCs. +the `ns10`/`ns20`/`ns30`/`ns40` test VPCs. `task verify` (and its constituent `task verify:scenarios`) also pings every -VPC's pods end-to-end via `task verify:ns50`/`ns10`/`ns20`/`ns30`/`ns40` — +VPC's pods end-to-end via `task verify:ns10`/`ns20`/`ns30`/`ns40` — full site-pair mesh for the 3-site VPCs, both-direction pod-to-pod for the single-site `ns30`/`ns40`. Run one on its own after redeploying a single scenario, e.g. `task verify:ns30` after `task deploy:ns30`. diff --git a/deploy/containerlab/Taskfile.yaml b/deploy/containerlab/Taskfile.yaml index 7e927203..0c04d711 100644 --- a/deploy/containerlab/Taskfile.yaml +++ b/deploy/containerlab/Taskfile.yaml @@ -20,6 +20,7 @@ tasks: - "build:node" - "build:fabric-router" - "build:galactic-router" + - "build:galactic-gateway" - "build:galactic-cni" "build:node": @@ -37,6 +38,11 @@ tasks: cmds: - docker build --network=host -t galactic-router:latest -f ../../containers/galactic-router/Dockerfile ../.. + "build:galactic-gateway": + desc: Build the galactic-gateway container image + cmds: + - docker build --network=host -t galactic-gateway:latest -f ../../containers/galactic-gateway/Dockerfile ../.. + "build:galactic-cni": desc: Build the galactic-cni installer image cmds: @@ -72,9 +78,11 @@ tasks: - sudo containerlab deploy -t {{.TOPO}} "deploy:rename-control": - desc: Rename iad-worker2 to iad-worker-control + desc: Rename iad-worker2 to iad-worker-control, iad-worker3/4 to iad-gateway1/2 cmds: - docker rename iad-worker2 iad-worker-control + - docker rename iad-worker3 iad-gateway1 + - docker rename iad-worker4 iad-gateway2 "load-image": internal: true @@ -91,6 +99,26 @@ tasks: vars: {IMAGE: "{{.FABRIC_ROUTER_IMAGE}}", NODE: iad-worker} - task: load-image vars: {IMAGE: "{{.FABRIC_ROUTER_IMAGE}}", NODE: iad-worker-control} + # iad-gateway1/2 need fabric-router (the underlay eBGP session the + # gateway XDP datapath's uplink depends on -- see config/fabric/'s + # affinity, extended to the gateway role), galactic-router (tenant + # BGP only, one container in config/gateway/base's two-container + # pod), and galactic-gateway (the edge XDP NAT+LB engine, the other + # container in that pod), but NOT galactic-cni: they never run the + # CNI DaemonSet at all (it only + # targets the edge role). + - task: load-image + vars: {IMAGE: "{{.FABRIC_ROUTER_IMAGE}}", NODE: iad-gateway1} + - task: load-image + vars: {IMAGE: "{{.FABRIC_ROUTER_IMAGE}}", NODE: iad-gateway2} + - task: load-image + vars: {IMAGE: galactic-router:latest, NODE: iad-gateway1} + - task: load-image + vars: {IMAGE: galactic-router:latest, NODE: iad-gateway2} + - task: load-image + vars: {IMAGE: galactic-gateway:latest, NODE: iad-gateway1} + - task: load-image + vars: {IMAGE: galactic-gateway:latest, NODE: iad-gateway2} - task: load-image vars: {IMAGE: "{{.FABRIC_ROUTER_IMAGE}}", NODE: sjc-worker} - task: load-image @@ -170,16 +198,11 @@ tasks: "deploy:scenarios": desc: Deploy all VPC test scenarios cmds: - - task: "deploy:ns50" - task: "deploy:ns10" - task: "deploy:ns20" - task: "deploy:ns30" - task: "deploy:ns40" - - "deploy:ns50": - desc: Deploy ns50 test VPC (IPv4, 3-site) - cmds: - - ./scripts/deploy-ns.sh ns50 dfw sjc iad + - task: "deploy:ns60" "deploy:ns10": desc: Deploy ns10 test VPC (IPv6-only, fd20 ULA) @@ -201,6 +224,14 @@ tasks: cmds: - ./scripts/deploy-ns.sh ns40 iad + "deploy:ns60": + desc: >- + Deploy ns60 test VPC (iad only, nginx backend for the edge XDP + NAT+LB gateway canary -- see resources/galactic-router-gateway/iad/ + networkrule-ns60.yaml, applied earlier by deploy:galactic-router) + cmds: + - ./scripts/deploy-ns.sh ns60 iad + verify: desc: Run all verification checks cmds: @@ -209,6 +240,7 @@ tasks: - task: "verify:bgp-peers" - task: "verify:srv6" - task: "verify:evpn" + - task: "verify:gateway" - task: "verify:scenarios" "verify:bgp-transit": @@ -261,20 +293,24 @@ tasks: - docker exec sjc-control-plane kubectl get bgprouters -A - docker exec dfw-control-plane kubectl get bgprouters -A + "verify:gateway": + desc: >- + Verify iad's edge XDP NAT+LB gateway canary CRDs exist. BGP peering + and XDP attach are now live, but this task itself only checks CRD + presence, not end-to-end traffic -- a veth-specific XDP_TX behavior + in this lab environment still blocks that, not a code bug. + cmds: + - docker exec iad-control-plane kubectl get networkgateways,networkrules -n galactic-system + - docker exec iad-control-plane kubectl get daemonset -n galactic-system -l app.kubernetes.io/name=galactic-gateway + "verify:scenarios": desc: Verify ping across all VPC test scenarios cmds: - - task: "verify:ns50" - task: "verify:ns10" - task: "verify:ns20" - task: "verify:ns30" - task: "verify:ns40" - "verify:ns50": - desc: Verify ns50 ping (IPv4, 3-site mesh) - cmds: - - ./scripts/verify-ns50.sh - "verify:ns10": desc: Verify ns10 ping (IPv6-only, 3-site mesh) cmds: @@ -301,6 +337,7 @@ tasks: - task: destroy - docker rmi kindest/node:galactic || true - docker rmi galactic-router:latest || true + - docker rmi galactic-gateway:latest || true - docker rmi galactic-cni:latest || true - docker rmi {{.FABRIC_ROUTER_IMAGE}} || true - rm -rf clab-{{.LAB}} build/ diff --git a/deploy/containerlab/docs/tenants.md b/deploy/containerlab/docs/tenants.md index d79ab39f..5a8fa7e3 100644 --- a/deploy/containerlab/docs/tenants.md +++ b/deploy/containerlab/docs/tenants.md @@ -1,8 +1,8 @@ -# Tenant Test VPCs (ns10–ns50) +# Tenant Test VPCs (ns10–ns40) ## Overview -The lab deploys five test VPCs — `ns50`, `ns10`, `ns20`, `ns30`, `ns40` — to +The lab deploys four test VPCs — `ns10`, `ns20`, `ns30`, `ns40` — to exercise different corners of the CNI/BGP/SRv6 path: single- vs. dual-stack addressing, 3-site cross-site EVPN reachability, and same-node/same-VRF pod-to-pod connectivity. Every one of them follows the same mechanism: Multus @@ -21,7 +21,6 @@ They differ only in scope and addressing: | VPC | Sites | Address families | VRF interface | Notes | |--------|------------------------------|-------------------|-------------------|-------| -| `ns50` | dfw, sjc, iad (3-site) | IPv4 + public IPv6 ptp | `G000000050V` | Also defines a `public` NAD with an IPv6 IPAM pool for external-connectivity testing. | | `ns10` | dfw, sjc, iad (3-site) | IPv6-only (fd20 ULA) | `G000000010V` | No `ipv4_subnet` at all. | | `ns20` | dfw, sjc, iad (3-site) | Dual-stack (fd20 ULA + IPv4) | `G000000020V` | Both families active; exercises the dual-stack IPAM path. | | `ns30` | dfw only, 2 attachments | IPv6-only (fd20 ULA) | `G000000030V` | Two distinct attachments (`private`/`private-b`, distinct `vpcattachment` values, same `vpc`), each its own single-replica Deployment, both land on `dfw-worker` and share one VRF — same-node connectivity, no cross-site hop. `verify:ns30` asserts the two pods share a node. | @@ -63,8 +62,8 @@ sharing a VPC on a node (`ns30`'s and `ns40`'s two apiece) resolves to the same `BGPVRFInstance` and therefore the same Argument, so each VPC still consumes exactly one slot per node regardless of how many attachments land on it. Concretely, expect hextets in the `0xe001`–`0xefff` range; the exact -value depends on allocation order (`ns50` is provisioned first in `task -deploy`, then `ns10`, `ns20`, `ns30`, `ns40` in that order, each consuming the +value depends on allocation order (`ns10` is provisioned first in `task +deploy`, then `ns20`, `ns30`, `ns40` in that order, each consuming the next free slot on each node it lands on). Always confirm the live value rather than trusting a table: @@ -74,27 +73,24 @@ docker exec dfw-control-plane kubectl get bgpvrfinstances -A ## Addressing reference -| Site | VPC | IPv6 pool (fd20 ULA) | IPv4 subnet | Public IPv6 pool | -|------|--------|------------------------|--------------------|---------------------| -| dfw | `ns50` | — | `172.20.1.0/24` | `2001:db8:1::/64` | -| sjc | `ns50` | — | `172.20.20.0/24` | `2001:db8:20::/64` | -| iad | `ns50` | — | `172.20.10.0/24` | `2001:db8:10::/64` | -| dfw | `ns10` | `fd20:10:ff01::/48` | none | — | -| sjc | `ns10` | `fd20:10:ff02::/48` | none | — | -| iad | `ns10` | `fd20:10:ff03::/48` | none | — | -| dfw | `ns20` | `fd20:20:ff01::/48` | `172.21.1.0/24` | — | -| sjc | `ns20` | `fd20:20:ff02::/48` | `172.21.20.0/24` | — | -| iad | `ns20` | `fd20:20:ff03::/48` | `172.21.10.0/24` | — | -| dfw | `ns30` (`private`) | `fd20:30:ff01::/48` | none | — | -| dfw | `ns30` (`private-b`) | `fd20:30:ff02::/48` | none | — | -| iad | `ns40` (`private`) | none | `172.40.10.0/24` | — | -| iad | `ns40` (`private-b`) | none | `172.40.20.0/24` | — | - -`ns50`'s IPv4 pools and `ns20`'s are deliberately from distinct `/16` blocks -(`172.20.0.0/16` vs. `172.21.0.0/16`) so the two VPCs' addressing never -overlaps; `ns40`'s `172.40.0.0/16` is separate again, split further into a -`.10.0/24`/`.20.0/24` pair between its two attachments (same pattern `ns20` -uses per-site). The IPv6 pools for `ns10`/`ns20`/`ns30` share the `fd20` ULA +| Site | VPC | IPv6 pool (fd20 ULA) | IPv4 subnet | +|------|--------|------------------------|--------------------| +| dfw | `ns10` | `fd20:10:ff01::/48` | none | +| sjc | `ns10` | `fd20:10:ff02::/48` | none | +| iad | `ns10` | `fd20:10:ff03::/48` | none | +| dfw | `ns20` | `fd20:20:ff01::/48` | `172.21.1.0/24` | +| sjc | `ns20` | `fd20:20:ff02::/48` | `172.21.20.0/24` | +| iad | `ns20` | `fd20:20:ff03::/48` | `172.21.10.0/24` | +| dfw | `ns30` (`private`) | `fd20:30:ff01::/48` | none | +| dfw | `ns30` (`private-b`) | `fd20:30:ff02::/48` | none | +| iad | `ns40` (`private`) | none | `172.40.10.0/24` | +| iad | `ns40` (`private-b`) | none | `172.40.20.0/24` | + +`ns20`'s IPv4 pool (`172.21.0.0/16`) and `ns40`'s (`172.40.0.0/16`) are +deliberately from distinct `/16` blocks so the two VPCs' addressing never +overlaps; `ns40`'s is split further into a `.10.0/24`/`.20.0/24` pair +between its two attachments (same pattern `ns20` uses per-site). The IPv6 +pools for `ns10`/`ns20`/`ns30` share the `fd20` ULA prefix, distinguished by the second hextet (`10`/`20`/`30`); `ns30`'s two attachments split further into `ff01`/`ff02` (same pattern `ns10`/`ns20` use per-site). @@ -119,13 +115,12 @@ docker exec sjc-control-plane kubectl get pods -n galactic-system ## Deploying a VPC's workloads -`task deploy` already runs all five as its final steps. To (re-)apply just +`task deploy` already runs all four as its final steps. To (re-)apply just one VPC's workloads on its own — e.g. after the lab was restarted — use its `task deploy:nsNN` target, which wraps `scripts/deploy-ns.sh `: ```bash -task deploy:ns50 # -> deploy-ns.sh ns50 dfw sjc iad task deploy:ns10 # -> deploy-ns.sh ns10 dfw sjc iad task deploy:ns20 # -> deploy-ns.sh ns20 dfw sjc iad task deploy:ns30 # -> deploy-ns.sh ns30 dfw @@ -145,7 +140,7 @@ docker exec sjc-control-plane kubectl get pods -n -o wide docker exec iad-control-plane kubectl get pods -n -o wide ``` -For the 3-site VPCs (`ns50`, `ns10`, `ns20`), expect one `Running` pod per +For the 3-site VPCs (`ns10`, `ns20`), expect one `Running` pod per site. For the single-site VPCs, expect **two** `Running` pods — one per attachment (`private`/`private-b`, distinct Deployments) — both on the one site's worker (`dfw-worker` for `ns30`, `iad-worker` for `ns40` — not @@ -175,7 +170,6 @@ resolves every pod's address and pings across every applicable pair, in both directions: ```bash -task verify:ns50 # 3-site mesh, IPv4 task verify:ns10 # 3-site mesh, IPv6 task verify:ns20 # 3-site mesh, both families task verify:ns30 # same-node pod-to-pod, IPv6 (dfw) @@ -190,7 +184,7 @@ above](#overview) for what actually keeps them there today — and a split pair would still ping fine over the cross-site path, so without the assertion these VPCs would silently stop exercising the same-node/same-VRF case they exist for. -`task verify` (via `task verify:scenarios`) runs all five. Use the scripts as +`task verify` (via `task verify:scenarios`) runs all four. Use the scripts as the reference for how to resolve pod names/addresses by hand (`lib.sh`'s `pod_name`/`pod_scheduling_node`/`pod_ip4`/`pod_ip6`/`ping_pod` helpers) if you need to reproduce a step manually while debugging — e.g. to ping the `ns30` @@ -224,7 +218,7 @@ affected VPC's `deploy:nsNN`: ```bash task deploy:cni -task deploy:ns50 # or whichever VPC's pods aren't getting IPs +task deploy:ns10 # or whichever VPC's pods aren't getting IPs ``` ### BGPAdvertisements not created @@ -267,7 +261,6 @@ docker exec dfw-worker dmesg | grep galactic interface name from the [Overview](#overview) table: ```bash - docker exec dfw-worker ip -4 route show table G000000050V # ns50 docker exec dfw-worker ip -6 route show table G000000010V # ns10 docker exec dfw-worker ip -6 route show table G000000020V # ns20 (IPv6 leg) docker exec dfw-worker ip -4 route show table G000000020V # ns20 (IPv4 leg) diff --git a/deploy/containerlab/docs/verification.md b/deploy/containerlab/docs/verification.md index e0049fd1..80f1b7a9 100644 --- a/deploy/containerlab/docs/verification.md +++ b/deploy/containerlab/docs/verification.md @@ -1,7 +1,7 @@ # Verification Run these checks after `task deploy` to confirm the lab is healthy end-to-end. For -deploying and verifying the `ns50`/`ns10`/`ns20`/`ns30`/`ns40` test workloads, +deploying and verifying the `ns10`/`ns20`/`ns30`/`ns40` test workloads, see [docs/tenants.md](tenants.md). ## Transit fabric @@ -11,7 +11,7 @@ see [docs/tenants.md](tenants.md). docker exec clab-gvpc-tr1 vtysh -c "show bgp ipv6 unicast summary" # Each site's per-node /56 SRv6 locator block should be present on all TR nodes -# (covers ns50's USID on that node — see docs/tenants.md) +# (covers ns10's USID on that node — see docs/tenants.md) docker exec clab-gvpc-tr1 vtysh -c "show bgp ipv6 unicast 2001:db8:ff01:100::/56" docker exec clab-gvpc-tr1 vtysh -c "show bgp ipv6 unicast 2001:db8:ff02:100::/56" docker exec clab-gvpc-tr1 vtysh -c "show bgp ipv6 unicast 2001:db8:ff03:100::/56" diff --git a/deploy/containerlab/gvpc.clab.yaml b/deploy/containerlab/gvpc.clab.yaml index cb1988d0..b58b58b5 100644 --- a/deploy/containerlab/gvpc.clab.yaml +++ b/deploy/containerlab/gvpc.clab.yaml @@ -145,6 +145,55 @@ topology: exec: - /galactic/scripts/install.sh + # Kind names these two containers iad-worker3/iad-worker4 (sequential, + # matching declaration order in node_files/iad/config.yaml) even though + # their Kubernetes node names are iad-gateway1/iad-gateway2 (set via + # that file's kubeadmConfigPatches, same indirection iad-worker2/ + # iad-worker-control already uses) -- Taskfile.yaml's deploy:rename-* + # step renames both to match after cluster creation, purely cosmetic + # (doesn't affect the links below, which are wired to the container + # before the rename happens). No blackhole route in `exec`: unlike + # iad-worker, these nodes never run galactic-veth's usid.c decap + # program at all (config/cni's DaemonSet only targets the edge role), + # so the blackhole-route workaround it needs doesn't apply here. + iad-worker3: + kind: ext-container + labels: + cluster: iad + graph-group: kind-worker + graph-level: 1 + graph-icon: server + platform: kind + reload: disabled + worker-index: "3" + mgmt-ipv6: fc00:10::115 + sysctls: + fs.inotify.max_user_instances: "256" + net.ipv4.conf.all.arp_announce: "2" + net.ipv4.ip_forward: "1" + net.ipv6.conf.all.forwarding: "1" + exec: + - /galactic/scripts/install.sh + + iad-worker4: + kind: ext-container + labels: + cluster: iad + graph-group: kind-worker + graph-level: 1 + graph-icon: server + platform: kind + reload: disabled + worker-index: "4" + mgmt-ipv6: fc00:10::116 + sysctls: + fs.inotify.max_user_instances: "256" + net.ipv4.conf.all.arp_announce: "2" + net.ipv4.ip_forward: "1" + net.ipv6.conf.all.forwarding: "1" + exec: + - /galactic/scripts/install.sh + tr1: group: transit labels: @@ -200,6 +249,36 @@ topology: - endpoints: ["iad-worker:eth1", "tr3:eth5"] - endpoints: ["iad-worker2:eth1", "tr3:eth4"] + # Gateway node uplinks (edge XDP NAT+LB DaemonSet; see + # config/gateway/base/ and deploy/containerlab/resources/ + # galactic-router-gateway/). tr3 (iad's transit router) already + # consumed eth1/eth2/eth3 for the TR mesh and eth4/eth5 for + # iad-worker/iad-worker2, so these take the next free ports. BGP + # peering on these links is now configured (node_files/tr3/frr.conf, + # plus two BGPPeer objects in resources/galactic-control/iad/ for the + # route reflector side) and the fabric fully converges over them, but + # real end-to-end ingress traffic through the datapath still doesn't + # reach a backend -- a veth-specific XDP_TX behavior in this lab + # environment, not a code bug. + # + # mtu: 1500 on both ends, unlike every other link in this topology + # (which gets containerlab's jumbo-frame veth default, ~9500) -- + # required, not cosmetic: galactic-router's edge NAT+LB datapath + # attaches its XDP program to this link's gateway-side interface in + # native/driver mode (edgeattach.go, XDPDriverMode; generic/SKB mode + # is deliberately not attempted, see that package's doc comment), and + # the kernel's veth_xdp_set() rejects native XDP with -ERANGE + # whenever the *peer's* MTU exceeds what fits in one page-sized + # buffer (SKB_WITH_OVERHEAD(PAGE_SIZE - VETH_XDP_HEADROOM), ~3.5-3.8KB + # on a 4K-page host) -- 9500 blows well past that regardless of which + # end asks. Confirmed live: galactic-router-gateway{1,2} crash-looped + # with exactly this ERANGE until both ends of these two links were + # dropped to 1500. + - endpoints: ["iad-worker3:eth1", "tr3:eth6"] + mtu: 1500 + - endpoints: ["iad-worker4:eth1", "tr3:eth7"] + mtu: 1500 + # TR full mesh (iBGP within AS 65100) - endpoints: ["tr1:eth2", "tr2:eth2"] - endpoints: ["tr1:eth3", "tr3:eth1"] diff --git a/deploy/containerlab/node_files/iad/config.yaml b/deploy/containerlab/node_files/iad/config.yaml index 215f3ab5..3c92222b 100644 --- a/deploy/containerlab/node_files/iad/config.yaml +++ b/deploy/containerlab/node_files/iad/config.yaml @@ -19,6 +19,42 @@ nodes: - key: galactic.datumapis.com/node value: control effect: NoSchedule + # Dedicated edge XDP NAT+LB gateway nodes -- canary for + # config/gateway/base/. Two nodes, active-active, matching the design's + # own topology requirement (see + # internal/gateway/doc.go): a single gateway node would still work but + # would never actually exercise the primary/secondary BGP local-pref + # failover path. Tainted the same way the control role is above, to + # keep ordinary tenant pods (resources/tenants/*/base/pod.yaml's + # netshoot Deployments) off these dedicated nodes -- only DaemonSets + # with a blanket toleration (config/router/base/daemonset.yaml, + # config/fabric/daemonset.yaml) tolerate it. + - role: worker + labels: + topology.kubernetes.io/region: iad + galactic.datumapis.com/node: gateway + kubeadmConfigPatches: + - | + kind: JoinConfiguration + nodeRegistration: + name: iad-gateway1 + taints: + - key: galactic.datumapis.com/node + value: gateway + effect: NoSchedule + - role: worker + labels: + topology.kubernetes.io/region: iad + galactic.datumapis.com/node: gateway + kubeadmConfigPatches: + - | + kind: JoinConfiguration + nodeRegistration: + name: iad-gateway2 + taints: + - key: galactic.datumapis.com/node + value: gateway + effect: NoSchedule networking: ipFamily: ipv6 disableDefaultCNI: true diff --git a/deploy/containerlab/node_files/tr3/frr.conf b/deploy/containerlab/node_files/tr3/frr.conf index 066605ce..2c7f8b44 100644 --- a/deploy/containerlab/node_files/tr3/frr.conf +++ b/deploy/containerlab/node_files/tr3/frr.conf @@ -25,6 +25,14 @@ interface eth5 description iad-worker-facing ipv6 address 2001:db8:1:30::1/64 ! +interface eth6 + description iad-gateway1-facing + ipv6 address 2001:db8:1:32::1/64 +! +interface eth7 + description iad-gateway2-facing + ipv6 address 2001:db8:1:33::1/64 +! ipv6 prefix-list MIN48 seq 5 prefix ::/0 le 48 permit ipv6 prefix-list MIN48 seq 10 deny @@ -39,6 +47,8 @@ router bgp 65100 bgp log-neighbor-changes neighbor 2001:db8:1:30::2 remote-as 65000 neighbor 2001:db8:1:31::2 remote-as 65000 + neighbor 2001:db8:1:32::2 remote-as 65000 + neighbor 2001:db8:1:33::2 remote-as 65000 neighbor eth1 interface remote-as 65100 neighbor eth2 interface remote-as 65100 neighbor eth3 interface remote-as 65100 @@ -48,6 +58,10 @@ router bgp 65100 neighbor 2001:db8:1:30::2 route-map FILTER_PREFIX in neighbor 2001:db8:1:31::2 activate neighbor 2001:db8:1:31::2 route-map FILTER_PREFIX in + neighbor 2001:db8:1:32::2 activate + neighbor 2001:db8:1:32::2 route-map FILTER_PREFIX in + neighbor 2001:db8:1:33::2 activate + neighbor 2001:db8:1:33::2 route-map FILTER_PREFIX in neighbor eth1 activate neighbor eth1 next-hop-self neighbor eth1 route-map FILTER_PREFIX in diff --git a/deploy/containerlab/resources/fabric-router/base/fabric-lab-patch.yaml b/deploy/containerlab/resources/fabric-router/base/fabric-lab-patch.yaml index fd5e33e6..11c96b8f 100644 --- a/deploy/containerlab/resources/fabric-router/base/fabric-lab-patch.yaml +++ b/deploy/containerlab/resources/fabric-router/base/fabric-lab-patch.yaml @@ -5,11 +5,19 @@ metadata: spec: template: spec: - # Narrower than config/fabric/'s default (edge or control): iad's - # control-node fabric pod is deployed separately by + # Narrower than config/fabric/'s default (edge, control, or gateway): + # iad's control-node fabric pod is deployed separately by # resources/fabric-control/iad/, with its own frr.conf for that - # node's distinct peering — so this per-site overlay sticks to edge only, - # or the control node would get two competing fabric pods. + # node's distinct peering — so this per-site overlay sticks to edge + # and gateway, or the control node would get two competing fabric + # pods. edge and gateway share this one DaemonSet because both just + # need an underlay eBGP session brought up before galactic-router + # starts (config/fabric/daemonset.yaml's own doc comment) — nothing + # about FRR itself differs by role, only the per-node frr.conf key + # each pod selects via NODE_NAME (../iad/kustomization.yaml's + # configMapGenerator carries both iad-worker's and iad-gateway1/2's + # keys in the same fabric-config ConfigMap). dfw/sjc have no + # gateway-labeled nodes, so this widening is a no-op there. affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: @@ -21,6 +29,7 @@ spec: operator: In values: - edge + - gateway initContainers: - name: frr-init image: fabric-router:latest diff --git a/deploy/containerlab/resources/fabric-router/dfw/frr.conf b/deploy/containerlab/resources/fabric-router/dfw/frr.conf.dfw-worker similarity index 100% rename from deploy/containerlab/resources/fabric-router/dfw/frr.conf rename to deploy/containerlab/resources/fabric-router/dfw/frr.conf.dfw-worker diff --git a/deploy/containerlab/resources/fabric-router/dfw/kustomization.yaml b/deploy/containerlab/resources/fabric-router/dfw/kustomization.yaml index 12355271..5db09fc3 100644 --- a/deploy/containerlab/resources/fabric-router/dfw/kustomization.yaml +++ b/deploy/containerlab/resources/fabric-router/dfw/kustomization.yaml @@ -5,4 +5,4 @@ resources: configMapGenerator: - name: fabric-config files: - - frr.conf.dfw-worker=frr.conf + - frr.conf.dfw-worker diff --git a/deploy/containerlab/resources/fabric-router/iad/frr.conf.iad-gateway1 b/deploy/containerlab/resources/fabric-router/iad/frr.conf.iad-gateway1 new file mode 100644 index 00000000..3ca05fba --- /dev/null +++ b/deploy/containerlab/resources/fabric-router/iad/frr.conf.iad-gateway1 @@ -0,0 +1,42 @@ +frr defaults traditional +hostname iad-fabric-gateway1 +log syslog informational + +interface lo + ! /128 provides a reachable address for this node's own outbound BGP + ! session to the route reflector (fc00:0:8::1:1790, GALACTIC_ROUTER_ + ! BGP_LOCAL_ADDRESS auto-detected from lo, matching iad-worker/ + ! iad-worker-control's own convention). + ipv6 address fc00:0:9::1/128 +! +interface eth1 + description tr3-facing + ipv6 address 2001:db8:1:32::2/64 +! + +router bgp 65000 + bgp router-id 10.255.255.5 + no bgp default ipv4-unicast + no bgp ebgp-requires-policy + bgp log-neighbor-changes + neighbor 2001:db8:1:32::1 remote-as 65100 + + address-family ipv6 unicast + neighbor 2001:db8:1:32::1 activate + neighbor 2001:db8:1:32::1 allowas-in 1 + # /128, not /48 -- matching every other node's own-loopback + # origination (e.g. tr3's "network fc00:0:6::1/128"). Every transit + # router's inbound FILTER_PREFIX route-map denies anything <=/48 + # (node_files/tr3/frr.conf's MIN48 prefix-list) specifically so a + # site's aggregate locator/loopback block never leaks onto the + # transit as a single de-aggregated route; only a per-node /128 + # peering loopback is meant to cross it. The original "network + # fc00:0:9::/48" (backed by "ipv6 route fc00:0:9::/48 Null0", removed + # here) got silently filtered by that same rule on every hop back to + # the route reflector, leaving this node's outbound BGPPeer session + # stuck in Active forever -- caught only once this link's underlay + # eBGP was actually brought up live (see node_files/tr3/frr.conf). + network fc00:0:9::1/128 + exit-address-family + +ipv6 forwarding diff --git a/deploy/containerlab/resources/fabric-router/iad/frr.conf.iad-gateway2 b/deploy/containerlab/resources/fabric-router/iad/frr.conf.iad-gateway2 new file mode 100644 index 00000000..11237e79 --- /dev/null +++ b/deploy/containerlab/resources/fabric-router/iad/frr.conf.iad-gateway2 @@ -0,0 +1,31 @@ +frr defaults traditional +hostname iad-fabric-gateway2 +log syslog informational + +interface lo + ! /128 provides a reachable address for this node's own outbound BGP + ! session to the route reflector (fc00:0:8::1:1790, GALACTIC_ROUTER_ + ! BGP_LOCAL_ADDRESS auto-detected from lo, matching iad-worker/ + ! iad-worker-control's own convention). + ipv6 address fc00:0:a::1/128 +! +interface eth1 + description tr3-facing + ipv6 address 2001:db8:1:33::2/64 +! + +router bgp 65000 + bgp router-id 10.255.255.6 + no bgp default ipv4-unicast + no bgp ebgp-requires-policy + bgp log-neighbor-changes + neighbor 2001:db8:1:33::1 remote-as 65100 + + address-family ipv6 unicast + neighbor 2001:db8:1:33::1 activate + neighbor 2001:db8:1:33::1 allowas-in 1 + # /128, not /48 -- see frr.conf.iad-gateway1's identical comment. + network fc00:0:a::1/128 + exit-address-family + +ipv6 forwarding diff --git a/deploy/containerlab/resources/fabric-router/iad/frr.conf b/deploy/containerlab/resources/fabric-router/iad/frr.conf.iad-worker similarity index 100% rename from deploy/containerlab/resources/fabric-router/iad/frr.conf rename to deploy/containerlab/resources/fabric-router/iad/frr.conf.iad-worker diff --git a/deploy/containerlab/resources/fabric-router/iad/kustomization.yaml b/deploy/containerlab/resources/fabric-router/iad/kustomization.yaml index 42c38ecb..9b9af7e3 100644 --- a/deploy/containerlab/resources/fabric-router/iad/kustomization.yaml +++ b/deploy/containerlab/resources/fabric-router/iad/kustomization.yaml @@ -5,4 +5,6 @@ resources: configMapGenerator: - name: fabric-config files: - - frr.conf.iad-worker=frr.conf + - frr.conf.iad-worker + - frr.conf.iad-gateway1 + - frr.conf.iad-gateway2 diff --git a/deploy/containerlab/resources/fabric-router/sjc/frr.conf b/deploy/containerlab/resources/fabric-router/sjc/frr.conf.sjc-worker similarity index 100% rename from deploy/containerlab/resources/fabric-router/sjc/frr.conf rename to deploy/containerlab/resources/fabric-router/sjc/frr.conf.sjc-worker diff --git a/deploy/containerlab/resources/fabric-router/sjc/kustomization.yaml b/deploy/containerlab/resources/fabric-router/sjc/kustomization.yaml index 405b88e3..1810dc83 100644 --- a/deploy/containerlab/resources/fabric-router/sjc/kustomization.yaml +++ b/deploy/containerlab/resources/fabric-router/sjc/kustomization.yaml @@ -5,4 +5,4 @@ resources: configMapGenerator: - name: fabric-config files: - - frr.conf.sjc-worker=frr.conf + - frr.conf.sjc-worker diff --git a/deploy/containerlab/resources/galactic-control/iad/bgppeer-gateway1.yaml b/deploy/containerlab/resources/galactic-control/iad/bgppeer-gateway1.yaml new file mode 100644 index 00000000..23010f5f --- /dev/null +++ b/deploy/containerlab/resources/galactic-control/iad/bgppeer-gateway1.yaml @@ -0,0 +1,13 @@ +apiVersion: network.datumapis.com/v1alpha1 +kind: BGPPeer +metadata: + name: iad-control-tenant-to-gateway1 + namespace: galactic-system +spec: + routerRef: + name: iad-control-tenant + peerASN: 65000 + address: "fc00:0:9::1" + addressFamilies: + - afi: l2vpn + safi: evpn diff --git a/deploy/containerlab/resources/galactic-control/iad/bgppeer-gateway2.yaml b/deploy/containerlab/resources/galactic-control/iad/bgppeer-gateway2.yaml new file mode 100644 index 00000000..dc736707 --- /dev/null +++ b/deploy/containerlab/resources/galactic-control/iad/bgppeer-gateway2.yaml @@ -0,0 +1,13 @@ +apiVersion: network.datumapis.com/v1alpha1 +kind: BGPPeer +metadata: + name: iad-control-tenant-to-gateway2 + namespace: galactic-system +spec: + routerRef: + name: iad-control-tenant + peerASN: 65000 + address: "fc00:0:a::1" + addressFamilies: + - afi: l2vpn + safi: evpn diff --git a/deploy/containerlab/resources/galactic-control/iad/kustomization.yaml b/deploy/containerlab/resources/galactic-control/iad/kustomization.yaml index d20a470a..66df9221 100644 --- a/deploy/containerlab/resources/galactic-control/iad/kustomization.yaml +++ b/deploy/containerlab/resources/galactic-control/iad/kustomization.yaml @@ -5,6 +5,8 @@ resources: - bgppeer-dfw.yaml - bgppeer-iad.yaml - bgppeer-sjc.yaml + - bgppeer-gateway1.yaml + - bgppeer-gateway2.yaml patches: - path: router-lab-patch.yaml target: diff --git a/deploy/containerlab/resources/galactic-router-gateway/base/kustomization.yaml b/deploy/containerlab/resources/galactic-router-gateway/base/kustomization.yaml new file mode 100644 index 00000000..9e0009ed --- /dev/null +++ b/deploy/containerlab/resources/galactic-router-gateway/base/kustomization.yaml @@ -0,0 +1,15 @@ +# Mirrors resources/galactic-router/base/'s pattern exactly, pointed at +# config/gateway/base instead of config/router/base: "gateway" is copied +# onto the node at deploy time (see scripts/deploy-galactic-router.sh's +# copy_router_gateway_config), nested here so this kustomization's +# "gateway" resource reference resolves. Unlike resources/galactic-router/ +# base/'s "tenant" resource, config/gateway/base is self-contained (its own +# full two-container DaemonSet spec, not a patch onto config/router/base), +# so there is no separate "base" resource to nest alongside it here. +resources: + - gateway +patches: + - path: router-lab-patch.yaml + target: + kind: DaemonSet + name: galactic-gateway diff --git a/deploy/containerlab/resources/galactic-router-gateway/base/router-lab-patch.yaml b/deploy/containerlab/resources/galactic-router-gateway/base/router-lab-patch.yaml new file mode 100644 index 00000000..90277821 --- /dev/null +++ b/deploy/containerlab/resources/galactic-router-gateway/base/router-lab-patch.yaml @@ -0,0 +1,14 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: galactic-gateway +spec: + template: + spec: + containers: + - name: galactic-router + image: galactic-router:latest + imagePullPolicy: Never + - name: galactic-gateway + image: galactic-gateway:latest + imagePullPolicy: Never diff --git a/deploy/containerlab/resources/galactic-router-gateway/iad-gateway1/bgppeer.yaml b/deploy/containerlab/resources/galactic-router-gateway/iad-gateway1/bgppeer.yaml new file mode 100644 index 00000000..4115c62e --- /dev/null +++ b/deploy/containerlab/resources/galactic-router-gateway/iad-gateway1/bgppeer.yaml @@ -0,0 +1,14 @@ +apiVersion: network.datumapis.com/v1alpha1 +kind: BGPPeer +metadata: + name: iad-gateway1-tenant-to-rr + namespace: galactic-system +spec: + routerRef: + name: iad-gateway1-tenant + peerASN: 65000 + address: "fc00:0:8::1" + remotePort: 1790 + addressFamilies: + - afi: l2vpn + safi: evpn diff --git a/deploy/containerlab/resources/galactic-router-gateway/iad-gateway1/bgprouter.yaml b/deploy/containerlab/resources/galactic-router-gateway/iad-gateway1/bgprouter.yaml new file mode 100644 index 00000000..6af6cf9f --- /dev/null +++ b/deploy/containerlab/resources/galactic-router-gateway/iad-gateway1/bgprouter.yaml @@ -0,0 +1,18 @@ +apiVersion: network.datumapis.com/v1alpha1 +kind: BGPRouter +metadata: + name: iad-gateway1-tenant + namespace: galactic-system +spec: + targetRef: + kind: Node + name: iad-gateway1 + roles: + - tenant + localASN: 65000 + routerID: "10.0.2.2" + srv6Locator: "2001:db8:ff03::/48" + nodeID: 2 + addressFamilies: + - afi: l2vpn + safi: evpn diff --git a/deploy/containerlab/resources/galactic-router-gateway/iad-gateway1/kustomization.yaml b/deploy/containerlab/resources/galactic-router-gateway/iad-gateway1/kustomization.yaml new file mode 100644 index 00000000..d6130417 --- /dev/null +++ b/deploy/containerlab/resources/galactic-router-gateway/iad-gateway1/kustomization.yaml @@ -0,0 +1,29 @@ +# iad-gateway1's instantiation of config/gateway/base (see that dir's +# kustomization.yaml doc comment for why this role needs one instance per +# gateway node rather than one shared DaemonSet). iad-gateway2/ is the +# sibling instantiation. +namespace: galactic-system +resources: + - ../base + - bgprouter.yaml + - bgppeer.yaml + - networkgateway.yaml +patches: + - path: node-patch.yaml + target: + kind: DaemonSet + name: galactic-gateway + # Strategic-merge patches can't rename a resource (Kustomize keeps the + # target's original identity regardless of what metadata.name the patch + # body says), so the rename to "galactic-gateway1" needs a JSON6902 + # patch -- same reason resources/fabric-control/iad/ renames its own + # DaemonSet: without a distinct name, this and iad-gateway2/'s DaemonSet + # would both be named "galactic-gateway" in the same namespace and + # silently clobber each other. + - target: + kind: DaemonSet + name: galactic-gateway + patch: |- + - op: replace + path: /metadata/name + value: galactic-gateway1 diff --git a/deploy/containerlab/resources/galactic-router-gateway/iad-gateway1/networkgateway.yaml b/deploy/containerlab/resources/galactic-router-gateway/iad-gateway1/networkgateway.yaml new file mode 100644 index 00000000..801662bb --- /dev/null +++ b/deploy/containerlab/resources/galactic-router-gateway/iad-gateway1/networkgateway.yaml @@ -0,0 +1,9 @@ +apiVersion: network.datumapis.com/v1alpha1 +kind: NetworkGateway +metadata: + name: iad-gateway1 + namespace: galactic-system +spec: + targetRef: + kind: Node + name: iad-gateway1 diff --git a/deploy/containerlab/resources/galactic-router-gateway/iad-gateway1/node-patch.yaml b/deploy/containerlab/resources/galactic-router-gateway/iad-gateway1/node-patch.yaml new file mode 100644 index 00000000..ae2ae383 --- /dev/null +++ b/deploy/containerlab/resources/galactic-router-gateway/iad-gateway1/node-patch.yaml @@ -0,0 +1,52 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: galactic-gateway +spec: + template: + spec: + # kubernetes.io/hostname pins this DaemonSet instance to exactly one + # node: GALACTIC_GATEWAY_SRV6_ADDRESS below must be unique per + # gateway node (see config/gateway/base/kustomization.yaml's doc + # comment), so galactic-gateway1/galactic-gateway2 are two separate + # DaemonSets rather than one DaemonSet matching both gateway-labeled + # nodes with identical env. + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: DoesNotExist + - key: galactic.datumapis.com/node + operator: In + values: + - gateway + - key: kubernetes.io/hostname + operator: In + values: + - iad-gateway1 + containers: + - name: galactic-gateway + env: + # eth1 is this lab's dedicated transit-fabric-facing uplink on + # every node (see resources/galactic-cni/daemonset-patch.yaml's + # comment) -- the same interface edgenat's XDP program attaches + # to for both underlay BGP and (once wired) ingress traffic. + - name: GALACTIC_GATEWAY_PUBLIC_INTERFACE + value: eth1 + # uFMT 48+16 uSID over this PoP's shared locator + # (2001:db8:ff03::/48, see bgprouter.yaml's srv6Locator). + # nodeID=2, Function=End.DT46 (arbitrary -- Argument 0 always + # misses vrf_table, so no Function value is ever consulted for + # it; see internal/plumbing/ebpf/prog/usid.c's Step 5-6 + # comment), Argument=0 (reserved, PR #740). Computed via + # internal/plumbing/ebpf/uformat.Encode directly, bypassing + # srv6.ComputeSID's argument==0 guard (that guard exists for + # tenant-VRF SID derivation specifically -- see + # internal/controller/networkgateway_controller.go's + # publishSelfAddress doc comment for why this value is + # supplied statically rather than computed by that + # reconciler today). + - name: GALACTIC_GATEWAY_SRV6_ADDRESS + value: "2001:db8:ff03:2:e000::" diff --git a/deploy/containerlab/resources/galactic-router-gateway/iad-gateway2/bgppeer.yaml b/deploy/containerlab/resources/galactic-router-gateway/iad-gateway2/bgppeer.yaml new file mode 100644 index 00000000..6f82f845 --- /dev/null +++ b/deploy/containerlab/resources/galactic-router-gateway/iad-gateway2/bgppeer.yaml @@ -0,0 +1,14 @@ +apiVersion: network.datumapis.com/v1alpha1 +kind: BGPPeer +metadata: + name: iad-gateway2-tenant-to-rr + namespace: galactic-system +spec: + routerRef: + name: iad-gateway2-tenant + peerASN: 65000 + address: "fc00:0:8::1" + remotePort: 1790 + addressFamilies: + - afi: l2vpn + safi: evpn diff --git a/deploy/containerlab/resources/galactic-router-gateway/iad-gateway2/bgprouter.yaml b/deploy/containerlab/resources/galactic-router-gateway/iad-gateway2/bgprouter.yaml new file mode 100644 index 00000000..e06e8efc --- /dev/null +++ b/deploy/containerlab/resources/galactic-router-gateway/iad-gateway2/bgprouter.yaml @@ -0,0 +1,18 @@ +apiVersion: network.datumapis.com/v1alpha1 +kind: BGPRouter +metadata: + name: iad-gateway2-tenant + namespace: galactic-system +spec: + targetRef: + kind: Node + name: iad-gateway2 + roles: + - tenant + localASN: 65000 + routerID: "10.0.2.3" + srv6Locator: "2001:db8:ff03::/48" + nodeID: 3 + addressFamilies: + - afi: l2vpn + safi: evpn diff --git a/deploy/containerlab/resources/galactic-router-gateway/iad-gateway2/kustomization.yaml b/deploy/containerlab/resources/galactic-router-gateway/iad-gateway2/kustomization.yaml new file mode 100644 index 00000000..391ab18d --- /dev/null +++ b/deploy/containerlab/resources/galactic-router-gateway/iad-gateway2/kustomization.yaml @@ -0,0 +1,21 @@ +# iad-gateway2's instantiation of config/gateway/base — the sibling of +# ../iad-gateway1/; see that dir's kustomization.yaml for the shared doc +# comment on why this role needs one instance per gateway node. +namespace: galactic-system +resources: + - ../base + - bgprouter.yaml + - bgppeer.yaml + - networkgateway.yaml +patches: + - path: node-patch.yaml + target: + kind: DaemonSet + name: galactic-gateway + - target: + kind: DaemonSet + name: galactic-gateway + patch: |- + - op: replace + path: /metadata/name + value: galactic-gateway2 diff --git a/deploy/containerlab/resources/galactic-router-gateway/iad-gateway2/networkgateway.yaml b/deploy/containerlab/resources/galactic-router-gateway/iad-gateway2/networkgateway.yaml new file mode 100644 index 00000000..45f22591 --- /dev/null +++ b/deploy/containerlab/resources/galactic-router-gateway/iad-gateway2/networkgateway.yaml @@ -0,0 +1,9 @@ +apiVersion: network.datumapis.com/v1alpha1 +kind: NetworkGateway +metadata: + name: iad-gateway2 + namespace: galactic-system +spec: + targetRef: + kind: Node + name: iad-gateway2 diff --git a/deploy/containerlab/resources/galactic-router-gateway/iad-gateway2/node-patch.yaml b/deploy/containerlab/resources/galactic-router-gateway/iad-gateway2/node-patch.yaml new file mode 100644 index 00000000..84182ba1 --- /dev/null +++ b/deploy/containerlab/resources/galactic-router-gateway/iad-gateway2/node-patch.yaml @@ -0,0 +1,52 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: galactic-gateway +spec: + template: + spec: + # kubernetes.io/hostname pins this DaemonSet instance to exactly one + # node: GALACTIC_GATEWAY_SRV6_ADDRESS below must be unique per + # gateway node (see config/gateway/base/kustomization.yaml's doc + # comment), so galactic-gateway1/galactic-gateway2 are two separate + # DaemonSets rather than one DaemonSet matching both gateway-labeled + # nodes with identical env. + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: DoesNotExist + - key: galactic.datumapis.com/node + operator: In + values: + - gateway + - key: kubernetes.io/hostname + operator: In + values: + - iad-gateway2 + containers: + - name: galactic-gateway + env: + # eth1 is this lab's dedicated transit-fabric-facing uplink on + # every node (see resources/galactic-cni/daemonset-patch.yaml's + # comment) -- the same interface edgenat's XDP program attaches + # to for both underlay BGP and (once wired) ingress traffic. + - name: GALACTIC_GATEWAY_PUBLIC_INTERFACE + value: eth1 + # uFMT 48+16 uSID over this PoP's shared locator + # (2001:db8:ff03::/48, see bgprouter.yaml's srv6Locator). + # nodeID=3, Function=End.DT46 (arbitrary -- Argument 0 always + # misses vrf_table, so no Function value is ever consulted for + # it; see internal/plumbing/ebpf/prog/usid.c's Step 5-6 + # comment), Argument=0 (reserved, PR #740). Computed via + # internal/plumbing/ebpf/uformat.Encode directly, bypassing + # srv6.ComputeSID's argument==0 guard (that guard exists for + # tenant-VRF SID derivation specifically -- see + # internal/controller/networkgateway_controller.go's + # publishSelfAddress doc comment for why this value is + # supplied statically rather than computed by that + # reconciler today). + - name: GALACTIC_GATEWAY_SRV6_ADDRESS + value: "2001:db8:ff03:3:e000::" diff --git a/deploy/containerlab/resources/galactic-router-gateway/iad/kustomization.yaml b/deploy/containerlab/resources/galactic-router-gateway/iad/kustomization.yaml new file mode 100644 index 00000000..1b99a04c --- /dev/null +++ b/deploy/containerlab/resources/galactic-router-gateway/iad/kustomization.yaml @@ -0,0 +1,9 @@ +# The single entry point for iad's gateway-role canary: both gateway +# nodes' DaemonSet+BGP+NetworkGateway instantiations, plus the ns60 +# NetworkRule. scripts/deploy-galactic-router.sh applies this one +# path rather than ../iad-gateway1/ and ../iad-gateway2/ separately. +namespace: galactic-system +resources: + - ../iad-gateway1 + - ../iad-gateway2 + - networkrule-ns60.yaml diff --git a/deploy/containerlab/resources/galactic-router-gateway/iad/networkrule-ns60.yaml b/deploy/containerlab/resources/galactic-router-gateway/iad/networkrule-ns60.yaml new file mode 100644 index 00000000..f058b2b8 --- /dev/null +++ b/deploy/containerlab/resources/galactic-router-gateway/iad/networkrule-ns60.yaml @@ -0,0 +1,30 @@ +# ns60: a live web service (resources/tenants/ns60/, vpc=60/vpcattachment=60, +# fd20:60:ff03::/48 -- see ns60/iad/nad.yaml) exposed through the edge XDP +# NAT+LB gateway on a real, routable ingress VIP. This rule's backend +# address is the nginx Deployment's actual live pod address +# (`kubectl get pod -n ns60 -o wide`) and its VIP is IPv6, matching the +# datapath's current scope: IPv6-only, plain TCP/UDP, no IPv4 support yet +# (see edgenat.c). +# 2001:db8:6060::1 is a documentation-range (RFC 3849, 2001:db8::/32) +# address picked to avoid colliding with any address already in use +# elsewhere in this topology (transit mesh: 2001:db8:0:*::/64 and +# 2001:db8:1:*::/64; SRv6 locators: 2001:db8:ff0X::/48) -- there is no +# real "public internet" in this lab, so this rule's own BGPAdvertisement +# (originated by both iad-gateway1 and iad-gateway2, active-active) is what +# makes it reachable from any other site's node once the underlay carries +# it, exactly like a real anycast ingress VIP would be. +apiVersion: network.datumapis.com/v1alpha1 +kind: NetworkRule +metadata: + name: ns60-web + namespace: galactic-system +spec: + vpcRef: "60" + vpcAttachmentRef: "60" + vipAddresses: + - 2001:db8:6060::1 + protocol: tcp + port: 80 + backends: + - address: fd20:60:ff03::100:0 + port: 80 diff --git a/deploy/containerlab/resources/tenants/ns50/base/kustomization.yaml b/deploy/containerlab/resources/tenants/ns50/base/kustomization.yaml deleted file mode 100644 index 3a32763c..00000000 --- a/deploy/containerlab/resources/tenants/ns50/base/kustomization.yaml +++ /dev/null @@ -1,17 +0,0 @@ -namespace: ns50 -resources: - - ../../base -patches: - - target: - kind: Deployment - name: netshoot - patch: | - apiVersion: apps/v1 - kind: Deployment - metadata: - name: netshoot - spec: - template: - metadata: - annotations: - v1.multus-cni.io/default-network: ns50/private diff --git a/deploy/containerlab/resources/tenants/ns50/dfw/nad.yaml b/deploy/containerlab/resources/tenants/ns50/dfw/nad.yaml deleted file mode 100644 index df701ef3..00000000 --- a/deploy/containerlab/resources/tenants/ns50/dfw/nad.yaml +++ /dev/null @@ -1,52 +0,0 @@ ---- -apiVersion: k8s.cni.cncf.io/v1 -kind: NetworkAttachmentDefinition -metadata: - name: private - namespace: ns50 -spec: - config: |- - { - "cniVersion": "1.0.0", - "name": "private", - "plugins": [ - { - "type": "galactic-veth", - "vpc": "50", - "vpcattachment": "50", - "namespace": "galactic-system", - "ipam": { - "type": "galactic-ipam", - "ipv4_subnet": "172.20.1.0/24", - "address_families": ["ipv4"] - } - }, - { - "type": "galactic-bgp", - "vpc": "50", - "vpcattachment": "50", - "namespace": "galactic-system" - } - ] - } - ---- -apiVersion: k8s.cni.cncf.io/v1 -kind: NetworkAttachmentDefinition -metadata: - name: igw - namespace: ns50 -spec: - config: |- - { - "cniVersion": "1.0.0", - "name": "public", - "type": "ptp", - "ipMasq": false, - "mtu": 1500, - "ipam": { - "type": "host-local", - "subnet": "2001:db8:1::/64", - "gateway": "2001:db8:1::1" - } - } diff --git a/deploy/containerlab/resources/tenants/ns50/iad/kustomization.yaml b/deploy/containerlab/resources/tenants/ns50/iad/kustomization.yaml deleted file mode 100644 index ee81d25b..00000000 --- a/deploy/containerlab/resources/tenants/ns50/iad/kustomization.yaml +++ /dev/null @@ -1,3 +0,0 @@ -resources: - - ../base - - nad.yaml diff --git a/deploy/containerlab/resources/tenants/ns50/iad/nad.yaml b/deploy/containerlab/resources/tenants/ns50/iad/nad.yaml deleted file mode 100644 index 54774955..00000000 --- a/deploy/containerlab/resources/tenants/ns50/iad/nad.yaml +++ /dev/null @@ -1,31 +0,0 @@ ---- -apiVersion: k8s.cni.cncf.io/v1 -kind: NetworkAttachmentDefinition -metadata: - name: private - namespace: ns50 -spec: - config: |- - { - "cniVersion": "1.0.0", - "name": "private", - "plugins": [ - { - "type": "galactic-veth", - "vpc": "50", - "vpcattachment": "50", - "namespace": "galactic-system", - "ipam": { - "type": "galactic-ipam", - "ipv4_subnet": "172.20.10.0/24", - "address_families": ["ipv4"] - } - }, - { - "type": "galactic-bgp", - "vpc": "50", - "vpcattachment": "50", - "namespace": "galactic-system" - } - ] - } diff --git a/deploy/containerlab/resources/tenants/ns50/sjc/kustomization.yaml b/deploy/containerlab/resources/tenants/ns50/sjc/kustomization.yaml deleted file mode 100644 index ee81d25b..00000000 --- a/deploy/containerlab/resources/tenants/ns50/sjc/kustomization.yaml +++ /dev/null @@ -1,3 +0,0 @@ -resources: - - ../base - - nad.yaml diff --git a/deploy/containerlab/resources/tenants/ns60/base/kustomization.yaml b/deploy/containerlab/resources/tenants/ns60/base/kustomization.yaml new file mode 100644 index 00000000..34add7b0 --- /dev/null +++ b/deploy/containerlab/resources/tenants/ns60/base/kustomization.yaml @@ -0,0 +1,13 @@ +# ns60's own base — unlike every other tenant, ns60 doesn't reuse +# ../../base's netshoot Deployment: it exists to expose a real web service +# through the edge XDP NAT+LB gateway (see +# resources/galactic-router-gateway/iad/networkrule-ns60.yaml), so its pod +# is an nginx server instead. namespace.yaml is a local duplicate of +# ../../base/namespace.yaml rather than a direct reference to it — see that +# file's own doc comment for why. pod.yaml here is ns60's own, already +# namespace-qualified in its multus annotation, so no patches: block is +# needed the way the netshoot-based tenants require. +namespace: ns60 +resources: + - namespace.yaml + - pod.yaml diff --git a/deploy/containerlab/resources/tenants/ns60/base/namespace.yaml b/deploy/containerlab/resources/tenants/ns60/base/namespace.yaml new file mode 100644 index 00000000..77e0761b --- /dev/null +++ b/deploy/containerlab/resources/tenants/ns60/base/namespace.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Namespace +metadata: + # Overwritten by this directory's kustomization.yaml `namespace:` field, + # exactly like ../../base/namespace.yaml (the shared placeholder every + # netshoot-based tenant reuses). Duplicated here rather than referenced + # directly (`../../base/namespace.yaml`) because kustomize's default load + # restrictor only allows a raw resource *file* reference to resolve + # within the referencing kustomization's own root — unlike referencing an + # entire sibling *kustomization directory* (what every other tenant's + # base/kustomization.yaml does via `../../base`), which ns60 can't do + # here without also pulling in that shared kustomization's netshoot + # pod.yaml. + name: tenant-placeholder diff --git a/deploy/containerlab/resources/tenants/ns60/base/pod.yaml b/deploy/containerlab/resources/tenants/ns60/base/pod.yaml new file mode 100644 index 00000000..bdf3b14d --- /dev/null +++ b/deploy/containerlab/resources/tenants/ns60/base/pod.yaml @@ -0,0 +1,35 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nginx + labels: + app: private +spec: + replicas: 1 + selector: + matchLabels: + app: private + template: + metadata: + labels: + app: private + annotations: + # Namespace-qualified for the same reason as tenants/base/pod.yaml's + # identical annotation: Multus resolves an unqualified + # default-network name against kube-system in this lab, not the + # pod's own namespace (verified empirically). + v1.multus-cni.io/default-network: ns60/private + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: DoesNotExist + containers: + - name: nginx + image: nginx:latest + imagePullPolicy: IfNotPresent + ports: + - containerPort: 80 diff --git a/deploy/containerlab/resources/tenants/ns50/dfw/kustomization.yaml b/deploy/containerlab/resources/tenants/ns60/iad/kustomization.yaml similarity index 100% rename from deploy/containerlab/resources/tenants/ns50/dfw/kustomization.yaml rename to deploy/containerlab/resources/tenants/ns60/iad/kustomization.yaml diff --git a/deploy/containerlab/resources/tenants/ns50/sjc/nad.yaml b/deploy/containerlab/resources/tenants/ns60/iad/nad.yaml similarity index 68% rename from deploy/containerlab/resources/tenants/ns50/sjc/nad.yaml rename to deploy/containerlab/resources/tenants/ns60/iad/nad.yaml index 5a6b8c5a..ce088bbf 100644 --- a/deploy/containerlab/resources/tenants/ns50/sjc/nad.yaml +++ b/deploy/containerlab/resources/tenants/ns60/iad/nad.yaml @@ -3,7 +3,7 @@ apiVersion: k8s.cni.cncf.io/v1 kind: NetworkAttachmentDefinition metadata: name: private - namespace: ns50 + namespace: ns60 spec: config: |- { @@ -12,19 +12,19 @@ spec: "plugins": [ { "type": "galactic-veth", - "vpc": "50", - "vpcattachment": "50", + "vpc": "60", + "vpcattachment": "60", "namespace": "galactic-system", "ipam": { "type": "galactic-ipam", - "ipv4_subnet": "172.20.20.0/24", - "address_families": ["ipv4"] + "ipv6_subnet": "fd20:60:ff03::/48", + "address_families": ["ipv6"] } }, { "type": "galactic-bgp", - "vpc": "50", - "vpcattachment": "50", + "vpc": "60", + "vpcattachment": "60", "namespace": "galactic-system" } ] diff --git a/deploy/containerlab/scripts/deploy-fabric.sh b/deploy/containerlab/scripts/deploy-fabric.sh index 6d1bd267..a6da303e 100755 --- a/deploy/containerlab/scripts/deploy-fabric.sh +++ b/deploy/containerlab/scripts/deploy-fabric.sh @@ -7,14 +7,18 @@ SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd) source "${SCRIPT_DIR}/lib.sh" # config/fabric/ (shared with production) is a single DaemonSet whose -# affinity allows both the edge and control node labels. resources/fabric-router/ -# base/ and resources/fabric-control/iad/ each build on a copy of it and -# patch in the lab-only image/imagePullPolicy plus a narrower affinity -# (edge-only / control-only respectively) — iad needs the two split back -# apart because its two nodes need different frr.conf. Copied onto the node -# at deploy time nested under each consuming overlay's own root so its -# "fabric" resource reference resolves (kustomize requires resources in or -# below the overlay root). +# affinity allows the edge, control, and gateway node labels. +# resources/fabric-router/base/ and resources/fabric-control/iad/ each +# build on a copy of it and patch in the lab-only image/imagePullPolicy +# plus a narrower affinity (edge+gateway / control-only respectively) — +# iad needs the control node split back out because it needs its own +# frr.conf and would otherwise get two competing fabric pods; edge and +# gateway share one DaemonSet since neither needs anything but a +# per-node frr.conf key (fabric-router/iad/kustomization.yaml's +# configMapGenerator carries both). Copied onto the node at deploy time +# nested under each consuming overlay's own root so its "fabric" resource +# reference resolves (kustomize requires resources in or below the +# overlay root). FABRIC_DIR=$(cd "${SCRIPT_DIR}/../../../config/fabric" && pwd) # copy_fabric_config NODE copies config/fabric/ onto NODE, nested under @@ -42,7 +46,9 @@ for site in dfw sjc; do done # iad-control-plane needs fabric + fabric-control + galactic-router — -# batch all copies together. +# batch all copies together. fabric-router/iad/ alone covers both +# iad-worker and iad-gateway1/2 (its ConfigMap carries all three +# frr.conf. keys; ../base/'s affinity matches both roles). node=$(control_plane iad) echo "Copying resources to ${node}..." copy_to "${node}" fabric-router @@ -52,7 +58,8 @@ copy_fabric_control_config "${node}" copy_to "${node}" galactic-router copy_to "${node}" galactic-control -# Both fabric overlays (per-site and iad's control role) are kustomize now. +# Both fabric overlays (per-site/gateway, and iad's control role) are +# kustomize now. apply_k "${node}" /galactic/resources/fabric-router/iad/ apply_k "${node}" /galactic/resources/fabric-control/iad/ diff --git a/deploy/containerlab/scripts/deploy-galactic-router.sh b/deploy/containerlab/scripts/deploy-galactic-router.sh index 576fe326..88acd532 100755 --- a/deploy/containerlab/scripts/deploy-galactic-router.sh +++ b/deploy/containerlab/scripts/deploy-galactic-router.sh @@ -18,12 +18,24 @@ source "${SCRIPT_DIR}/lib.sh" GALACTIC_ROUTER_BASE_DIR=$(cd "${SCRIPT_DIR}/../../../config/router/base" && pwd) GALACTIC_ROUTER_TENANT_DIR=$(cd "${SCRIPT_DIR}/../../../config/router/tenant" && pwd) GALACTIC_ROUTER_TENANT_CONTROL_DIR=$(cd "${SCRIPT_DIR}/../../../config/router/tenant-control" && pwd) +# config/gateway/base is the edge XDP NAT+LB gateway's own two-container +# pod base (galactic-router + galactic-gateway) -- self-contained, unlike +# config/router/{tenant,tenant-control}, so no matching +# config/router/base copy is needed alongside it the way copy_router_config/ +# copy_router_control_config need one. +GALACTIC_GATEWAY_BASE_DIR=$(cd "${SCRIPT_DIR}/../../../config/gateway/base" && pwd) # copy_router_config NODE copies config/router/{base,tenant} onto NODE, # nested under resources/galactic-router/base/ so the overlay's "../base" -# resource reference resolves. +# resource reference resolves. rm -rf first: like deploy-cni.sh's +# GALACTIC_CNI_DIR copy, docker cp nests SRC inside an already-existing +# DEST dir instead of overwriting it, so a rerun against an +# already-provisioned node would silently keep serving the prior copy +# from underneath the new one -- kubectl would then report the DaemonSet +# "unchanged" even after a real manifest edit. copy_router_config() { local node="$1" + docker exec "${node}" rm -rf /galactic/resources/galactic-router/base/base /galactic/resources/galactic-router/base/tenant docker cp "${GALACTIC_ROUTER_BASE_DIR}" "${node}:/galactic/resources/galactic-router/base/base" docker cp "${GALACTIC_ROUTER_TENANT_DIR}" "${node}:/galactic/resources/galactic-router/base/tenant" } @@ -32,17 +44,31 @@ copy_router_config() { # onto NODE, nested under resources/galactic-control/iad/ so the # tenant-control overlay's "../base" resource reference resolves. Its node # affinity (route-reflector role, control node only) applies as-is; the -# lab only needs to patch in the image and BGP address/port. +# lab only needs to patch in the image and BGP address/port. rm -rf first +# -- see copy_router_config's comment. copy_router_control_config() { local node="$1" + docker exec "${node}" rm -rf /galactic/resources/galactic-control/iad/base /galactic/resources/galactic-control/iad/tenant-control docker cp "${GALACTIC_ROUTER_BASE_DIR}" "${node}:/galactic/resources/galactic-control/iad/base" docker cp "${GALACTIC_ROUTER_TENANT_CONTROL_DIR}" "${node}:/galactic/resources/galactic-control/iad/tenant-control" } +# copy_router_gateway_config NODE copies config/gateway/base onto NODE, +# nested under resources/galactic-router-gateway/base/ so that overlay's +# "gateway" resource reference resolves. Each per-node overlay +# (iad-gateway1/, iad-gateway2/) references ../base, so this is only +# copied once regardless of how many gateway nodes exist. rm -rf first -- +# see copy_router_config's comment. +copy_router_gateway_config() { + local node="$1" + docker exec "${node}" rm -rf /galactic/resources/galactic-router-gateway/base/gateway + docker cp "${GALACTIC_GATEWAY_BASE_DIR}" "${node}:/galactic/resources/galactic-router-gateway/base/gateway" +} + # apply_galactic_router applies the site's galactic-router overlay (DaemonSet # + BGP CRDs). Shared by all three sites; iad layers its route-reflector on # top after calling this. NADs and test workloads live under -# resources/tenants/ns50/ and are applied by deploy-ns.sh. +# resources/tenants/ns10/ and are applied by deploy-ns.sh. apply_galactic_router() { local node="$1" site="$2" apply_k "${node}" "/galactic/resources/galactic-router/${site}/" @@ -51,6 +77,7 @@ apply_galactic_router() { for site in dfw sjc; do node=$(control_plane "${site}") echo "Applying galactic-router/${site} to ${node}..." + docker exec "${node}" rm -rf /galactic/resources/galactic-router copy_to "${node}" galactic-router copy_router_config "${node}" apply_galactic_router "${node}" "${site}" @@ -65,4 +92,23 @@ copy_router_control_config "${node}" apply_galactic_router "${node}" iad apply_k "${node}" /galactic/resources/galactic-control/iad/ +# iad's gateway-role canary: two dedicated nodes, iad-gateway1/ +# iad-gateway2, each running its own two-container +# pod instance (config/gateway/base's per-node-unique +# GALACTIC_GATEWAY_SRV6_ADDRESS -- see config/gateway/base/kustomization. +# yaml's doc comment). galactic-gateway's own ServiceAccount/ClusterRole +# (config/gateway/{serviceaccount,rbac}.yaml, already copied onto this +# node by deploy-system.sh's copy_config) must be applied once before the +# per-node overlays below, same as galactic-cni/galactic-router's RBAC is +# applied by deploy-system.sh itself rather than by this script. +echo "Applying galactic-gateway RBAC to ${node}..." +apply_f "${node}" /galactic/config/gateway/serviceaccount.yaml +apply_f "${node}" /galactic/config/gateway/rbac.yaml + +echo "Applying galactic-router-gateway/iad to ${node}..." +docker exec "${node}" rm -rf /galactic/resources/galactic-router-gateway +copy_to "${node}" galactic-router-gateway +copy_router_gateway_config "${node}" +apply_k "${node}" /galactic/resources/galactic-router-gateway/iad/ + echo "Done." diff --git a/deploy/containerlab/scripts/deploy-system.sh b/deploy/containerlab/scripts/deploy-system.sh index a4abeff5..147b9b06 100755 --- a/deploy/containerlab/scripts/deploy-system.sh +++ b/deploy/containerlab/scripts/deploy-system.sh @@ -11,8 +11,9 @@ source "${SCRIPT_DIR}/lib.sh" # Extract the datum-cloud/network commit SHA from go.mod (pseudo-version # suffix after the last hyphen, e.g. v0.0.0-20260708202618-77cf276d17f1 → -# 77cf276d17f1). -NETWORK_SHA=$(awk '/go\.datum\.net\/network/ {print $2}' "${SCRIPT_DIR}/../../../go.mod" | sed 's/.*-//') +# 77cf276d17f1). $1 must match the require line's module path exactly, not +# just a substring, so an unrelated line can't corrupt NETWORK_SHA. +NETWORK_SHA=$(awk '$1 == "go.datum.net/network" {print $2}' "${SCRIPT_DIR}/../../../go.mod" | sed 's/.*-//') NETWORK_CRD_URL="https://raw.githubusercontent.com/datum-cloud/network/${NETWORK_SHA}/config/crd" # VPC/VPCAttachment CRDs come from the separate companion VPC operator, @@ -29,6 +30,8 @@ network_crds=( network.datumapis.com_bgppolicies.yaml network.datumapis.com_bgprouters.yaml network.datumapis.com_bgpvrfinstances.yaml + network.datumapis.com_networkgateways.yaml + network.datumapis.com_networkrules.yaml ) cloud_crds=( diff --git a/deploy/containerlab/scripts/verify-ns50.sh b/deploy/containerlab/scripts/verify-ns50.sh deleted file mode 100755 index 69e91f4d..00000000 --- a/deploy/containerlab/scripts/verify-ns50.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash -# verify-ns50.sh — confirm IPv4 ping connectivity between ns50 pods on every -# pair of sites, in both directions (full mesh). ns50 is IPv4, 3-site. -set -euo pipefail - -SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd) -source "${SCRIPT_DIR}/lib.sh" - -NS=ns50 -SITES=(dfw sjc iad) - -declare -A NODE POD IP4 -for s in "${SITES[@]}"; do - NODE[$s]=$(control_plane "${s}") - POD[$s]=$(pod_name "${NODE[$s]}" "${NS}") - IP4[$s]=$(pod_ip4 "${NODE[$s]}" "${NS}" "${POD[$s]}") - echo "${s}: pod=${POD[$s]} ip=${IP4[$s]}" -done - -for src in "${SITES[@]}"; do - for dst in "${SITES[@]}"; do - [ "${src}" = "${dst}" ] && continue - echo "--- ${src} -> ${dst} (${IP4[$dst]}) ---" - ping_pod "${NODE[$src]}" "${NS}" "${POD[$src]}" -4 "${IP4[$dst]}" - done -done - -echo "ns50: ping OK between all site pairs"