diff --git a/.github/workflows/test-e2e.yml b/.github/workflows/test-e2e.yml index 420fe171..cc9bf2cd 100644 --- a/.github/workflows/test-e2e.yml +++ b/.github/workflows/test-e2e.yml @@ -37,8 +37,11 @@ jobs: # was removed in helm 3.13, so pin to the last release that accepts it. version: v3.12.3 + # The same installer the federated workflow uses. An explicit `make` list + # here meant the two workflows disagreed about which binaries exist, and + # a tool added to the test-infra tools task reached only one of them. - name: Install pinned tools - run: make kind kustomize cmctl chainsaw + run: task test-infra:tools - name: Bring up the prod-fidelity env run: task test-infra:up @@ -62,6 +65,22 @@ jobs: KUBECONFIG="${UKC}" kubectl -n cert-manager get pods -o wide KUBECONFIG="${UKC}" kubectl -n network-services-operator-system get certificate,secret echo "::endgroup::" + echo "::group::IPAM (upstream project control plane)" + KUBECONFIG="${UKC}" kubectl get apiservice v1alpha1.ipam.miloapis.com -o yaml + KUBECONFIG="${UKC}" kubectl -n ipam-system get pods,certificate,configmap -o wide + KUBECONFIG="${UKC}" kubectl -n ipam-system logs -l app=ipam-apiserver --tail=200 --all-containers + KUBECONFIG="${UKC}" kubectl -n ipam-system get cluster.postgresql.cnpg.io -o wide + KUBECONFIG="${UKC}" kubectl -n ipam-system describe cluster.postgresql.cnpg.io ipam-db + KUBECONFIG="${UKC}" kubectl -n ipam-system logs -l cnpg.io/cluster=ipam-db --tail=100 --all-containers + echo "::endgroup::" + echo "::group::CloudNativePG operator" + KUBECONFIG="${UKC}" kubectl -n cnpg-system get pods -o wide + KUBECONFIG="${UKC}" kubectl -n cnpg-system logs deploy/cnpg-cloudnative-pg --tail=200 + echo "::endgroup::" + echo "::group::IPAM fixtures" + KUBECONFIG=${TMPDIR}/.ipam-tenant-impersonation.yaml kubectl --context tenant-project-alpha get ipclasses,ippools -o wide + KUBECONFIG=${TMPDIR}/.ipam-tenant-impersonation.yaml kubectl --context tenant-project-beta get ipclasses,ippools -o wide + echo "::endgroup::" KC=${TMPDIR}/.kind-nso-downstream.yaml bin/kind-v0.32.0 get kubeconfig --name nso-downstream > "${KC}" export KUBECONFIG="${KC}" diff --git a/PROJECT b/PROJECT index 59ebc91d..706f9797 100644 --- a/PROJECT +++ b/PROJECT @@ -185,4 +185,22 @@ resources: kind: ConnectorClass path: go.datum.net/network-services-operator/api/v1alpha1 version: v1alpha1 +- api: + crdVersion: v1 + namespaced: true + controller: true + domain: datumapis.com + group: networking + kind: NetworkInterface + path: go.datum.net/network-services-operator/api/v1alpha + version: v1alpha +- api: + crdVersion: v1 + namespaced: true + controller: true + domain: datumapis.com + group: networking + kind: NetworkInterfaceClaim + path: go.datum.net/network-services-operator/api/v1alpha + version: v1alpha version: "3" diff --git a/Taskfile.test-infra.yml b/Taskfile.test-infra.yml index 6236e44d..71215f36 100644 --- a/Taskfile.test-infra.yml +++ b/Taskfile.test-infra.yml @@ -54,10 +54,38 @@ vars: IMG: sh: echo "ghcr.io/datum-cloud/network-services-operator:$(git rev-parse --short HEAD)" + # IPAM runs on the upstream cluster as the project control plane NSO claims + # addresses from. Its manifests come from the bundle milo-os/ipam publishes on + # every push to main, and its image from ghcr.io — both at the commit go.mod + # pins, so the apiserver and the Go client NSO compiles against stay in + # lockstep. Nothing is built locally: the image is pulled from ghcr by + # digest, multi-arch since upstream 13bd2c8 fixed the arm64 build. + # + # Digest, not tag: the release workflow republishes on every push, so the tag + # can move. The tag is kept alongside for legibility. Update all three + # together, and keep IPAM_COMMIT equal to the go.mod replace directive — + # ipam-bundle fails if the artifact does not carry that commit. + IPAM_BUNDLE_REPO: ghcr.io/milo-os/ipam-kustomize + IPAM_BUNDLE_TAG: v0.0.0-13bd2c8 + IPAM_BUNDLE_DIGEST: sha256:08418654958ea6861d54fe5f9e87d9cf306b87429b29fc41a7876dc143a13bf5 + IPAM_COMMIT: 13bd2c8c077b12983fa16a8888dcd117b65a14d2 + IPAM_STAGE: '{{.TMP_DIR}}/.ipam-bundle' + + # The dns-operator CRDs come from its published bundle at the version go.mod + # pins, so the CRDs installed and the types NSO compiles against match. + DNS_BUNDLE_REPO: ghcr.io/datum-cloud/dns-operator-kustomize + DNS_BUNDLE_TAG: v0.5.1 + DNS_BUNDLE_DIGEST: sha256:cd7dd3a0d2fbbbaa029f2b5e083723a675e88c89e56f999257ff4bd87221cd4a + DNS_STAGE: '{{.TMP_DIR}}/.dns-bundle' + CRANE: '{{.REPO}}/bin/crane' + CRANE_VERSION: v0.21.7 + IPAM_KUBECONFIG: '{{.TMP_DIR}}/.ipam-tenant-impersonation.yaml' + IPAM_PROJECTS: project-alpha project-beta + tasks: tools: - desc: "Install the pinned tool binaries this env needs into bin/ (kind v0.32.0, karmadactl, kustomize, cmctl, chainsaw). Idempotent — skips any already present. Lets CI and a clean checkout bring up the env without hand-placed binaries." + desc: "Install the pinned tool binaries this env needs into bin/ (kind v0.32.0, karmadactl, kustomize, cmctl, chainsaw, crane). Idempotent — skips any already present. Lets CI and a clean checkout bring up the env without hand-placed binaries." vars: KIND_TOOL_VERSION: v0.32.0 KARMADACTL_VERSION: v1.15.2 @@ -75,6 +103,11 @@ tasks: - '[ -x {{.REPO}}/bin/kustomize ] || GOBIN={{.REPO}}/bin go install sigs.k8s.io/kustomize/kustomize/v5@{{.KUSTOMIZE_VERSION}}' - '[ -x {{.REPO}}/bin/cmctl ] || GOBIN={{.REPO}}/bin go install github.com/cert-manager/cmctl/v2@{{.CMCTL_VERSION}}' - '[ -x {{.REPO}}/bin/chainsaw ] || GOBIN={{.REPO}}/bin go install github.com/kyverno/chainsaw@{{.CHAINSAW_VERSION}}' + # crane reads the published IPAM manifest bundle out of ghcr.io. The flux + # CLI can do this too, but its go.mod replace directives stop `go install` + # building it, and crane is a much smaller dependency than all of flux for + # one registry read. + - '[ -x {{.REPO}}/bin/crane ] || GOBIN={{.REPO}}/bin go install github.com/google/go-containerregistry/cmd/crane@{{.CRANE_VERSION}}' # karmadactl carries replace directives in its go.mod, so `go install` can't # build it; pull the published release binary for this OS/arch instead. - | @@ -98,9 +131,23 @@ tasks: - task: downstream-namespaces - task: cert-manager-upstream - task: cert-manager-downstream + # Fetched early: it depends on nothing in the cluster, and pulling it here + # surfaces a registry problem before the long build below. + - task: ipam-bundle - task: nso-image - task: dns-crds + # The manager mounts this secret, so it has to exist before the pod does. + - task: ipam-kubeconfig - task: prepare-upstream + # IPAM's serving certificate cannot issue until upstream cert-manager is + # running, and upstream cert-manager crashloops with "the Gateway API CRDs + # do not seem to be present" until prepare-upstream installs them. Every + # IPAM step that touches the cluster therefore has to follow it. + - task: cnpg-operator + - task: ipam-deploy + - task: ipam-wait + - task: ipam-fixtures + - task: ipam-namespaces - task: eg-downstream - task: extension-server - task: billing-usage-collector @@ -120,6 +167,10 @@ tasks: - '{{.KIND}} delete cluster --name {{.UPSTREAM_CLUSTER}} || true' - '{{.KIND}} delete cluster --name {{.DOWNSTREAM_CLUSTER}} || true' - rm -f {{.TMP_DIR}}/.kind-{{.DOWNSTREAM_CLUSTER}}-internal.yaml {{.TMP_DIR}}/.kind-{{.UPSTREAM_CLUSTER}}.yaml {{.TMP_DIR}}/.kind-{{.DOWNSTREAM_CLUSTER}}.yaml + # The IPAM fixtures and their database die with the upstream cluster; only + # the impersonation kubeconfig outlives it. + - rm -f {{.IPAM_KUBECONFIG}} + - rm -rf {{.IPAM_STAGE}} {{.DNS_STAGE}} - echo "✨ done." clusters: @@ -159,17 +210,25 @@ tasks: - '{{.KIND}} load docker-image {{.IMG}} --name {{.DOWNSTREAM_CLUSTER}}' dns-crds: - desc: "Install DNSZone / DNSRecordSet / DNSZoneClass CRDs from the dns-operator Go module on the upstream cluster. Required before prepare-upstream when gateway.enableDNSIntegration is true (the operator indexes and watches these types at startup)." - vars: - # Escape {{.Dir}} so Task does not treat it as a Task template var; - # go list must receive the literal {{.Dir}} format string. - DNS_OPERATOR_CRD_DIR: - sh: go list -m -f '{{`{{.Dir}}`}}' go.miloapis.com/dns-operator + desc: "Install the dns-operator CRDs (DNSZone / DNSRecordSet / DNSZoneClass) on the upstream cluster from the operator's published bundle, pinned by digest at the version go.mod pins. Required before prepare-upstream when gateway.enableDNSIntegration is true, since the operator indexes and watches these types at startup." cmds: - - echo "📜 installing dns-operator CRDs (upstream) from {{.DNS_OPERATOR_CRD_DIR}}" - - kubectl --context {{.UPSTREAM_CTX}} apply -f {{.DNS_OPERATOR_CRD_DIR}}/config/crd/bases/dns.networking.miloapis.com_dnszoneclasses.yaml - - kubectl --context {{.UPSTREAM_CTX}} apply -f {{.DNS_OPERATOR_CRD_DIR}}/config/crd/bases/dns.networking.miloapis.com_dnszones.yaml - - kubectl --context {{.UPSTREAM_CTX}} apply -f {{.DNS_OPERATOR_CRD_DIR}}/config/crd/bases/dns.networking.miloapis.com_dnsrecordsets.yaml + - | + set -e + stage="{{.DNS_STAGE}}" + if [ -f "$stage/.digest" ] && [ "$(cat "$stage/.digest")" = "{{.DNS_BUNDLE_DIGEST}}" ]; then + echo "📜 dns-operator bundle already staged ({{.DNS_BUNDLE_TAG}})" >&2 + else + echo "📜 fetching {{.DNS_BUNDLE_REPO}}:{{.DNS_BUNDLE_TAG}}" >&2 + rev=$({{.REPO}}/hack/fetch-oci-bundle.sh \ + {{.CRANE}} {{.CRANE_VERSION}} \ + {{.DNS_BUNDLE_REPO}} {{.DNS_BUNDLE_DIGEST}} {{.DNS_BUNDLE_TAG}} "$stage/bundle") + echo "✅ fetched $rev" >&2 + echo "{{.DNS_BUNDLE_DIGEST}}" > "$stage/.digest" + fi + - echo "📜 installing dns-operator CRDs (upstream)" + - kubectl --context {{.UPSTREAM_CTX}} apply -f {{.DNS_STAGE}}/bundle/crd/bases/dns.networking.miloapis.com_dnszoneclasses.yaml + - kubectl --context {{.UPSTREAM_CTX}} apply -f {{.DNS_STAGE}}/bundle/crd/bases/dns.networking.miloapis.com_dnszones.yaml + - kubectl --context {{.UPSTREAM_CTX}} apply -f {{.DNS_STAGE}}/bundle/crd/bases/dns.networking.miloapis.com_dnsrecordsets.yaml prepare-upstream: desc: "Deploy the NSO manager + webhook (config/e2e) on the upstream cluster, with the prod-base memory profile." @@ -358,6 +417,230 @@ tasks: - kubectl --context {{.DOWNSTREAM_CTX}} wait gatewayclass datum-downstream-gateway-e2e --for=condition=Accepted --timeout=120s - echo "✅ core components ready." + # ---- IPAM --------------------------------------------------------------- + # + # IPAM is the upstream PROJECT control plane: NSO's manager, running on the + # same kind cluster, plays the cell that claims addresses from it. The two + # roles are separated by kubeconfig and impersonation, not by cluster + # boundary, so colocating them costs no fidelity — the cells that provision + # network interfaces run single-cluster in production and reach IPAM exactly + # this way. + + cnpg-operator: + desc: "Install the CloudNativePG operator (chart 0.29.0 / CNPG 1.30.0) on the upstream cluster and wait until it can actually admit a Cluster. IPAM's bundle ships CNPG as a Flux HelmRelease; this env has no Flux, so it takes the same chart through kustomize --enable-helm, the way cert-manager is installed here." + cmds: + - echo "🐘 installing CloudNativePG (upstream)" + - '{{.KUSTOMIZE}} build --enable-helm config/dependencies/postgres-operator | kubectl --context {{.UPSTREAM_CTX}} apply --server-side=true --force-conflicts -f -' + - echo "⏳ waiting for the CNPG CRDs to be established" + - kubectl --context {{.UPSTREAM_CTX}} wait --for=condition=Established crd/clusters.postgresql.cnpg.io --timeout=120s + - echo "⏳ waiting for the CNPG operator" + - kubectl --context {{.UPSTREAM_CTX}} -n cnpg-system rollout status deploy/cnpg-cloudnative-pg --timeout=300s + # Deployment Available is not the same as "the webhook answers": the + # operator injects its own CA into the webhook configuration after it + # starts, and a Cluster applied into that window is rejected with a TLS + # or connection error that reads exactly like a flaky apply. Block until a + # server-side dry run is genuinely admitted, the same way wait-ready does + # for the NSO webhook. + - echo "⏳ waiting for the CNPG webhook to admit" + - | + PROBE='{"apiVersion":"postgresql.cnpg.io/v1","kind":"Cluster","metadata":{"name":"admit-probe","namespace":"default"},"spec":{"instances":1,"storage":{"size":"128Mi"}}}' + for i in $(seq 1 60); do + if echo "$PROBE" | kubectl --context {{.UPSTREAM_CTX}} create --dry-run=server -f - >/dev/null 2>&1; then + echo "✅ CNPG admitting"; break + fi + if [ "$i" -eq 60 ]; then echo "❌ CNPG webhook never became ready"; exit 1; fi + sleep 2 + done + + ipam-bundle: + desc: "Fetch the published IPAM manifest bundle from ghcr.io and assemble the composed deployment in a staging directory. Pinned by digest, and verified to carry the commit go.mod pins, so a rebuild of that commit cannot silently change what is deployed. Anonymous pull — CI needs no registry credentials." + cmds: + - | + set -e + stage="{{.IPAM_STAGE}}" + if [ -f "$stage/.digest" ] && [ "$(cat "$stage/.digest")" = "{{.IPAM_BUNDLE_DIGEST}}" ]; then + echo "📦 IPAM bundle already staged ({{.IPAM_BUNDLE_TAG}})" >&2 + else + echo "📦 fetching {{.IPAM_BUNDLE_REPO}}:{{.IPAM_BUNDLE_TAG}}" >&2 + rev=$({{.REPO}}/hack/fetch-oci-bundle.sh \ + {{.CRANE}} {{.CRANE_VERSION}} \ + {{.IPAM_BUNDLE_REPO}} {{.IPAM_BUNDLE_DIGEST}} {{.IPAM_COMMIT}} "$stage/bundle") + echo "✅ fetched $rev" >&2 + fi + # Refreshed every run: these come from the working tree, so an edit to + # the overlay or a patch must reach the staged build without re-pulling. + rm -rf "$stage/overlay" "$stage/patches" + cp -R "{{.REPO}}/config/dependencies/ipam/overlay" "$stage/overlay" + cp -R "{{.REPO}}/config/dependencies/ipam/patches" "$stage/patches" + cp "{{.REPO}}/config/dependencies/ipam/root-kustomization.yaml" "$stage/kustomization.yaml" + echo "{{.IPAM_BUNDLE_DIGEST}}" > "$stage/.digest" + + ipam-render: + desc: "Render the composed IPAM deployment to stdout. What ipam-deploy applies, without applying it — the composed root is assembled at task time, so this is the only way to review it." + deps: + - ipam-bundle + cmds: + - '{{.KUSTOMIZE}} build {{.IPAM_STAGE}}' + + ipam-deploy: + deps: + - ipam-bundle + - cnpg-operator + desc: "Apply the IPAM apiserver, its Postgres, and the tenant RBAC to the upstream cluster (config/dependencies/ipam). Applies only — ipam-wait is what gates on readiness. Must follow prepare-upstream: upstream cert-manager crashloops until the Gateway-API CRDs it probes at startup exist, and until it runs, ipam-tls is never issued." + cmds: + - echo "🔧 deploying IPAM (upstream)" + - task: ipam-certmanager-ready + # The aggregated apiserver verifies the front proxy with + # --requestheader-client-ca-file. In kind that CA lives in the host's + # extension-apiserver-authentication ConfigMap, so it has to be copied + # into ipam-system before the pod starts — without it every impersonated + # extra arrives unauthenticated and every claim reads an empty tenant. + - kubectl --context {{.UPSTREAM_CTX}} create namespace ipam-system --dry-run=client -o yaml | kubectl --context {{.UPSTREAM_CTX}} apply -f - + - | + kubectl --context {{.UPSTREAM_CTX}} -n kube-system get configmap extension-apiserver-authentication \ + -o jsonpath='{.data.requestheader-client-ca-file}' > {{.TMP_DIR}}/.ipam-requestheader-ca.crt + kubectl --context {{.UPSTREAM_CTX}} -n ipam-system create configmap control-plane-ca \ + --from-file=ca.crt={{.TMP_DIR}}/.ipam-requestheader-ca.crt \ + --dry-run=client -o yaml | kubectl --context {{.UPSTREAM_CTX}} apply -f - + - '{{.KUSTOMIZE}} build {{.IPAM_STAGE}} | kubectl --context {{.UPSTREAM_CTX}} apply --server-side=true --force-conflicts -f -' + + ipam-certmanager-ready: + desc: "Make sure upstream cert-manager is actually running before anything asks it for a certificate. It exits at startup when the Gateway-API CRDs are absent, so on a fresh env it is in CrashLoopBackOff by the time prepare-upstream installs them — and the backoff can be minutes long. Restart it rather than wait the backoff out." + cmds: + - | + if kubectl --context {{.UPSTREAM_CTX}} -n cert-manager wait deploy cert-manager \ + --for=condition=Available --timeout=15s >/dev/null 2>&1; then + echo "✅ upstream cert-manager already Available" + exit 0 + fi + echo "♻️ upstream cert-manager is not Available; restarting it" + kubectl --context {{.UPSTREAM_CTX}} -n cert-manager rollout restart deploy cert-manager + kubectl --context {{.UPSTREAM_CTX}} -n cert-manager rollout status deploy cert-manager --timeout=240s + + ipam-wait: + desc: "Gate on IPAM being genuinely servable: Postgres, the apiserver rollout, then the APIService itself. The certificate gate is on the internal CA, not on a serving cert: the CSI driver issues serving certs per pod at mount time, so a CA problem surfaces as a pod that never leaves Init. The APIService gate is not redundant — a suite that starts while aggregation is still settling fails with ServiceUnavailable, which reads like a test bug and is not one." + cmds: + # The serving cert is issued per pod by the cert-manager CSI driver, from + # the CA this gate waits on. Without the CA the mount never completes and + # the pod sits in Init with nothing in its logs to explain why. + - echo "⏳ waiting for the IPAM internal CA" + - kubectl --context {{.UPSTREAM_CTX}} -n ipam-system wait certificate/ipam-ca --for=condition=Ready --timeout=180s + - echo "⏳ waiting for Postgres (CNPG cluster ipam-db)" + # The apiserver's migrate init container runs `ipam migrate up` against + # this database, so it crashloops until Postgres accepts connections. + # CNPG reports Ready only once the instance is serving, which is what the + # init container needs — a scheduled pod is not enough. This replaced a + # `wait pod -l app.kubernetes.io/name=postgresql` gate when the plain + # Deployment gave way to the operator. + - kubectl --context {{.UPSTREAM_CTX}} -n ipam-system wait cluster.postgresql.cnpg.io/ipam-db --for=condition=Ready --timeout=420s + # Ready is the operator's opinion; the init container's is the one that + # matters. Ask the database directly, as the role and database IPAM + # actually connects as, so a cluster that reports Ready while still + # finishing bootstrap fails here rather than as a crashlooping migration. + - | + for i in $(seq 1 45); do + POD=$(kubectl --context {{.UPSTREAM_CTX}} -n ipam-system get pod \ + -l cnpg.io/cluster=ipam-db -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true) + if [ -n "$POD" ] && kubectl --context {{.UPSTREAM_CTX}} -n ipam-system exec "$POD" -c postgres -- \ + pg_isready -U ipam -d ipam >/dev/null 2>&1; then + echo "✅ postgres accepting connections as ipam/ipam"; break + fi + if [ "$i" -eq 45 ]; then echo "❌ postgres never accepted a connection"; exit 1; fi + sleep 2 + done + - echo "⏳ waiting for the IPAM apiserver (readiness probe has a 60s initial delay)" + # rollout status, not `wait pod -l`: a label selector also matches pods + # left over from a previous revision that are still terminating, and the + # wait then fails on a pod that is on its way out. + - kubectl --context {{.UPSTREAM_CTX}} -n ipam-system rollout status deploy/ipam-apiserver --timeout=300s + - echo "⏳ waiting for the aggregated APIService" + - kubectl --context {{.UPSTREAM_CTX}} wait --for=condition=Available apiservice/v1alpha1.ipam.miloapis.com --timeout=180s + - echo "✅ IPAM serving ipam.miloapis.com/v1alpha1" + + ipam-kubeconfig: + desc: "Wire the NSO manager to IPAM: mint a token for its ServiceAccount and publish it as the ipam-cluster-kubeconfig secret on the upstream cluster. The manager authenticates with this and sets the project impersonation per request; the RBAC that lets it do so ships in config/dependencies/ipam/rbac.yaml." + cmds: + - echo "🔗 wiring NSO manager -> IPAM (secret ipam-cluster-kubeconfig)" + - kubectl --context {{.UPSTREAM_CTX}} create namespace network-services-operator-system --dry-run=client -o yaml | kubectl --context {{.UPSTREAM_CTX}} apply -f - + # The ServiceAccount is created by config/e2e, which is applied later, so + # ensure it exists before minting a token against it. Applying the same + # object twice is harmless. + - | + kubectl --context {{.UPSTREAM_CTX}} -n network-services-operator-system create serviceaccount \ + network-services-operator-controller-manager --dry-run=client -o yaml \ + | kubectl --context {{.UPSTREAM_CTX}} apply -f - + # The manager runs inside the upstream cluster, so the kubeconfig must + # name the in-cluster service address rather than the host-mapped one. + - | + CA=$(kubectl --context {{.UPSTREAM_CTX}} -n kube-system get configmap kube-root-ca.crt -o jsonpath='{.data.ca\.crt}' | base64 | tr -d '\n') + TOKEN=$(kubectl --context {{.UPSTREAM_CTX}} -n network-services-operator-system create token \ + network-services-operator-controller-manager --duration=24h) + cat > {{.TMP_DIR}}/.ipam-cluster-kubeconfig.yaml <&2 + echo " Are both clusters up? Run 'task test-infra:clusters' first." >&2 + exit 1 + fi - echo "🌐 karmada init (apiserver {{.KARMADA_APISERVER_VERSION}}, advertise 127.0.0.1:32443, cert-ip incl {{.HOST_IP}}) on {{.UPSTREAM_CLUSTER}}" - mkdir -p {{.TMP_DIR}}/karmada-{{.UPSTREAM_CLUSTER}} - | @@ -429,7 +722,7 @@ tasks: - '{{.CHAINSAW}} test ./test/e2e-edge/extension-server-smoke --cluster {{.UPSTREAM_CLUSTER}}={{.TMP_DIR}}/.kind-{{.UPSTREAM_CLUSTER}}.yaml --cluster {{.DOWNSTREAM_CLUSTER}}={{.TMP_DIR}}/.kind-{{.DOWNSTREAM_CLUSTER}}.yaml' e2e: - desc: "Run chainsaw e2e scenarios against the live two-cluster env. Pass a scenario name or path after -- (e.g. `task test-infra:e2e -- waf-enforcement`); with no arg, runs every ext-server-path scenario that targets nso-upstream/nso-downstream. Use SCENARIOS=... to override the default set." + desc: "Run chainsaw e2e scenarios against the live two-cluster env. Pass a scenario name or path after -- (e.g. `task test-infra:e2e -- waf-enforcement`); with no arg, runs every ext-server-path scenario that targets nso-upstream/nso-downstream. Use SCENARIOS=... to override the default set. NOT safe to run while anything else is touching the env. The interface suites use fixed object names in the shared ipam-e2e-* namespaces (the project routing lives on those namespace labels), and this task clears and re-seeds the cluster-scoped fixtures at its head — which also republishes the tenant RBAC. So a second invocation, or a bare ipam-fixtures / ipam-fixtures-clear / ipam-tenant-rbac run, will pull the fixtures and the tenant's access out from under a suite already in flight. Run one at a time." vars: # Scenarios authored against this env's cluster names and downstream path. # Older fixtures targeting the previous cluster names are excluded here. @@ -438,7 +731,14 @@ tasks: SELECTED: '{{.CLI_ARGS | default .SCENARIOS | default .DEFAULT_SCENARIOS}}' deps: - kubeconfigs + # Only the suites still need this; see the task's own note. + - ipam-impersonation-kubeconfig cmds: + # Reset the cluster-scoped fixtures before the first suite. A run that + # died mid-suite leaves pools and classes that chainsaw's namespace + # teardown cannot reach, and the next run would fail on "already exists" + # rather than on what it was testing. + - task: ipam-fixtures - | set -e for s in {{.SELECTED}}; do @@ -449,13 +749,17 @@ tasks: *) dir="./test/e2e-edge/$s" ;; esac echo "🧪 chainsaw: $dir" + # --parallel 1 for the same reason test-e2e sets it: suites share + # cluster-scoped IPPools and IPClasses, so running two at once has + # them carving the same address space. {{.CHAINSAW}} test "$dir" \ + --parallel 1 \ --cluster {{.UPSTREAM_CLUSTER}}={{.TMP_DIR}}/.kind-{{.UPSTREAM_CLUSTER}}.yaml \ --cluster {{.DOWNSTREAM_CLUSTER}}={{.TMP_DIR}}/.kind-{{.DOWNSTREAM_CLUSTER}}.yaml done test-e2e: - desc: "Run the full control-plane e2e suite (test/e2e) against the live two-cluster env. The suite's chainsaw aliases nso-standard/nso-infra are mapped to this env's upstream/downstream kubeconfigs at invocation. Pass a scenario path after -- to run a single test." + desc: "Run the full control-plane e2e suite (test/e2e) against the live two-cluster env. The suite's chainsaw aliases nso-standard/nso-infra are mapped to this env's upstream/downstream kubeconfigs at invocation. Pass a scenario path after -- to run a single test. NOT safe to run while anything else is touching the env. The interface suites use fixed object names in the shared ipam-e2e-* namespaces (the project routing lives on those namespace labels), and this task clears and re-seeds the cluster-scoped fixtures at its head — which also republishes the tenant RBAC. So a second invocation, or a bare ipam-fixtures / ipam-fixtures-clear / ipam-tenant-rbac run, will pull the fixtures and the tenant's access out from under a suite already in flight. Run one at a time." vars: # CLI_ARGS (after --) wins so a single test can be selected; else the whole # suite directory. @@ -466,7 +770,12 @@ tasks: REPORT_NAME: '{{.REPORT_NAME | default "chainsaw-report"}}' deps: - kubeconfigs + # Only the suites still need this; see the task's own note. + - ipam-impersonation-kubeconfig cmds: + # See the note in `e2e`: cluster-scoped fixtures outlive a failed run, so + # the suite starts by putting them back to a known state. + - task: ipam-fixtures - echo "🧪 chainsaw suite {{.TARGET}} (nso-standard={{.UPSTREAM_CLUSTER}}, nso-infra={{.DOWNSTREAM_CLUSTER}}); report {{.REPORT_PATH}}/{{.REPORT_NAME}}.json" - | {{.CHAINSAW}} test {{.TARGET}} \ diff --git a/api/v1alpha/groupversion_info.go b/api/v1alpha/groupversion_info.go index 5d70324a..bf4230a5 100644 --- a/api/v1alpha/groupversion_info.go +++ b/api/v1alpha/groupversion_info.go @@ -38,6 +38,10 @@ func addKnownTypes(scheme *runtime.Scheme) error { &NetworkBindingList{}, &NetworkContext{}, &NetworkContextList{}, + &NetworkInterface{}, + &NetworkInterfaceList{}, + &NetworkInterfaceClaim{}, + &NetworkInterfaceClaimList{}, &NetworkPolicy{}, &NetworkPolicyList{}, &Subnet{}, diff --git a/api/v1alpha/networkinterface_types.go b/api/v1alpha/networkinterface_types.go new file mode 100644 index 00000000..4d3563f8 --- /dev/null +++ b/api/v1alpha/networkinterface_types.go @@ -0,0 +1,329 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package v1alpha + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// NetworkInterfaceReclaimPolicy decides what becomes of an interface, and the +// addresses it holds, when the claim bound to it is deleted. +// +// The two policies differ only when a workload goes away: on scale-down, on +// deletion, or whenever a claim is removed. While a workload is running, or +// while an instance is being replaced, both policies keep the addresses. +// +// +kubebuilder:validation:Enum=Delete;Retain +type NetworkInterfaceReclaimPolicy string + +const ( + // NetworkInterfaceReclaimPolicyDelete deletes the interface and returns its + // addresses to IPAM. A workload recreated later gets new addresses. + NetworkInterfaceReclaimPolicyDelete NetworkInterfaceReclaimPolicy = "Delete" + + // NetworkInterfaceReclaimPolicyRetain keeps the interface and its addresses + // after the claim is gone. The interface returns to the Available phase and + // keeps holding the addresses, so a later claim of the same name binds the + // same interface and the workload comes back on the same addresses. The + // addresses stay reserved, and billable, for as long as the interface + // exists. Deleting the interface releases its claim on them but does not + // return them to the pool today, so a retained address is reclaimed by an + // operator rather than automatically. + NetworkInterfaceReclaimPolicyRetain NetworkInterfaceReclaimPolicy = "Retain" +) + +// NetworkInterfacePhase reports whether an interface is held by a claim. +// +// +kubebuilder:validation:Enum=Available;Bound +type NetworkInterfacePhase string + +const ( + // NetworkInterfacePhaseAvailable means the interface still holds its addresses + // but no claim is bound to it. Retained interfaces wait here for a claim of + // the matching name. + NetworkInterfacePhaseAvailable NetworkInterfacePhase = "Available" + + // NetworkInterfacePhaseBound means the claim named in spec.claimRef holds the + // interface. + NetworkInterfacePhaseBound NetworkInterfacePhase = "Bound" +) + +const ( + // NetworkInterfaceAllocated reports that every address the interface must + // carry is allocated and recorded in spec. + NetworkInterfaceAllocated = "Allocated" + + // NetworkInterfaceProgrammed reports that the data plane carries the + // interface's addresses. Traffic flows only once this is true. + NetworkInterfaceProgrammed = "Programmed" +) + +// NetworkInterfaceAddress is an address the interface holds inside its network. +// These are the addresses configured on the NIC itself, and they always carry a +// prefix length. +type NetworkInterfaceAddress struct { + // family is the address family of this entry. + // + // +kubebuilder:validation:Required + Family IPFamily `json:"family"` + + // address is the address the interface holds, in CIDR notation, such as + // 10.128.0.2/32 or 2001:db8:a001::1/128. + // + // For IPv6 this may be a block delegated to the interface rather than a + // single address, such as 2001:db8:a001::/96. The interface owns the whole + // block and assigns within it. + // + // +kubebuilder:validation:Required + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:MaxLength=45 + Address string `json:"address"` + + // gateway is the next hop the interface routes through for this family, such + // as 10.128.0.1. It is resolved from the subnet backing the network in this + // location, so nothing has to read the subnet to configure the NIC. It is + // empty until that subnet exists. + // + // +kubebuilder:validation:Optional + // +kubebuilder:validation:MaxLength=45 + Gateway string `json:"gateway,omitempty"` + + // primary marks the address projected into single-address fields, such as an + // instance's reported network IP. + // + // Exactly one address is primary for the interface as a whole, not one per + // family. It is the address of the first family the claim listed in + // spec.ipFamilies. + // + // +kubebuilder:validation:Optional + Primary bool `json:"primary,omitempty"` + + // class is the IPAM class this address was allocated from, such as + // private-ipv6. It is empty for the addresses a claim requests by family + // rather than by class. + // + // +kubebuilder:validation:Optional + // +kubebuilder:validation:MaxLength=63 + Class string `json:"class,omitempty"` +} + +// NetworkInterfaceExternalAddress is an address reachable from outside the +// network, mapped onto an address the interface holds inside it. A public IPv4 +// address in front of a private address is the usual case. +// +// Unlike an interface address, an external address is a bare address with no +// prefix length, such as 203.0.113.10, because nothing configures it on the +// NIC. The data plane maps it onto the interface address of the same family. +type NetworkInterfaceExternalAddress struct { + // family is the address family of this entry. + // + // +kubebuilder:validation:Required + Family IPFamily `json:"family"` + + // address is the externally reachable address, such as 203.0.113.10. It + // carries no prefix length. + // + // +kubebuilder:validation:Required + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:MaxLength=45 + Address string `json:"address"` + + // class is the IPAM class this address was allocated from, such as + // public-ipv4. It matches the class the claim requested in spec.addresses. + // + // +kubebuilder:validation:Required + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:MaxLength=63 + Class string `json:"class"` +} + +// NetworkInterfaceClaimRef identifies the claim holding an interface. +type NetworkInterfaceClaimRef struct { + // name is the name of the NetworkInterfaceClaim, in the same namespace as the + // interface. A claim name stays with the workload slot it serves, so a + // replacement instance binds this same interface and its addresses. + // + // +kubebuilder:validation:Required + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:MaxLength=253 + Name string `json:"name"` +} + +// LocalNetworkInterfaceRef references a NetworkInterface in the same namespace. +type LocalNetworkInterfaceRef struct { + // name is the network interface name. + // + // +kubebuilder:validation:Required + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:MaxLength=253 + Name string `json:"name"` +} + +// NetworkInterfaceAttachmentRef references the provider resource realizing an +// interface on the data plane, such as an instance NIC attachment. It tells an +// operator what is carrying the interface, and it is written by the provider +// rather than by a user. +type NetworkInterfaceAttachmentRef struct { + // apiGroup is the API group of the referent, such as + // compute.datumapis.com. + // + // +kubebuilder:validation:Required + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:MaxLength=253 + APIGroup string `json:"apiGroup"` + + // kind is the kind of the referent. + // + // +kubebuilder:validation:Required + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:MaxLength=63 + Kind string `json:"kind"` + + // name is the name of the referent. + // + // +kubebuilder:validation:Required + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:MaxLength=253 + Name string `json:"name"` +} + +// NetworkInterfaceSpec defines the desired state of NetworkInterface. It is +// written by the operator when a claim is fulfilled, and it carries everything +// a provider needs to configure a NIC without reading any other resource. +type NetworkInterfaceSpec struct { + // network is the network this interface belongs to, in the same namespace as + // the interface. It comes from the claim and does not change. + // + // +kubebuilder:validation:Required + // +kubebuilder:validation:XValidation:message="network is immutable and cannot be changed after creation",rule="self == oldSelf" + Network LocalNetworkRef `json:"network"` + + // claimRef is the claim currently holding this interface. It is empty while a + // retained interface waits, unbound, for a claim of its name to return. + // + // +kubebuilder:validation:Optional + ClaimRef *NetworkInterfaceClaimRef `json:"claimRef,omitempty"` + + // interfaceName is the device name the interface presents to the guest + // operating system, such as eth0 or eth1. It comes from the claim. + // + // +kubebuilder:validation:Optional + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:MaxLength=15 + // +kubebuilder:default="eth0" + InterfaceName string `json:"interfaceName,omitempty"` + + // mtu is the MTU, in bytes, the interface must be configured with. It is + // resolved from the network, so a provider never has to read the network to + // configure the NIC. + // + // +kubebuilder:validation:Optional + // +kubebuilder:validation:Minimum=1300 + // +kubebuilder:validation:Maximum=8856 + MTU int32 `json:"mtu,omitempty"` + + // addresses are the addresses the interface holds inside its network, at most + // one per address family, exactly one of them primary. Each carries a prefix + // length and, once the location has a subnet, the gateway to route through. + // + // +kubebuilder:validation:Optional + // +kubebuilder:validation:MaxItems=4 + // +kubebuilder:validation:XValidation:message="Exactly one address must be primary",rule="size(self) == 0 || self.filter(a, has(a.primary) && a.primary).size() == 1" + // +kubebuilder:validation:XValidation:message="Only one address may be held per address family",rule="self.all(a, self.exists_one(b, b.family == a.family))" + Addresses []NetworkInterfaceAddress `json:"addresses,omitempty"` + + // externalAddresses are the addresses the interface is reachable at from + // outside the network, each mapped onto the interface address of the same + // family. They come from the classes the claim requested, and they are absent + // for a workload that only needs private addressing. + // + // +kubebuilder:validation:Optional + // +kubebuilder:validation:MaxItems=4 + // +kubebuilder:validation:XValidation:message="External addresses must be unique",rule="self.all(a, self.exists_one(b, b.address == a.address))" + // +kubebuilder:validation:XValidation:message="Only one external address may be held per address class",rule="self.all(a, self.exists_one(b, b.class == a.class))" + ExternalAddresses []NetworkInterfaceExternalAddress `json:"externalAddresses,omitempty"` + + // reclaimPolicy decides what becomes of this interface, and its addresses, + // when the claim holding it is deleted. It comes from the claim, and a claim + // asking for a different policy cannot bind this interface. + // + // +kubebuilder:validation:Optional + // +kubebuilder:default="Delete" + ReclaimPolicy NetworkInterfaceReclaimPolicy `json:"reclaimPolicy,omitempty"` +} + +// NetworkInterfaceStatus defines the observed state of NetworkInterface: which +// claim holds it, what realizes it on the data plane, and whether programming +// has succeeded. +type NetworkInterfaceStatus struct { + // phase reports whether a claim holds the interface. Bound means the claim in + // spec.claimRef holds it. Available means it is retained and holding its + // addresses with no claim bound. + // + // +kubebuilder:validation:Optional + Phase NetworkInterfacePhase `json:"phase,omitempty"` + + // networkContextRef is the network's presence in this location, resolved or + // created while fulfilling the claim. It is a breadcrumb for operators + // tracing where a network landed, and nothing needs it to configure a NIC. + // + // +kubebuilder:validation:Optional + NetworkContextRef *LocalNetworkContextRef `json:"networkContextRef,omitempty"` + + // attachmentRef is the data-plane resource realizing this interface. The + // provider sets it once an attachment exists. + // + // +kubebuilder:validation:Optional + AttachmentRef *NetworkInterfaceAttachmentRef `json:"attachmentRef,omitempty"` + + // vpc is the base62 identifier of the VPC backing this network in this + // location, matching the identifier the fabric keys on. The provider records + // it when the attachment is programmed. + // + // +kubebuilder:validation:Optional + VPC string `json:"vpc,omitempty"` + + // conditions report the current state of the interface. Allocated means every + // address is held. Programmed means the data plane carries them. + Conditions []metav1.Condition `json:"conditions,omitempty"` +} + +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status + +// NetworkInterface is an interface on a network, together with the addresses it +// holds. It is the unit that owns addresses: as long as the interface exists, +// its addresses stay allocated to it. +// +// You do not create a NetworkInterface. Ask for one with a +// NetworkInterfaceClaim, and the operator creates the interface, allocates its +// addresses, and binds the two. A provider then reads the interface to +// configure a NIC, and reports what it programmed in status. +// +// An interface outlives the instance using it. Whether it outlives the claim +// that asked for it depends on spec.reclaimPolicy. +// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp" +// +kubebuilder:printcolumn:name="Network",type=string,JSONPath=".spec.network.name" +// +kubebuilder:printcolumn:name="Phase",type=string,JSONPath=".status.phase" +// +kubebuilder:printcolumn:name="Claim",type=string,JSONPath=".spec.claimRef.name" +// +kubebuilder:printcolumn:name="Allocated",type=string,JSONPath=`.status.conditions[?(@.type=="Allocated")].status` +// +kubebuilder:printcolumn:name="Programmed",type=string,JSONPath=`.status.conditions[?(@.type=="Programmed")].status` +type NetworkInterface struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + // +kubebuilder:validation:Required + Spec NetworkInterfaceSpec `json:"spec,omitempty"` + + // +kubebuilder:default={conditions:{{type:"Allocated",status:"Unknown",reason:"Pending", message:"Waiting for controller", lastTransitionTime: "1970-01-01T00:00:00Z"},{type:"Programmed",status:"Unknown",reason:"Pending", message:"Waiting for controller", lastTransitionTime: "1970-01-01T00:00:00Z"}}} + Status NetworkInterfaceStatus `json:"status,omitempty"` +} + +// +kubebuilder:object:root=true + +// NetworkInterfaceList contains a list of NetworkInterface. +type NetworkInterfaceList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []NetworkInterface `json:"items"` +} diff --git a/api/v1alpha/networkinterfaceclaim_types.go b/api/v1alpha/networkinterfaceclaim_types.go new file mode 100644 index 00000000..b30fa394 --- /dev/null +++ b/api/v1alpha/networkinterfaceclaim_types.go @@ -0,0 +1,253 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package v1alpha + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + // NetworkInterfaceClaimBound reports that an interface is bound to the claim + // and named in status.networkInterfaceRef. + NetworkInterfaceClaimBound = "Bound" + + // NetworkInterfaceClaimAllocated reports that every requested address family, + // and every requested class, holds an address. + NetworkInterfaceClaimAllocated = "Allocated" + + // NetworkInterfaceClaimProgrammed reports that the data plane carries the + // claimed addresses. + NetworkInterfaceClaimProgrammed = "Programmed" + + // NetworkInterfaceClaimReady reports that the claim is bound, allocated, and + // programmed. A workload that needs the network should wait on this one + // condition rather than on the three it summarizes. + NetworkInterfaceClaimReady = "Ready" +) + +// NetworkInterfaceAddressRequest asks for one address beyond the ones the +// interface holds inside its network, such as a public IPv4 address in front of +// a private one. +type NetworkInterfaceAddressRequest struct { + // class is the IPAM class to allocate from, such as public-ipv4. + // + // A class names a kind of address, and the platform decides which pool and + // prefix length serve it. A class never names a pool, a prefix length, or a + // CIDR, so a class cannot be used to ask for a particular address. + // + // +kubebuilder:validation:Required + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:MaxLength=63 + Class string `json:"class"` +} + +// NetworkInterfaceClaimSpec defines the desired state of NetworkInterfaceClaim. +// Every field states what the interface must be able to do, never which +// interface or which address to use. +// +// Most of the spec is immutable, because the addresses are allocated against +// it. To change one of those fields, delete the claim and create a new one, +// accepting that the workload gets new addresses unless the interface is +// retained. +// +// +kubebuilder:validation:XValidation:message="networkInterfaceName is immutable and cannot be set, changed, or cleared after creation",rule="has(self.networkInterfaceName) == has(oldSelf.networkInterfaceName) && (!has(self.networkInterfaceName) || self.networkInterfaceName == oldSelf.networkInterfaceName)" +// +kubebuilder:validation:XValidation:message="addresses is immutable and cannot be set, changed, or cleared after creation",rule="has(self.addresses) == has(oldSelf.addresses) && (!has(self.addresses) || self.addresses == oldSelf.addresses)" +type NetworkInterfaceClaimSpec struct { + // network is the network the interface attaches to. The network must already + // exist in the same namespace as the claim. + // + // Immutable. An interface that changed network would hold addresses from a + // space it no longer belongs to, so move a workload by recreating the claim + // against the other network. + // + // +kubebuilder:validation:Required + // +kubebuilder:validation:XValidation:message="network is immutable and cannot be changed after creation",rule="self == oldSelf" + Network LocalNetworkRef `json:"network"` + + // interfaceName is the device name the interface presents to the guest + // operating system, such as eth0 or eth1. Set it when a workload has more + // than one interface and the guest configuration names them. + // + // Immutable, because the guest is configured against it. + // + // +kubebuilder:validation:Optional + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:MaxLength=15 + // +kubebuilder:default="eth0" + // +kubebuilder:validation:XValidation:message="interfaceName is immutable and cannot be changed after creation",rule="self == oldSelf" + InterfaceName string `json:"interfaceName,omitempty"` + + // ipFamilies are the address families the interface must carry, in priority + // order. List [IPv6, IPv4] for a dual-stack interface. The first family + // listed holds the interface's primary address, which is the one reported in + // single-address fields such as an instance's network IP. + // + // Every family listed must be satisfiable or the claim does not bind. Asking + // for a family the network does not carry fails the claim outright rather + // than leaving it pending, and no partially addressed interface is ever + // published. + // + // +kubebuilder:validation:Optional + // +kubebuilder:validation:MinItems=1 + // +kubebuilder:validation:MaxItems=2 + // +kubebuilder:default={IPv6} + // +kubebuilder:validation:XValidation:message="Each address family may be requested at most once",rule="self.all(f, self.exists_one(g, g == f))" + // +kubebuilder:validation:XValidation:message="ipFamilies is immutable and cannot be changed after creation",rule="self == oldSelf" + IPFamilies []IPFamily `json:"ipFamilies,omitempty"` + + // addresses request extra addresses by class, beyond the ones the interface + // holds inside its network. Each appears in status.externalAddresses as a + // bare address, mapped onto the interface address of the same family. + // + // Omit this field for ordinary private addressing, which is the common case. + // + // +kubebuilder:validation:Optional + // +kubebuilder:validation:MinItems=1 + // +kubebuilder:validation:MaxItems=4 + // +kubebuilder:validation:XValidation:message="Each address class may be requested at most once",rule="self.all(a, self.exists_one(b, b.class == a.class))" + Addresses []NetworkInterfaceAddressRequest `json:"addresses,omitempty"` + + // reclaimPolicy decides what becomes of the bound interface, and its + // addresses, when this claim is deleted. + // + // Delete deletes the interface and returns its addresses to IPAM. A workload + // recreated later comes back on different addresses. + // + // Retain keeps the interface, unbound and still holding its addresses, so a + // later claim of this name binds it again and the workload returns to the + // same addresses. Choose Retain when an address is published in DNS, allowed + // through a firewall, or otherwise depended on from outside. + // + // A retained address is reserved, and billable, for as long as the interface + // exists. Deleting the interface does not return it to the pool today, so + // choose Retain for addresses worth holding rather than as a default. + // + // Both policies keep the addresses while the claim exists, including across + // instance replacement. They differ only on scale-down and deletion. + // + // Immutable. An address keeps the policy it was allocated under, and a claim + // asking for a policy the interface was not allocated under cannot bind it. + // + // +kubebuilder:validation:Optional + // +kubebuilder:default="Delete" + // +kubebuilder:validation:XValidation:message="reclaimPolicy is immutable and cannot be changed after creation",rule="self == oldSelf" + ReclaimPolicy NetworkInterfaceReclaimPolicy `json:"reclaimPolicy,omitempty"` + + // networkInterfaceName binds one specific interface by name, instead of the + // interface named after this claim. The named interface must already carry + // every family and class this claim asks for, under the same reclaim policy, + // and must not be held by another claim. + // + // Leave it empty, which is the normal case. The claim then binds the + // interface of its own name, retained by an earlier claim, or creates one. + // + // Immutable, including from empty to set. Rebinding a workload to a different + // interface means a new claim. + // + // +kubebuilder:validation:Optional + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:MaxLength=253 + NetworkInterfaceName string `json:"networkInterfaceName,omitempty"` +} + +// NetworkInterfaceClaimStatus defines the observed state of +// NetworkInterfaceClaim. It repeats the bound interface's addresses so a +// consumer reads one object rather than following the reference. +type NetworkInterfaceClaimStatus struct { + // networkInterfaceRef is the interface bound to this claim, in the same + // namespace. Read it to reach fields the claim does not repeat, such as the + // MTU and the data-plane attachment. + // + // +kubebuilder:validation:Optional + NetworkInterfaceRef *LocalNetworkInterfaceRef `json:"networkInterfaceRef,omitempty"` + + // addresses are the addresses the bound interface holds inside its network, + // each with its prefix length and, once the location has a subnet, its + // gateway. They are copied from the interface, which remains the source of + // truth. + // + // +kubebuilder:validation:Optional + Addresses []NetworkInterfaceAddress `json:"addresses,omitempty"` + + // externalAddresses are the addresses the bound interface is reachable at from + // outside the network, one per class the claim requested. Each is a bare + // address with no prefix length. They are copied from the interface. + // + // +kubebuilder:validation:Optional + ExternalAddresses []NetworkInterfaceExternalAddress `json:"externalAddresses,omitempty"` + + // conditions report the current state of the claim. Wait on Ready, which is + // true once the claim is bound, its addresses are allocated, and the data + // plane carries them. + Conditions []metav1.Condition `json:"conditions,omitempty"` +} + +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status + +// NetworkInterfaceClaim asks for an interface on a network. It is the resource +// a user creates. The operator finds or creates a NetworkInterface that +// satisfies it, allocates the addresses, and reports them in status. +// +// A claim describes what the interface must be able to do, never which +// interface or address to use. One claim holds at most one interface, and one +// interface is held by at most one claim. +// +// A claim's name is what makes addresses stable. It names the slot in a +// workload rather than the instance filling it, so an instance replaced by +// another that asks for the same claim name comes back on the same interface +// and the same addresses. What happens when the claim itself is deleted is +// spec.reclaimPolicy. +// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp" +// +kubebuilder:printcolumn:name="Network",type=string,JSONPath=".spec.network.name" +// +kubebuilder:printcolumn:name="Interface",type=string,JSONPath=".status.networkInterfaceRef.name" +// +kubebuilder:printcolumn:name="Ready",type=string,JSONPath=`.status.conditions[?(@.type=="Ready")].status` +// +kubebuilder:printcolumn:name="Reason",type=string,JSONPath=`.status.conditions[?(@.type=="Ready")].reason` +type NetworkInterfaceClaim struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + // +kubebuilder:validation:Required + Spec NetworkInterfaceClaimSpec `json:"spec,omitempty"` + + // +kubebuilder:default={conditions:{{type:"Bound",status:"Unknown",reason:"Pending", message:"Waiting for controller", lastTransitionTime: "1970-01-01T00:00:00Z"},{type:"Allocated",status:"Unknown",reason:"Pending", message:"Waiting for controller", lastTransitionTime: "1970-01-01T00:00:00Z"},{type:"Programmed",status:"Unknown",reason:"Pending", message:"Waiting for controller", lastTransitionTime: "1970-01-01T00:00:00Z"},{type:"Ready",status:"Unknown",reason:"Pending", message:"Waiting for controller", lastTransitionTime: "1970-01-01T00:00:00Z"}}} + Status NetworkInterfaceClaimStatus `json:"status,omitempty"` +} + +// +kubebuilder:object:root=true + +// NetworkInterfaceClaimList contains a list of NetworkInterfaceClaim. +type NetworkInterfaceClaimList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []NetworkInterfaceClaim `json:"items"` +} + +// NetworkInterfaceClaimTemplate describes an interface a workload needs, from +// which one NetworkInterfaceClaim is created per slot. It is embedded in a +// workload API the way a StatefulSet embeds volumeClaimTemplates, and serves +// the same purpose: the workload declares the interface once, and every slot +// gets a claim of its own that outlives the instance filling it. +// +// Each claim is named from the template and the slot, and both parts stay the +// same for the life of that slot, so a replacement instance derives the same +// name, finds the claim already there, and returns to the addresses it was +// already holding. +type NetworkInterfaceClaimTemplate struct { + // metadata is the standard object metadata for the claims this template + // produces. + // + // The name distinguishes one interface of a slot from another and becomes + // part of every claim's name. Labels and annotations are copied onto every + // claim produced. No other field is honoured. + // + // +kubebuilder:validation:Optional + metav1.ObjectMeta `json:"metadata,omitempty"` + + // spec is the claim created for each slot. It is copied onto every claim the + // template produces, so every slot gets the same network, families, classes, + // and reclaim policy. + // + // +kubebuilder:validation:Required + Spec NetworkInterfaceClaimSpec `json:"spec"` +} diff --git a/api/v1alpha/zz_generated.deepcopy.go b/api/v1alpha/zz_generated.deepcopy.go index aceb7595..406fb91b 100644 --- a/api/v1alpha/zz_generated.deepcopy.go +++ b/api/v1alpha/zz_generated.deepcopy.go @@ -607,6 +607,21 @@ func (in *LocalNetworkContextRef) DeepCopy() *LocalNetworkContextRef { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LocalNetworkInterfaceRef) DeepCopyInto(out *LocalNetworkInterfaceRef) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LocalNetworkInterfaceRef. +func (in *LocalNetworkInterfaceRef) DeepCopy() *LocalNetworkInterfaceRef { + if in == nil { + return nil + } + out := new(LocalNetworkInterfaceRef) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *LocalNetworkRef) DeepCopyInto(out *LocalNetworkRef) { *out = *in @@ -1203,6 +1218,342 @@ func (in *NetworkIPAM) DeepCopy() *NetworkIPAM { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetworkInterface) DeepCopyInto(out *NetworkInterface) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkInterface. +func (in *NetworkInterface) DeepCopy() *NetworkInterface { + if in == nil { + return nil + } + out := new(NetworkInterface) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *NetworkInterface) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetworkInterfaceAddress) DeepCopyInto(out *NetworkInterfaceAddress) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkInterfaceAddress. +func (in *NetworkInterfaceAddress) DeepCopy() *NetworkInterfaceAddress { + if in == nil { + return nil + } + out := new(NetworkInterfaceAddress) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetworkInterfaceAddressRequest) DeepCopyInto(out *NetworkInterfaceAddressRequest) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkInterfaceAddressRequest. +func (in *NetworkInterfaceAddressRequest) DeepCopy() *NetworkInterfaceAddressRequest { + if in == nil { + return nil + } + out := new(NetworkInterfaceAddressRequest) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetworkInterfaceAttachmentRef) DeepCopyInto(out *NetworkInterfaceAttachmentRef) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkInterfaceAttachmentRef. +func (in *NetworkInterfaceAttachmentRef) DeepCopy() *NetworkInterfaceAttachmentRef { + if in == nil { + return nil + } + out := new(NetworkInterfaceAttachmentRef) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetworkInterfaceClaim) DeepCopyInto(out *NetworkInterfaceClaim) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkInterfaceClaim. +func (in *NetworkInterfaceClaim) DeepCopy() *NetworkInterfaceClaim { + if in == nil { + return nil + } + out := new(NetworkInterfaceClaim) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *NetworkInterfaceClaim) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetworkInterfaceClaimList) DeepCopyInto(out *NetworkInterfaceClaimList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]NetworkInterfaceClaim, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkInterfaceClaimList. +func (in *NetworkInterfaceClaimList) DeepCopy() *NetworkInterfaceClaimList { + if in == nil { + return nil + } + out := new(NetworkInterfaceClaimList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *NetworkInterfaceClaimList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetworkInterfaceClaimRef) DeepCopyInto(out *NetworkInterfaceClaimRef) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkInterfaceClaimRef. +func (in *NetworkInterfaceClaimRef) DeepCopy() *NetworkInterfaceClaimRef { + if in == nil { + return nil + } + out := new(NetworkInterfaceClaimRef) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetworkInterfaceClaimSpec) DeepCopyInto(out *NetworkInterfaceClaimSpec) { + *out = *in + out.Network = in.Network + if in.IPFamilies != nil { + in, out := &in.IPFamilies, &out.IPFamilies + *out = make([]IPFamily, len(*in)) + copy(*out, *in) + } + if in.Addresses != nil { + in, out := &in.Addresses, &out.Addresses + *out = make([]NetworkInterfaceAddressRequest, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkInterfaceClaimSpec. +func (in *NetworkInterfaceClaimSpec) DeepCopy() *NetworkInterfaceClaimSpec { + if in == nil { + return nil + } + out := new(NetworkInterfaceClaimSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetworkInterfaceClaimStatus) DeepCopyInto(out *NetworkInterfaceClaimStatus) { + *out = *in + if in.NetworkInterfaceRef != nil { + in, out := &in.NetworkInterfaceRef, &out.NetworkInterfaceRef + *out = new(LocalNetworkInterfaceRef) + **out = **in + } + if in.Addresses != nil { + in, out := &in.Addresses, &out.Addresses + *out = make([]NetworkInterfaceAddress, len(*in)) + copy(*out, *in) + } + if in.ExternalAddresses != nil { + in, out := &in.ExternalAddresses, &out.ExternalAddresses + *out = make([]NetworkInterfaceExternalAddress, len(*in)) + copy(*out, *in) + } + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]v1.Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkInterfaceClaimStatus. +func (in *NetworkInterfaceClaimStatus) DeepCopy() *NetworkInterfaceClaimStatus { + if in == nil { + return nil + } + out := new(NetworkInterfaceClaimStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetworkInterfaceClaimTemplate) DeepCopyInto(out *NetworkInterfaceClaimTemplate) { + *out = *in + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkInterfaceClaimTemplate. +func (in *NetworkInterfaceClaimTemplate) DeepCopy() *NetworkInterfaceClaimTemplate { + if in == nil { + return nil + } + out := new(NetworkInterfaceClaimTemplate) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetworkInterfaceExternalAddress) DeepCopyInto(out *NetworkInterfaceExternalAddress) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkInterfaceExternalAddress. +func (in *NetworkInterfaceExternalAddress) DeepCopy() *NetworkInterfaceExternalAddress { + if in == nil { + return nil + } + out := new(NetworkInterfaceExternalAddress) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetworkInterfaceList) DeepCopyInto(out *NetworkInterfaceList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]NetworkInterface, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkInterfaceList. +func (in *NetworkInterfaceList) DeepCopy() *NetworkInterfaceList { + if in == nil { + return nil + } + out := new(NetworkInterfaceList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *NetworkInterfaceList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetworkInterfaceSpec) DeepCopyInto(out *NetworkInterfaceSpec) { + *out = *in + out.Network = in.Network + if in.ClaimRef != nil { + in, out := &in.ClaimRef, &out.ClaimRef + *out = new(NetworkInterfaceClaimRef) + **out = **in + } + if in.Addresses != nil { + in, out := &in.Addresses, &out.Addresses + *out = make([]NetworkInterfaceAddress, len(*in)) + copy(*out, *in) + } + if in.ExternalAddresses != nil { + in, out := &in.ExternalAddresses, &out.ExternalAddresses + *out = make([]NetworkInterfaceExternalAddress, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkInterfaceSpec. +func (in *NetworkInterfaceSpec) DeepCopy() *NetworkInterfaceSpec { + if in == nil { + return nil + } + out := new(NetworkInterfaceSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetworkInterfaceStatus) DeepCopyInto(out *NetworkInterfaceStatus) { + *out = *in + if in.NetworkContextRef != nil { + in, out := &in.NetworkContextRef, &out.NetworkContextRef + *out = new(LocalNetworkContextRef) + **out = **in + } + if in.AttachmentRef != nil { + in, out := &in.AttachmentRef, &out.AttachmentRef + *out = new(NetworkInterfaceAttachmentRef) + **out = **in + } + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]v1.Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkInterfaceStatus. +func (in *NetworkInterfaceStatus) DeepCopy() *NetworkInterfaceStatus { + if in == nil { + return nil + } + out := new(NetworkInterfaceStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *NetworkList) DeepCopyInto(out *NetworkList) { *out = *in diff --git a/config/crd/bases/networking.datumapis.com_networkinterfaceclaims.yaml b/config/crd/bases/networking.datumapis.com_networkinterfaceclaims.yaml new file mode 100644 index 00000000..0b587210 --- /dev/null +++ b/config/crd/bases/networking.datumapis.com_networkinterfaceclaims.yaml @@ -0,0 +1,437 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.16.4 + name: networkinterfaceclaims.networking.datumapis.com +spec: + group: networking.datumapis.com + names: + kind: NetworkInterfaceClaim + listKind: NetworkInterfaceClaimList + plural: networkinterfaceclaims + singular: networkinterfaceclaim + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + - jsonPath: .spec.network.name + name: Network + type: string + - jsonPath: .status.networkInterfaceRef.name + name: Interface + type: string + - jsonPath: .status.conditions[?(@.type=="Ready")].status + name: Ready + type: string + - jsonPath: .status.conditions[?(@.type=="Ready")].reason + name: Reason + type: string + name: v1alpha + schema: + openAPIV3Schema: + description: |- + NetworkInterfaceClaim asks for an interface on a network. It is the resource + a user creates. The operator finds or creates a NetworkInterface that + satisfies it, allocates the addresses, and reports them in status. + + A claim describes what the interface must be able to do, never which + interface or address to use. One claim holds at most one interface, and one + interface is held by at most one claim. + + A claim's name is what makes addresses stable. It names the slot in a + workload rather than the instance filling it, so an instance replaced by + another that asks for the same claim name comes back on the same interface + and the same addresses. What happens when the claim itself is deleted is + spec.reclaimPolicy. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: |- + NetworkInterfaceClaimSpec defines the desired state of NetworkInterfaceClaim. + Every field states what the interface must be able to do, never which + interface or which address to use. + + Most of the spec is immutable, because the addresses are allocated against + it. To change one of those fields, delete the claim and create a new one, + accepting that the workload gets new addresses unless the interface is + retained. + properties: + addresses: + description: |- + addresses request extra addresses by class, beyond the ones the interface + holds inside its network. Each appears in status.externalAddresses as a + bare address, mapped onto the interface address of the same family. + + Omit this field for ordinary private addressing, which is the common case. + items: + description: |- + NetworkInterfaceAddressRequest asks for one address beyond the ones the + interface holds inside its network, such as a public IPv4 address in front of + a private one. + properties: + class: + description: |- + class is the IPAM class to allocate from, such as public-ipv4. + + A class names a kind of address, and the platform decides which pool and + prefix length serve it. A class never names a pool, a prefix length, or a + CIDR, so a class cannot be used to ask for a particular address. + maxLength: 63 + minLength: 1 + type: string + required: + - class + type: object + maxItems: 4 + minItems: 1 + type: array + x-kubernetes-validations: + - message: Each address class may be requested at most once + rule: self.all(a, self.exists_one(b, b.class == a.class)) + interfaceName: + default: eth0 + description: |- + interfaceName is the device name the interface presents to the guest + operating system, such as eth0 or eth1. Set it when a workload has more + than one interface and the guest configuration names them. + + Immutable, because the guest is configured against it. + maxLength: 15 + minLength: 1 + type: string + x-kubernetes-validations: + - message: interfaceName is immutable and cannot be changed after + creation + rule: self == oldSelf + ipFamilies: + default: + - IPv6 + description: |- + ipFamilies are the address families the interface must carry, in priority + order. List [IPv6, IPv4] for a dual-stack interface. The first family + listed holds the interface's primary address, which is the one reported in + single-address fields such as an instance's network IP. + + Every family listed must be satisfiable or the claim does not bind. Asking + for a family the network does not carry fails the claim outright rather + than leaving it pending, and no partially addressed interface is ever + published. + items: + enum: + - IPv4 + - IPv6 + type: string + maxItems: 2 + minItems: 1 + type: array + x-kubernetes-validations: + - message: Each address family may be requested at most once + rule: self.all(f, self.exists_one(g, g == f)) + - message: ipFamilies is immutable and cannot be changed after creation + rule: self == oldSelf + network: + description: |- + network is the network the interface attaches to. The network must already + exist in the same namespace as the claim. + + Immutable. An interface that changed network would hold addresses from a + space it no longer belongs to, so move a workload by recreating the claim + against the other network. + properties: + name: + description: The network name + type: string + required: + - name + type: object + x-kubernetes-validations: + - message: network is immutable and cannot be changed after creation + rule: self == oldSelf + networkInterfaceName: + description: |- + networkInterfaceName binds one specific interface by name, instead of the + interface named after this claim. The named interface must already carry + every family and class this claim asks for, under the same reclaim policy, + and must not be held by another claim. + + Leave it empty, which is the normal case. The claim then binds the + interface of its own name, retained by an earlier claim, or creates one. + + Immutable, including from empty to set. Rebinding a workload to a different + interface means a new claim. + maxLength: 253 + minLength: 1 + type: string + reclaimPolicy: + default: Delete + description: |- + reclaimPolicy decides what becomes of the bound interface, and its + addresses, when this claim is deleted. + + Delete deletes the interface and returns its addresses to IPAM. A workload + recreated later comes back on different addresses. + + Retain keeps the interface, unbound and still holding its addresses, so a + later claim of this name binds it again and the workload returns to the + same addresses. Choose Retain when an address is published in DNS, allowed + through a firewall, or otherwise depended on from outside. + + A retained address is reserved, and billable, for as long as the interface + exists. Deleting the interface does not return it to the pool today, so + choose Retain for addresses worth holding rather than as a default. + + Both policies keep the addresses while the claim exists, including across + instance replacement. They differ only on scale-down and deletion. + + Immutable. An address keeps the policy it was allocated under, and a claim + asking for a policy the interface was not allocated under cannot bind it. + enum: + - Delete + - Retain + type: string + x-kubernetes-validations: + - message: reclaimPolicy is immutable and cannot be changed after + creation + rule: self == oldSelf + required: + - network + type: object + x-kubernetes-validations: + - message: networkInterfaceName is immutable and cannot be set, changed, + or cleared after creation + rule: has(self.networkInterfaceName) == has(oldSelf.networkInterfaceName) + && (!has(self.networkInterfaceName) || self.networkInterfaceName == + oldSelf.networkInterfaceName) + - message: addresses is immutable and cannot be set, changed, or cleared + after creation + rule: has(self.addresses) == has(oldSelf.addresses) && (!has(self.addresses) + || self.addresses == oldSelf.addresses) + status: + default: + conditions: + - lastTransitionTime: "1970-01-01T00:00:00Z" + message: Waiting for controller + reason: Pending + status: Unknown + type: Bound + - lastTransitionTime: "1970-01-01T00:00:00Z" + message: Waiting for controller + reason: Pending + status: Unknown + type: Allocated + - lastTransitionTime: "1970-01-01T00:00:00Z" + message: Waiting for controller + reason: Pending + status: Unknown + type: Programmed + - lastTransitionTime: "1970-01-01T00:00:00Z" + message: Waiting for controller + reason: Pending + status: Unknown + type: Ready + description: |- + NetworkInterfaceClaimStatus defines the observed state of + NetworkInterfaceClaim. It repeats the bound interface's addresses so a + consumer reads one object rather than following the reference. + properties: + addresses: + description: |- + addresses are the addresses the bound interface holds inside its network, + each with its prefix length and, once the location has a subnet, its + gateway. They are copied from the interface, which remains the source of + truth. + items: + description: |- + NetworkInterfaceAddress is an address the interface holds inside its network. + These are the addresses configured on the NIC itself, and they always carry a + prefix length. + properties: + address: + description: |- + address is the address the interface holds, in CIDR notation, such as + 10.128.0.2/32 or 2001:db8:a001::1/128. + + For IPv6 this may be a block delegated to the interface rather than a + single address, such as 2001:db8:a001::/96. The interface owns the whole + block and assigns within it. + maxLength: 45 + minLength: 1 + type: string + class: + description: |- + class is the IPAM class this address was allocated from, such as + private-ipv6. It is empty for the addresses a claim requests by family + rather than by class. + maxLength: 63 + type: string + family: + description: family is the address family of this entry. + enum: + - IPv4 + - IPv6 + type: string + gateway: + description: |- + gateway is the next hop the interface routes through for this family, such + as 10.128.0.1. It is resolved from the subnet backing the network in this + location, so nothing has to read the subnet to configure the NIC. It is + empty until that subnet exists. + maxLength: 45 + type: string + primary: + description: |- + primary marks the address projected into single-address fields, such as an + instance's reported network IP. + + Exactly one address is primary for the interface as a whole, not one per + family. It is the address of the first family the claim listed in + spec.ipFamilies. + type: boolean + required: + - address + - family + type: object + type: array + conditions: + description: |- + conditions report the current state of the claim. Wait on Ready, which is + true once the claim is bound, its addresses are allocated, and the data + plane carries them. + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + externalAddresses: + description: |- + externalAddresses are the addresses the bound interface is reachable at from + outside the network, one per class the claim requested. Each is a bare + address with no prefix length. They are copied from the interface. + items: + description: |- + NetworkInterfaceExternalAddress is an address reachable from outside the + network, mapped onto an address the interface holds inside it. A public IPv4 + address in front of a private address is the usual case. + + Unlike an interface address, an external address is a bare address with no + prefix length, such as 203.0.113.10, because nothing configures it on the + NIC. The data plane maps it onto the interface address of the same family. + properties: + address: + description: |- + address is the externally reachable address, such as 203.0.113.10. It + carries no prefix length. + maxLength: 45 + minLength: 1 + type: string + class: + description: |- + class is the IPAM class this address was allocated from, such as + public-ipv4. It matches the class the claim requested in spec.addresses. + maxLength: 63 + minLength: 1 + type: string + family: + description: family is the address family of this entry. + enum: + - IPv4 + - IPv6 + type: string + required: + - address + - class + - family + type: object + type: array + networkInterfaceRef: + description: |- + networkInterfaceRef is the interface bound to this claim, in the same + namespace. Read it to reach fields the claim does not repeat, such as the + MTU and the data-plane attachment. + properties: + name: + description: name is the network interface name. + maxLength: 253 + minLength: 1 + type: string + required: + - name + type: object + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} diff --git a/config/crd/bases/networking.datumapis.com_networkinterfaces.yaml b/config/crd/bases/networking.datumapis.com_networkinterfaces.yaml new file mode 100644 index 00000000..2fd787b7 --- /dev/null +++ b/config/crd/bases/networking.datumapis.com_networkinterfaces.yaml @@ -0,0 +1,385 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.16.4 + name: networkinterfaces.networking.datumapis.com +spec: + group: networking.datumapis.com + names: + kind: NetworkInterface + listKind: NetworkInterfaceList + plural: networkinterfaces + singular: networkinterface + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + - jsonPath: .spec.network.name + name: Network + type: string + - jsonPath: .status.phase + name: Phase + type: string + - jsonPath: .spec.claimRef.name + name: Claim + type: string + - jsonPath: .status.conditions[?(@.type=="Allocated")].status + name: Allocated + type: string + - jsonPath: .status.conditions[?(@.type=="Programmed")].status + name: Programmed + type: string + name: v1alpha + schema: + openAPIV3Schema: + description: |- + NetworkInterface is an interface on a network, together with the addresses it + holds. It is the unit that owns addresses: as long as the interface exists, + its addresses stay allocated to it. + + You do not create a NetworkInterface. Ask for one with a + NetworkInterfaceClaim, and the operator creates the interface, allocates its + addresses, and binds the two. A provider then reads the interface to + configure a NIC, and reports what it programmed in status. + + An interface outlives the instance using it. Whether it outlives the claim + that asked for it depends on spec.reclaimPolicy. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: |- + NetworkInterfaceSpec defines the desired state of NetworkInterface. It is + written by the operator when a claim is fulfilled, and it carries everything + a provider needs to configure a NIC without reading any other resource. + properties: + addresses: + description: |- + addresses are the addresses the interface holds inside its network, at most + one per address family, exactly one of them primary. Each carries a prefix + length and, once the location has a subnet, the gateway to route through. + items: + description: |- + NetworkInterfaceAddress is an address the interface holds inside its network. + These are the addresses configured on the NIC itself, and they always carry a + prefix length. + properties: + address: + description: |- + address is the address the interface holds, in CIDR notation, such as + 10.128.0.2/32 or 2001:db8:a001::1/128. + + For IPv6 this may be a block delegated to the interface rather than a + single address, such as 2001:db8:a001::/96. The interface owns the whole + block and assigns within it. + maxLength: 45 + minLength: 1 + type: string + class: + description: |- + class is the IPAM class this address was allocated from, such as + private-ipv6. It is empty for the addresses a claim requests by family + rather than by class. + maxLength: 63 + type: string + family: + description: family is the address family of this entry. + enum: + - IPv4 + - IPv6 + type: string + gateway: + description: |- + gateway is the next hop the interface routes through for this family, such + as 10.128.0.1. It is resolved from the subnet backing the network in this + location, so nothing has to read the subnet to configure the NIC. It is + empty until that subnet exists. + maxLength: 45 + type: string + primary: + description: |- + primary marks the address projected into single-address fields, such as an + instance's reported network IP. + + Exactly one address is primary for the interface as a whole, not one per + family. It is the address of the first family the claim listed in + spec.ipFamilies. + type: boolean + required: + - address + - family + type: object + maxItems: 4 + type: array + x-kubernetes-validations: + - message: Exactly one address must be primary + rule: size(self) == 0 || self.filter(a, has(a.primary) && a.primary).size() + == 1 + - message: Only one address may be held per address family + rule: self.all(a, self.exists_one(b, b.family == a.family)) + claimRef: + description: |- + claimRef is the claim currently holding this interface. It is empty while a + retained interface waits, unbound, for a claim of its name to return. + properties: + name: + description: |- + name is the name of the NetworkInterfaceClaim, in the same namespace as the + interface. A claim name stays with the workload slot it serves, so a + replacement instance binds this same interface and its addresses. + maxLength: 253 + minLength: 1 + type: string + required: + - name + type: object + externalAddresses: + description: |- + externalAddresses are the addresses the interface is reachable at from + outside the network, each mapped onto the interface address of the same + family. They come from the classes the claim requested, and they are absent + for a workload that only needs private addressing. + items: + description: |- + NetworkInterfaceExternalAddress is an address reachable from outside the + network, mapped onto an address the interface holds inside it. A public IPv4 + address in front of a private address is the usual case. + + Unlike an interface address, an external address is a bare address with no + prefix length, such as 203.0.113.10, because nothing configures it on the + NIC. The data plane maps it onto the interface address of the same family. + properties: + address: + description: |- + address is the externally reachable address, such as 203.0.113.10. It + carries no prefix length. + maxLength: 45 + minLength: 1 + type: string + class: + description: |- + class is the IPAM class this address was allocated from, such as + public-ipv4. It matches the class the claim requested in spec.addresses. + maxLength: 63 + minLength: 1 + type: string + family: + description: family is the address family of this entry. + enum: + - IPv4 + - IPv6 + type: string + required: + - address + - class + - family + type: object + maxItems: 4 + type: array + x-kubernetes-validations: + - message: External addresses must be unique + rule: self.all(a, self.exists_one(b, b.address == a.address)) + - message: Only one external address may be held per address class + rule: self.all(a, self.exists_one(b, b.class == a.class)) + interfaceName: + default: eth0 + description: |- + interfaceName is the device name the interface presents to the guest + operating system, such as eth0 or eth1. It comes from the claim. + maxLength: 15 + minLength: 1 + type: string + mtu: + description: |- + mtu is the MTU, in bytes, the interface must be configured with. It is + resolved from the network, so a provider never has to read the network to + configure the NIC. + format: int32 + maximum: 8856 + minimum: 1300 + type: integer + network: + description: |- + network is the network this interface belongs to, in the same namespace as + the interface. It comes from the claim and does not change. + properties: + name: + description: The network name + type: string + required: + - name + type: object + x-kubernetes-validations: + - message: network is immutable and cannot be changed after creation + rule: self == oldSelf + reclaimPolicy: + default: Delete + description: |- + reclaimPolicy decides what becomes of this interface, and its addresses, + when the claim holding it is deleted. It comes from the claim, and a claim + asking for a different policy cannot bind this interface. + enum: + - Delete + - Retain + type: string + required: + - network + type: object + status: + default: + conditions: + - lastTransitionTime: "1970-01-01T00:00:00Z" + message: Waiting for controller + reason: Pending + status: Unknown + type: Allocated + - lastTransitionTime: "1970-01-01T00:00:00Z" + message: Waiting for controller + reason: Pending + status: Unknown + type: Programmed + description: |- + NetworkInterfaceStatus defines the observed state of NetworkInterface: which + claim holds it, what realizes it on the data plane, and whether programming + has succeeded. + properties: + attachmentRef: + description: |- + attachmentRef is the data-plane resource realizing this interface. The + provider sets it once an attachment exists. + properties: + apiGroup: + description: |- + apiGroup is the API group of the referent, such as + compute.datumapis.com. + maxLength: 253 + minLength: 1 + type: string + kind: + description: kind is the kind of the referent. + maxLength: 63 + minLength: 1 + type: string + name: + description: name is the name of the referent. + maxLength: 253 + minLength: 1 + type: string + required: + - apiGroup + - kind + - name + type: object + conditions: + description: |- + conditions report the current state of the interface. Allocated means every + address is held. Programmed means the data plane carries them. + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + networkContextRef: + description: |- + networkContextRef is the network's presence in this location, resolved or + created while fulfilling the claim. It is a breadcrumb for operators + tracing where a network landed, and nothing needs it to configure a NIC. + properties: + name: + description: The network context name + type: string + required: + - name + type: object + phase: + description: |- + phase reports whether a claim holds the interface. Bound means the claim in + spec.claimRef holds it. Available means it is retained and holding its + addresses with no claim bound. + enum: + - Available + - Bound + type: string + vpc: + description: |- + vpc is the base62 identifier of the VPC backing this network in this + location, matching the identifier the fabric keys on. The provider records + it when the attachment is programmed. + type: string + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index 1dad8cb9..3b8099b0 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -16,6 +16,8 @@ resources: - bases/networking.datumapis.com_connectors.yaml - bases/networking.datumapis.com_connectoradvertisements.yaml - bases/networking.datumapis.com_connectorclasses.yaml +- bases/networking.datumapis.com_networkinterfaces.yaml +- bases/networking.datumapis.com_networkinterfaceclaims.yaml # +kubebuilder:scaffold:crdkustomizeresource patches: diff --git a/config/dependencies/README.md b/config/dependencies/README.md new file mode 100644 index 00000000..18d218ee --- /dev/null +++ b/config/dependencies/README.md @@ -0,0 +1,49 @@ +# Dependencies + +Deployment configuration for services NSO depends on but does not own — other +teams' components, deployed here so the test environment can exercise the real +integration instead of a stand-in. + +One named subdirectory per dependency. + +| directory | what it is | +|---|---| +| `ipam/` | The Milo IPAM aggregated apiserver (`ipam.miloapis.com`), the project control plane NSO claims addresses from. | + +## How this differs from the neighbours + +- **`config/tools/`** — things that support the environment rather than + participate in the feature under test: cert-manager, Envoy Gateway, + external-dns, kind cluster configs. A dependency here is something NSO's own + controllers talk to at runtime; a tool is scaffolding. +- **`config/e2e/`, `config/dev/`** — NSO's *own* deployment, per environment. + +## What belongs here, and what does not + +Deployment configuration only: what it takes to stand the dependency up. If a +file exists to make a test assert something, it is test data and lives in +`test/fixtures//` instead. + +The line is easy to blur, because the only consumer of this directory today is +the test environment — which makes everything in it feel like test scaffolding. +It is not. The test: + +> If the suites were deleted tomorrow, would this file still be needed to run +> the dependency? + +For `ipam/`, `overlay/` and `patches/` survive that question and stay; the +IPClass/IPPool seeds, the `ipam-e2e-*` namespaces, and the `e2e-tenant-tester` +binding do not, and live in `test/e2e/fixtures/ipam/`. + +Identities are split on the same line. `overlay/rbac.yaml` grants what the +operator needs to assert a project; the identity the suites impersonate is +bound to that same role from `test/e2e/fixtures/ipam/rbac.yaml`. + +## Convention + +A dependency is consumed at the version the Go module graph already pins, so +the manifests deployed and the client NSO compiles against cannot drift apart. +Prefer the upstream project's own published manifests over copies kept here: +`ipam/` pulls a digest-pinned OCI bundle at task time and keeps only this repo's +additions in `overlay/` and `patches/`. The name mirrors IPAM's own +`config/dependencies/postgres-operator`. diff --git a/config/dependencies/ipam/overlay/cluster-issuer.yaml b/config/dependencies/ipam/overlay/cluster-issuer.yaml new file mode 100644 index 00000000..cc23a476 --- /dev/null +++ b/config/dependencies/ipam/overlay/cluster-issuer.yaml @@ -0,0 +1,6 @@ +apiVersion: cert-manager.io/v1 +kind: ClusterIssuer +metadata: + name: selfsigned-cluster-issuer +spec: + selfSigned: {} diff --git a/config/dependencies/ipam/overlay/kustomization.yaml b/config/dependencies/ipam/overlay/kustomization.yaml new file mode 100644 index 00000000..ad70bc85 --- /dev/null +++ b/config/dependencies/ipam/overlay/kustomization.yaml @@ -0,0 +1,18 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +# Everything this env adds to the published IPAM bundle, and nothing from it. +# Kept self-contained so `task validate-kustomizations` can build it on a clean +# checkout, where the bundle has not been fetched yet. +# +# The composed deployment — this plus the bundle's base and components, with the +# patches in ../patches — is assembled at task time by test-infra:ipam-bundle +# from ../root-kustomization.yaml. + +namespace: ipam-system + +resources: + - cluster-issuer.yaml + - secret.yaml + - postgres-cluster.yaml + - rbac.yaml diff --git a/config/dependencies/ipam/overlay/postgres-cluster.yaml b/config/dependencies/ipam/overlay/postgres-cluster.yaml new file mode 100644 index 00000000..00af77af --- /dev/null +++ b/config/dependencies/ipam/overlay/postgres-cluster.yaml @@ -0,0 +1,55 @@ +# IPAM's database, declared for the CloudNativePG operator. +# +# The operator ships with IPAM's own bundle; the Cluster does not — IPAM +# installs CNPG and leaves the database to the consumer, so this is ours to +# author. It replaces the plain Deployment this env ran before. +# +# CNPG must be fully live before this applies: the CRD has to be established +# and the webhook has to be answering, or the apply fails in a way that reads +# like a flake. task test-infra:cnpg-operator gates on exactly that. +apiVersion: postgresql.cnpg.io/v1 +kind: Cluster +metadata: + name: ipam-db + namespace: ipam-system +spec: + # One instance: a single kind node cannot honour anti-affinity across + # replicas, and a failover this env can never exercise is not worth the + # startup time. + instances: 1 + imageName: ghcr.io/cloudnative-pg/postgresql:17.0 + + bootstrap: + initdb: + database: ipam + owner: ipam + # Credentials come from the secret above rather than being generated, so + # the DSN IPAM reads is a constant. + secret: + name: postgres-credentials + + storage: + size: 1Gi + # Unset storageClass: take the cluster default, which on kind is the + # built-in local-path provisioner. + + # Matches what the retired Deployment ran with. synchronous_commit=off is + # safe here and meaningfully faster: the suites do many small writes, and a + # lost transaction on a database that is destroyed with the cluster costs + # nothing. + postgresql: + parameters: + synchronous_commit: "off" + max_connections: "200" + shared_buffers: 192MB + + # The memory request must exceed shared_buffers, and CNPG's webhook enforces + # it. The retired Deployment asked for 128Mi while telling Postgres to reserve + # 192MB of it, which nothing validated — the operator refuses the same spec. + resources: + requests: + cpu: 50m + memory: 256Mi + limits: + cpu: "2" + memory: 512Mi diff --git a/config/dependencies/ipam/overlay/rbac.yaml b/config/dependencies/ipam/overlay/rbac.yaml new file mode 100644 index 00000000..4bfc4212 --- /dev/null +++ b/config/dependencies/ipam/overlay/rbac.yaml @@ -0,0 +1,77 @@ +# Two RBAC layers are required to reach IPAM as a project. +# +# 1. The caller impersonates a user plus the three iam.miloapis.com extras that +# carry the project. The kube-apiserver front proxy re-emits those extras as +# X-Remote-Extra-*, which is where IPAM reads the tenant from. +# 2. IPAM authorizes through a delegated SubjectAccessReview against the host +# apiserver, so the IMPERSONATED user needs ordinary RBAC of its own. +# +# The kind admin credential is in system:masters and is authorized for +# everything already, so only the NSO manager's ServiceAccount needs the +# impersonation grant below. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: nso-ipam-impersonator +rules: + - apiGroups: [""] + resources: ["users"] + verbs: ["impersonate"] + resourceNames: ["nso-ipam-agent"] + - apiGroups: ["authentication.k8s.io"] + resources: + - "userextras/iam.miloapis.com/parent-name" + - "userextras/iam.miloapis.com/parent-type" + - "userextras/iam.miloapis.com/parent-api-group" + verbs: ["impersonate"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: nso-ipam-impersonator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: nso-ipam-impersonator +subjects: + - kind: ServiceAccount + name: network-services-operator-controller-manager + namespace: network-services-operator-system +--- +# ipclasses is not optional: every claim resolves its class on create, so a +# subject without it gets a 403 on every allocation. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: nso-ipam-tenant +rules: + - apiGroups: [""] + resources: ["namespaces"] + verbs: ["get", "list", "create", "delete"] + - apiGroups: ["ipam.miloapis.com"] + resources: + - "ippools" + - "ippools/status" + - "ipclasses" + - "ipclasses/status" + - "ipclaims" + - "ipclaims/status" + - "ipallocations" + - "ipallocations/status" + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: nso-ipam-tenant +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: nso-ipam-tenant +subjects: + # The identity the NSO manager impersonates. The test suites impersonate a + # different one, bound to this same role from test/e2e/fixtures/ipam/rbac.yaml — + # test identities do not belong in the deployment. + - kind: User + apiGroup: rbac.authorization.k8s.io + name: nso-ipam-agent diff --git a/config/dependencies/ipam/overlay/secret.yaml b/config/dependencies/ipam/overlay/secret.yaml new file mode 100644 index 00000000..6c155c4d --- /dev/null +++ b/config/dependencies/ipam/overlay/secret.yaml @@ -0,0 +1,29 @@ +# One secret serves two consumers, so the password exists in exactly one place. +# +# IPAM reads `dsn` and passes it to both the migrate init container and the +# apiserver. CNPG reads `username`/`password` to create the application role +# when it bootstraps the cluster, which is why the type is basic-auth — CNPG +# rejects anything else for a bootstrap secret. Extra keys are ignored by both. +# +# Bootstrapping from a secret we author, rather than letting CNPG generate one, +# is what keeps the DSN a static string. CNPG's generated `-app` secret has a +# random password and would force the DSN to be assembled at deploy time from +# values that change on every rebuild. +# +# The host is the read-write service CNPG creates for the cluster below: +# -rw. sslmode=disable because CNPG serves TLS with its own internal +# CA, which the client would have to be taught to trust for no benefit here. +apiVersion: v1 +kind: Secret +metadata: + name: postgres-credentials + namespace: ipam-system + labels: + app.kubernetes.io/name: postgres + app.kubernetes.io/component: database + app.kubernetes.io/part-of: ipam.miloapis.com +type: kubernetes.io/basic-auth +stringData: + username: "ipam" + password: "devpassword" + dsn: "postgres://ipam:devpassword@ipam-db-rw.ipam-system.svc.cluster.local:5432/ipam?sslmode=disable" diff --git a/config/dependencies/ipam/patches/apiservice-patch.yaml b/config/dependencies/ipam/patches/apiservice-patch.yaml new file mode 100644 index 00000000..65d4b02f --- /dev/null +++ b/config/dependencies/ipam/patches/apiservice-patch.yaml @@ -0,0 +1,16 @@ +--- +# Trust the CA that signs the serving certificate, rather than skipping +# verification. +# +# cert-manager's cainjector reads the ipam-ca Certificate and writes its CA into +# spec.caBundle. That is the same CA the CSI driver issues each pod's serving +# cert from, so aggregation verifies a real chain. This replaced +# insecureSkipTLSVerify: true, which was needed only while the serving cert was +# self-signed and chained to nothing — the bundle's own component already sets +# that field to false, so this patch no longer has to touch it. +apiVersion: apiregistration.k8s.io/v1 +kind: APIService +metadata: + name: v1alpha1.ipam.miloapis.com + annotations: + cert-manager.io/inject-ca-from: ipam-system/ipam-ca diff --git a/config/dependencies/ipam/patches/deployment-patch.yaml b/config/dependencies/ipam/patches/deployment-patch.yaml new file mode 100644 index 00000000..e28cc10c --- /dev/null +++ b/config/dependencies/ipam/patches/deployment-patch.yaml @@ -0,0 +1,24 @@ +--- +# The image is pulled from ghcr by digest, so IfNotPresent: the node fetches it +# once and reuses it. Never would fail here — nothing side-loads the image any +# more, and a digest-pinned reference cannot go stale, so there is nothing for +# Always to re-check. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: ipam-apiserver +spec: + template: + spec: + initContainers: + - name: migrate + imagePullPolicy: IfNotPresent + containers: + - name: apiserver + imagePullPolicy: IfNotPresent + # No Milo quota backend in the kind/test-infra cluster — the + # quota.miloapis.com CRDs don't exist, so the quota plugin's informers + # would never sync and readyz would hang. Disable quota here. + env: + - name: ENABLE_QUOTA + value: "false" diff --git a/config/dependencies/ipam/patches/networkpolicy-apiserver-egress-patch.yaml b/config/dependencies/ipam/patches/networkpolicy-apiserver-egress-patch.yaml new file mode 100644 index 00000000..47e0c332 --- /dev/null +++ b/config/dependencies/ipam/patches/networkpolicy-apiserver-egress-patch.yaml @@ -0,0 +1,26 @@ +# Let the apiserver reach the kube-apiserver by address rather than by +# namespace. +# +# The base policy allows egress to :443/:6443 selected by +# namespaceSelector (kube-system, default). That never matches in kind: a +# connection to kubernetes.default.svc is DNATed to the node's own address, +# which belongs to no pod and therefore to no namespace, so the rule selects +# nothing and the traffic is dropped. +# +# The symptom is not a connection error. The apiserver comes up, serves TLS, +# and then sits at "informer-sync failed: 2 informers not started yet: +# [*v1.PriorityLevelConfiguration *v1.FlowSchema]" forever — readiness never +# passes and the APIService stays MissingEndpoints. It presents as a flake +# because a pod that happens to start before kindnet programs the policy syncs +# its informers and stays healthy for the rest of its life. +- op: add + path: /spec/egress/- + value: + to: + - ipBlock: + cidr: 0.0.0.0/0 + ports: + - protocol: TCP + port: 6443 + - protocol: TCP + port: 443 diff --git a/config/dependencies/ipam/patches/single-replica-patch.yaml b/config/dependencies/ipam/patches/single-replica-patch.yaml new file mode 100644 index 00000000..0d749b72 --- /dev/null +++ b/config/dependencies/ipam/patches/single-replica-patch.yaml @@ -0,0 +1,14 @@ +# One replica, env-only. The IPAM base runs two so a kubelet eviction during a +# claim CREATE has a second pod to absorb the request — a real concern in +# production and one this single-node kind cluster cannot exercise anyway. +# +# It is not merely unnecessary here, it is actively broken: the second replica +# reliably hangs with "informer-sync failed: 2 informers not started yet: +# [*v1.FlowSchema *v1.PriorityLevelConfiguration]" and never becomes Ready, +# which wedges the readiness gate while the first replica serves fine. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: ipam-apiserver +spec: + replicas: 1 diff --git a/config/dependencies/ipam/root-kustomization.yaml b/config/dependencies/ipam/root-kustomization.yaml new file mode 100644 index 00000000..fc5e2c08 --- /dev/null +++ b/config/dependencies/ipam/root-kustomization.yaml @@ -0,0 +1,70 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +# The composed IPAM deployment: the published bundle plus this env's additions. +# +# NOT named kustomization.yaml on purpose. It only builds inside the staging +# directory test-infra:ipam-bundle assembles, where `bundle/` is the extracted +# OCI artifact; in the repo it would be a kustomization that cannot build, and +# `task validate-kustomizations` discovers every kustomization.yaml there is. +# The self-contained half lives in overlay/ and is validated there. + +namespace: ipam-system + +resources: + - bundle/base + - overlay + +components: + - bundle/components/namespace + - bundle/components/api-registration + # Gives the base's CSI volume something it can actually issue from. The base + # points it at selfsigned-cluster-issuer, and a SelfSigned issuer signs every + # certificate with a freshly generated key — so each pod would get an + # unrelated cert chained to nothing and the APIService would have no stable CA + # to trust. This component bootstraps a real CA from that selfsigned issuer + # and repoints the volume at it, which is what makes per-pod issuance work. + # + # It lives here rather than in overlay/ because it comes from the fetched + # bundle, which does not exist on a clean checkout — overlay/ has to stay + # self-contained so validate-kustomizations can build it. + - bundle/components/cert-manager-ca + +# Pulled from ghcr by digest, not by the bundle's own tag: the release workflow +# republishes on every push, so a tag can move under us. The tag is recorded for +# legibility. Update it, the digest, and the bundle pins in +# Taskfile.test-infra.yml together. +# +# This is an index digest, so the node resolves its own platform. Genuinely +# multi-arch since upstream 13bd2c8: before that, both platform entries of the +# index pointed at one layer holding an x86-64 binary, which ran under emulation +# on arm64 and panicked inside pgx mid-suite. The two platforms now carry +# distinct layers and the arm64 entry is a real AArch64 binary — verified by +# reading the ELF header, not by trusting the manifest, because the manifest is +# exactly what was wrong before. +images: + - name: ghcr.io/milo-os/ipam + newName: ghcr.io/milo-os/ipam + # tag v0.0.0-13bd2c8 + digest: sha256:4fb11adf3f748d41f5e3d7f9d3357eb7aafe211e356297a0d845d579dacffe8f + +patches: + - path: patches/apiservice-patch.yaml + - path: patches/deployment-patch.yaml + target: + kind: Deployment + name: ipam-apiserver + - path: patches/single-replica-patch.yaml + target: + kind: Deployment + name: ipam-apiserver + - path: patches/networkpolicy-apiserver-egress-patch.yaml + target: + kind: NetworkPolicy + name: ipam-apiserver + +labels: + - includeSelectors: false + includeTemplates: true + pairs: + environment: test-infra diff --git a/config/dependencies/postgres-operator/kustomization.yaml b/config/dependencies/postgres-operator/kustomization.yaml new file mode 100644 index 00000000..c190dc6a --- /dev/null +++ b/config/dependencies/postgres-operator/kustomization.yaml @@ -0,0 +1,43 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +# The CloudNativePG operator, which runs IPAM's database. +# +# IPAM's own bundle installs CNPG at dependencies/postgres-operator as a Flux +# HelmRelease. This env runs no Flux, so it takes the same chart through +# kustomize --enable-helm — the way cert-manager is installed here — and skips +# the HelmRelease. The name mirrors IPAM's so the correspondence is obvious. +# +# The operator installs CRDs and a webhook; nothing may apply a Cluster until +# all three are live. task test-infra:cnpg-operator gates on that. +# +# Pinned rather than floating (IPAM's HelmRelease tracks "0.x"): a CI run should +# not change underneath us because upstream published a release. + +resources: + - namespace.yaml + +helmCharts: + - name: cloudnative-pg + repo: https://cloudnative-pg.github.io/charts + # chart 0.29.0 == CloudNativePG 1.30.0 + version: 0.29.0 + releaseName: cnpg + namespace: cnpg-system + # The chart declares kubeVersion >=1.29. Rendering happens offline, where + # helm assumes a default Kubernetes far older than that and refuses the + # chart — so state the version this env actually runs (K8S_NODE_IMAGE in + # Taskfile.test-infra.yml). Bump both together. + kubeVersion: 1.35.5 + valuesInline: + # One operator replica on a single-node kind; the default requests are + # sized for a real cluster and crowd the node alongside everything else + # this env runs. + replicaCount: 1 + resources: + requests: + cpu: 50m + memory: 100Mi + limits: + cpu: 500m + memory: 512Mi diff --git a/config/dependencies/postgres-operator/namespace.yaml b/config/dependencies/postgres-operator/namespace.yaml new file mode 100644 index 00000000..8deac4c7 --- /dev/null +++ b/config/dependencies/postgres-operator/namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: cnpg-system diff --git a/config/e2e/config.yaml b/config/e2e/config.yaml index 812afa1a..b11fb7a1 100644 --- a/config/e2e/config.yaml +++ b/config/e2e/config.yaml @@ -16,3 +16,11 @@ gateway: gateway.networking.datumapis.com/certificate-issuer: [] downstreamResourceManagement: kubeconfigPath: /etc/downstream-cluster/kubeconfig +ipam: + kubeconfigPath: /etc/ipam-cluster/kubeconfig + impersonateUsername: nso-ipam-agent +networkInterface: + enabled: true + location: + name: us-central-1 + namespace: default diff --git a/config/e2e/kustomization.yaml b/config/e2e/kustomization.yaml index 9e9744c4..3b02e6c0 100644 --- a/config/e2e/kustomization.yaml +++ b/config/e2e/kustomization.yaml @@ -36,10 +36,24 @@ patches: volumeMounts: - name: downstream-cluster-kubeconfig mountPath: /etc/downstream-cluster + - name: ipam-cluster-kubeconfig + mountPath: /etc/ipam-cluster volumes: - name: downstream-cluster-kubeconfig secret: secretName: downstream-cluster-kubeconfig + # Optional only so a missing secret fails in the manager rather + # than in the kubelet. It does NOT make the manager tolerate the + # secret's absence: with networkInterface.enabled true, loading + # the IPAM kubeconfig is part of setup, and failing it exits the + # process — so the manager crashloops and takes every other + # controller with it. Deliberate: a clear "unable to load IPAM + # kubeconfig" beats a manager that comes up half-functional. + # test-infra:up creates the secret before prepare-upstream. + - name: ipam-cluster-kubeconfig + secret: + secretName: ipam-cluster-kubeconfig + optional: true replacements: - source: # Add cert-manager annotation to ValidatingWebhookConfiguration, MutatingWebhookConfiguration and CRDs diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index fd1e8b98..cf59916f 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -125,6 +125,13 @@ rules: - get - list - watch +- apiGroups: + - events.k8s.io + resources: + - events + verbs: + - create + - patch - apiGroups: - externaldns.k8s.io resources: @@ -229,6 +236,8 @@ rules: - httpproxies - networkbindings - networkcontexts + - networkinterfaceclaims + - networkinterfaces - networkpolicies - networks - subnetclaims @@ -251,6 +260,8 @@ rules: - httpproxies/finalizers - networkbindings/finalizers - networkcontexts/finalizers + - networkinterfaceclaims/finalizers + - networkinterfaces/finalizers - networkpolicies/finalizers - networks/finalizers - subnetclaims/finalizers @@ -267,6 +278,8 @@ rules: - httpproxies/status - networkbindings/status - networkcontexts/status + - networkinterfaceclaims/status + - networkinterfaces/status - networkpolicies/status - networks/status - subnetclaims/status diff --git a/docs/api/connectoradvertisements.md b/docs/api/connectoradvertisements.md new file mode 100644 index 00000000..e5ae48bb --- /dev/null +++ b/docs/api/connectoradvertisements.md @@ -0,0 +1,360 @@ +# API Reference + +Packages: + +- [networking.datumapis.com/v1alpha1](#networkingdatumapiscomv1alpha1) + +# networking.datumapis.com/v1alpha1 + +Resource Types: + +- [ConnectorAdvertisement](#connectoradvertisement) + + + + +## ConnectorAdvertisement +[↩ Parent](#networkingdatumapiscomv1alpha1 ) + + + + + + +ConnectorAdvertisement is the Schema for the connectoradvertisements API. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
apiVersionstringnetworking.datumapis.com/v1alpha1true
kindstringConnectorAdvertisementtrue
metadataobjectRefer to the Kubernetes API documentation for the fields of the `metadata` field.true
specobject + Spec defines the desired state of a ConnectorAdvertisement
+ 		true
statusobject + Status defines the observed state of a ConnectorAdvertisement
+
+ Default: map[conditions:[map[lastTransitionTime:1970-01-01T00:00:00Z message:Waiting for controller reason:Pending status:Unknown type:Accepted]]]
+ 		false
+ + +### ConnectorAdvertisement.spec +[↩ Parent](#connectoradvertisement) + + + +Spec defines the desired state of a ConnectorAdvertisement + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
connectorRefobject + ConnectorRef references the Connector being advertised.
+ 		true
layer4[]object + Layer 4 services being advertised.
+ 		false
+ + +### ConnectorAdvertisement.spec.connectorRef +[↩ Parent](#connectoradvertisementspec) + + + +ConnectorRef references the Connector being advertised. + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
namestring + Name of the referenced Connector.
+ 		true
+ + +### ConnectorAdvertisement.spec.layer4[index] +[↩ Parent](#connectoradvertisementspec) + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
namestring + Name of the advertisement.
+ 		true
services[]object + Layer 4 services being advertised.
+ 		true
+ + +### ConnectorAdvertisement.spec.layer4[index].services[index] +[↩ Parent](#connectoradvertisementspeclayer4index) + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
addressstring + Address of the service. + +Can be an IPv4, IPv6, or a DNS address. A DNS address may contain +wildcards. A DNS address acts as an allow list for what addresses the +connector will allow to be requested through it. + +DNS resolution is the responsibility of the connector.
+ 		true
ports[]object + Ports of the service.
+ 		true
+ + +### ConnectorAdvertisement.spec.layer4[index].services[index].ports[index] +[↩ Parent](#connectoradvertisementspeclayer4indexservicesindex) + + + +Layer4ServicePort represents a port for a Layer 4 service. + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
namestring + Named port for the service.
+ 		true
portinteger + Port number for the service.
+
+ Format: int32
+ Minimum: 1
+ Maximum: 65535
+ 		true
protocolstring + Protocol for port. Must be TCP or UDP, defaults to "TCP".
+
+ Default: TCP
+ 		true
+ + +### ConnectorAdvertisement.status +[↩ Parent](#connectoradvertisement) + + + +Status defines the observed state of a ConnectorAdvertisement + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
conditions[]object + Conditions describe the current conditions of the ConnectorAdvertisement. + +Known conditions: +- Accepted: indicates whether the referenced Connector has been resolved. + When Accepted is False, the reason will explain why the reference + could not be resolved (for example, ConnectorNotFound).
+ 		false
+ + +### ConnectorAdvertisement.status.conditions[index] +[↩ Parent](#connectoradvertisementstatus) + + + +Condition contains details for one aspect of the current state of this API Resource. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
lastTransitionTimestring + lastTransitionTime is the last time the condition transitioned from one status to another. +This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable.
+
+ Format: date-time
+ 		true
messagestring + message is a human readable message indicating details about the transition. +This may be an empty string.
+ 		true
reasonstring + reason contains a programmatic identifier indicating the reason for the condition's last transition. +Producers of specific condition types may define expected values and meanings for this field, +and whether the values are considered a guaranteed API. +The value should be a CamelCase string. +This field may not be empty.
+ 		true
statusenum + status of the condition, one of True, False, Unknown.
+
+ Enum: True, False, Unknown
+ 		true
typestring + type of condition in CamelCase or in foo.example.com/CamelCase.
+ 		true
observedGenerationinteger + observedGeneration represents the .metadata.generation that the condition was set based upon. +For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date +with respect to the current state of the instance.
+
+ Format: int64
+ Minimum: 0
+ 		false
diff --git a/docs/api/connectorclasses.md b/docs/api/connectorclasses.md new file mode 100644 index 00000000..98306332 --- /dev/null +++ b/docs/api/connectorclasses.md @@ -0,0 +1,96 @@ +# API Reference + +Packages: + +- [networking.datumapis.com/v1alpha1](#networkingdatumapiscomv1alpha1) + +# networking.datumapis.com/v1alpha1 + +Resource Types: + +- [ConnectorClass](#connectorclass) + + + + +## ConnectorClass +[↩ Parent](#networkingdatumapiscomv1alpha1 ) + + + + + + +ConnectorClass is the Schema for the connectorclasses API. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
apiVersionstringnetworking.datumapis.com/v1alpha1true
kindstringConnectorClasstrue
metadataobjectRefer to the Kubernetes API documentation for the fields of the `metadata` field.true
specobject + Spec defines the desired state of a ConnectorClass
+ 		true
statusobject + Status defines the observed state of a ConnectorClass
+ 		false
+ + +### ConnectorClass.spec +[↩ Parent](#connectorclass) + + + +Spec defines the desired state of a ConnectorClass + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
controllerNamestring + ControllerName is the name of the controller responsible for this ConnectorClass.
+
+ Default: networking.datumapis.com/datum-connect
+ 		true
diff --git a/docs/api/connectors.md b/docs/api/connectors.md new file mode 100644 index 00000000..6cb3d00d --- /dev/null +++ b/docs/api/connectors.md @@ -0,0 +1,579 @@ +# API Reference + +Packages: + +- [networking.datumapis.com/v1alpha1](#networkingdatumapiscomv1alpha1) + +# networking.datumapis.com/v1alpha1 + +Resource Types: + +- [Connector](#connector) + + + + +## Connector +[↩ Parent](#networkingdatumapiscomv1alpha1 ) + + + + + + +Connector is the Schema for the connectors API. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
apiVersionstringnetworking.datumapis.com/v1alpha1true
kindstringConnectortrue
metadataobjectRefer to the Kubernetes API documentation for the fields of the `metadata` field.true
specobject + Spec defines the desired state of a Connector
+ 		true
statusobject + Status defines the observed state of a Connector
+
+ Default: map[conditions:[map[lastTransitionTime:1970-01-01T00:00:00Z message:Waiting for controller reason:Pending status:Unknown type:Accepted]]]
+ 		false
+ + +### Connector.spec +[↩ Parent](#connector) + + + +Spec defines the desired state of a Connector + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
connectorClassNamestring +
+ 		true
capabilities[]object + Capabilities desired to be supported by the connector. + +A connector may choose to not support all requested capabilities, and may +also choose to support additional capabilities not requested here. The +condition of each capability will reflect whether the capability is supported +or not in the ConnectorStatus.
+ 		false
+ + +### Connector.spec.capabilities[index] +[↩ Parent](#connectorspec) + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
typestring + Type of capability
+ 		true
connectTCPobject +
+ 		false
+ + +### Connector.spec.capabilities[index].connectTCP +[↩ Parent](#connectorspeccapabilitiesindex) + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
disabledboolean +
+ 		false
+ + +### Connector.status +[↩ Parent](#connector) + + + +Status defines the observed state of a Connector + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
capabilities[]object + Capabilities describe the status of each capability of the connector.
+ 		false
conditions[]object + Conditions describe the current conditions of the HTTPProxy.
+ 		false
connectionDetailsobject + ConnectionDetails provide details on how to connect to the connector.
+
+ Validations:
  • !(self.type != 'PublicKey' && has(self.publicKey)): publicKey field must be nil if the type is not PublicKey
  • self.type == 'PublicKey' && has(self.publicKey): publicKey field must be specified if the type is PublicKey
    • +     		false
    leaseRefobject + LeaseRef references the Lease used to report connector liveness. + +The connector controller creates the Lease when a Connector is created +and records it here. Connector implementations (agents) are expected to +periodically renew the Lease to indicate liveness.
    +     		false
    + + +### Connector.status.capabilities[index] +[↩ Parent](#connectorstatus) + + + + + + + + + + + + + + + + + + + + + + + + + +
    NameTypeDescriptionRequired
    typestring + Type of capability
    +     		true
    conditions[]object + Conditions describe the current conditions of the capability.
    +     		false
    + + +### Connector.status.capabilities[index].conditions[index] +[↩ Parent](#connectorstatuscapabilitiesindex) + + + +Condition contains details for one aspect of the current state of this API Resource. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    NameTypeDescriptionRequired
    lastTransitionTimestring + lastTransitionTime is the last time the condition transitioned from one status to another. +This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable.
    +
    + Format: date-time
    +     		true
    messagestring + message is a human readable message indicating details about the transition. +This may be an empty string.
    +     		true
    reasonstring + reason contains a programmatic identifier indicating the reason for the condition's last transition. +Producers of specific condition types may define expected values and meanings for this field, +and whether the values are considered a guaranteed API. +The value should be a CamelCase string. +This field may not be empty.
    +     		true
    statusenum + status of the condition, one of True, False, Unknown.
    +
    + Enum: True, False, Unknown
    +     		true
    typestring + type of condition in CamelCase or in foo.example.com/CamelCase.
    +     		true
    observedGenerationinteger + observedGeneration represents the .metadata.generation that the condition was set based upon. +For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date +with respect to the current state of the instance.
    +
    + Format: int64
    + Minimum: 0
    +     		false
    + + +### Connector.status.conditions[index] +[↩ Parent](#connectorstatus) + + + +Condition contains details for one aspect of the current state of this API Resource. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    NameTypeDescriptionRequired
    lastTransitionTimestring + lastTransitionTime is the last time the condition transitioned from one status to another. +This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable.
    +
    + Format: date-time
    +     		true
    messagestring + message is a human readable message indicating details about the transition. +This may be an empty string.
    +     		true
    reasonstring + reason contains a programmatic identifier indicating the reason for the condition's last transition. +Producers of specific condition types may define expected values and meanings for this field, +and whether the values are considered a guaranteed API. +The value should be a CamelCase string. +This field may not be empty.
    +     		true
    statusenum + status of the condition, one of True, False, Unknown.
    +
    + Enum: True, False, Unknown
    +     		true
    typestring + type of condition in CamelCase or in foo.example.com/CamelCase.
    +     		true
    observedGenerationinteger + observedGeneration represents the .metadata.generation that the condition was set based upon. +For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date +with respect to the current state of the instance.
    +
    + Format: int64
    + Minimum: 0
    +     		false
    + + +### Connector.status.connectionDetails +[↩ Parent](#connectorstatus) + + + +ConnectionDetails provide details on how to connect to the connector. + + + + + + + + + + + + + + + + + + + + + +
    NameTypeDescriptionRequired
    typeenum + Type of connection details provided.
    +
    + Enum: PublicKey
    +     		true
    publicKeyobject + PublicKey connection details
    +     		false
    + + +### Connector.status.connectionDetails.publicKey +[↩ Parent](#connectorstatusconnectiondetails) + + + +PublicKey connection details + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    NameTypeDescriptionRequired
    addresses[]object + Addresses where the connector can be reached
    +     		true
    homeRelaystring + Home Relay server of the connector + +Must be a valid URL
    +
    + Validations:
  • isURL(self): Must be a URL.
    • +     		true
    discoveryModeenum + The mode used to discover the public key
    +
    + Enum: DNS
    + Default: DNS
    +     		false
    idstring + The public key to dial and connect to
    +     		false
    + + +### Connector.status.connectionDetails.publicKey.addresses[index] +[↩ Parent](#connectorstatusconnectiondetailspublickey) + + + +PublicKeyConnectorAddress defines an address and port for a connector. + + + + + + + + + + + + + + + + + + + + + +
    NameTypeDescriptionRequired
    addressstring + IPv4 or IPv6 address.
    +
    + Validations:
  • isIP(self): Must be an IP address.
    • +     		true
    portinteger + Port where the connector can be reached.
    +
    + Format: int32
    + Minimum: 1
    + Maximum: 65535
    +     		true
    + + +### Connector.status.leaseRef +[↩ Parent](#connectorstatus) + + + +LeaseRef references the Lease used to report connector liveness. + +The connector controller creates the Lease when a Connector is created +and records it here. Connector implementations (agents) are expected to +periodically renew the Lease to indicate liveness. + + + + + + + + + + + + + + + + +
    NameTypeDescriptionRequired
    namestring + Name of the referent. +This field is effectively required, but due to backwards compatibility is +allowed to be empty. Instances of this type with an empty value here are +almost certainly wrong. +More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
    +
    + Default:
    +     		false
    diff --git a/docs/api/domains.md b/docs/api/domains.md index 78119a9d..6793b1db 100644 --- a/docs/api/domains.md +++ b/docs/api/domains.md @@ -94,6 +94,26 @@ DomainSpec defines the desired state of Domain Validations:
  • oldSelf == '' || self == oldSelf: A domain name is immutable and cannot be changed after creation
  • self.indexOf('.') != -1: Must have at least two segments separated by dots
    • true + + desiredRegistrationRefreshAttempt + string + + DesiredRegistrationRefreshAttempt is the desired time of the next registration refresh attempt.
    +
    + Validations:
  • oldSelf == null || self == null || self == oldSelf || self >= oldSelf + duration('5m'): must be at least 5m after the previous desiredRegistrationRefreshAttempt when changed
    • + Format: date-time
    + + false + + desiredVerificationRefreshAttempt + string + + DesiredVerificationRefreshAttempt is the desired time of the next verification refresh attempt.
    +
    + Validations:
  • oldSelf == null || self == null || self == oldSelf || self >= oldSelf + duration('5m'): must be at least 5m after the previous desiredVerificationRefreshAttempt when changed
    • + Format: date-time
    + + false @@ -369,6 +389,15 @@ Registration represents the registration information for a domain
    false + + lastRefreshAttempt + string + +
    +
    + Format: date-time
    + + false nextRefreshAttempt string @@ -811,6 +840,15 @@ DomainVerificationStatus represents the verification status of a domain
    false + + lastVerificationAttempt + string + +
    +
    + Format: date-time
    + + false nextVerificationAttempt string diff --git a/docs/api/httpproxies.md b/docs/api/httpproxies.md index 826c7aa0..633faf77 100644 --- a/docs/api/httpproxies.md +++ b/docs/api/httpproxies.md @@ -234,6 +234,16 @@ Supports http and https protocols, IPs or DNS addresses in the host, custom ports, and paths.
    true + + connector + object + + Connector references the Connector that should be used for this backend. + +For now, only a name reference is supported. In the future this can be +extended to selector-based matching to allow multiple connectors.
    + + false filters []object @@ -244,6 +254,46 @@ request is being forwarded to the backend defined here.
    Validations:
  • !(self.exists(f, f.type == 'RequestRedirect') && self.exists(f, f.type == 'URLRewrite')): May specify either requestRedirect or urlRewrite, but not both
  • self.filter(f, f.type == 'RequestHeaderModifier').size() <= 1: RequestHeaderModifier filter cannot be repeated
  • self.filter(f, f.type == 'ResponseHeaderModifier').size() <= 1: ResponseHeaderModifier filter cannot be repeated
  • self.filter(f, f.type == 'RequestRedirect').size() <= 1: RequestRedirect filter cannot be repeated
  • self.filter(f, f.type == 'URLRewrite').size() <= 1: URLRewrite filter cannot be repeated
    • false + + tls + object + + TLS contains backend TLS configuration. + +When the backend endpoint uses HTTPS with an IP address, the Hostname field +must be specified for TLS certificate validation.
    + + false + + + + +### HTTPProxy.spec.rules[index].backends[index].connector +[↩ Parent](#httpproxyspecrulesindexbackendsindex) + + + +Connector references the Connector that should be used for this backend. + +For now, only a name reference is supported. In the future this can be +extended to selector-based matching to allow multiple connectors. + + + + + + + + + + + + + + +
    NameTypeDescriptionRequired
    namestring + Name of the referenced Connector.
    +     		true
    @@ -260,8 +310,8 @@ examples include request or response modification, implementing authentication strategies, rate-limiting, and traffic shaping. API guarantee/conformance is defined based on the type of the filter. - - + + @@ -309,9 +359,9 @@ Unknown values here must result in the implementation setting the Accepted Condition for the Route to `status: False`, with a Reason of `UnsupportedValue`. -
    +

    - Enum: RequestHeaderModifier, ResponseHeaderModifier, RequestMirror, RequestRedirect, URLRewrite, ExtensionRef
    + Enum: RequestHeaderModifier, ResponseHeaderModifier, RequestMirror, RequestRedirect, URLRewrite, ExtensionRef, CORS
     @@ -321,9 +371,7 @@ Reason of `UnsupportedValue`. CORS defines a schema for a filter that responds to the cross-origin request based on HTTP response header. -Support: Extended - -
    +Support: Extended
     @@ -340,6 +388,24 @@ This filter can be used multiple times within the same rule. Support: Implementation-specific
    + + + + + @@ -410,8 +476,6 @@ cross-origin request based on HTTP response header. Support: Extended - -
    true
    false
    false
    externalAuthobject + ExternalAuth configures settings related to sending request details +to an external auth service. The external service MUST authenticate +the request, and MAY authorize the request as well. + +If there is any problem communicating with the external service, +this filter MUST fail closed. + +Support: Extended + +
    +
    + Validations:
  • self.protocol == 'GRPC' ? has(self.grpc) : true: grpc must be specified when protocol is set to 'GRPC'
  • has(self.grpc) ? self.protocol == 'GRPC' : true: protocol must be 'GRPC' when grpc is set
  • self.protocol == 'HTTP' ? has(self.http) : true: http must be specified when protocol is set to 'HTTP'
  • has(self.http) ? self.protocol == 'HTTP' : true: protocol must be 'HTTP' when http is set
    • +     		false
    requestHeaderModifier object
    @@ -428,16 +492,14 @@ Support: Extended AllowCredentials indicates whether the actual cross-origin request allows to include credentials. -The only valid value for the `Access-Control-Allow-Credentials` response -header is true (case-sensitive). +When set to true, the gateway will include the `Access-Control-Allow-Credentials` +response header with value true (case-sensitive). -If the credentials are not allowed in cross-origin requests, the gateway -will omit the header `Access-Control-Allow-Credentials` entirely rather -than setting its value to false. +When set to false or omitted the gateway will omit the header +`Access-Control-Allow-Credentials` entirely (this is the standard CORS +behavior). Support: Extended
    -
    - Enum: true
    @@ -447,7 +509,7 @@ Support: Extended
    AllowHeaders indicates which HTTP request headers are supported for accessing the requested resource. -Header names are not case sensitive. +Header names are not case-sensitive. Multiple header names in the value of the `Access-Control-Allow-Headers` response header are separated by a comma (","). @@ -466,20 +528,25 @@ does not recognize by the client, it will also occur an error on the client side. A wildcard indicates that the requests with all HTTP headers are allowed. -The `Access-Control-Allow-Headers` response header can only use `*` -wildcard as value when the `AllowCredentials` field is unspecified. - -When the `AllowCredentials` field is specified and `AllowHeaders` field -specified with the `*` wildcard, the gateway must specify one or more +If config contains the wildcard "*" in allowHeaders and the request is +not credentialed, the `Access-Control-Allow-Headers` response header +can either use the `*` wildcard or the value of +Access-Control-Request-Headers from the request. + +When the request is credentialed, the gateway must not specify the `*` +wildcard in the `Access-Control-Allow-Headers` response header. When +also the `AllowCredentials` field is true and `AllowHeaders` field +is specified with the `*` wildcard, the gateway must specify one or more HTTP headers in the value of the `Access-Control-Allow-Headers` response header. The value of the header `Access-Control-Allow-Headers` is same as the `Access-Control-Request-Headers` header provided by the client. If the header `Access-Control-Request-Headers` is not included in the request, the gateway will omit the `Access-Control-Allow-Headers` -response header, instead of specifying the `*` wildcard. A Gateway -implementation may choose to add implementation-specific default headers. +response header, instead of specifying the `*` wildcard. Support: Extended
    +
    + Validations:
  • !('*' in self && self.size() > 1): AllowHeaders cannot contain '*' alongside other methods
    • @@ -492,7 +559,7 @@ requested resource. Valid values are any method defined by RFC9110, along with the special value `*`, which represents all HTTP methods are allowed. -Method names are case sensitive, so these values are also case-sensitive. +Method names are case-sensitive, so these values are also case-sensitive. (See https://www.rfc-editor.org/rfc/rfc2616#section-5.1.1) Multiple method names in the value of the `Access-Control-Allow-Methods` @@ -512,18 +579,21 @@ is not included in the list of methods specified by the response header `Access-Control-Allow-Methods`, it will present an error on the client side. -The `Access-Control-Allow-Methods` response header can only use `*` -wildcard as value when the `AllowCredentials` field is unspecified. +If config contains the wildcard "*" in allowMethods and the request is +not credentialed, the `Access-Control-Allow-Methods` response header +can either use the `*` wildcard or the value of +Access-Control-Request-Method from the request. -When the `AllowCredentials` field is specified and `AllowMethods` field +When the request is credentialed, the gateway must not specify the `*` +wildcard in the `Access-Control-Allow-Methods` response header. When +also the `AllowCredentials` field is true and `AllowMethods` field specified with the `*` wildcard, the gateway must specify one HTTP method in the value of the Access-Control-Allow-Methods response header. The value of the header `Access-Control-Allow-Methods` is same as the `Access-Control-Request-Method` header provided by the client. If the header `Access-Control-Request-Method` is not included in the request, the gateway will omit the `Access-Control-Allow-Methods` response header, -instead of specifying the `*` wildcard. A Gateway implementation may -choose to add implementation-specific default methods. +instead of specifying the `*` wildcard. Support: Extended

    @@ -577,10 +647,19 @@ cross-origin response headers. Alternatively, the gateway responds with the CORS headers. The cross-origin request fails on the client side. Therefore, the client doesn't attempt the actual cross-origin request. -The `Access-Control-Allow-Origin` response header can only use `*` -wildcard as value when the `AllowCredentials` field is unspecified. +Conversely, if the request `Origin` matches one of the configured +allowed origins, the gateway sets the response header +`Access-Control-Allow-Origin` to the same value as the `Origin` +header provided by the client. + +When config has the wildcard ("*") in allowOrigins, and the request +is not credentialed (e.g., it is a preflight request), the +`Access-Control-Allow-Origin` response header either contains the +wildcard as well or the Origin from the request. -When the `AllowCredentials` field is specified and `AllowOrigins` field +When the request is credentialed, the gateway must not specify the `*` +wildcard in the `Access-Control-Allow-Origin` response header. When +also the `AllowCredentials` field is true and `AllowOrigins` field specified with the `*` wildcard, the gateway must return a single origin in the value of the `Access-Control-Allow-Origin` response header, instead of specifying the `*` wildcard. The value of the header @@ -588,6 +667,8 @@ instead of specifying the `*` wildcard. The value of the header the client. Support: Extended
    +
    + Validations:
  • !('*' in self && self.size() > 1): AllowOrigins cannot contain '*' alongside other origins
    • @@ -614,15 +695,18 @@ When an HTTP header name is specified using the `ExposeHeaders` field, this additional header will be exposed as part of the response to the client. -Header names are not case sensitive. +Header names are not case-sensitive. Multiple header names in the value of the `Access-Control-Expose-Headers` response header are separated by a comma (","). A wildcard indicates that the responses with all HTTP headers are exposed to clients. The `Access-Control-Expose-Headers` response header can only -use `*` wildcard as value when the `AllowCredentials` field is -unspecified. +use `*` wildcard as value when the request is not credentialed. + +When the `exposeHeaders` config field contains the "*" wildcard and +the request is credentialed, the gateway cannot use the `*` wildcard in +the `Access-Control-Expose-Headers` response header. Support: Extended
    @@ -639,7 +723,10 @@ The information provided by the `Access-Control-Allow-Methods` and client until the time specified by `Access-Control-Max-Age` elapses. The default value of `Access-Control-Max-Age` response header is 5 -(seconds).
    +(seconds). + +When the `MaxAge` field is unspecified, the gateway sets the response +header "Access-Control-Max-Age: 5" by default.

    Format: int32
    Default: 5
    @@ -699,15 +786,21 @@ When unspecified or empty string, core API group is inferred.
    false
    false
    false
    -### HTTPProxy.spec.rules[index].backends[index].filters[index].requestHeaderModifier +### HTTPProxy.spec.rules[index].backends[index].filters[index].externalAuth [↩ Parent](#httpproxyspecrulesindexbackendsindexfiltersindex) -RequestHeaderModifier defines a schema for a filter that modifies request -headers. +ExternalAuth configures settings related to sending request details +to an external auth service. The external service MUST authenticate +the request, and MAY authorize the request as well. -Support: Core +If there is any problem communicating with the external service, +this filter MUST fail closed. + +Support: Extended + + @@ -719,81 +812,104 @@ Support: Core - - + + + + + + + - + - - + + - - + + + + + + +
    add[]objectbackendRefobject - Add adds the given header(s) (name, value) to the request -before the action. It appends to any existing values associated -with the header name. + BackendRef is a reference to a backend to send authorization +requests to. -Input: - GET /foo HTTP/1.1 - my-header: foo +The backend must speak the selected protocol (GRPC or HTTP) on the +referenced port. -Config: - add: - - name: "my-header" - value: "bar,baz" +If the backend service requires TLS, use BackendTLSPolicy to tell the +implementation to supply the TLS details to be used to connect to that +backend.
    +
    + Validations:
  • (size(self.group) == 0 && self.kind == 'Service') ? has(self.port) : true: Must have port for Service reference
    • +     		true
    protocolenum + ExternalAuthProtocol describes which protocol to use when communicating with an +ext_authz authorization server. -Output: - GET /foo HTTP/1.1 - my-header: foo,bar,baz
    +When this is set to GRPC, each backend must use the Envoy ext_authz protocol +on the port specified in `backendRefs`. Requests and responses are defined +in the protobufs explained at: +https://www.envoyproxy.io/docs/envoy/latest/api-v3/service/auth/v3/external_auth.proto + +When this is set to HTTP, each backend must respond with a `200` status +code in on a successful authorization. Any other code is considered +an authorization failure. + +Feature Names: +GRPC Support - HTTPRouteExternalAuthGRPC +HTTP Support - HTTPRouteExternalAuthHTTP
    +
    + Enum: HTTP, GRPC
    		falsetrue
    remove[]stringforwardBodyobject - Remove the given header(s) from the HTTP request before the action. The -value of Remove is a list of HTTP header names. Note that the header -names are case-insensitive (see -https://datatracker.ietf.org/doc/html/rfc2616#section-4.2). + ForwardBody controls if requests to the authorization server should include +the body of the client request; and if so, how big that body is allowed +to be. -Input: - GET /foo HTTP/1.1 - my-header1: foo - my-header2: bar - my-header3: baz +It is expected that implementations will buffer the request body up to +`forwardBody.maxSize` bytes. Bodies over that size must be rejected with a +4xx series error (413 or 403 are common examples), and fail processing +of the filter. -Config: - remove: ["my-header1", "my-header3"] +If unset, or `forwardBody.maxSize` is set to `0`, then the body will not +be forwarded. -Output: - GET /foo HTTP/1.1 - my-header2: bar
    +Feature Name: HTTPRouteExternalAuthForwardBody
    		 false
    set[]objectgrpcobject - Set overwrites the request with the given header (name, value) -before the action. - -Input: - GET /foo HTTP/1.1 - my-header: foo + GRPCAuthConfig contains configuration for communication with ext_authz +protocol-speaking backends. -Config: - set: - - name: "my-header" - value: "bar" +If unset, implementations must assume the default behavior for each +included field is intended.
    +     		false
    httpobject + HTTPAuthConfig contains configuration for communication with HTTP-speaking +backends. -Output: - GET /foo HTTP/1.1 - my-header: bar
    +If unset, implementations must assume the default behavior for each +included field is intended.
    		 false
    -### HTTPProxy.spec.rules[index].backends[index].filters[index].requestHeaderModifier.add[index] -[↩ Parent](#httpproxyspecrulesindexbackendsindexfiltersindexrequestheadermodifier) +### HTTPProxy.spec.rules[index].backends[index].filters[index].externalAuth.backendRef +[↩ Parent](#httpproxyspecrulesindexbackendsindexfiltersindexexternalauth) -HTTPHeader represents an HTTP Header name and value as defined by RFC 7230. +BackendRef is a reference to a backend to send authorization +requests to. + +The backend must speak the selected protocol (GRPC or HTTP) on the +referenced port. + +If the backend service requires TLS, use BackendTLSPolicy to tell the +implementation to supply the TLS details to be used to connect to that +backend. @@ -808,33 +924,93 @@ HTTPHeader represents an HTTP Header name and value as defined by RFC 7230. - + - + + + + + + + + + + + + + + + +
    name string - Name is the name of the HTTP Header to be matched. Name matching MUST be -case-insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). - -If multiple entries specify equivalent header names, the first entry with -an equivalent name MUST be considered for a match. Subsequent entries -with an equivalent header name MUST be ignored. Due to the -case-insensitivity of header names, "foo" and "Foo" are considered -equivalent.
    + Name is the name of the referent.
    		 true
    valuegroup string - Value is the value of HTTP Header to be matched.
    + Group is the group of the referent. For example, "gateway.networking.k8s.io". +When unspecified or empty string, core API group is inferred.
    +
    + Default:
    		truefalse
    kindstring + Kind is the Kubernetes resource kind of the referent. For example +"Service". + +Defaults to "Service" when not specified. + +ExternalName services can refer to CNAME DNS records that may live +outside of the cluster and as such are difficult to reason about in +terms of conformance. They also may not be safe to forward to (see +CVE-2021-25740 for more information). Implementations SHOULD NOT +support ExternalName Services. + +Support: Core (Services with a type other than ExternalName) + +Support: Implementation-specific (Services with type ExternalName)
    +
    + Default: Service
    +     		false
    namespacestring + Namespace is the namespace of the backend. When unspecified, the local +namespace is inferred. + +Note that when a namespace different than the local namespace is specified, +a ReferenceGrant object is required in the referent namespace to allow that +namespace's owner to accept the reference. See the ReferenceGrant +documentation for details. + +Support: Core
    +     		false
    portinteger + Port specifies the destination port number to use for this resource. +Port is required when the referent is a Kubernetes Service. In this +case, the port number is the service port number, not the target port. +For other resources, destination port might be derived from the referent +resource or this field.
    +
    + Format: int32
    + Minimum: 1
    + Maximum: 65535
    +     		false
    -### HTTPProxy.spec.rules[index].backends[index].filters[index].requestHeaderModifier.set[index] -[↩ Parent](#httpproxyspecrulesindexbackendsindexfiltersindexrequestheadermodifier) +### HTTPProxy.spec.rules[index].backends[index].filters[index].externalAuth.forwardBody +[↩ Parent](#httpproxyspecrulesindexbackendsindexfiltersindexexternalauth) -HTTPHeader represents an HTTP Header name and value as defined by RFC 7230. +ForwardBody controls if requests to the authorization server should include +the body of the client request; and if so, how big that body is allowed +to be. + +It is expected that implementations will buffer the request body up to +`forwardBody.maxSize` bytes. Bodies over that size must be rejected with a +4xx series error (413 or 403 are common examples), and fail processing +of the filter. + +If unset, or `forwardBody.maxSize` is set to `0`, then the body will not +be forwarded. + +Feature Name: HTTPRouteExternalAuthForwardBody @@ -846,44 +1022,36 @@ HTTPHeader represents an HTTP Header name and value as defined by RFC 7230. - - + + - - - - - - +
    namestringmaxSizeinteger - Name is the name of the HTTP Header to be matched. Name matching MUST be -case-insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). + MaxSize specifies how large in bytes the largest body that will be buffered +and sent to the authorization server. If the body size is larger than +`maxSize`, then the body sent to the authorization server must be +truncated to `maxSize` bytes. -If multiple entries specify equivalent header names, the first entry with -an equivalent name MUST be considered for a match. Subsequent entries -with an equivalent header name MUST be ignored. Due to the -case-insensitivity of header names, "foo" and "Foo" are considered -equivalent.
    -     		true
    valuestring - Value is the value of HTTP Header to be matched.
    +Experimental note: This behavior needs to be checked against +various dataplanes; it may need to be changed. +See https://github.com/kubernetes-sigs/gateway-api/pull/4001#discussion_r2291405746 +for more. + +If 0, the body will not be sent to the authorization server.
    		truefalse
    -### HTTPProxy.spec.rules[index].backends[index].filters[index].requestMirror -[↩ Parent](#httpproxyspecrulesindexbackendsindexfiltersindex) - +### HTTPProxy.spec.rules[index].backends[index].filters[index].externalAuth.grpc +[↩ Parent](#httpproxyspecrulesindexbackendsindexfiltersindexexternalauth) -RequestMirror defines a schema for a filter that mirrors requests. -Requests are sent to the specified destination, but responses from -that destination are ignored. -This filter can be used multiple times within the same rule. Note that -not all implementations will be able to support mirroring to multiple -backends. +GRPCAuthConfig contains configuration for communication with ext_authz +protocol-speaking backends. -Support: Extended +If unset, implementations must assume the default behavior for each +included field is intended. @@ -895,97 +1063,109 @@ Support: Extended - - + + + + +
    backendRefobjectallowedHeaders[]string - BackendRef references a resource where mirrored requests are sent. + AllowedRequestHeaders specifies what headers from the client request +will be sent to the authorization server. -Mirrored requests must be sent only to a single destination endpoint -within this BackendRef, irrespective of how many endpoints are present -within this BackendRef. +If this list is empty, then all headers must be sent. -If the referent cannot be found, this BackendRef is invalid and must be -dropped from the Gateway. The controller must ensure the "ResolvedRefs" -condition on the Route status is set to `status: False` and not configure -this backend in the underlying implementation. +If the list has entries, only those entries must be sent.
    +     		false
    -If there is a cross-namespace reference to an *existing* object -that is not allowed by a ReferenceGrant, the controller must ensure the -"ResolvedRefs" condition on the Route is set to `status: False`, -with the "RefNotPermitted" reason and not configure this backend in the -underlying implementation. -In either error case, the Message of the `ResolvedRefs` Condition -should be used to provide more detail about the problem. +### HTTPProxy.spec.rules[index].backends[index].filters[index].externalAuth.http +[↩ Parent](#httpproxyspecrulesindexbackendsindexfiltersindexexternalauth) -Support: Extended for Kubernetes Service -Support: Implementation-specific for any other resource
    -
    - Validations:
  • (size(self.group) == 0 && self.kind == 'Service') ? has(self.port) : true: Must have port for Service reference
    • + +HTTPAuthConfig contains configuration for communication with HTTP-speaking +backends. + +If unset, implementations must assume the default behavior for each +included field is intended. + + + + + + + + + + + + + + - + - - + + - - + +
    NameTypeDescriptionRequired
    allowedHeaders[]string + AllowedRequestHeaders specifies what additional headers from the client request +will be sent to the authorization server. + +The following headers must always be sent to the authorization server, +regardless of this setting: + +* `Host` +* `Method` +* `Path` +* `Content-Length` +* `Authorization` + +If this list is empty, then only those headers must be sent. + +Note that `Content-Length` has a special behavior, in that the length +sent must be correct for the actual request to the external authorization +server - that is, it must reflect the actual number of bytes sent in the +body of the request to the authorization server. + +So if the `forwardBody` stanza is unset, or `forwardBody.maxSize` is set +to `0`, then `Content-Length` must be `0`. If `forwardBody.maxSize` is set +to anything other than `0`, then the `Content-Length` of the authorization +request must be set to the actual number of bytes forwarded.
    		truefalse
    fractionobjectallowedResponseHeaders[]string - Fraction represents the fraction of requests that should be -mirrored to BackendRef. + AllowedResponseHeaders specifies what headers from the authorization response +will be copied into the request to the backend. -Only one of Fraction or Percent may be specified. If neither field -is specified, 100% of requests will be mirrored.
    -
    - Validations:
  • self.numerator <= self.denominator: numerator must be less than or equal to denominator
    • +If this list is empty, then all headers from the authorization server +except Authority or Host must be copied.
    		 false
    percentintegerpathstring - Percent represents the percentage of requests that should be -mirrored to BackendRef. Its minimum value is 0 (indicating 0% of -requests) and its maximum value is 100 (indicating 100% of requests). + Path sets the prefix that paths from the client request will have added +when forwarded to the authorization server. -Only one of Fraction or Percent may be specified. If neither field -is specified, 100% of requests will be mirrored.
    -
    - Format: int32
    - Minimum: 0
    - Maximum: 100
    +When empty or unspecified, no prefix is added. + +Valid values are the same as the "value" regex for path values in the `match` +stanza, and the validation regex will screen out invalid paths in the same way. +Even with the validation, implementations MUST sanitize this input before using it +directly.
    		 false
    -### HTTPProxy.spec.rules[index].backends[index].filters[index].requestMirror.backendRef -[↩ Parent](#httpproxyspecrulesindexbackendsindexfiltersindexrequestmirror) - - - -BackendRef references a resource where mirrored requests are sent. - -Mirrored requests must be sent only to a single destination endpoint -within this BackendRef, irrespective of how many endpoints are present -within this BackendRef. - -If the referent cannot be found, this BackendRef is invalid and must be -dropped from the Gateway. The controller must ensure the "ResolvedRefs" -condition on the Route status is set to `status: False` and not configure -this backend in the underlying implementation. +### HTTPProxy.spec.rules[index].backends[index].filters[index].requestHeaderModifier +[↩ Parent](#httpproxyspecrulesindexbackendsindexfiltersindex) -If there is a cross-namespace reference to an *existing* object -that is not allowed by a ReferenceGrant, the controller must ensure the -"ResolvedRefs" condition on the Route is set to `status: False`, -with the "RefNotPermitted" reason and not configure this backend in the -underlying implementation. -In either error case, the Message of the `ResolvedRefs` Condition -should be used to provide more detail about the problem. -Support: Extended for Kubernetes Service +RequestHeaderModifier defines a schema for a filter that modifies request +headers. -Support: Implementation-specific for any other resource +Support: Core @@ -997,88 +1177,81 @@ Support: Implementation-specific for any other resource - - - - - - - - - - - - + + - - + + - - + +
    namestring - Name is the name of the referent.
    -     		true
    groupstring - Group is the group of the referent. For example, "gateway.networking.k8s.io". -When unspecified or empty string, core API group is inferred.
    -
    - Default:
    -     		false
    kindstringadd[]object - Kind is the Kubernetes resource kind of the referent. For example -"Service". - -Defaults to "Service" when not specified. + Add adds the given header(s) (name, value) to the request +before the action. It appends to any existing values associated +with the header name. -ExternalName services can refer to CNAME DNS records that may live -outside of the cluster and as such are difficult to reason about in -terms of conformance. They also may not be safe to forward to (see -CVE-2021-25740 for more information). Implementations SHOULD NOT -support ExternalName Services. +Input: + GET /foo HTTP/1.1 + my-header: foo -Support: Core (Services with a type other than ExternalName) +Config: + add: + - name: "my-header" + value: "bar,baz" -Support: Implementation-specific (Services with type ExternalName)
    -
    - Default: Service
    +Output: + GET /foo HTTP/1.1 + my-header: foo,bar,baz
    		 false
    namespacestringremove[]string - Namespace is the namespace of the backend. When unspecified, the local -namespace is inferred. + Remove the given header(s) from the HTTP request before the action. The +value of Remove is a list of HTTP header names. Note that the header +names are case-insensitive (see +https://datatracker.ietf.org/doc/html/rfc2616#section-4.2). -Note that when a namespace different than the local namespace is specified, -a ReferenceGrant object is required in the referent namespace to allow that -namespace's owner to accept the reference. See the ReferenceGrant -documentation for details. +Input: + GET /foo HTTP/1.1 + my-header1: foo + my-header2: bar + my-header3: baz -Support: Core
    +Config: + remove: ["my-header1", "my-header3"] + +Output: + GET /foo HTTP/1.1 + my-header2: bar
    		 false
    portintegerset[]object - Port specifies the destination port number to use for this resource. -Port is required when the referent is a Kubernetes Service. In this -case, the port number is the service port number, not the target port. -For other resources, destination port might be derived from the referent -resource or this field.
    -
    - Format: int32
    - Minimum: 1
    - Maximum: 65535
    + Set overwrites the request with the given header (name, value) +before the action. + +Input: + GET /foo HTTP/1.1 + my-header: foo + +Config: + set: + - name: "my-header" + value: "bar" + +Output: + GET /foo HTTP/1.1 + my-header: bar
    		 false
    -### HTTPProxy.spec.rules[index].backends[index].filters[index].requestMirror.fraction -[↩ Parent](#httpproxyspecrulesindexbackendsindexfiltersindexrequestmirror) - +### HTTPProxy.spec.rules[index].backends[index].filters[index].requestHeaderModifier.add[index] +[↩ Parent](#httpproxyspecrulesindexbackendsindexfiltersindexrequestheadermodifier) -Fraction represents the fraction of requests that should be -mirrored to BackendRef. -Only one of Fraction or Percent may be specified. If neither field -is specified, 100% of requests will be mirrored. +HTTPHeader represents an HTTP Header name and value as defined by RFC 7230. @@ -1090,39 +1263,42 @@ is specified, 100% of requests will be mirrored. - - + + - - + + - +
    numeratorintegernamestring -
    -
    - Format: int32
    - Minimum: 0
    + Name is the name of the HTTP Header to be matched. Name matching MUST be +case-insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). + +If multiple entries specify equivalent header names, the first entry with +an equivalent name MUST be considered for a match. Subsequent entries +with an equivalent header name MUST be ignored. Due to the +case-insensitivity of header names, "foo" and "Foo" are considered +equivalent.
    		 true
    denominatorintegervaluestring -
    -
    - Format: int32
    - Default: 100
    - Minimum: 1
    + Value is the value of HTTP Header to be matched. + +Must consist of printable US-ASCII characters, optionally separated +by single tabs or spaces. See: https://tools.ietf.org/html/rfc7230#section-3.2 + + +
    		falsetrue
    -### HTTPProxy.spec.rules[index].backends[index].filters[index].requestRedirect -[↩ Parent](#httpproxyspecrulesindexbackendsindexfiltersindex) - +### HTTPProxy.spec.rules[index].backends[index].filters[index].requestHeaderModifier.set[index] +[↩ Parent](#httpproxyspecrulesindexbackendsindexfiltersindexrequestheadermodifier) -RequestRedirect defines a schema for a filter that responds to the -request with an HTTP redirection. -Support: Core +HTTPHeader represents an HTTP Header name and value as defined by RFC 7230. @@ -1134,116 +1310,152 @@ Support: Core - + - + - - + + - - - - + + +
    hostnamename string - Hostname is the hostname to be used in the value of the `Location` -header in the response. -When empty, the hostname in the `Host` header of the request is used. + Name is the name of the HTTP Header to be matched. Name matching MUST be +case-insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). -Support: Core
    +If multiple entries specify equivalent header names, the first entry with +an equivalent name MUST be considered for a match. Subsequent entries +with an equivalent header name MUST be ignored. Due to the +case-insensitivity of header names, "foo" and "Foo" are considered +equivalent.
    		falsetrue
    pathobjectvaluestring - Path defines parameters used to modify the path of the incoming request. -The modified path is then used to construct the `Location` header. When -empty, the request path is used as-is. + Value is the value of HTTP Header to be matched. + +Must consist of printable US-ASCII characters, optionally separated +by single tabs or spaces. See: https://tools.ietf.org/html/rfc7230#section-3.2 + -Support: Extended
    -
    - Validations:
  • self.type == 'ReplaceFullPath' ? has(self.replaceFullPath) : true: replaceFullPath must be specified when type is set to 'ReplaceFullPath'
  • has(self.replaceFullPath) ? self.type == 'ReplaceFullPath' : true: type must be 'ReplaceFullPath' when replaceFullPath is set
  • self.type == 'ReplacePrefixMatch' ? has(self.replacePrefixMatch) : true: replacePrefixMatch must be specified when type is set to 'ReplacePrefixMatch'
  • has(self.replacePrefixMatch) ? self.type == 'ReplacePrefixMatch' : true: type must be 'ReplacePrefixMatch' when replacePrefixMatch is set
    • +
    		false
    portintegertrue
    + + +### HTTPProxy.spec.rules[index].backends[index].filters[index].requestMirror +[↩ Parent](#httpproxyspecrulesindexbackendsindexfiltersindex) + + + +RequestMirror defines a schema for a filter that mirrors requests. +Requests are sent to the specified destination, but responses from +that destination are ignored. + +This filter can be used multiple times within the same rule. Note that +not all implementations will be able to support mirroring to multiple +backends. + +Support: Extended + + + + + + + + + + + + + - + - - + + - +
    NameTypeDescriptionRequired
    backendRefobject - Port is the port to be used in the value of the `Location` -header in the response. + BackendRef references a resource where mirrored requests are sent. -If no port is specified, the redirect port MUST be derived using the -following rules: +Mirrored requests must be sent only to a single destination endpoint +within this BackendRef, irrespective of how many endpoints are present +within this BackendRef. -* If redirect scheme is not-empty, the redirect port MUST be the well-known - port associated with the redirect scheme. Specifically "http" to port 80 - and "https" to port 443. If the redirect scheme does not have a - well-known port, the listener port of the Gateway SHOULD be used. -* If redirect scheme is empty, the redirect port MUST be the Gateway - Listener port. +If the referent cannot be found, this BackendRef is invalid and must be +dropped from the Gateway. The controller must ensure the "ResolvedRefs" +condition on the Route status is set to `status: False` and not configure +this backend in the underlying implementation. -Implementations SHOULD NOT add the port number in the 'Location' -header in the following cases: +If there is a cross-namespace reference to an *existing* object +that is not allowed by a ReferenceGrant, the controller must ensure the +"ResolvedRefs" condition on the Route is set to `status: False`, +with the "RefNotPermitted" reason and not configure this backend in the +underlying implementation. -* A Location header that will use HTTP (whether that is determined via - the Listener protocol or the Scheme field) _and_ use port 80. -* A Location header that will use HTTPS (whether that is determined via - the Listener protocol or the Scheme field) _and_ use port 443. +In either error case, the Message of the `ResolvedRefs` Condition +should be used to provide more detail about the problem. -Support: Extended
    +Support: Extended for Kubernetes Service + +Support: Implementation-specific for any other resource

    - Format: int32
    - Minimum: 1
    - Maximum: 65535
    + Validations:
  • (size(self.group) == 0 && self.kind == 'Service') ? has(self.port) : true: Must have port for Service reference
    		• falsetrue
    schemeenumfractionobject - Scheme is the scheme to be used in the value of the `Location` header in -the response. When empty, the scheme of the request is used. - -Scheme redirects can affect the port of the redirect, for more information, -refer to the documentation for the port field of this filter. - -Note that values may be added to this enum, implementations -must ensure that unknown values will not cause a crash. - -Unknown values here must result in the implementation setting the -Accepted Condition for the Route to `status: False`, with a -Reason of `UnsupportedValue`. + Fraction represents the fraction of requests that should be +mirrored to BackendRef. -Support: Extended
    +Only one of Fraction or Percent may be specified. If neither field +is specified, 100% of requests will be mirrored.

    - Enum: http, https
    + Validations:
  • self.numerator <= self.denominator: numerator must be less than or equal to denominator
    		•  false
    statusCodepercent integer - StatusCode is the HTTP status code to be used in response. - -Note that values may be added to this enum, implementations -must ensure that unknown values will not cause a crash. - -Unknown values here must result in the implementation setting the -Accepted Condition for the Route to `status: False`, with a -Reason of `UnsupportedValue`. + Percent represents the percentage of requests that should be +mirrored to BackendRef. Its minimum value is 0 (indicating 0% of +requests) and its maximum value is 100 (indicating 100% of requests). -Support: Core
    +Only one of Fraction or Percent may be specified. If neither field +is specified, 100% of requests will be mirrored.

    - Enum: 301, 302
    - Default: 302
    + Format: int32
    + Minimum: 0
    + Maximum: 100
    		 false
    -### HTTPProxy.spec.rules[index].backends[index].filters[index].requestRedirect.path -[↩ Parent](#httpproxyspecrulesindexbackendsindexfiltersindexrequestredirect) +### HTTPProxy.spec.rules[index].backends[index].filters[index].requestMirror.backendRef +[↩ Parent](#httpproxyspecrulesindexbackendsindexfiltersindexrequestmirror) -Path defines parameters used to modify the path of the incoming request. -The modified path is then used to construct the `Location` header. When -empty, the request path is used as-is. +BackendRef references a resource where mirrored requests are sent. -Support: Extended +Mirrored requests must be sent only to a single destination endpoint +within this BackendRef, irrespective of how many endpoints are present +within this BackendRef. + +If the referent cannot be found, this BackendRef is invalid and must be +dropped from the Gateway. The controller must ensure the "ResolvedRefs" +condition on the Route status is set to `status: False` and not configure +this backend in the underlying implementation. + +If there is a cross-namespace reference to an *existing* object +that is not allowed by a ReferenceGrant, the controller must ensure the +"ResolvedRefs" condition on the Route is set to `status: False`, +with the "RefNotPermitted" reason and not configure this backend in the +underlying implementation. + +In either error case, the Message of the `ResolvedRefs` Condition +should be used to provide more detail about the problem. + +Support: Extended for Kubernetes Service + +Support: Implementation-specific for any other resource @@ -1255,65 +1467,88 @@ Support: Extended - - + + - + - + + + + + + + + + + +
    typeenumnamestring - Type defines the type of path modifier. Additional types may be -added in a future release of the API. - -Note that values may be added to this enum, implementations -must ensure that unknown values will not cause a crash. - -Unknown values here must result in the implementation setting the -Accepted Condition for the Route to `status: False`, with a -Reason of `UnsupportedValue`.
    -
    - Enum: ReplaceFullPath, ReplacePrefixMatch
    + Name is the name of the referent.
    		 true
    replaceFullPathgroup string - ReplaceFullPath specifies the value with which to replace the full path -of a request during a rewrite or redirect.
    + Group is the group of the referent. For example, "gateway.networking.k8s.io". +When unspecified or empty string, core API group is inferred.
    +
    + Default:
    		false
    replacePrefixMatchkind string - ReplacePrefixMatch specifies the value with which to replace the prefix -match of a request during a rewrite or redirect. For example, a request -to "/foo/bar" with a prefix match of "/foo" and a ReplacePrefixMatch -of "/xyz" would be modified to "/xyz/bar". + Kind is the Kubernetes resource kind of the referent. For example +"Service". -Note that this matches the behavior of the PathPrefix match type. This -matches full path elements. A path element refers to the list of labels -in the path split by the `/` separator. When specified, a trailing `/` is -ignored. For example, the paths `/abc`, `/abc/`, and `/abc/def` would all -match the prefix `/abc`, but the path `/abcd` would not. +Defaults to "Service" when not specified. -ReplacePrefixMatch is only compatible with a `PathPrefix` HTTPRouteMatch. -Using any other HTTPRouteMatch type on the same HTTPRouteRule will result in -the implementation setting the Accepted Condition for the Route to `status: False`. +ExternalName services can refer to CNAME DNS records that may live +outside of the cluster and as such are difficult to reason about in +terms of conformance. They also may not be safe to forward to (see +CVE-2021-25740 for more information). Implementations SHOULD NOT +support ExternalName Services. -Request Path | Prefix Match | Replace Prefix | Modified Path
    +Support: Core (Services with a type other than ExternalName) + +Support: Implementation-specific (Services with type ExternalName)
    +
    + Default: Service
    +     		false
    namespacestring + Namespace is the namespace of the backend. When unspecified, the local +namespace is inferred. + +Note that when a namespace different than the local namespace is specified, +a ReferenceGrant object is required in the referent namespace to allow that +namespace's owner to accept the reference. See the ReferenceGrant +documentation for details. + +Support: Core
    +     		false
    portinteger + Port specifies the destination port number to use for this resource. +Port is required when the referent is a Kubernetes Service. In this +case, the port number is the service port number, not the target port. +For other resources, destination port might be derived from the referent +resource or this field.
    +
    + Format: int32
    + Minimum: 1
    + Maximum: 65535
    		 false
    -### HTTPProxy.spec.rules[index].backends[index].filters[index].responseHeaderModifier -[↩ Parent](#httpproxyspecrulesindexbackendsindexfiltersindex) +### HTTPProxy.spec.rules[index].backends[index].filters[index].requestMirror.fraction +[↩ Parent](#httpproxyspecrulesindexbackendsindexfiltersindexrequestmirror) -ResponseHeaderModifier defines a schema for a filter that modifies response -headers. +Fraction represents the fraction of requests that should be +mirrored to BackendRef. -Support: Extended +Only one of Fraction or Percent may be specified. If neither field +is specified, 100% of requests will be mirrored. @@ -1325,12 +1560,247 @@ Support: Extended - - + + + + + + + + + +
    add[]objectnumeratorinteger - Add adds the given header(s) (name, value) to the request -before the action. It appends to any existing values associated -with the header name. +
    +
    + Format: int32
    + Minimum: 0
    +     		true
    denominatorinteger +
    +
    + Format: int32
    + Default: 100
    + Minimum: 1
    +     		false
    + + +### HTTPProxy.spec.rules[index].backends[index].filters[index].requestRedirect +[↩ Parent](#httpproxyspecrulesindexbackendsindexfiltersindex) + + + +RequestRedirect defines a schema for a filter that responds to the +request with an HTTP redirection. + +Support: Core + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    NameTypeDescriptionRequired
    hostnamestring + Hostname is the hostname to be used in the value of the `Location` +header in the response. +When empty, the hostname in the `Host` header of the request is used. + +Support: Core
    +     		false
    pathobject + Path defines parameters used to modify the path of the incoming request. +The modified path is then used to construct the `Location` header. When +empty, the request path is used as-is. + +Support: Extended
    +
    + Validations:
  • self.type == 'ReplaceFullPath' ? has(self.replaceFullPath) : true: replaceFullPath must be specified when type is set to 'ReplaceFullPath'
  • has(self.replaceFullPath) ? self.type == 'ReplaceFullPath' : true: type must be 'ReplaceFullPath' when replaceFullPath is set
  • self.type == 'ReplacePrefixMatch' ? has(self.replacePrefixMatch) : true: replacePrefixMatch must be specified when type is set to 'ReplacePrefixMatch'
  • has(self.replacePrefixMatch) ? self.type == 'ReplacePrefixMatch' : true: type must be 'ReplacePrefixMatch' when replacePrefixMatch is set
    • +     		false
    portinteger + Port is the port to be used in the value of the `Location` +header in the response. + +If no port is specified, the redirect port MUST be derived using the +following rules: + +* If redirect scheme is not-empty, the redirect port MUST be the well-known + port associated with the redirect scheme. Specifically "http" to port 80 + and "https" to port 443. If the redirect scheme does not have a + well-known port, the listener port of the Gateway SHOULD be used. +* If redirect scheme is empty, the redirect port MUST be the Gateway + Listener port. + +Implementations SHOULD NOT add the port number in the 'Location' +header in the following cases: + +* A Location header that will use HTTP (whether that is determined via + the Listener protocol or the Scheme field) _and_ use port 80. +* A Location header that will use HTTPS (whether that is determined via + the Listener protocol or the Scheme field) _and_ use port 443. + +Support: Extended
    +
    + Format: int32
    + Minimum: 1
    + Maximum: 65535
    +     		false
    schemeenum + Scheme is the scheme to be used in the value of the `Location` header in +the response. When empty, the scheme of the request is used. + +Scheme redirects can affect the port of the redirect, for more information, +refer to the documentation for the port field of this filter. + +Note that values may be added to this enum, implementations +must ensure that unknown values will not cause a crash. + +Unknown values here must result in the implementation setting the +Accepted Condition for the Route to `status: False`, with a +Reason of `UnsupportedValue`. + +Support: Extended
    +
    + Enum: http, https
    +     		false
    statusCodeinteger + StatusCode is the HTTP status code to be used in response. + +Note that values may be added to this enum, implementations +must ensure that unknown values will not cause a crash. + +Unknown values here must result in the implementation setting the +Accepted Condition for the Route to `status: False`, with a +Reason of `UnsupportedValue`. + +Support: Core
    +
    + Enum: 301, 302, 303, 307, 308
    + Default: 302
    +     		false
    + + +### HTTPProxy.spec.rules[index].backends[index].filters[index].requestRedirect.path +[↩ Parent](#httpproxyspecrulesindexbackendsindexfiltersindexrequestredirect) + + + +Path defines parameters used to modify the path of the incoming request. +The modified path is then used to construct the `Location` header. When +empty, the request path is used as-is. + +Support: Extended + + + + + + + + + + + + + + + + + + + + + + + + + + +
    NameTypeDescriptionRequired
    typeenum + Type defines the type of path modifier. Additional types may be +added in a future release of the API. + +Note that values may be added to this enum, implementations +must ensure that unknown values will not cause a crash. + +Unknown values here must result in the implementation setting the +Accepted Condition for the Route to `status: False`, with a +Reason of `UnsupportedValue`.
    +
    + Enum: ReplaceFullPath, ReplacePrefixMatch
    +     		true
    replaceFullPathstring + ReplaceFullPath specifies the value with which to replace the full path +of a request during a rewrite or redirect.
    +     		false
    replacePrefixMatchstring + ReplacePrefixMatch specifies the value with which to replace the prefix +match of a request during a rewrite or redirect. For example, a request +to "/foo/bar" with a prefix match of "/foo" and a ReplacePrefixMatch +of "/xyz" would be modified to "/xyz/bar". + +Note that this matches the behavior of the PathPrefix match type. This +matches full path elements. A path element refers to the list of labels +in the path split by the `/` separator. When specified, a trailing `/` is +ignored. For example, the paths `/abc`, `/abc/`, and `/abc/def` would all +match the prefix `/abc`, but the path `/abcd` would not. + +ReplacePrefixMatch is only compatible with a `PathPrefix` HTTPRouteMatch. +Using any other HTTPRouteMatch type on the same HTTPRouteRule will result in +the implementation setting the Accepted Condition for the Route to `status: False`. + +Request Path | Prefix Match | Replace Prefix | Modified Path
    +     		false
    + + +### HTTPProxy.spec.rules[index].backends[index].filters[index].responseHeaderModifier +[↩ Parent](#httpproxyspecrulesindexbackendsindexfiltersindex) + + + +ResponseHeaderModifier defines a schema for a filter that modifies response +headers. + +Support: Extended + + + + + + + + + + + + + + @@ -1469,7 +1945,13 @@ equivalent.
    @@ -1583,25 +2065,554 @@ the implementation setting the Accepted Condition for the Route to `status: Fals Request Path | Prefix Match | Replace Prefix | Modified Path
    - + + +
    NameTypeDescriptionRequired
    add[]object + Add adds the given header(s) (name, value) to the request +before the action. It appends to any existing values associated +with the header name. Input: GET /foo HTTP/1.1 @@ -1428,7 +1898,13 @@ equivalent.
    		value string - Value is the value of HTTP Header to be matched.
    + Value is the value of HTTP Header to be matched. + +Must consist of printable US-ASCII characters, optionally separated +by single tabs or spaces. See: https://tools.ietf.org/html/rfc7230#section-3.2 + + +
    		 true
    value string - Value is the value of HTTP Header to be matched.
    + Value is the value of HTTP Header to be matched. + +Must consist of printable US-ASCII characters, optionally separated +by single tabs or spaces. See: https://tools.ietf.org/html/rfc7230#section-3.2 + + +
    		 true
    falsefalse
    + + +### HTTPProxy.spec.rules[index].backends[index].tls +[↩ Parent](#httpproxyspecrulesindexbackendsindex) + + + +TLS contains backend TLS configuration. + +When the backend endpoint uses HTTPS with an IP address, the Hostname field +must be specified for TLS certificate validation. + + + + + + + + + + + + + + + + +
    NameTypeDescriptionRequired
    hostnamestring + Hostname is used for TLS certificate validation when connecting to an +HTTPS backend. This hostname is used for: + +1. SNI (Server Name Indication) during the TLS handshake +2. Certificate validation - the certificate must be valid for this hostname + +This field is required when the backend endpoint uses HTTPS with an IP +address, as there is no hostname to extract from the endpoint URL. + +When the backend endpoint uses HTTPS with a DNS hostname, this field is +optional and defaults to the hostname from the endpoint URL.
    +     		false
    + + +### HTTPProxy.spec.rules[index].filters[index] +[↩ Parent](#httpproxyspecrulesindex) + + + +HTTPRouteFilter defines processing steps that must be completed during the +request or response lifecycle. HTTPRouteFilters are meant as an extension +point to express processing that may be done in Gateway implementations. Some +examples include request or response modification, implementing +authentication strategies, rate-limiting, and traffic shaping. API +guarantee/conformance is defined based on the type of the filter. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    NameTypeDescriptionRequired
    typeenum + Type identifies the type of filter to apply. As with other API fields, +types are classified into three conformance levels: + +- Core: Filter types and their corresponding configuration defined by + "Support: Core" in this package, e.g. "RequestHeaderModifier". All + implementations must support core filters. + +- Extended: Filter types and their corresponding configuration defined by + "Support: Extended" in this package, e.g. "RequestMirror". Implementers + are encouraged to support extended filters. + +- Implementation-specific: Filters that are defined and supported by + specific vendors. + In the future, filters showing convergence in behavior across multiple + implementations will be considered for inclusion in extended or core + conformance levels. Filter-specific configuration for such filters + is specified using the ExtensionRef field. `Type` should be set to + "ExtensionRef" for custom filters. + +Implementers are encouraged to define custom implementation types to +extend the core API with implementation-specific behavior. + +If a reference to a custom filter type cannot be resolved, the filter +MUST NOT be skipped. Instead, requests that would have been processed by +that filter MUST receive a HTTP error response. + +Note that values may be added to this enum, implementations +must ensure that unknown values will not cause a crash. + +Unknown values here must result in the implementation setting the +Accepted Condition for the Route to `status: False`, with a +Reason of `UnsupportedValue`. + +
    +
    + Enum: RequestHeaderModifier, ResponseHeaderModifier, RequestMirror, RequestRedirect, URLRewrite, ExtensionRef, CORS
    +     		true
    corsobject + CORS defines a schema for a filter that responds to the +cross-origin request based on HTTP response header. + +Support: Extended
    +     		false
    extensionRefobject + ExtensionRef is an optional, implementation-specific extension to the +"filter" behavior. For example, resource "myroutefilter" in group +"networking.example.net"). ExtensionRef MUST NOT be used for core and +extended filters. + +This filter can be used multiple times within the same rule. + +Support: Implementation-specific
    +     		false
    externalAuthobject + ExternalAuth configures settings related to sending request details +to an external auth service. The external service MUST authenticate +the request, and MAY authorize the request as well. + +If there is any problem communicating with the external service, +this filter MUST fail closed. + +Support: Extended + +
    +
    + Validations:
  • self.protocol == 'GRPC' ? has(self.grpc) : true: grpc must be specified when protocol is set to 'GRPC'
  • has(self.grpc) ? self.protocol == 'GRPC' : true: protocol must be 'GRPC' when grpc is set
  • self.protocol == 'HTTP' ? has(self.http) : true: http must be specified when protocol is set to 'HTTP'
  • has(self.http) ? self.protocol == 'HTTP' : true: protocol must be 'HTTP' when http is set
    • +     		false
    requestHeaderModifierobject + RequestHeaderModifier defines a schema for a filter that modifies request +headers. + +Support: Core
    +     		false
    requestMirrorobject + RequestMirror defines a schema for a filter that mirrors requests. +Requests are sent to the specified destination, but responses from +that destination are ignored. + +This filter can be used multiple times within the same rule. Note that +not all implementations will be able to support mirroring to multiple +backends. + +Support: Extended
    +
    + Validations:
  • !(has(self.percent) && has(self.fraction)): Only one of percent or fraction may be specified in HTTPRequestMirrorFilter
    • +     		false
    requestRedirectobject + RequestRedirect defines a schema for a filter that responds to the +request with an HTTP redirection. + +Support: Core
    +     		false
    responseHeaderModifierobject + ResponseHeaderModifier defines a schema for a filter that modifies response +headers. + +Support: Extended
    +     		false
    urlRewriteobject + URLRewrite defines a schema for a filter that modifies a request during forwarding. + +Support: Extended
    +     		false
    + + +### HTTPProxy.spec.rules[index].filters[index].cors +[↩ Parent](#httpproxyspecrulesindexfiltersindex) + + + +CORS defines a schema for a filter that responds to the +cross-origin request based on HTTP response header. + +Support: Extended + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    NameTypeDescriptionRequired
    allowCredentialsboolean + AllowCredentials indicates whether the actual cross-origin request allows +to include credentials. + +When set to true, the gateway will include the `Access-Control-Allow-Credentials` +response header with value true (case-sensitive). + +When set to false or omitted the gateway will omit the header +`Access-Control-Allow-Credentials` entirely (this is the standard CORS +behavior). + +Support: Extended
    +     		false
    allowHeaders[]string + AllowHeaders indicates which HTTP request headers are supported for +accessing the requested resource. + +Header names are not case-sensitive. + +Multiple header names in the value of the `Access-Control-Allow-Headers` +response header are separated by a comma (","). + +When the `AllowHeaders` field is configured with one or more headers, the +gateway must return the `Access-Control-Allow-Headers` response header +which value is present in the `AllowHeaders` field. + +If any header name in the `Access-Control-Request-Headers` request header +is not included in the list of header names specified by the response +header `Access-Control-Allow-Headers`, it will present an error on the +client side. + +If any header name in the `Access-Control-Allow-Headers` response header +does not recognize by the client, it will also occur an error on the +client side. + +A wildcard indicates that the requests with all HTTP headers are allowed. +If config contains the wildcard "*" in allowHeaders and the request is +not credentialed, the `Access-Control-Allow-Headers` response header +can either use the `*` wildcard or the value of +Access-Control-Request-Headers from the request. + +When the request is credentialed, the gateway must not specify the `*` +wildcard in the `Access-Control-Allow-Headers` response header. When +also the `AllowCredentials` field is true and `AllowHeaders` field +is specified with the `*` wildcard, the gateway must specify one or more +HTTP headers in the value of the `Access-Control-Allow-Headers` response +header. The value of the header `Access-Control-Allow-Headers` is same as +the `Access-Control-Request-Headers` header provided by the client. If +the header `Access-Control-Request-Headers` is not included in the +request, the gateway will omit the `Access-Control-Allow-Headers` +response header, instead of specifying the `*` wildcard. + +Support: Extended
    +
    + Validations:
  • !('*' in self && self.size() > 1): AllowHeaders cannot contain '*' alongside other methods
    • +     		false
    allowMethods[]enum + AllowMethods indicates which HTTP methods are supported for accessing the +requested resource. + +Valid values are any method defined by RFC9110, along with the special +value `*`, which represents all HTTP methods are allowed. + +Method names are case-sensitive, so these values are also case-sensitive. +(See https://www.rfc-editor.org/rfc/rfc2616#section-5.1.1) + +Multiple method names in the value of the `Access-Control-Allow-Methods` +response header are separated by a comma (","). + +A CORS-safelisted method is a method that is `GET`, `HEAD`, or `POST`. +(See https://fetch.spec.whatwg.org/#cors-safelisted-method) The +CORS-safelisted methods are always allowed, regardless of whether they +are specified in the `AllowMethods` field. + +When the `AllowMethods` field is configured with one or more methods, the +gateway must return the `Access-Control-Allow-Methods` response header +which value is present in the `AllowMethods` field. + +If the HTTP method of the `Access-Control-Request-Method` request header +is not included in the list of methods specified by the response header +`Access-Control-Allow-Methods`, it will present an error on the client +side. + +If config contains the wildcard "*" in allowMethods and the request is +not credentialed, the `Access-Control-Allow-Methods` response header +can either use the `*` wildcard or the value of +Access-Control-Request-Method from the request. + +When the request is credentialed, the gateway must not specify the `*` +wildcard in the `Access-Control-Allow-Methods` response header. When +also the `AllowCredentials` field is true and `AllowMethods` field +specified with the `*` wildcard, the gateway must specify one HTTP method +in the value of the Access-Control-Allow-Methods response header. The +value of the header `Access-Control-Allow-Methods` is same as the +`Access-Control-Request-Method` header provided by the client. If the +header `Access-Control-Request-Method` is not included in the request, +the gateway will omit the `Access-Control-Allow-Methods` response header, +instead of specifying the `*` wildcard. + +Support: Extended
    +
    + Validations:
  • !('*' in self && self.size() > 1): AllowMethods cannot contain '*' alongside other methods
    • + Enum: GET, HEAD, POST, PUT, DELETE, CONNECT, OPTIONS, TRACE, PATCH, *
    +     		false
    allowOrigins[]string + AllowOrigins indicates whether the response can be shared with requested +resource from the given `Origin`. + +The `Origin` consists of a scheme and a host, with an optional port, and +takes the form `://(:)`. + +Valid values for scheme are: `http` and `https`. + +Valid values for port are any integer between 1 and 65535 (the list of +available TCP/UDP ports). Note that, if not included, port `80` is +assumed for `http` scheme origins, and port `443` is assumed for `https` +origins. This may affect origin matching. + +The host part of the origin may contain the wildcard character `*`. These +wildcard characters behave as follows: + +* `*` is a greedy match to the _left_, including any number of + DNS labels to the left of its position. This also means that + `*` will include any number of period `.` characters to the + left of its position. +* A wildcard by itself matches all hosts. + +An origin value that includes _only_ the `*` character indicates requests +from all `Origin`s are allowed. + +When the `AllowOrigins` field is configured with multiple origins, it +means the server supports clients from multiple origins. If the request +`Origin` matches the configured allowed origins, the gateway must return +the given `Origin` and sets value of the header +`Access-Control-Allow-Origin` same as the `Origin` header provided by the +client. + +The status code of a successful response to a "preflight" request is +always an OK status (i.e., 204 or 200). + +If the request `Origin` does not match the configured allowed origins, +the gateway returns 204/200 response but doesn't set the relevant +cross-origin response headers. Alternatively, the gateway responds with +403 status to the "preflight" request is denied, coupled with omitting +the CORS headers. The cross-origin request fails on the client side. +Therefore, the client doesn't attempt the actual cross-origin request. + +Conversely, if the request `Origin` matches one of the configured +allowed origins, the gateway sets the response header +`Access-Control-Allow-Origin` to the same value as the `Origin` +header provided by the client. + +When config has the wildcard ("*") in allowOrigins, and the request +is not credentialed (e.g., it is a preflight request), the +`Access-Control-Allow-Origin` response header either contains the +wildcard as well or the Origin from the request. + +When the request is credentialed, the gateway must not specify the `*` +wildcard in the `Access-Control-Allow-Origin` response header. When +also the `AllowCredentials` field is true and `AllowOrigins` field +specified with the `*` wildcard, the gateway must return a single origin +in the value of the `Access-Control-Allow-Origin` response header, +instead of specifying the `*` wildcard. The value of the header +`Access-Control-Allow-Origin` is same as the `Origin` header provided by +the client. + +Support: Extended
    +
    + Validations:
  • !('*' in self && self.size() > 1): AllowOrigins cannot contain '*' alongside other origins
    • +     		false
    exposeHeaders[]string + ExposeHeaders indicates which HTTP response headers can be exposed +to client-side scripts in response to a cross-origin request. + +A CORS-safelisted response header is an HTTP header in a CORS response +that it is considered safe to expose to the client scripts. +The CORS-safelisted response headers include the following headers: +`Cache-Control` +`Content-Language` +`Content-Length` +`Content-Type` +`Expires` +`Last-Modified` +`Pragma` +(See https://fetch.spec.whatwg.org/#cors-safelisted-response-header-name) +The CORS-safelisted response headers are exposed to client by default. + +When an HTTP header name is specified using the `ExposeHeaders` field, +this additional header will be exposed as part of the response to the +client. + +Header names are not case-sensitive. + +Multiple header names in the value of the `Access-Control-Expose-Headers` +response header are separated by a comma (","). + +A wildcard indicates that the responses with all HTTP headers are exposed +to clients. The `Access-Control-Expose-Headers` response header can only +use `*` wildcard as value when the request is not credentialed. + +When the `exposeHeaders` config field contains the "*" wildcard and +the request is credentialed, the gateway cannot use the `*` wildcard in +the `Access-Control-Expose-Headers` response header. + +Support: Extended
    +     		false
    maxAgeinteger + MaxAge indicates the duration (in seconds) for the client to cache the +results of a "preflight" request. + +The information provided by the `Access-Control-Allow-Methods` and +`Access-Control-Allow-Headers` response headers can be cached by the +client until the time specified by `Access-Control-Max-Age` elapses. + +The default value of `Access-Control-Max-Age` response header is 5 +(seconds). + +When the `MaxAge` field is unspecified, the gateway sets the response +header "Access-Control-Max-Age: 5" by default.
    +
    + Format: int32
    + Default: 5
    + Minimum: 1
    +     		false
    + + +### HTTPProxy.spec.rules[index].filters[index].extensionRef +[↩ Parent](#httpproxyspecrulesindexfiltersindex) + + + +ExtensionRef is an optional, implementation-specific extension to the +"filter" behavior. For example, resource "myroutefilter" in group +"networking.example.net"). ExtensionRef MUST NOT be used for core and +extended filters. + +This filter can be used multiple times within the same rule. + +Support: Implementation-specific + + + + + + + + + + + + + + + + + + + + + + + + +
    NameTypeDescriptionRequired
    groupstring + Group is the group of the referent. For example, "gateway.networking.k8s.io". +When unspecified or empty string, core API group is inferred.
    +     		true
    kindstring + Kind is kind of the referent. For example "HTTPRoute" or "Service".
    +     		true
    namestring + Name is the name of the referent.
    +     		true
    -### HTTPProxy.spec.rules[index].filters[index] -[↩ Parent](#httpproxyspecrulesindex) +### HTTPProxy.spec.rules[index].filters[index].externalAuth +[↩ Parent](#httpproxyspecrulesindexfiltersindex) -HTTPRouteFilter defines processing steps that must be completed during the -request or response lifecycle. HTTPRouteFilters are meant as an extension -point to express processing that may be done in Gateway implementations. Some -examples include request or response modification, implementing -authentication strategies, rate-limiting, and traffic shaping. API -guarantee/conformance is defined based on the type of the filter. +ExternalAuth configures settings related to sending request details +to an external auth service. The external service MUST authenticate +the request, and MAY authorize the request as well. + +If there is any problem communicating with the external service, +this filter MUST fail closed. + +Support: Extended - - + @@ -1613,144 +2624,104 @@ guarantee/conformance is defined based on the type of the filter. - - + + - - - - - - - + + - - - - - - + - + - - - - - - + - +
    typeenumbackendRefobject - Type identifies the type of filter to apply. As with other API fields, -types are classified into three conformance levels: - -- Core: Filter types and their corresponding configuration defined by - "Support: Core" in this package, e.g. "RequestHeaderModifier". All - implementations must support core filters. - -- Extended: Filter types and their corresponding configuration defined by - "Support: Extended" in this package, e.g. "RequestMirror". Implementers - are encouraged to support extended filters. - -- Implementation-specific: Filters that are defined and supported by - specific vendors. - In the future, filters showing convergence in behavior across multiple - implementations will be considered for inclusion in extended or core - conformance levels. Filter-specific configuration for such filters - is specified using the ExtensionRef field. `Type` should be set to - "ExtensionRef" for custom filters. - -Implementers are encouraged to define custom implementation types to -extend the core API with implementation-specific behavior. - -If a reference to a custom filter type cannot be resolved, the filter -MUST NOT be skipped. Instead, requests that would have been processed by -that filter MUST receive a HTTP error response. - -Note that values may be added to this enum, implementations -must ensure that unknown values will not cause a crash. + BackendRef is a reference to a backend to send authorization +requests to. -Unknown values here must result in the implementation setting the -Accepted Condition for the Route to `status: False`, with a -Reason of `UnsupportedValue`. +The backend must speak the selected protocol (GRPC or HTTP) on the +referenced port. -
    +If the backend service requires TLS, use BackendTLSPolicy to tell the +implementation to supply the TLS details to be used to connect to that +backend.

    - Enum: RequestHeaderModifier, ResponseHeaderModifier, RequestMirror, RequestRedirect, URLRewrite, ExtensionRef
    + Validations:
  • (size(self.group) == 0 && self.kind == 'Service') ? has(self.port) : true: Must have port for Service reference
    		•  true
    corsobject - CORS defines a schema for a filter that responds to the -cross-origin request based on HTTP response header. - -Support: Extended - -
    -     		false
    extensionRefobjectprotocolenum - ExtensionRef is an optional, implementation-specific extension to the -"filter" behavior. For example, resource "myroutefilter" in group -"networking.example.net"). ExtensionRef MUST NOT be used for core and -extended filters. + ExternalAuthProtocol describes which protocol to use when communicating with an +ext_authz authorization server. -This filter can be used multiple times within the same rule. +When this is set to GRPC, each backend must use the Envoy ext_authz protocol +on the port specified in `backendRefs`. Requests and responses are defined +in the protobufs explained at: +https://www.envoyproxy.io/docs/envoy/latest/api-v3/service/auth/v3/external_auth.proto -Support: Implementation-specific
    -     		false
    requestHeaderModifierobject - RequestHeaderModifier defines a schema for a filter that modifies request -headers. +When this is set to HTTP, each backend must respond with a `200` status +code in on a successful authorization. Any other code is considered +an authorization failure. -Support: Core
    +Feature Names: +GRPC Support - HTTPRouteExternalAuthGRPC +HTTP Support - HTTPRouteExternalAuthHTTP
    +
    + Enum: HTTP, GRPC
    		falsetrue
    requestMirrorforwardBody object - RequestMirror defines a schema for a filter that mirrors requests. -Requests are sent to the specified destination, but responses from -that destination are ignored. + ForwardBody controls if requests to the authorization server should include +the body of the client request; and if so, how big that body is allowed +to be. -This filter can be used multiple times within the same rule. Note that -not all implementations will be able to support mirroring to multiple -backends. +It is expected that implementations will buffer the request body up to +`forwardBody.maxSize` bytes. Bodies over that size must be rejected with a +4xx series error (413 or 403 are common examples), and fail processing +of the filter. -Support: Extended
    -
    - Validations:
  • !(has(self.percent) && has(self.fraction)): Only one of percent or fraction may be specified in HTTPRequestMirrorFilter
    • -     		false
    requestRedirectobject - RequestRedirect defines a schema for a filter that responds to the -request with an HTTP redirection. +If unset, or `forwardBody.maxSize` is set to `0`, then the body will not +be forwarded. -Support: Core
    +Feature Name: HTTPRouteExternalAuthForwardBody
    		 false
    responseHeaderModifiergrpc object - ResponseHeaderModifier defines a schema for a filter that modifies response -headers. + GRPCAuthConfig contains configuration for communication with ext_authz +protocol-speaking backends. -Support: Extended
    +If unset, implementations must assume the default behavior for each +included field is intended.
    		 false
    urlRewritehttp object - URLRewrite defines a schema for a filter that modifies a request during forwarding. + HTTPAuthConfig contains configuration for communication with HTTP-speaking +backends. -Support: Extended
    +If unset, implementations must assume the default behavior for each +included field is intended.
    		 false
    -### HTTPProxy.spec.rules[index].filters[index].cors -[↩ Parent](#httpproxyspecrulesindexfiltersindex) +### HTTPProxy.spec.rules[index].filters[index].externalAuth.backendRef +[↩ Parent](#httpproxyspecrulesindexfiltersindexexternalauth) -CORS defines a schema for a filter that responds to the -cross-origin request based on HTTP response header. +BackendRef is a reference to a backend to send authorization +requests to. -Support: Extended +The backend must speak the selected protocol (GRPC or HTTP) on the +referenced port. - +If the backend service requires TLS, use BackendTLSPolicy to tell the +implementation to supply the TLS details to be used to connect to that +backend. @@ -1762,247 +2733,173 @@ Support: Extended - - + + + + + + + - - + + - - + + - - + + + + +
    allowCredentialsbooleannamestring - AllowCredentials indicates whether the actual cross-origin request allows -to include credentials. - -The only valid value for the `Access-Control-Allow-Credentials` response -header is true (case-sensitive). - -If the credentials are not allowed in cross-origin requests, the gateway -will omit the header `Access-Control-Allow-Credentials` entirely rather -than setting its value to false. - -Support: Extended
    + Name is the name of the referent.
    +     		true
    groupstring + Group is the group of the referent. For example, "gateway.networking.k8s.io". +When unspecified or empty string, core API group is inferred.

    - Enum: true
    + Default:
    		false
    allowHeaders[]stringkindstring - AllowHeaders indicates which HTTP request headers are supported for -accessing the requested resource. - -Header names are not case sensitive. - -Multiple header names in the value of the `Access-Control-Allow-Headers` -response header are separated by a comma (","). - -When the `AllowHeaders` field is configured with one or more headers, the -gateway must return the `Access-Control-Allow-Headers` response header -which value is present in the `AllowHeaders` field. - -If any header name in the `Access-Control-Request-Headers` request header -is not included in the list of header names specified by the response -header `Access-Control-Allow-Headers`, it will present an error on the -client side. + Kind is the Kubernetes resource kind of the referent. For example +"Service". -If any header name in the `Access-Control-Allow-Headers` response header -does not recognize by the client, it will also occur an error on the -client side. +Defaults to "Service" when not specified. -A wildcard indicates that the requests with all HTTP headers are allowed. -The `Access-Control-Allow-Headers` response header can only use `*` -wildcard as value when the `AllowCredentials` field is unspecified. +ExternalName services can refer to CNAME DNS records that may live +outside of the cluster and as such are difficult to reason about in +terms of conformance. They also may not be safe to forward to (see +CVE-2021-25740 for more information). Implementations SHOULD NOT +support ExternalName Services. -When the `AllowCredentials` field is specified and `AllowHeaders` field -specified with the `*` wildcard, the gateway must specify one or more -HTTP headers in the value of the `Access-Control-Allow-Headers` response -header. The value of the header `Access-Control-Allow-Headers` is same as -the `Access-Control-Request-Headers` header provided by the client. If -the header `Access-Control-Request-Headers` is not included in the -request, the gateway will omit the `Access-Control-Allow-Headers` -response header, instead of specifying the `*` wildcard. A Gateway -implementation may choose to add implementation-specific default headers. +Support: Core (Services with a type other than ExternalName) -Support: Extended
    +Support: Implementation-specific (Services with type ExternalName)
    +
    + Default: Service
    		 false
    allowMethods[]enumnamespacestring - AllowMethods indicates which HTTP methods are supported for accessing the -requested resource. - -Valid values are any method defined by RFC9110, along with the special -value `*`, which represents all HTTP methods are allowed. - -Method names are case sensitive, so these values are also case-sensitive. -(See https://www.rfc-editor.org/rfc/rfc2616#section-5.1.1) - -Multiple method names in the value of the `Access-Control-Allow-Methods` -response header are separated by a comma (","). - -A CORS-safelisted method is a method that is `GET`, `HEAD`, or `POST`. -(See https://fetch.spec.whatwg.org/#cors-safelisted-method) The -CORS-safelisted methods are always allowed, regardless of whether they -are specified in the `AllowMethods` field. - -When the `AllowMethods` field is configured with one or more methods, the -gateway must return the `Access-Control-Allow-Methods` response header -which value is present in the `AllowMethods` field. - -If the HTTP method of the `Access-Control-Request-Method` request header -is not included in the list of methods specified by the response header -`Access-Control-Allow-Methods`, it will present an error on the client -side. - -The `Access-Control-Allow-Methods` response header can only use `*` -wildcard as value when the `AllowCredentials` field is unspecified. + Namespace is the namespace of the backend. When unspecified, the local +namespace is inferred. -When the `AllowCredentials` field is specified and `AllowMethods` field -specified with the `*` wildcard, the gateway must specify one HTTP method -in the value of the Access-Control-Allow-Methods response header. The -value of the header `Access-Control-Allow-Methods` is same as the -`Access-Control-Request-Method` header provided by the client. If the -header `Access-Control-Request-Method` is not included in the request, -the gateway will omit the `Access-Control-Allow-Methods` response header, -instead of specifying the `*` wildcard. A Gateway implementation may -choose to add implementation-specific default methods. +Note that when a namespace different than the local namespace is specified, +a ReferenceGrant object is required in the referent namespace to allow that +namespace's owner to accept the reference. See the ReferenceGrant +documentation for details. -Support: Extended
    -
    - Validations:
  • !('*' in self && self.size() > 1): AllowMethods cannot contain '*' alongside other methods
    • - Enum: GET, HEAD, POST, PUT, DELETE, CONNECT, OPTIONS, TRACE, PATCH, *
    +Support: Core
    		 false
    allowOrigins[]stringportinteger - AllowOrigins indicates whether the response can be shared with requested -resource from the given `Origin`. - -The `Origin` consists of a scheme and a host, with an optional port, and -takes the form `://(:)`. + Port specifies the destination port number to use for this resource. +Port is required when the referent is a Kubernetes Service. In this +case, the port number is the service port number, not the target port. +For other resources, destination port might be derived from the referent +resource or this field.
    +
    + Format: int32
    + Minimum: 1
    + Maximum: 65535
    +     		false
    -Valid values for scheme are: `http` and `https`. -Valid values for port are any integer between 1 and 65535 (the list of -available TCP/UDP ports). Note that, if not included, port `80` is -assumed for `http` scheme origins, and port `443` is assumed for `https` -origins. This may affect origin matching. +### HTTPProxy.spec.rules[index].filters[index].externalAuth.forwardBody +[↩ Parent](#httpproxyspecrulesindexfiltersindexexternalauth) -The host part of the origin may contain the wildcard character `*`. These -wildcard characters behave as follows: -* `*` is a greedy match to the _left_, including any number of - DNS labels to the left of its position. This also means that - `*` will include any number of period `.` characters to the - left of its position. -* A wildcard by itself matches all hosts. -An origin value that includes _only_ the `*` character indicates requests -from all `Origin`s are allowed. +ForwardBody controls if requests to the authorization server should include +the body of the client request; and if so, how big that body is allowed +to be. -When the `AllowOrigins` field is configured with multiple origins, it -means the server supports clients from multiple origins. If the request -`Origin` matches the configured allowed origins, the gateway must return -the given `Origin` and sets value of the header -`Access-Control-Allow-Origin` same as the `Origin` header provided by the -client. +It is expected that implementations will buffer the request body up to +`forwardBody.maxSize` bytes. Bodies over that size must be rejected with a +4xx series error (413 or 403 are common examples), and fail processing +of the filter. -The status code of a successful response to a "preflight" request is -always an OK status (i.e., 204 or 200). +If unset, or `forwardBody.maxSize` is set to `0`, then the body will not +be forwarded. -If the request `Origin` does not match the configured allowed origins, -the gateway returns 204/200 response but doesn't set the relevant -cross-origin response headers. Alternatively, the gateway responds with -403 status to the "preflight" request is denied, coupled with omitting -the CORS headers. The cross-origin request fails on the client side. -Therefore, the client doesn't attempt the actual cross-origin request. +Feature Name: HTTPRouteExternalAuthForwardBody -The `Access-Control-Allow-Origin` response header can only use `*` -wildcard as value when the `AllowCredentials` field is unspecified. + + + + + + + + + + + + + - - - - +
    NameTypeDescriptionRequired
    maxSizeinteger + MaxSize specifies how large in bytes the largest body that will be buffered +and sent to the authorization server. If the body size is larger than +`maxSize`, then the body sent to the authorization server must be +truncated to `maxSize` bytes. -When the `AllowCredentials` field is specified and `AllowOrigins` field -specified with the `*` wildcard, the gateway must return a single origin -in the value of the `Access-Control-Allow-Origin` response header, -instead of specifying the `*` wildcard. The value of the header -`Access-Control-Allow-Origin` is same as the `Origin` header provided by -the client. +Experimental note: This behavior needs to be checked against +various dataplanes; it may need to be changed. +See https://github.com/kubernetes-sigs/gateway-api/pull/4001#discussion_r2291405746 +for more. -Support: Extended
    +If 0, the body will not be sent to the authorization server.
    		 false
    exposeHeaders[]string - ExposeHeaders indicates which HTTP response headers can be exposed -to client-side scripts in response to a cross-origin request. +
    -A CORS-safelisted response header is an HTTP header in a CORS response -that it is considered safe to expose to the client scripts. -The CORS-safelisted response headers include the following headers: -`Cache-Control` -`Content-Language` -`Content-Length` -`Content-Type` -`Expires` -`Last-Modified` -`Pragma` -(See https://fetch.spec.whatwg.org/#cors-safelisted-response-header-name) -The CORS-safelisted response headers are exposed to client by default. -When an HTTP header name is specified using the `ExposeHeaders` field, -this additional header will be exposed as part of the response to the -client. +### HTTPProxy.spec.rules[index].filters[index].externalAuth.grpc +[↩ Parent](#httpproxyspecrulesindexfiltersindexexternalauth) -Header names are not case sensitive. -Multiple header names in the value of the `Access-Control-Expose-Headers` -response header are separated by a comma (","). -A wildcard indicates that the responses with all HTTP headers are exposed -to clients. The `Access-Control-Expose-Headers` response header can only -use `*` wildcard as value when the `AllowCredentials` field is -unspecified. +GRPCAuthConfig contains configuration for communication with ext_authz +protocol-speaking backends. -Support: Extended
    - - false - - maxAge - integer +If unset, implementations must assume the default behavior for each +included field is intended. + + + + + + + + + + + + +
    NameTypeDescriptionRequired
    allowedHeaders[]string - MaxAge indicates the duration (in seconds) for the client to cache the -results of a "preflight" request. + AllowedRequestHeaders specifies what headers from the client request +will be sent to the authorization server. -The information provided by the `Access-Control-Allow-Methods` and -`Access-Control-Allow-Headers` response headers can be cached by the -client until the time specified by `Access-Control-Max-Age` elapses. +If this list is empty, then all headers must be sent. -The default value of `Access-Control-Max-Age` response header is 5 -(seconds).
    -
    - Format: int32
    - Default: 5
    - Minimum: 1
    +If the list has entries, only those entries must be sent.
    		 false
    -### HTTPProxy.spec.rules[index].filters[index].extensionRef -[↩ Parent](#httpproxyspecrulesindexfiltersindex) - +### HTTPProxy.spec.rules[index].filters[index].externalAuth.http +[↩ Parent](#httpproxyspecrulesindexfiltersindexexternalauth) -ExtensionRef is an optional, implementation-specific extension to the -"filter" behavior. For example, resource "myroutefilter" in group -"networking.example.net"). ExtensionRef MUST NOT be used for core and -extended filters. -This filter can be used multiple times within the same rule. +HTTPAuthConfig contains configuration for communication with HTTP-speaking +backends. -Support: Implementation-specific +If unset, implementations must assume the default behavior for each +included field is intended. @@ -2014,27 +2911,60 @@ Support: Implementation-specific - - + + - + - - + + - + - + - +
    groupstringallowedHeaders[]string - Group is the group of the referent. For example, "gateway.networking.k8s.io". -When unspecified or empty string, core API group is inferred.
    + AllowedRequestHeaders specifies what additional headers from the client request +will be sent to the authorization server. + +The following headers must always be sent to the authorization server, +regardless of this setting: + +* `Host` +* `Method` +* `Path` +* `Content-Length` +* `Authorization` + +If this list is empty, then only those headers must be sent. + +Note that `Content-Length` has a special behavior, in that the length +sent must be correct for the actual request to the external authorization +server - that is, it must reflect the actual number of bytes sent in the +body of the request to the authorization server. + +So if the `forwardBody` stanza is unset, or `forwardBody.maxSize` is set +to `0`, then `Content-Length` must be `0`. If `forwardBody.maxSize` is set +to anything other than `0`, then the `Content-Length` of the authorization +request must be set to the actual number of bytes forwarded.
    		truefalse
    kindstringallowedResponseHeaders[]string - Kind is kind of the referent. For example "HTTPRoute" or "Service".
    + AllowedResponseHeaders specifies what headers from the authorization response +will be copied into the request to the backend. + +If this list is empty, then all headers from the authorization server +except Authority or Host must be copied.
    		truefalse
    namepath string - Name is the name of the referent.
    + Path sets the prefix that paths from the client request will have added +when forwarded to the authorization server. + +When empty or unspecified, no prefix is added. + +Valid values are the same as the "value" regex for path values in the `match` +stanza, and the validation regex will screen out invalid paths in the same way. +Even with the validation, implementations MUST sanitize this input before using it +directly.
    		truefalse
    @@ -2162,7 +3092,13 @@ equivalent.
    value string - Value is the value of HTTP Header to be matched.
    + Value is the value of HTTP Header to be matched. + +Must consist of printable US-ASCII characters, optionally separated +by single tabs or spaces. See: https://tools.ietf.org/html/rfc7230#section-3.2 + + +
    true @@ -2203,7 +3139,13 @@ equivalent.
    value string - Value is the value of HTTP Header to be matched.
    + Value is the value of HTTP Header to be matched. + +Must consist of printable US-ASCII characters, optionally separated +by single tabs or spaces. See: https://tools.ietf.org/html/rfc7230#section-3.2 + + +
    true @@ -2566,7 +3508,7 @@ Reason of `UnsupportedValue`. Support: Core

    - Enum: 301, 302
    + Enum: 301, 302, 303, 307, 308
    Default: 302
    false @@ -2768,7 +3710,13 @@ equivalent.
    value string - Value is the value of HTTP Header to be matched.
    + Value is the value of HTTP Header to be matched. + +Must consist of printable US-ASCII characters, optionally separated +by single tabs or spaces. See: https://tools.ietf.org/html/rfc7230#section-3.2 + + +
    true @@ -2809,7 +3757,13 @@ equivalent.
    value string - Value is the value of HTTP Header to be matched.
    + Value is the value of HTTP Header to be matched. + +Must consist of printable US-ASCII characters, optionally separated +by single tabs or spaces. See: https://tools.ietf.org/html/rfc7230#section-3.2 + + +
    true @@ -3049,7 +4003,13 @@ processing a repeated header, with special handling for "Set-Cookie".
    value string - Value is the value of HTTP Header to be matched.
    + Value is the value of HTTP Header to be matched. + +Must consist of printable US-ASCII characters, optionally separated +by single tabs or spaces. See: https://tools.ietf.org/html/rfc7230#section-3.2 + + +
    true @@ -3216,6 +4176,17 @@ This field will not contain custom hostnames defined in the HTTPProxy. See the `hostnames` field
    false + + canonicalHostname + string + + CanonicalHostname is the platform-managed stable hostname assigned to this +HTTPProxy (e.g., ".datumproxy.net"). Users may create external CNAME +or ALIAS records pointing to this hostname to route traffic through the +platform. The platform manages A/AAAA records for this hostname in the +datumproxy.net zone.
    + + false conditions []object @@ -3223,6 +4194,16 @@ the `hostnames` field
    Conditions describe the current conditions of the HTTPProxy.
    false + + hostnameStatuses + []object + + HostnameStatuses lists the per-hostname status for each hostname configured +on this HTTPProxy. Each entry includes verification and DNS record +programming conditions. Use this field instead of the deprecated Hostnames +field for detailed per-hostname lifecycle information.
    + + false hostnames []string @@ -3230,7 +4211,10 @@ the `hostnames` field
    Hostnames lists the hostnames that have been bound to the HTTPProxy. If this list does not match that defined in the HTTPProxy, see the -`HostnamesVerified` condition message for details.
    +`HostnamesVerified` condition message for details. + +Deprecated: Use HostnameStatuses for detailed per-hostname status. +This field will be removed in a future API version.
    false @@ -3281,6 +4265,121 @@ Examples: `1.2.3.4`, `128::1`, `my-ip-address`.
    +Condition contains details for one aspect of the current state of this API Resource. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    NameTypeDescriptionRequired
    lastTransitionTimestring + lastTransitionTime is the last time the condition transitioned from one status to another. +This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable.
    +
    + Format: date-time
    +     		true
    messagestring + message is a human readable message indicating details about the transition. +This may be an empty string.
    +     		true
    reasonstring + reason contains a programmatic identifier indicating the reason for the condition's last transition. +Producers of specific condition types may define expected values and meanings for this field, +and whether the values are considered a guaranteed API. +The value should be a CamelCase string. +This field may not be empty.
    +     		true
    statusenum + status of the condition, one of True, False, Unknown.
    +
    + Enum: True, False, Unknown
    +     		true
    typestring + type of condition in CamelCase or in foo.example.com/CamelCase.
    +     		true
    observedGenerationinteger + observedGeneration represents the .metadata.generation that the condition was set based upon. +For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date +with respect to the current state of the instance.
    +
    + Format: int64
    + Minimum: 0
    +     		false
    + + +### HTTPProxy.status.hostnameStatuses[index] +[↩ Parent](#httpproxystatus) + + + +HostnameStatus captures the per-hostname verification and DNS programming status. +Each hostname configured on an HTTPProxy has a corresponding entry tracking +its lifecycle from domain ownership verification through DNS record creation. + + + + + + + + + + + + + + + + + + + + + +
    NameTypeDescriptionRequired
    hostnamestring + Hostname is the fully qualified domain name being tracked. +Must be a valid RFC 1123 hostname without a trailing dot.
    +     		true
    conditions[]object + Conditions contains the current status conditions for this hostname. +Standard condition types include Verified and DNSRecordProgrammed.
    +     		false
    + + +### HTTPProxy.status.hostnameStatuses[index].conditions[index] +[↩ Parent](#httpproxystatushostnamestatusesindex) + + + Condition contains details for one aspect of the current state of this API Resource. diff --git a/docs/api/locationbindings.md b/docs/api/locationbindings.md new file mode 100644 index 00000000..d93dd036 --- /dev/null +++ b/docs/api/locationbindings.md @@ -0,0 +1,256 @@ +# API Reference + +Packages: + +- [networking.datumapis.com/v1alpha](#networkingdatumapiscomv1alpha) + +# networking.datumapis.com/v1alpha + +Resource Types: + +- [LocationBinding](#locationbinding) + + + + +## LocationBinding +[↩ Parent](#networkingdatumapiscomv1alpha ) + + + + + + +LocationBinding is the Schema for the locationbindings API. It is a +cluster-scoped projection of a cluster-scoped Location into a project's +virtual control plane, created once the location's class is supported, the +Location is Ready, and the corresponding ServiceAvailability is Available. + +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    NameTypeDescriptionRequired
    apiVersionstringnetworking.datumapis.com/v1alphatrue
    kindstringLocationBindingtrue
    metadataobjectRefer to the Kubernetes API documentation for the fields of the `metadata` field.true
    specobject + LocationBindingSpec defines the desired state of LocationBinding.
    +     		false
    statusobject + LocationBindingStatus defines the observed state of LocationBinding.
    +     		false
    + + +### LocationBinding.spec +[↩ Parent](#locationbinding) + + + +LocationBindingSpec defines the desired state of LocationBinding. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    NameTypeDescriptionRequired
    locationRefobject + LocationRef references the canonical cluster-scoped Location object.
    +     		true
    displayNamestring + DisplayName is a human-readable label for the location.
    +     		false
    locationClassNamestring + LocationClassName mirrors spec.locationClassName from the referenced Location.
    +     		false
    topologymap[string]string + Topology mirrors spec.topology from the referenced Location, containing +well-known keys like topology.datum.net/city-code and topology.datum.net/region.
    +     		false
    + + +### LocationBinding.spec.locationRef +[↩ Parent](#locationbindingspec) + + + +LocationRef references the canonical cluster-scoped Location object. + + + + + + + + + + + + + + + + +
    NameTypeDescriptionRequired
    namestring + Name of the referent. +This field is effectively required, but due to backwards compatibility is +allowed to be empty. Instances of this type with an empty value here are +almost certainly wrong. +More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
    +
    + Default:
    +     		false
    + + +### LocationBinding.status +[↩ Parent](#locationbinding) + + + +LocationBindingStatus defines the observed state of LocationBinding. + + + + + + + + + + + + + + + + +
    NameTypeDescriptionRequired
    conditions[]object +
    +     		false
    + + +### LocationBinding.status.conditions[index] +[↩ Parent](#locationbindingstatus) + + + +Condition contains details for one aspect of the current state of this API Resource. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    NameTypeDescriptionRequired
    lastTransitionTimestring + lastTransitionTime is the last time the condition transitioned from one status to another. +This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable.
    +
    + Format: date-time
    +     		true
    messagestring + message is a human readable message indicating details about the transition. +This may be an empty string.
    +     		true
    reasonstring + reason contains a programmatic identifier indicating the reason for the condition's last transition. +Producers of specific condition types may define expected values and meanings for this field, +and whether the values are considered a guaranteed API. +The value should be a CamelCase string. +This field may not be empty.
    +     		true
    statusenum + status of the condition, one of True, False, Unknown.
    +
    + Enum: True, False, Unknown
    +     		true
    typestring + type of condition in CamelCase or in foo.example.com/CamelCase.
    +     		true
    observedGenerationinteger + observedGeneration represents the .metadata.generation that the condition was set based upon. +For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date +with respect to the current state of the instance.
    +
    + Format: int64
    + Minimum: 0
    +     		false
    diff --git a/docs/api/networkinterfaceclaims.md b/docs/api/networkinterfaceclaims.md new file mode 100644 index 00000000..c5f1805c --- /dev/null +++ b/docs/api/networkinterfaceclaims.md @@ -0,0 +1,580 @@ +# API Reference + +Packages: + +- [networking.datumapis.com/v1alpha](#networkingdatumapiscomv1alpha) + +# networking.datumapis.com/v1alpha + +Resource Types: + +- [NetworkInterfaceClaim](#networkinterfaceclaim) + + + + +## NetworkInterfaceClaim +[↩ Parent](#networkingdatumapiscomv1alpha ) + + + + + + +NetworkInterfaceClaim asks for an interface on a network. It is the resource +a user creates. The operator finds or creates a NetworkInterface that +satisfies it, allocates the addresses, and reports them in status. + +A claim describes what the interface must be able to do, never which +interface or address to use. One claim holds at most one interface, and one +interface is held by at most one claim. + +A claim's name is what makes addresses stable. It names the slot in a +workload rather than the instance filling it, so an instance replaced by +another that asks for the same claim name comes back on the same interface +and the same addresses. What happens when the claim itself is deleted is +spec.reclaimPolicy. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    NameTypeDescriptionRequired
    apiVersionstringnetworking.datumapis.com/v1alphatrue
    kindstringNetworkInterfaceClaimtrue
    metadataobjectRefer to the Kubernetes API documentation for the fields of the `metadata` field.true
    specobject + NetworkInterfaceClaimSpec defines the desired state of NetworkInterfaceClaim. +Every field states what the interface must be able to do, never which +interface or which address to use. + +Most of the spec is immutable, because the addresses are allocated against +it. To change one of those fields, delete the claim and create a new one, +accepting that the workload gets new addresses unless the interface is +retained.
    +
    + Validations:
  • has(self.networkInterfaceName) == has(oldSelf.networkInterfaceName) && (!has(self.networkInterfaceName) || self.networkInterfaceName == oldSelf.networkInterfaceName): networkInterfaceName is immutable and cannot be set, changed, or cleared after creation
  • has(self.addresses) == has(oldSelf.addresses) && (!has(self.addresses) || self.addresses == oldSelf.addresses): addresses is immutable and cannot be set, changed, or cleared after creation
    • +     		true
    statusobject + NetworkInterfaceClaimStatus defines the observed state of +NetworkInterfaceClaim. It repeats the bound interface's addresses so a +consumer reads one object rather than following the reference.
    +
    + Default: map[conditions:[map[lastTransitionTime:1970-01-01T00:00:00Z message:Waiting for controller reason:Pending status:Unknown type:Bound] map[lastTransitionTime:1970-01-01T00:00:00Z message:Waiting for controller reason:Pending status:Unknown type:Allocated] map[lastTransitionTime:1970-01-01T00:00:00Z message:Waiting for controller reason:Pending status:Unknown type:Programmed] map[lastTransitionTime:1970-01-01T00:00:00Z message:Waiting for controller reason:Pending status:Unknown type:Ready]]]
    +     		false
    + + +### NetworkInterfaceClaim.spec +[↩ Parent](#networkinterfaceclaim) + + + +NetworkInterfaceClaimSpec defines the desired state of NetworkInterfaceClaim. +Every field states what the interface must be able to do, never which +interface or which address to use. + +Most of the spec is immutable, because the addresses are allocated against +it. To change one of those fields, delete the claim and create a new one, +accepting that the workload gets new addresses unless the interface is +retained. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    NameTypeDescriptionRequired
    networkobject + network is the network the interface attaches to. The network must already +exist in the same namespace as the claim. + +Immutable. An interface that changed network would hold addresses from a +space it no longer belongs to, so move a workload by recreating the claim +against the other network.
    +
    + Validations:
  • self == oldSelf: network is immutable and cannot be changed after creation
    • +     		true
    addresses[]object + addresses request extra addresses by class, beyond the ones the interface +holds inside its network. Each appears in status.externalAddresses as a +bare address, mapped onto the interface address of the same family. + +Omit this field for ordinary private addressing, which is the common case.
    +
    + Validations:
  • self.all(a, self.exists_one(b, b.class == a.class)): Each address class may be requested at most once
    • +     		false
    interfaceNamestring + interfaceName is the device name the interface presents to the guest +operating system, such as eth0 or eth1. Set it when a workload has more +than one interface and the guest configuration names them. + +Immutable, because the guest is configured against it.
    +
    + Validations:
  • self == oldSelf: interfaceName is immutable and cannot be changed after creation
    • + Default: eth0
    +     		false
    ipFamilies[]enum + ipFamilies are the address families the interface must carry, in priority +order. List [IPv6, IPv4] for a dual-stack interface. The first family +listed holds the interface's primary address, which is the one reported in +single-address fields such as an instance's network IP. + +Every family listed must be satisfiable or the claim does not bind. Asking +for a family the network does not carry fails the claim outright rather +than leaving it pending, and no partially addressed interface is ever +published.
    +
    + Validations:
  • self.all(f, self.exists_one(g, g == f)): Each address family may be requested at most once
  • self == oldSelf: ipFamilies is immutable and cannot be changed after creation
    • + Enum: IPv4, IPv6
    + Default: [IPv6]
    +     		false
    networkInterfaceNamestring + networkInterfaceName binds one specific interface by name, instead of the +interface named after this claim. The named interface must already carry +every family and class this claim asks for, under the same reclaim policy, +and must not be held by another claim. + +Leave it empty, which is the normal case. The claim then binds the +interface of its own name, retained by an earlier claim, or creates one. + +Immutable, including from empty to set. Rebinding a workload to a different +interface means a new claim.
    +     		false
    reclaimPolicyenum + reclaimPolicy decides what becomes of the bound interface, and its +addresses, when this claim is deleted. + +Delete deletes the interface and returns its addresses to IPAM. A workload +recreated later comes back on different addresses. + +Retain keeps the interface, unbound and still holding its addresses, so a +later claim of this name binds it again and the workload returns to the +same addresses. Choose Retain when an address is published in DNS, allowed +through a firewall, or otherwise depended on from outside. + +A retained address is reserved, and billable, for as long as the interface +exists. Deleting the interface does not return it to the pool today, so +choose Retain for addresses worth holding rather than as a default. + +Both policies keep the addresses while the claim exists, including across +instance replacement. They differ only on scale-down and deletion. + +Immutable. An address keeps the policy it was allocated under, and a claim +asking for a policy the interface was not allocated under cannot bind it.
    +
    + Validations:
  • self == oldSelf: reclaimPolicy is immutable and cannot be changed after creation
    • + Enum: Delete, Retain
    + Default: Delete
    +     		false
    + + +### NetworkInterfaceClaim.spec.network +[↩ Parent](#networkinterfaceclaimspec) + + + +network is the network the interface attaches to. The network must already +exist in the same namespace as the claim. + +Immutable. An interface that changed network would hold addresses from a +space it no longer belongs to, so move a workload by recreating the claim +against the other network. + + + + + + + + + + + + + + + + +
    NameTypeDescriptionRequired
    namestring + The network name
    +     		true
    + + +### NetworkInterfaceClaim.spec.addresses[index] +[↩ Parent](#networkinterfaceclaimspec) + + + +NetworkInterfaceAddressRequest asks for one address beyond the ones the +interface holds inside its network, such as a public IPv4 address in front of +a private one. + + + + + + + + + + + + + + + + +
    NameTypeDescriptionRequired
    classstring + class is the IPAM class to allocate from, such as public-ipv4. + +A class names a kind of address, and the platform decides which pool and +prefix length serve it. A class never names a pool, a prefix length, or a +CIDR, so a class cannot be used to ask for a particular address.
    +     		true
    + + +### NetworkInterfaceClaim.status +[↩ Parent](#networkinterfaceclaim) + + + +NetworkInterfaceClaimStatus defines the observed state of +NetworkInterfaceClaim. It repeats the bound interface's addresses so a +consumer reads one object rather than following the reference. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    NameTypeDescriptionRequired
    addresses[]object + addresses are the addresses the bound interface holds inside its network, +each with its prefix length and, once the location has a subnet, its +gateway. They are copied from the interface, which remains the source of +truth.
    +     		false
    conditions[]object + conditions report the current state of the claim. Wait on Ready, which is +true once the claim is bound, its addresses are allocated, and the data +plane carries them.
    +     		false
    externalAddresses[]object + externalAddresses are the addresses the bound interface is reachable at from +outside the network, one per class the claim requested. Each is a bare +address with no prefix length. They are copied from the interface.
    +     		false
    networkInterfaceRefobject + networkInterfaceRef is the interface bound to this claim, in the same +namespace. Read it to reach fields the claim does not repeat, such as the +MTU and the data-plane attachment.
    +     		false
    + + +### NetworkInterfaceClaim.status.addresses[index] +[↩ Parent](#networkinterfaceclaimstatus) + + + +NetworkInterfaceAddress is an address the interface holds inside its network. +These are the addresses configured on the NIC itself, and they always carry a +prefix length. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    NameTypeDescriptionRequired
    addressstring + address is the address the interface holds, in CIDR notation, such as +10.128.0.2/32 or 2001:db8:a001::1/128. + +For IPv6 this may be a block delegated to the interface rather than a +single address, such as 2001:db8:a001::/96. The interface owns the whole +block and assigns within it.
    +     		true
    familyenum + family is the address family of this entry.
    +
    + Enum: IPv4, IPv6
    +     		true
    classstring + class is the IPAM class this address was allocated from, such as +private-ipv6. It is empty for the addresses a claim requests by family +rather than by class.
    +     		false
    gatewaystring + gateway is the next hop the interface routes through for this family, such +as 10.128.0.1. It is resolved from the subnet backing the network in this +location, so nothing has to read the subnet to configure the NIC. It is +empty until that subnet exists.
    +     		false
    primaryboolean + primary marks the address projected into single-address fields, such as an +instance's reported network IP. + +Exactly one address is primary for the interface as a whole, not one per +family. It is the address of the first family the claim listed in +spec.ipFamilies.
    +     		false
    + + +### NetworkInterfaceClaim.status.conditions[index] +[↩ Parent](#networkinterfaceclaimstatus) + + + +Condition contains details for one aspect of the current state of this API Resource. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    NameTypeDescriptionRequired
    lastTransitionTimestring + lastTransitionTime is the last time the condition transitioned from one status to another. +This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable.
    +
    + Format: date-time
    +     		true
    messagestring + message is a human readable message indicating details about the transition. +This may be an empty string.
    +     		true
    reasonstring + reason contains a programmatic identifier indicating the reason for the condition's last transition. +Producers of specific condition types may define expected values and meanings for this field, +and whether the values are considered a guaranteed API. +The value should be a CamelCase string. +This field may not be empty.
    +     		true
    statusenum + status of the condition, one of True, False, Unknown.
    +
    + Enum: True, False, Unknown
    +     		true
    typestring + type of condition in CamelCase or in foo.example.com/CamelCase.
    +     		true
    observedGenerationinteger + observedGeneration represents the .metadata.generation that the condition was set based upon. +For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date +with respect to the current state of the instance.
    +
    + Format: int64
    + Minimum: 0
    +     		false
    + + +### NetworkInterfaceClaim.status.externalAddresses[index] +[↩ Parent](#networkinterfaceclaimstatus) + + + +NetworkInterfaceExternalAddress is an address reachable from outside the +network, mapped onto an address the interface holds inside it. A public IPv4 +address in front of a private address is the usual case. + +Unlike an interface address, an external address is a bare address with no +prefix length, such as 203.0.113.10, because nothing configures it on the +NIC. The data plane maps it onto the interface address of the same family. + + + + + + + + + + + + + + + + + + + + + + + + + + +
    NameTypeDescriptionRequired
    addressstring + address is the externally reachable address, such as 203.0.113.10. It +carries no prefix length.
    +     		true
    classstring + class is the IPAM class this address was allocated from, such as +public-ipv4. It matches the class the claim requested in spec.addresses.
    +     		true
    familyenum + family is the address family of this entry.
    +
    + Enum: IPv4, IPv6
    +     		true
    + + +### NetworkInterfaceClaim.status.networkInterfaceRef +[↩ Parent](#networkinterfaceclaimstatus) + + + +networkInterfaceRef is the interface bound to this claim, in the same +namespace. Read it to reach fields the claim does not repeat, such as the +MTU and the data-plane attachment. + + + + + + + + + + + + + + + + +
    NameTypeDescriptionRequired
    namestring + name is the network interface name.
    +     		true
    diff --git a/docs/api/networkinterfaces.md b/docs/api/networkinterfaces.md new file mode 100644 index 00000000..e67f22bf --- /dev/null +++ b/docs/api/networkinterfaces.md @@ -0,0 +1,580 @@ +# API Reference + +Packages: + +- [networking.datumapis.com/v1alpha](#networkingdatumapiscomv1alpha) + +# networking.datumapis.com/v1alpha + +Resource Types: + +- [NetworkInterface](#networkinterface) + + + + +## NetworkInterface +[↩ Parent](#networkingdatumapiscomv1alpha ) + + + + + + +NetworkInterface is an interface on a network, together with the addresses it +holds. It is the unit that owns addresses: as long as the interface exists, +its addresses stay allocated to it. + +You do not create a NetworkInterface. Ask for one with a +NetworkInterfaceClaim, and the operator creates the interface, allocates its +addresses, and binds the two. A provider then reads the interface to +configure a NIC, and reports what it programmed in status. + +An interface outlives the instance using it. Whether it outlives the claim +that asked for it depends on spec.reclaimPolicy. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    NameTypeDescriptionRequired
    apiVersionstringnetworking.datumapis.com/v1alphatrue
    kindstringNetworkInterfacetrue
    metadataobjectRefer to the Kubernetes API documentation for the fields of the `metadata` field.true
    specobject + NetworkInterfaceSpec defines the desired state of NetworkInterface. It is +written by the operator when a claim is fulfilled, and it carries everything +a provider needs to configure a NIC without reading any other resource.
    +     		true
    statusobject + NetworkInterfaceStatus defines the observed state of NetworkInterface: which +claim holds it, what realizes it on the data plane, and whether programming +has succeeded.
    +
    + Default: map[conditions:[map[lastTransitionTime:1970-01-01T00:00:00Z message:Waiting for controller reason:Pending status:Unknown type:Allocated] map[lastTransitionTime:1970-01-01T00:00:00Z message:Waiting for controller reason:Pending status:Unknown type:Programmed]]]
    +     		false
    + + +### NetworkInterface.spec +[↩ Parent](#networkinterface) + + + +NetworkInterfaceSpec defines the desired state of NetworkInterface. It is +written by the operator when a claim is fulfilled, and it carries everything +a provider needs to configure a NIC without reading any other resource. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    NameTypeDescriptionRequired
    networkobject + network is the network this interface belongs to, in the same namespace as +the interface. It comes from the claim and does not change.
    +
    + Validations:
  • self == oldSelf: network is immutable and cannot be changed after creation
    • +     		true
    addresses[]object + addresses are the addresses the interface holds inside its network, at most +one per address family, exactly one of them primary. Each carries a prefix +length and, once the location has a subnet, the gateway to route through.
    +
    + Validations:
  • size(self) == 0 || self.filter(a, has(a.primary) && a.primary).size() == 1: Exactly one address must be primary
  • self.all(a, self.exists_one(b, b.family == a.family)): Only one address may be held per address family
    • +     		false
    claimRefobject + claimRef is the claim currently holding this interface. It is empty while a +retained interface waits, unbound, for a claim of its name to return.
    +     		false
    externalAddresses[]object + externalAddresses are the addresses the interface is reachable at from +outside the network, each mapped onto the interface address of the same +family. They come from the classes the claim requested, and they are absent +for a workload that only needs private addressing.
    +
    + Validations:
  • self.all(a, self.exists_one(b, b.address == a.address)): External addresses must be unique
  • self.all(a, self.exists_one(b, b.class == a.class)): Only one external address may be held per address class
    • +     		false
    interfaceNamestring + interfaceName is the device name the interface presents to the guest +operating system, such as eth0 or eth1. It comes from the claim.
    +
    + Default: eth0
    +     		false
    mtuinteger + mtu is the MTU, in bytes, the interface must be configured with. It is +resolved from the network, so a provider never has to read the network to +configure the NIC.
    +
    + Format: int32
    + Minimum: 1300
    + Maximum: 8856
    +     		false
    reclaimPolicyenum + reclaimPolicy decides what becomes of this interface, and its addresses, +when the claim holding it is deleted. It comes from the claim, and a claim +asking for a different policy cannot bind this interface.
    +
    + Enum: Delete, Retain
    + Default: Delete
    +     		false
    + + +### NetworkInterface.spec.network +[↩ Parent](#networkinterfacespec) + + + +network is the network this interface belongs to, in the same namespace as +the interface. It comes from the claim and does not change. + + + + + + + + + + + + + + + + +
    NameTypeDescriptionRequired
    namestring + The network name
    +     		true
    + + +### NetworkInterface.spec.addresses[index] +[↩ Parent](#networkinterfacespec) + + + +NetworkInterfaceAddress is an address the interface holds inside its network. +These are the addresses configured on the NIC itself, and they always carry a +prefix length. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    NameTypeDescriptionRequired
    addressstring + address is the address the interface holds, in CIDR notation, such as +10.128.0.2/32 or 2001:db8:a001::1/128. + +For IPv6 this may be a block delegated to the interface rather than a +single address, such as 2001:db8:a001::/96. The interface owns the whole +block and assigns within it.
    +     		true
    familyenum + family is the address family of this entry.
    +
    + Enum: IPv4, IPv6
    +     		true
    classstring + class is the IPAM class this address was allocated from, such as +private-ipv6. It is empty for the addresses a claim requests by family +rather than by class.
    +     		false
    gatewaystring + gateway is the next hop the interface routes through for this family, such +as 10.128.0.1. It is resolved from the subnet backing the network in this +location, so nothing has to read the subnet to configure the NIC. It is +empty until that subnet exists.
    +     		false
    primaryboolean + primary marks the address projected into single-address fields, such as an +instance's reported network IP. + +Exactly one address is primary for the interface as a whole, not one per +family. It is the address of the first family the claim listed in +spec.ipFamilies.
    +     		false
    + + +### NetworkInterface.spec.claimRef +[↩ Parent](#networkinterfacespec) + + + +claimRef is the claim currently holding this interface. It is empty while a +retained interface waits, unbound, for a claim of its name to return. + + + + + + + + + + + + + + + + +
    NameTypeDescriptionRequired
    namestring + name is the name of the NetworkInterfaceClaim, in the same namespace as the +interface. A claim name stays with the workload slot it serves, so a +replacement instance binds this same interface and its addresses.
    +     		true
    + + +### NetworkInterface.spec.externalAddresses[index] +[↩ Parent](#networkinterfacespec) + + + +NetworkInterfaceExternalAddress is an address reachable from outside the +network, mapped onto an address the interface holds inside it. A public IPv4 +address in front of a private address is the usual case. + +Unlike an interface address, an external address is a bare address with no +prefix length, such as 203.0.113.10, because nothing configures it on the +NIC. The data plane maps it onto the interface address of the same family. + + + + + + + + + + + + + + + + + + + + + + + + + + +
    NameTypeDescriptionRequired
    addressstring + address is the externally reachable address, such as 203.0.113.10. It +carries no prefix length.
    +     		true
    classstring + class is the IPAM class this address was allocated from, such as +public-ipv4. It matches the class the claim requested in spec.addresses.
    +     		true
    familyenum + family is the address family of this entry.
    +
    + Enum: IPv4, IPv6
    +     		true
    + + +### NetworkInterface.status +[↩ Parent](#networkinterface) + + + +NetworkInterfaceStatus defines the observed state of NetworkInterface: which +claim holds it, what realizes it on the data plane, and whether programming +has succeeded. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    NameTypeDescriptionRequired
    attachmentRefobject + attachmentRef is the data-plane resource realizing this interface. The +provider sets it once an attachment exists.
    +     		false
    conditions[]object + conditions report the current state of the interface. Allocated means every +address is held. Programmed means the data plane carries them.
    +     		false
    networkContextRefobject + networkContextRef is the network's presence in this location, resolved or +created while fulfilling the claim. It is a breadcrumb for operators +tracing where a network landed, and nothing needs it to configure a NIC.
    +     		false
    phaseenum + phase reports whether a claim holds the interface. Bound means the claim in +spec.claimRef holds it. Available means it is retained and holding its +addresses with no claim bound.
    +
    + Enum: Available, Bound
    +     		false
    vpcstring + vpc is the base62 identifier of the VPC backing this network in this +location, matching the identifier the fabric keys on. The provider records +it when the attachment is programmed.
    +     		false
    + + +### NetworkInterface.status.attachmentRef +[↩ Parent](#networkinterfacestatus) + + + +attachmentRef is the data-plane resource realizing this interface. The +provider sets it once an attachment exists. + + + + + + + + + + + + + + + + + + + + + + + + + + +
    NameTypeDescriptionRequired
    apiGroupstring + apiGroup is the API group of the referent, such as +compute.datumapis.com.
    +     		true
    kindstring + kind is the kind of the referent.
    +     		true
    namestring + name is the name of the referent.
    +     		true
    + + +### NetworkInterface.status.conditions[index] +[↩ Parent](#networkinterfacestatus) + + + +Condition contains details for one aspect of the current state of this API Resource. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    NameTypeDescriptionRequired
    lastTransitionTimestring + lastTransitionTime is the last time the condition transitioned from one status to another. +This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable.
    +
    + Format: date-time
    +     		true
    messagestring + message is a human readable message indicating details about the transition. +This may be an empty string.
    +     		true
    reasonstring + reason contains a programmatic identifier indicating the reason for the condition's last transition. +Producers of specific condition types may define expected values and meanings for this field, +and whether the values are considered a guaranteed API. +The value should be a CamelCase string. +This field may not be empty.
    +     		true
    statusenum + status of the condition, one of True, False, Unknown.
    +
    + Enum: True, False, Unknown
    +     		true
    typestring + type of condition in CamelCase or in foo.example.com/CamelCase.
    +     		true
    observedGenerationinteger + observedGeneration represents the .metadata.generation that the condition was set based upon. +For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date +with respect to the current state of the instance.
    +
    + Format: int64
    + Minimum: 0
    +     		false
    + + +### NetworkInterface.status.networkContextRef +[↩ Parent](#networkinterfacestatus) + + + +networkContextRef is the network's presence in this location, resolved or +created while fulfilling the claim. It is a breadcrumb for operators +tracing where a network landed, and nothing needs it to configure a NIC. + + + + + + + + + + + + + + + + +
    NameTypeDescriptionRequired
    namestring + The network context name
    +     		true
    diff --git a/docs/api/trafficprotectionpolicies.md b/docs/api/trafficprotectionpolicies.md index d710060e..907fa747 100644 --- a/docs/api/trafficprotectionpolicies.md +++ b/docs/api/trafficprotectionpolicies.md @@ -192,7 +192,7 @@ Core Rule Set (CRS). paranoia levels to use.

    Validations:
  • self.detection >= self.blocking: detection paranoia level must be greater than or equal to blocking paranoia level
    • - Default: map[]
    + Default: map[blocking:1 detection:1]
    false @@ -527,6 +527,43 @@ with a composite key made up of the AncestorRef and the ControllerName. PolicyAncestorStatus struct describes the status of.
    true + + conditions + []object + + Conditions describes the status of the Policy with respect to the given Ancestor. + + + +Notes for implementors: + +Conditions are a listType `map`, which means that they function like a +map with a key of the `type` field _in the k8s apiserver_. + +This means that implementations must obey some rules when updating this +section. + +* Implementations MUST perform a read-modify-write cycle on this field + before modifying it. That is, when modifying this field, implementations + must be confident they have fetched the most recent version of this field, + and ensure that changes they make are on that recent version. +* Implementations MUST NOT remove or reorder Conditions that they are not + directly responsible for. For example, if an implementation sees a Condition + with type `special.io/SomeField`, it MUST NOT remove, change or update that + Condition. +* Implementations MUST always _merge_ changes into Conditions of the same Type, + rather than creating more than one Condition of the same Type. +* Implementations MUST always update the `observedGeneration` field of the + Condition to the `metadata.generation` of the Gateway at the time of update creation. +* If the `observedGeneration` of a Condition is _greater than_ the value the + implementation knows about, then it MUST NOT perform the update on that Condition, + but must wait for a future reconciliation and status update. (The assumption is that + the implementation's copy of the object is stale and an update will be re-triggered + if relevant.) + +
    + + true controllerName string @@ -546,13 +583,6 @@ entries to status populated with their ControllerName are cleaned up when they a longer necessary.
    true - - conditions - []object - - Conditions describes the status of the Policy with respect to the given Ancestor.
    - - false diff --git a/docs/enhancements/network-interfaces.md b/docs/enhancements/network-interfaces.md new file mode 100644 index 00000000..91a7fc83 --- /dev/null +++ b/docs/enhancements/network-interfaces.md @@ -0,0 +1,756 @@ +--- +status: provisional +stage: alpha +latest-milestone: "v0.x" +--- + +# A network interface a workload can be handed + +- [Summary](#summary) +- [Motivation](#motivation) + - [Goals](#goals) + - [Non-Goals](#non-goals) +- [Proposal](#proposal) + - [What it feels like](#what-it-feels-like) + - [Notes/Constraints/Caveats](#notesconstraintscaveats) +- [Design Details](#design-details) + - [NetworkInterfaceClaim](#networkinterfaceclaim) + - [NetworkInterface](#networkinterface) + - [Binding](#binding) + - [Fulfilling a claim](#fulfilling-a-claim) + - [Reaching the data plane](#reaching-the-data-plane) + - [What compute writes](#what-compute-writes) + - [A workload in two locations](#a-workload-in-two-locations) +- [What this depends on](#what-this-depends-on) +- [Drawbacks](#drawbacks) +- [Alternatives](#alternatives) +- [Open Questions](#open-questions) +- [References](#references) + +## Summary + +Compute asks for a network interface the way a pod asks for storage: it writes a +**`NetworkInterfaceClaim`** naming a network and what the interface needs, and NSO binds it +to a **`NetworkInterface`** carrying everything required to configure a NIC — addresses, +gateway, MTU, and the data-plane identity behind them. + +Two properties follow from that shape. **Compute stops reaching into networking +internals**: it no longer creates `NetworkBinding` and `SubnetClaim` objects, and no longer +needs to know that a `NetworkContext` exists. And **an interface becomes a thing that +outlives the instance using it**, which is what makes a per-instance address, a retained +address, and an address a consumer can actually see all possible at once. + +This document is written for the compute service, for infrastructure providers that +configure NICs, and for NSO, which fulfills claims. It defines the shape of the two +resources and the contract between the three. + +## Motivation + +The interface between compute and networking today is not an interface. It is compute +reaching through NSO's internals, and three problems come out of that. + +**An address is shared where it should be per-instance.** `WorkloadDeploymentReconciler` +creates one `SubnetClaim` per deployment and every instance in that deployment draws from +it. The code says so in a `TODO`. Instances in a deployment cannot have distinct addresses +because nothing per-instance exists to hold one. + +**Nobody can see their address.** `Instance.status.networkInterfaces[].assignments.networkIP` +is declared, printed as a column, and never written. A consumer running +`kubectl get instance` sees an empty field where the answer should be. + +**Every provider re-derives the same facts.** A provider configuring a NIC needs the +address, the gateway, the MTU, and which network it belongs to. Today it walks +`NetworkBinding` → `NetworkContext` → `SubnetClaim` → `Subnet` to assemble them, which +means NSO cannot change those objects without breaking every provider, and a new provider +must learn all four before it can bring up one interface. + +All three are the same missing noun. There is no object that means *this instance's +interface on this network in this location*, so the facts about it live scattered across +the objects that happen to produce them. + +[Compute PR #210](https://github.com/datum-cloud/compute/pull/210) settles where an address +comes from. This document settles what holds it. + +### Goals + +- Give compute one resource to create and one resource to read, with no knowledge of how + NSO satisfies it. +- Give every instance its own interface, and every interface its own addresses. +- Give infrastructure providers a single resource carrying everything needed to configure + a NIC. +- Let an interface — and therefore its addresses — survive the instance that used it, so a + replacement instance comes back to the same address. +- Report allocation and programming separately, so an instance is never told its network + is ready before it can carry traffic. +- Let NSO change how it allocates, and what it allocates from, without a compute release. + +### Non-Goals + +- **Choosing addresses.** Which address an interface gets, out of what space, under what + policy, is [compute PR #210](https://github.com/datum-cloud/compute/pull/210). This + document consumes that answer and never re-decides it. +- **Programming the data plane.** A `NetworkInterface` states what an interface must be; + `VPCAttachment` and the agents behind it make it so. +- **Retiring `NetworkBinding`, `NetworkContext`, `SubnetClaim`, or `Subnet`.** They remain + NSO's internals and keep doing what they do. What changes is that nothing outside NSO + writes them. +- **Interface-level network policy.** `InstanceNetworkInterface.networkPolicy` stays where + it is, on the compute side, and is not part of this contract. +- **Multiple interfaces per instance beyond the first.** The model is a list and carries + more than one without changes; whether compute exposes that at launch is a compute + product decision, not an API constraint here. +- **Interfaces for anything other than an instance.** Load balancers and gateways also + attach to networks, and the same pair of resources should serve them. Nothing here + assumes an instance, but nothing here is designed against a second consumer either. +- **Hot-attach and hot-detach.** Adding or removing an interface on a running instance is + additive to this model and out of scope. + +## Proposal + +Compute creates a claim. NSO binds an interface to it. Providers read the interface. +Nobody reads anything else. + +### What it feels like + +A consumer writes the same workload compute PR #210 describes — a network, a list of +families, and optionally a class of extra address: + +```yaml +apiVersion: compute.datumapis.com/v1alpha +kind: Workload +metadata: + name: hello-sandbox +spec: + template: + spec: + runtime: + sandbox: + containers: + - name: app + image: ghcr.io/datum-cloud/hello-unikraft:latest + networkInterfaces: + - name: eth0 + network: + name: default + ipFamilies: + - IPv6 + - IPv4 + reclaimPolicy: Retain + addresses: + - class: public-unicast-ipv4 + placements: + - name: default + locations: + - us-central-1 + scaleSettings: + minReplicas: 2 +``` + +Compute turns that into one claim per instance, per interface, in the cell the deployment +landed at: + +```yaml +apiVersion: networking.datumapis.com/v1alpha +kind: NetworkInterfaceClaim +metadata: + # Derived from the slot and the interface. Stable across every instance that + # ever fills this slot. + name: hello-sandbox-default-us-central-1-0-eth0 + ownerReferences: + - kind: Instance + name: hello-sandbox-default-us-central-1-0 +spec: + network: + name: default + interfaceName: eth0 + ipFamilies: + - IPv6 + - IPv4 + reclaimPolicy: Retain + addresses: + - class: public-unicast-ipv4 +``` + +NSO binds it, and the claim reports the answer: + +```console +$ kubectl get networkinterfaceclaim hello-sandbox-default-us-central-1-0-eth0 -o yaml +status: + networkInterfaceRef: + name: nic-4f2a9c1e + addresses: + - family: IPv6 address: fd20:a1b:2c3d:1:0:1::/96 primary: true + - family: IPv4 address: 10.128.0.2/32 + externalAddresses: + - family: IPv4 address: 198.51.100.11 class: public-unicast-ipv4 + conditions: + - type: Bound status: "True" + - type: Allocated status: "True" + - type: Programmed status: "True" + - type: Ready status: "True" +``` + +The addresses appear on the claim as well as on the interface, deliberately. Compute +watches one object per interface and never needs a second read to answer "what address did +this instance get" — the same reason a `PersistentVolumeClaim` reports its own capacity. + +A provider bringing up the NIC reads the interface, and reads nothing else: + +```console +$ kubectl get networkinterface nic-4f2a9c1e -o yaml +spec: + network: + name: default + claimRef: + name: hello-sandbox-default-us-central-1-0-eth0 + uid: 8c1d… + interfaceName: eth0 + mtu: 1460 + reclaimPolicy: Retain + addresses: + - family: IPv6 address: fd20:a1b:2c3d:1:0:1::/96 gateway: fd20:a1b:2c3d:1::1 primary: true + - family: IPv4 address: 10.128.0.2/32 gateway: 10.128.0.1 + externalAddresses: + - family: IPv4 address: 198.51.100.11 class: public-unicast-ipv4 +status: + phase: Bound + networkContextRef: + name: default-us-central-1 + attachmentRef: + apiGroup: cloud.datumapis.com + kind: VPCAttachment + name: nic-4f2a9c1e + vpc: 3kF9qP2x + conditions: + - type: Allocated status: "True" + - type: Programmed status: "True" +``` + +Everything a NIC needs is in `spec`. Nothing in `spec` requires a second lookup to +interpret. The `NetworkContext` a provider used to have to find appears in `status` as a +breadcrumb for operators — nothing reads it to do its job. + +### Notes/Constraints/Caveats + +- **The claim is the durable identity, not the instance.** Its name derives from the slot, + so a replacement instance finds a claim that already holds its addresses. +- **A claim binds exactly one interface, and an interface binds exactly one claim.** There + is no fan-out and no re-matching. +- **Consumers never write either resource.** Compute writes claims on their behalf; NSO + writes interfaces. +- **`Allocated` and `Programmed` are separate, and `Ready` requires both.** Allocation is + synchronous, programming is not, and an instance released on allocation alone comes up + before its packets can move. +- **Both resources live in the cell the deployment landed at.** Location is implicit in + where the claim exists; nobody writes it down. +- **The addresses on a claim's status are a copy, and the interface is the source of + truth.** They are written in the same reconcile that sets `Bound`. +- **A retained interface returns to `Available`, not to a dead end.** Compute PR #210 is + explicit that the storage `Released` state — where an operator must clear a stale + reference by hand — must not be copied, and it is not. + +## Design Details + +### NetworkInterfaceClaim + +A claim states what an interface must be able to do. Every field is intent; none of it +describes a result. + +```yaml +apiVersion: networking.datumapis.com/v1alpha +kind: NetworkInterfaceClaim +spec: + # The network this interface attaches to. A LocalNetworkRef — a claim and its + # network are always in the same namespace. + # Required. Immutable: an interface that changed network after allocation + # holds addresses from a space it no longer belongs to. + network: + name: default + + # The name the interface presents to the guest. Defaults to eth0. Immutable. + interfaceName: eth0 + + # The address families this interface must carry, in priority order. The first + # is the interface's primary address. Defaults to [IPv6] — the platform is + # IPv6-first. + # + # All requested families must be satisfiable, or the claim does not bind. A + # partially-addressed interface is not published. A family the network does + # not carry is a validation failure, not a pending condition. + ipFamilies: + - IPv6 + - IPv4 + + # Additional addresses beyond the network-internal ones, each named by class. + # A class is an IPAM concept: the consumer names a kind of address, never a + # pool, a prefix length, or a CIDR. Omitted entirely for ordinary private + # addressing, which is the common case. + addresses: + - class: public-unicast-ipv4 + + # What becomes of the bound interface when this claim is deleted. + # Delete — the interface is deleted and its addresses released. + # Retain — the interface survives, unbound, still holding its addresses, + # waiting for a claim of this name to come back. + # Defaults to Delete. The IP class may set a different default; this overrides + # it. + reclaimPolicy: Retain + + # Optional. Binds a specific existing interface by name, the part + # PersistentVolumeClaim.volumeName plays for storage. Left empty — the normal + # case — NSO chooses or creates one. + networkInterfaceName: "" +``` + +Status carries the binding, a copy of the result, and the conditions: + +```yaml +status: + # Set once, when the claim binds. Never recomputed. + networkInterfaceRef: + name: nic-4f2a9c1e + + # Copied from the bound interface so a consumer reads one object. + addresses: [...] + externalAddresses: [...] + + conditions: + # An interface has been bound to this claim. + - type: Bound + # Every requested family holds an address. + - type: Allocated + # The data plane can carry those addresses. + - type: Programmed + # Bound, Allocated, and Programmed are all true. + - type: Ready +``` + +`Allocated` and `Programmed` are surfaced on the claim rather than left on the interface +because they are the two facts compute gates an instance on, and compute should not have to +hold a second watch to learn them. `SubnetClaim` already defaults exactly this trio of +conditions, so the pattern is NSO's own. + +### NetworkInterface + +An interface is a result. It is written by NSO, read by providers, and never authored by +hand outside of an operator repairing something. + +```yaml +apiVersion: networking.datumapis.com/v1alpha +kind: NetworkInterface +spec: + # The network this interface belongs to. + network: + name: default + + # The claim currently holding this interface. Empty when the interface is + # retained and unbound. The uid is recorded so a claim deleted and recreated + # under the same name is recognised as a different claim. + claimRef: + name: hello-sandbox-default-us-central-1-0-eth0 + uid: 8c1d… + + interfaceName: eth0 + + # Resolved from Network.spec.mtu, so a provider never reads the network. + mtu: 1460 + + # The addresses inside the network. One entry per family, exactly one primary. + # For IPv6 this is the endpoint's whole /96 block, not a single address; the + # interface owns the block and assigns within it. + addresses: + - family: IPv6 + address: fd20:a1b:2c3d:1:0:1::/96 + gateway: fd20:a1b:2c3d:1::1 + primary: true + class: tenant-endpoint-ipv6 + - family: IPv4 + address: 10.128.0.2/32 + gateway: 10.128.0.1 + class: tenant-endpoint-ipv4 + + # Addresses reachable from outside the network, each mapped one-to-one onto + # the interface's address of the same family. + externalAddresses: + - family: IPv4 + address: 198.51.100.11 + class: public-unicast-ipv4 + + reclaimPolicy: Retain + +status: + # Available — allocated, holding addresses, bound to nothing. + # Bound — held by the claim in spec.claimRef. + phase: Bound + + # The network's presence in this location, which NSO resolved or created while + # fulfilling the claim. Recorded for operators; nothing depends on it. + networkContextRef: + name: default-us-central-1 + + # The data-plane realization, once one exists. + attachmentRef: + apiGroup: cloud.datumapis.com + kind: VPCAttachment + name: nic-4f2a9c1e + + # Base62 VPC identifier, matching VPC.status.vpc. The fabric keys on this. + vpc: 3kF9qP2x + + conditions: + - type: Allocated + - type: Programmed +``` + +**There is no `Released` phase.** A retained interface whose claim is deleted goes straight +back to `Available` with its addresses intact, and the next claim of the same name binds it. +Storage's `Released` state requires an operator to clear a stale reference before the volume +can be used again; an address held in a finite public range cannot wait on that. The `uid` +in `claimRef` is what makes this safe — it is the record that distinguishes the claim that +held the interface from the one asking for it now. + +**Addresses live in `spec`, not `status`.** They are the desired configuration of a NIC, +and a provider that has to read `status` to know what to configure has no way to tell a +requested address from an observed one. `status` holds only what NSO observed: where the +network is, what realizes the interface, and whether programming succeeded. + +### Binding + +Binding follows storage, because storage's model is the one that survives a holder being +replaced. + +**A claim binds once, at creation.** NSO either finds a retained interface whose name the +claim asks for, or allocates a new one. From then on the claim records the interface and +the interface records the claim, and nothing recomputes the pairing. There is no matching +pass, no window where an address is loose, and no way for a re-match to go wrong. + +**The claim's name is the durable identity.** Compute derives it from the slot — workload, +placement, location, ordinal — and the interface name, all of which are stable across every +instance that ever fills that slot. A replacement instance creates a claim, finds the name +already exists, and adopts it. + +**A claim ends when its slot does.** Deleting the workload deletes its claims through +ownership; scaling down deletes the claims of the slots it removes. `reclaimPolicy` then +decides what happens to the interface: + +| Event | `Delete` | `Retain` | +|---|---|---| +| Instance replaced or rescheduled | claim survives — same interface, same addresses | same | +| Instance redeployed with a new template | claim survives — same interface, same addresses | same | +| Scale down then back up | new interface, new addresses | same interface, same addresses | +| Workload deleted then recreated | new interface, new addresses | same interface, same addresses | + +The first two rows are the same in both columns and that is the point: **an interface +survives a redeploy on its own. Surviving a scale-down takes `Retain`.** + +A retained interface still holds its addresses, still counts against its holder's budget, +and carries a lease — otherwise a public address sits out of service indefinitely with +nothing pressuring anyone to release it. Expiry, its duration, and operator force-release +are IPAM's to define; this resource only records the policy that triggers them. + +### Fulfilling a claim + +What NSO does with a claim is the part compute stops doing. + +**Resolve the network's presence in this location.** The claim names a network and exists +in a cell, which is enough to find or create the `NetworkContext` for that pair. Compute +creates no `NetworkBinding` — that object is now NSO's business, created on the claim's +behalf and reported back only as a breadcrumb in `status.networkContextRef`. + +**Allocate an address per requested family.** Each becomes an `IPClaim` of the appropriate +class against the platform allocator, with the network and location supplied as scope, per +[compute PR #210](https://github.com/datum-cloud/compute/pull/210). Each entry in the +claim's `addresses` list becomes one more `IPClaim` of the class it names. Every one must +succeed before anything is published. + +**Create the interface and bind it.** The addresses, the gateways read from the location's +subnet, and the MTU read from the network land in `spec`. `Allocated` goes true, the claim +goes `Bound`, and compute can see an address. + +**Wait for the data plane.** `Programmed` follows separately, when the attachment realizing +the interface reports ready. + +The ordering matters for the failure case: an exhausted pool, a location with no space, or +a family the network does not carry all fail before an interface exists, so a claim that +cannot be satisfied reports why rather than binding to something incomplete. The condition +message should name what ran out and at which level, which is the property PR #210 asks +the allocator to preserve. + +### Reaching the data plane + +A `NetworkInterface` says what an interface must be. `VPCAttachment`, in the +[cloud](https://github.com/datum-cloud/cloud) API group, is where it becomes real, and the +split is deliberate: an interface is allocated as soon as a claim exists, before an instance +has been scheduled to any node, while an attachment cannot exist until a node, a container, +and a veth pair do. + +``` +NetworkInterfaceClaim compute's intent created per instance, per interface + │ binds + ▼ +NetworkInterface NSO's answer addresses, gateway, MTU + │ realized by + ▼ +VPCAttachment the node's reality node, containerID, VRF, veth, pod subnet + │ attaches to + ▼ +VPC the data plane base62 identity the fabric keys on +``` + +The agent on the node creates the `VPCAttachment` from the interface, copying +`spec.addresses` into `spec.interface.addresses` and naming the VPC backing this network in +this location. It reports back the facts only a node knows — the container ID, the host and +VRF device names, the pod subnet — and NSO sets `Programmed` on the interface when the +attachment reports ready, copying the VPC identifier onto `status.vpc`. + +Nothing in `VPCAttachment` changes to support this. It already requires the addresses to +have been decided elsewhere; this names the elsewhere. + +`Programmed` going false — an attachment lost, a node drained — does not release the +interface. The addresses stay allocated because the claim still exists, and the instance's +`Ready` condition reflects the loss without renumbering anything. + +### What compute writes + +Compute's changes are additive on the consumer-facing side and a removal on the internal +side. `InstanceNetworkInterface.network` keeps its existing `NetworkRef` type — the +consumer-facing reference does not move. + +**On `InstanceNetworkInterface`**, four fields join it: + +| Field | Meaning | Default | +|---|---|---| +| `name` | the interface name in the guest, and the claim-name suffix | `eth0` | +| `ipFamilies` | families to carry, in priority order | `[IPv6]` | +| `reclaimPolicy` | whether addresses survive a scale-down | `Delete` | +| `addresses[].class` | extra addresses by class | none | + +`ipFamilies` and `reclaimPolicy` are the two fields +[compute #112](https://github.com/datum-cloud/compute/issues/112) already calls for. +`addresses[].class` is compute PR #210's. `name` is new here, and exists because a claim +name has to be derived from something stable that a consumer chose. + +**On `InstanceNetworkInterfaceStatus`**, the single-address shape grows into the list the +interface actually holds: + +```yaml +status: + networkInterfaces: + - name: eth0 + addresses: + - family: IPv6 address: fd20:a1b:2c3d:1:0:1::/96 primary: true + - family: IPv4 address: 10.128.0.2/32 + external: + - family: IPv4 address: 198.51.100.11 + assignments: + # Retained: the primary address of each family, so existing print + # columns and clients keep working. + networkIP: fd20:a1b:2c3d:1:0:1:: + externalIP: 198.51.100.11 + conditions: + - type: Allocated status: "True" + - type: Programmed status: "True" +``` + +`assignments` keeps its shape and finally gets written. It becomes a projection of the +primary address rather than a field of its own, which means the print column on `Instance` +starts showing a value without anyone changing the column. + +**In `WorkloadDeploymentReconciler`**, the per-deployment `NetworkBinding` and `SubnetClaim` +are replaced by one `NetworkInterfaceClaim` per instance per interface. That is the fix for +the shared-allocation problem: the object that holds an address is now per-instance, so +addresses can be too. + +**The `network` scheduling gate becomes per-instance.** Today it is removed when a shared +allocation is ready, which releases every instance in the deployment at once. It should be +removed for an instance when that instance's own claims report `Ready` — meaning bound, +allocated, and programmed. An instance whose interface is allocated but not yet programmed +stays gated, which is the whole reason the two conditions are separate. + +Nothing changes in the federation path. Claims and interfaces are created in the POP cell +by NSO, which already runs there for exactly this reason. Addresses land on the local +`Instance`, and the existing write-back to Karmada and `InstanceProjector` mirror carry them +to the project. The consumer reads addresses in the same place they read everything else +about an instance. + +### A workload in two locations + +The same workload from compute PR #210 — one network, two locations, two replicas each, +dual-stack, a public address per instance. Every object this design causes to exist: + +**The consumer's project.** Two objects, both written by the consumer: the `Network` and +the `Workload`. Nothing about interfaces appears here at all. + +**Each POP cell.** The deployment arrives by placement; everything below it is created +locally. + +``` +us-central-1 eu-west-1 + WorkloadDeployment/…-americas WorkloadDeployment/…-europe + Instance/…-americas-us-central-1-{0,1} Instance/…-europe-eu-west-1-{0,1} + NetworkInterfaceClaim/…-{0,1}-eth0 NetworkInterfaceClaim/…-{0,1}-eth0 + NetworkInterface ×2 NetworkInterface ×2 + NetworkContext/default-us-central-1 NetworkContext/default-eu-west-1 + VPCAttachment ×2 VPCAttachment ×2 +``` + +Four claims, four interfaces, four attachments, and one `NetworkContext` per location that +the first claim in that location caused to exist. Each interface holds three addresses — an +IPv6 endpoint block, an IPv4 address, and a public IPv4 address — for the twelve +allocations compute PR #210 accounts for. + +**What each layer knows.** The claim knows the network, the interface name, the families, +and the policy. The interface adds the addresses, the gateways, and the MTU. The attachment +adds the node, the container, and the devices. No layer repeats the one below it, and no +consumer of a layer needs the one above it. + +**Scaling to one replica in `us-central-1`** deletes one instance, and with it one claim. +With `reclaimPolicy: Retain`, the interface stays, holding `fd20:a1b:2c3d:1:0:2::/96`, +`10.128.0.3/32`, and `198.51.100.12`, in phase `Available`. Scaling back up recreates a +claim with the identical name, which binds that interface, and the new instance comes back +on the address the old one had. With `Delete`, all three addresses go back to their pools +and the new instance gets whatever is next. + +Removing the `eu-west-1` placement releases the addresses its instances held, but not that +location's `NetworkContext` or subnet. Those belong to the network, and other workloads on +it draw from them. + +## What this depends on + +An interface resource is necessary and not sufficient. Each of the following is assumed and +none of it is provided here. + +- **[Compute PR #210](https://github.com/datum-cloud/compute/pull/210) must land first.** + This design consumes classes, `IPClaim`, retention, and the central allocator wholesale. + Without it there is no answer to what address an interface gets, and `spec.addresses` has + nothing to hold. +- **A network's default families and an interface's must agree.** `NetworkSpec.ipFamilies` + defaults to `[IPv4]` while a claim here defaults to `[IPv6]`, so the default workload on + the default network requests a family its network does not carry. PR #210 raises the same + mismatch as an open question; this design is where it surfaces as a hard failure, and it + needs settling before either ships. +- **A network needs one routing identity across every location it reaches**, unique + platform-wide, or the two halves of a multi-location workload are unrelated networks + sharing a name. That it is not the per-location forwarding-instance identifier needs to + stay true. +- **A moved instance needs its old route withdrawn before the new one is trusted.** + Retention makes this worse, not better: the address stays valid across the move, so both + advertisements look legitimate and traffic splits. +- **Subnets and gateways need programming, not just allocation.** + `spec.addresses[].gateway` is only useful if something answers at that address. Without + it, oversized packets are dropped silently — handshakes succeed and large transfers hang. + This is exactly why `Programmed` is a separate condition rather than folded into + `Allocated`. +- **Consuming a class must be a privilege.** `spec.addresses[].class` on a claim is written + by compute on a consumer's behalf. If the class name is the only authorization boundary, + the check on it must fail closed, and it must check the consumer's project rather than + the platform identity that made the call. +- **Providers must migrate.** The GCP and Unikraft providers read `NetworkBinding`, + `NetworkContext`, `SubnetClaim`, and `Subnet` today. They keep working until they are + moved to `NetworkInterface`, and compute's creation of the old objects cannot be removed + until every one of them has moved. + +## Drawbacks + +- **Two resources where consumers see none.** Every interface now costs a claim, an + interface, and an attachment where a deployment used to cost one shared subnet claim. At + four replicas that is twelve objects instead of one. The cost is per-instance object + count in the POP cell, and it buys the per-instance address that motivates the work. +- **A retained interface holds capacity nobody else can use.** That is the price of an + address that survives a scale-down, and on a finite public range it is the price that + matters. The lease is the mitigation, not the elimination. +- **The binding is invisible until it exists.** A claim that cannot bind — no space in the + location, a family the network does not carry — reports a condition and holds its instance + gated. That is correct and it is also a new way for an instance to be stuck, which needs + the failure to name what could not be satisfied rather than reporting "not ready". +- **Splitting `Allocated` from `Programmed` makes readiness slower to reach**, on purpose. + Instances that used to come up on allocation now wait for the data plane. Some workloads + will notice. + +## Alternatives + +- **Put the addresses on `Instance.status` and skip both resources.** Rejected: it is close + to the status quo, it gives providers nothing stable to watch, and an address on an + instance's status cannot outlive the instance, which forecloses retention entirely. +- **One resource instead of two — a `NetworkInterface` compute creates directly.** + Rejected: it merges intent with result, so a provider cannot tell a requested address + from an allocated one, and it removes the object that survives the instance. The claim + exists precisely so something outlives its holder. +- **Reuse `VPCAttachment` as the compute-facing resource.** Rejected: an attachment + requires a node, a container ID, and device names, none of which exist when the address + must be allocated. It is the realization of an interface, not the request for one. +- **Keep `NetworkBinding` and `SubnetClaim` and just add a per-instance `SubnetClaim`.** + Rejected: it fixes the sharing problem and none of the others. Compute stays coupled to + NSO internals, providers still walk four objects, and there is still nothing that survives + an instance. +- **Bind by re-matching rather than by never unbinding.** Rejected for the same reason + compute PR #210 rejects it: releasing an interface on instance deletion and re-matching it + later opens a window where the address is loose and needs a durable identity distinct from + the holder. The claim name is already that identity. +- **A `Released` phase, as storage has.** Rejected: it requires an operator to clear a stale + reference before the interface is usable, which for a public address means capacity held + hostage to a manual step. The `uid` on `claimRef` provides the same safety without the + dead end. +- **Let the claim carry the location explicitly**, as `SubnetClaim` does. Rejected: the + claim exists in the cell that serves the location, so the field would be a value the + writer copies from where it is already standing, and a value that can disagree with + reality. + +## Open Questions + +**What size block does an endpoint actually get?** The +[tenant addressing plan](https://github.com/datum-cloud/enhancements/blob/main/architecture/design/network/addressing/tenant.md) +specifies a `/96` per endpoint. `VPCAttachment.status.podSubnet` documents a `/80`. Both +cannot be right, and the answer changes what `spec.addresses[].address` holds for IPv6 and +how much space an interface can hand to containers without a control-plane round trip. + +**Where does a load balancer's interface come from?** Nothing in this design is +instance-specific, and a load balancer needs the same object with the same fields. Whether +it uses the same claim kind, or the shape simply gets copied, should be settled before a +second consumer arrives and settles it by accident. + +**Does the claim need to express bandwidth or queue policy?** Interface-level rate limits, +queue disciplines, and offload capabilities are properties of a NIC a consumer might +reasonably ask for. Adding them later is additive; deciding now whether they belong on the +claim or on the instance type keeps them from landing in both. + +**Should an interface be attachable to more than one network?** The model says no — one +interface, one network, and multi-network reachability comes from multiple interfaces. That +is the simpler answer and it should be confirmed against the connector and interconnect +work rather than assumed. + +**Does `NetworkInterface` need to name the VPC in `spec`?** Today it appears only in +`status`, discovered when the attachment is programmed. If a provider ever needs the VPC +identity before an attachment exists, it moves — and that would make the interface depend on +a location's data plane being resolved at allocation time, which it currently does not. + +## References + +**What this builds on** + +- [IP classes for workload address allocation](https://github.com/datum-cloud/compute/pull/210) + — where an address comes from, and the retention model this document reuses without + restating. +- [Tenant addressing](https://github.com/datum-cloud/enhancements/blob/main/architecture/design/network/addressing/tenant.md) + — the per-network IPv6 `/48`, the per-location `/64`, and the per-endpoint block an + interface holds. +- [Federated Deployment Scheduling](https://github.com/datum-cloud/compute/blob/main/docs/enhancements/federated-deployment-scheduling.md) + — which control plane each object lives in, and the write-back path addresses travel to + reach a consumer. + +**The work this serves** + +- [network-services-operator#164](https://github.com/datum-cloud/network-services-operator/issues/164) + — the issue calling for these two resources, and the decoupling they provide. +- [compute#112](https://github.com/datum-cloud/compute/issues/112) + — the compute-side issue: per-instance allocation, addresses that reach + `Instance.status`, and the `ipFamilies` and `reclaimPolicy` fields. + +**The types involved** + +- [`Network`, `NetworkContext`, `SubnetClaim`](../../api/v1alpha) — the network an interface + belongs to, its presence in a location, and the internals compute stops writing. +- [`VPC` and `VPCAttachment`](https://github.com/datum-cloud/cloud/tree/main/api/v1alpha1) + — the data plane an interface is realized on. +- [Compute API types](https://github.com/datum-cloud/compute/tree/main/api/v1alpha) + — `Instance`, `InstanceNetworkInterface`, and the status fields this fills in. diff --git a/go.mod b/go.mod index 2c461b2e..4f21ba48 100644 --- a/go.mod +++ b/go.mod @@ -25,21 +25,21 @@ require ( github.com/spf13/cobra v1.10.2 github.com/stretchr/testify v1.11.1 go.miloapis.com/dns-operator v0.5.1 - go.miloapis.com/milo v0.28.1 + go.miloapis.com/milo v0.28.4-0.20260629130346-79689376fe11 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0 go.opentelemetry.io/otel v1.44.0 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.44.0 go.opentelemetry.io/otel/sdk v1.44.0 go.opentelemetry.io/otel/trace v1.44.0 - golang.org/x/net v0.55.0 - golang.org/x/sync v0.20.0 + golang.org/x/net v0.56.0 + golang.org/x/sync v0.22.0 google.golang.org/grpc v1.81.1 google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af k8s.io/api v0.36.1 k8s.io/apiextensions-apiserver v0.36.1 k8s.io/apimachinery v0.36.1 k8s.io/client-go v0.36.1 - k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2 + k8s.io/utils v0.0.0-20260319190234-28399d86e0b5 sigs.k8s.io/controller-runtime v0.24.1 sigs.k8s.io/gateway-api v1.5.1 sigs.k8s.io/gateway-api/conformance v1.5.1 @@ -47,6 +47,11 @@ require ( sigs.k8s.io/yaml v1.6.0 ) +require ( + github.com/go-openapi/swag/cmdutils v0.25.4 // indirect + github.com/go-openapi/swag/netutils v0.25.4 // indirect +) + require ( cel.dev/expr v0.25.2 // indirect dario.cat/mergo v1.0.2 // indirect @@ -120,7 +125,7 @@ require ( github.com/docker/go-events v0.0.0-20250808211157-605354379745 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/dominikbraun/graph v0.23.0 // indirect - github.com/ebitengine/purego v0.10.0 // indirect + github.com/ebitengine/purego v0.10.1 // indirect github.com/emicklei/go-restful/v3 v3.13.0 // indirect github.com/envoyproxy/go-control-plane/ratelimit v0.1.1-0.20260409050421-3f47accd6e14 // indirect github.com/envoyproxy/protoc-gen-validate v1.3.3 // indirect @@ -128,7 +133,7 @@ require ( github.com/evanphx/json-patch/v5 v5.9.11 // indirect github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f // indirect github.com/fatih/color v1.19.0 // indirect - github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/felixge/httpsnoop v1.1.0 // indirect github.com/fsnotify/fsnotify v1.10.1 // indirect github.com/fxamacker/cbor/v2 v2.9.0 // indirect github.com/go-errors/errors v1.4.2 // indirect @@ -146,7 +151,7 @@ require ( github.com/go-openapi/loads v0.23.3 // indirect github.com/go-openapi/spec v0.22.5 // indirect github.com/go-openapi/strfmt v0.26.3 // indirect - github.com/go-openapi/swag v0.23.1 // indirect + github.com/go-openapi/swag v0.25.4 // indirect github.com/go-openapi/swag/conv v0.26.0 // indirect github.com/go-openapi/swag/fileutils v0.26.0 // indirect github.com/go-openapi/swag/jsonname v0.26.0 // indirect @@ -167,7 +172,7 @@ require ( github.com/golang-sql/sqlexp v0.1.0 // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/google/btree v1.1.3 // indirect - github.com/google/gnostic-models v0.7.0 // indirect + github.com/google/gnostic-models v0.7.1 // indirect github.com/google/go-containerregistry v0.21.6 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/google/pprof v0.0.0-20260115054156-294ebfa9ad83 // indirect @@ -183,7 +188,6 @@ require ( github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect github.com/jackc/pgx/v5 v5.9.2 // indirect github.com/jmoiron/sqlx v1.4.0 // indirect - github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.18.6 // indirect github.com/kortschak/goroutine v1.1.3 // indirect @@ -193,8 +197,7 @@ require ( github.com/lib/pq v1.11.2 // indirect github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect github.com/longhorn/go-iscsi-helper v0.0.0-20210330030558-49a327fb024e // indirect - github.com/lufia/plan9stats v0.0.0-20251013123823-9fd1530e3ec3 // indirect - github.com/mailru/easyjson v0.9.1 // indirect + github.com/lufia/plan9stats v0.0.0-20260330125221-c963978e514e // indirect github.com/mattn/go-colorable v0.1.14 // indirect github.com/mattn/go-isatty v0.0.22 // indirect github.com/mattn/go-runewidth v0.0.16 // indirect @@ -206,11 +209,11 @@ require ( github.com/mitchellh/reflectwalk v1.0.2 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect github.com/moby/go-archive v0.2.0 // indirect - github.com/moby/moby/api v1.54.2 // indirect - github.com/moby/moby/client v0.4.1 // indirect - github.com/moby/patternmatcher v0.6.0 // indirect + github.com/moby/moby/api v1.55.0 // indirect + github.com/moby/moby/client v0.5.0 // indirect + github.com/moby/patternmatcher v0.6.1 // indirect github.com/moby/spdystream v0.5.1 // indirect - github.com/moby/sys/sequential v0.6.0 // indirect + github.com/moby/sys/sequential v0.7.0 // indirect github.com/moby/sys/user v0.4.0 // indirect github.com/moby/sys/userns v0.1.0 // indirect github.com/moby/term v0.5.2 // indirect @@ -247,7 +250,7 @@ require ( github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 // indirect github.com/segmentio/asm v1.2.1 // indirect github.com/segmentio/ksuid v1.0.4 // indirect - github.com/shirou/gopsutil/v4 v4.26.4 // indirect + github.com/shirou/gopsutil/v4 v4.26.6 // indirect github.com/shopspring/decimal v1.4.0 // indirect github.com/sirupsen/logrus v1.9.4 // indirect github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect @@ -257,8 +260,8 @@ require ( github.com/spf13/viper v1.21.0 // indirect github.com/stoewer/go-strcase v1.3.1 // indirect github.com/subosito/gotenv v1.6.0 // indirect - github.com/tklauser/go-sysconf v0.3.16 // indirect - github.com/tklauser/numcpus v0.11.0 // indirect + github.com/tklauser/go-sysconf v0.4.0 // indirect + github.com/tklauser/numcpus v0.12.0 // indirect github.com/vishvananda/netlink v1.3.1 // indirect github.com/vishvananda/netns v0.0.5 // indirect github.com/x448/float16 v0.8.4 // indirect @@ -270,9 +273,10 @@ require ( go.etcd.io/etcd/api/v3 v3.6.8 // indirect go.etcd.io/etcd/client/pkg/v3 v3.6.8 // indirect go.etcd.io/etcd/client/v3 v3.6.8 // indirect + go.miloapis.com/ipam v0.3.2-0.20260813020700-13bd2c8c077b go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/contrib/exporters/autoexport v0.67.0 // indirect - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.68.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.44.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.44.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.44.0 // indirect @@ -284,16 +288,16 @@ require ( go.uber.org/zap v1.28.0 // indirect go.yaml.in/yaml/v2 v2.4.4 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect - golang.org/x/crypto v0.52.0 // indirect + golang.org/x/crypto v0.54.0 // indirect golang.org/x/crypto/x509roots/fallback v0.0.0-20250406160420-959f8f3db0fb // indirect golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f // indirect - golang.org/x/mod v0.36.0 // indirect + golang.org/x/mod v0.37.0 // indirect golang.org/x/oauth2 v0.36.0 // indirect - golang.org/x/sys v0.45.0 // indirect - golang.org/x/term v0.43.0 // indirect - golang.org/x/text v0.37.0 // indirect + golang.org/x/sys v0.47.0 // indirect + golang.org/x/term v0.45.0 // indirect + golang.org/x/text v0.40.0 // indirect golang.org/x/time v0.15.0 // indirect - golang.org/x/tools v0.45.0 // indirect + golang.org/x/tools v0.47.0 // indirect gomodules.xyz/jsonpatch/v2 v2.5.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa // indirect @@ -306,7 +310,7 @@ require ( k8s.io/cli-runtime v0.36.1 // indirect k8s.io/component-base v0.36.1 // indirect k8s.io/klog/v2 v2.140.0 // indirect - k8s.io/kube-openapi v0.0.0-20260317180543-43fb72c5454a // indirect + k8s.io/kube-openapi v0.0.0-20260330154417-16be699c7b31 // indirect k8s.io/kubectl v0.36.1 // indirect k8s.io/metrics v0.36.1 // indirect k8s.io/streaming v0.36.1 // indirect @@ -321,3 +325,5 @@ require ( sigs.k8s.io/randfill v1.0.0 // indirect sigs.k8s.io/structured-merge-diff/v6 v6.3.2 // indirect ) + +replace go.miloapis.com/ipam => github.com/milo-os/ipam v0.3.2-0.20260813020700-13bd2c8c077b diff --git a/go.sum b/go.sum index 644b3719..6d66fd3d 100644 --- a/go.sum +++ b/go.sum @@ -190,8 +190,8 @@ github.com/dominikbraun/graph v0.23.0 h1:TdZB4pPqCLFxYhdyMFb1TBdFxp8XLcJfTTBQucV github.com/dominikbraun/graph v0.23.0/go.mod h1:yOjYyogZLY1LSG9E33JWZJiq5k83Qy2C6POAuiViluc= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= -github.com/ebitengine/purego v0.10.0 h1:QIw4xfpWT6GWTzaW5XEKy3HXoqrJGx1ijYHzTF0/ISU= -github.com/ebitengine/purego v0.10.0/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ= +github.com/ebitengine/purego v0.10.1 h1:dewVBCBT2GaMu1SrNTYxQhgQBethzfhiwvZiLGP/qyY= +github.com/ebitengine/purego v0.10.1/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ= github.com/emicklei/go-restful/v3 v3.13.0 h1:C4Bl2xDndpU6nJ4bc1jXd+uTmYPVUwkD6bFY/oTyCes= github.com/emicklei/go-restful/v3 v3.13.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/envoyproxy/gateway v1.8.1 h1:i8POtrPtel3n0iGP55MNy6U1rTW4msyhkEWjXAF09ts= @@ -214,8 +214,8 @@ github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f h1:Wl78ApPPB2 github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f/go.mod h1:OSYXu++VVOHnXeitef/D8n/6y4QV8uLHSFXX4NeXMGc= github.com/fatih/color v1.19.0 h1:Zp3PiM21/9Ld6FzSKyL5c/BULoe/ONr9KlbYVOfG8+w= github.com/fatih/color v1.19.0/go.mod h1:zNk67I0ZUT1bEGsSGyCZYZNrHuTkJJB+r6Q9VuMi0LE= -github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= -github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/felixge/httpsnoop v1.1.0 h1:3YtUj32ZZkqZtt3sZZsClsymw/QDuVfpNhoA31zeORc= +github.com/felixge/httpsnoop v1.1.0/go.mod h1:Zqxgdd+1Rkcz8euOqdr7lqgCRJztwr5hp9vDSi5UZCE= github.com/foxcpp/go-mockdns v1.2.0 h1:omK3OrHRD1IWJz1FuFBCFquhXslXoF17OvBS6JPzZF0= github.com/foxcpp/go-mockdns v1.2.0/go.mod h1:IhLeSFGed3mJIAXPH2aiRQB+kqz7oqu8ld2qVbOu7Wk= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= @@ -266,8 +266,10 @@ github.com/go-openapi/spec v0.22.5 h1:KhO7RBlKQfonUWX2WzQCoLIXVA6AcNqDGZ3a1Dutdl github.com/go-openapi/spec v0.22.5/go.mod h1:vxpOtMya5TXtENXKE5bKqv5NjocVhyhxHrlZfvKnZ74= github.com/go-openapi/strfmt v0.26.3 h1:rzmslHarJgBbf2qfGge+X3htclQfmXqBZMm0Too0HhU= github.com/go-openapi/strfmt v0.26.3/go.mod h1:a5nsUw0oRpQzZeOwx8bi6cKbzFZslpbCKt1LEot+KnQ= -github.com/go-openapi/swag v0.23.1 h1:lpsStH0n2ittzTnbaSloVZLuB5+fvSY/+hnagBjSNZU= -github.com/go-openapi/swag v0.23.1/go.mod h1:STZs8TbRvEQQKUA+JZNAm3EWlgaOBGpyFDqQnDHMef0= +github.com/go-openapi/swag v0.25.4 h1:OyUPUFYDPDBMkqyxOTkqDYFnrhuhi9NR6QVUvIochMU= +github.com/go-openapi/swag v0.25.4/go.mod h1:zNfJ9WZABGHCFg2RnY0S4IOkAcVTzJ6z2Bi+Q4i6qFQ= +github.com/go-openapi/swag/cmdutils v0.25.4 h1:8rYhB5n6WawR192/BfUu2iVlxqVR9aRgGJP6WaBoW+4= +github.com/go-openapi/swag/cmdutils v0.25.4/go.mod h1:pdae/AFo6WxLl5L0rq87eRzVPm/XRHM3MoYgRMvG4A0= github.com/go-openapi/swag/conv v0.26.0 h1:5yGGsPYI1ZCva93U0AoKi/iZrNhaJEjr324YVsiD89I= github.com/go-openapi/swag/conv v0.26.0/go.mod h1:tpAmIL7X58VPnHHiSO4uE3jBeRamGsFsfdDeDtb5ECE= github.com/go-openapi/swag/fileutils v0.26.0 h1:WJoPRvsA7QRiiWluowkLJa9jaYR7FCuxmDvnCgaRRxU= @@ -282,6 +284,8 @@ github.com/go-openapi/swag/loading v0.26.0 h1:Apg6zaKhCJurpJer0DCxq99qwmhFddBhaM github.com/go-openapi/swag/loading v0.26.0/go.mod h1:dBxQ/6V2uBaAQdevN18VELE6xSpJWZxLX4txe12JwDg= github.com/go-openapi/swag/mangling v0.26.0 h1:Du2YC4YLA/Y5m/YKQd7AnY5qq0wRKSFZTTt8ktFaXcQ= github.com/go-openapi/swag/mangling v0.26.0/go.mod h1:jifS7W9vbg+pw63bT+GI53otluMQL3CeemuyCHKwVx0= +github.com/go-openapi/swag/netutils v0.25.4 h1:Gqe6K71bGRb3ZQLusdI8p/y1KLgV4M/k+/HzVSqT8H0= +github.com/go-openapi/swag/netutils v0.25.4/go.mod h1:m2W8dtdaoX7oj9rEttLyTeEFFEBvnAx9qHd5nJEBzYg= github.com/go-openapi/swag/stringutils v0.26.0 h1:qZQngLxs5s7SLijc3N2ZO+fUq2o8LjuWAASSrJuh+xg= github.com/go-openapi/swag/stringutils v0.26.0/go.mod h1:sWn5uY+QIIspwPhvgnqJsH8xqFT2ZbYcvbcFanRyhFE= github.com/go-openapi/swag/typeutils v0.26.0 h1:2kdEwdiNWy+JJdOvu5MA2IIg2SylWAFuuyQIKYybfq4= @@ -329,8 +333,8 @@ github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg= github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= github.com/google/cel-go v0.26.0 h1:DPGjXackMpJWH680oGY4lZhYjIameYmR+/6RBdDGmaI= github.com/google/cel-go v0.26.0/go.mod h1:A9O8OU9rdvrK5MQyrqfIxo1a0u4g3sF8KB6PUIaryMM= -github.com/google/gnostic-models v0.7.0 h1:qwTtogB15McXDaNqTZdzPJRHvaVJlAl+HVQnLmJEJxo= -github.com/google/gnostic-models v0.7.0/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ= +github.com/google/gnostic-models v0.7.1 h1:SisTfuFKJSKM5CPZkffwi6coztzzeYUhc3v4yxLWH8c= +github.com/google/gnostic-models v0.7.1/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= @@ -386,8 +390,6 @@ github.com/jmoiron/sqlx v1.4.0 h1:1PLqN7S1UYp5t4SrVVnt4nUVNemrDAtxlulVe+Qgm3o= github.com/jmoiron/sqlx v1.4.0/go.mod h1:ZrZ7UsYB/weZdl2Bxg6jCRO9c3YHl8r3ahlKmRT4JLY= github.com/jonboulle/clockwork v0.5.0 h1:Hyh9A8u51kptdkR+cqRpT1EebBwTn1oK9YfGYbdFz6I= github.com/jonboulle/clockwork v0.5.0/go.mod h1:3mZlmanh0g2NDKO5TWZVJAfofYk64M7XN3SzBPjZF60= -github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= -github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/josharian/native v1.1.0 h1:uuaP0hAbW7Y4l0ZRQ6C9zfb7Mg1mbFKry/xzDAfmtLA= github.com/josharian/native v1.1.0/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w= github.com/joshdk/go-junit v1.0.0 h1:S86cUKIdwBHWwA6xCmFlf3RTLfVXYQfvanM5Uh+K6GE= @@ -427,10 +429,8 @@ github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9 github.com/longhorn/go-iscsi-helper v0.0.0-20210330030558-49a327fb024e h1:hz4quJkaJWDo+xW+G6wTF6d6/95QvJ+o2D0+bB/tJ1U= github.com/longhorn/go-iscsi-helper v0.0.0-20210330030558-49a327fb024e/go.mod h1:9z/y9glKmWEdV50tjlUPxFwi1goQfIrrsoZbnMyIZbY= github.com/longhorn/nsfilelock v0.0.0-20200723175406-fa7c83ad0003/go.mod h1:0CLeXlf59Lg6C0kjLSDf47ft73Dh37CwymYRKWwAn04= -github.com/lufia/plan9stats v0.0.0-20251013123823-9fd1530e3ec3 h1:PwQumkgq4/acIiZhtifTV5OUqqiP82UAl0h87xj/l9k= -github.com/lufia/plan9stats v0.0.0-20251013123823-9fd1530e3ec3/go.mod h1:autxFIvghDt3jPTLoqZ9OZ7s9qTGNAWmYCjVFWPX/zg= -github.com/mailru/easyjson v0.9.1 h1:LbtsOm5WAswyWbvTEOqhypdPeZzHavpZx96/n553mR8= -github.com/mailru/easyjson v0.9.1/go.mod h1:1+xMtQp2MRNVL/V1bOzuP3aP8VNwRW55fQUto+XFtTU= +github.com/lufia/plan9stats v0.0.0-20260330125221-c963978e514e h1:Q6MvJtQK/iRcRtzAscm/zF23XxJlbECiGPyRicsX+Ak= +github.com/lufia/plan9stats v0.0.0-20260330125221-c963978e514e/go.mod h1:autxFIvghDt3jPTLoqZ9OZ7s9qTGNAWmYCjVFWPX/zg= github.com/maruel/natural v1.1.1 h1:Hja7XhhmvEFhcByqDoHz9QZbkWey+COd9xWfCfn1ioo= github.com/maruel/natural v1.1.1/go.mod h1:v+Rfd79xlw1AgVBjbO0BEQmptqb5HvL/k9GRHB7ZKEg= github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= @@ -453,6 +453,8 @@ github.com/microsoft/go-mssqldb v1.10.0/go.mod h1:mnG7lGa9iYJbzJqGCXyuQCegStKMr3 github.com/miekg/dns v1.1.46/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME= github.com/miekg/dns v1.1.72 h1:vhmr+TF2A3tuoGNkLDFK9zi36F2LS+hKTRW0Uf8kbzI= github.com/miekg/dns v1.1.72/go.mod h1:+EuEPhdHOsfk6Wk5TT2CzssZdqkmFhf8r+aVyDEToIs= +github.com/milo-os/ipam v0.3.2-0.20260813020700-13bd2c8c077b h1:iJ0zDKVRgC4cz0/eOX8dp+wBNw8iqaGI87zwgqdBZH8= +github.com/milo-os/ipam v0.3.2-0.20260813020700-13bd2c8c077b/go.mod h1:Jj7xg4lJi9psE0+4PuOg/GQOG8rG13h112xYoM994rc= github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= @@ -465,16 +467,16 @@ github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3N github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= github.com/moby/go-archive v0.2.0 h1:zg5QDUM2mi0JIM9fdQZWC7U8+2ZfixfTYoHL7rWUcP8= github.com/moby/go-archive v0.2.0/go.mod h1:mNeivT14o8xU+5q1YnNrkQVpK+dnNe/K6fHqnTg4qPU= -github.com/moby/moby/api v1.54.2 h1:wiat9QAhnDQjA7wk1kh/TqHz2I1uUA7M7t9SAl/JNXg= -github.com/moby/moby/api v1.54.2/go.mod h1:+RQ6wluLwtYaTd1WnPLykIDPekkuyD/ROWQClE83pzs= -github.com/moby/moby/client v0.4.1 h1:DMQgisVoMkmMs7fp3ROSdiBnoAu8+vo3GggFl06M/wY= -github.com/moby/moby/client v0.4.1/go.mod h1:z52C9O2POPOsnxZAy//WtKcQ32P+jT/NGeXu/7nfjGQ= -github.com/moby/patternmatcher v0.6.0 h1:GmP9lR19aU5GqSSFko+5pRqHi+Ohk1O69aFiKkVGiPk= -github.com/moby/patternmatcher v0.6.0/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= +github.com/moby/moby/api v1.55.0 h1:2/sexvQyqIWS8pRSCFddBfpW2qE7vR7FCL+vN8pxwMc= +github.com/moby/moby/api v1.55.0/go.mod h1:+RQ6wluLwtYaTd1WnPLykIDPekkuyD/ROWQClE83pzs= +github.com/moby/moby/client v0.5.0 h1:5XhyPk2fuOWf6RlSFa3MkIIgDZkF25xToXW8Q/BH7cc= +github.com/moby/moby/client v0.5.0/go.mod h1:rcVpF8ncl9vo5gaIBdol6CnbEtSj1uxMvEV/UrykF/s= +github.com/moby/patternmatcher v0.6.1 h1:qlhtafmr6kgMIJjKJMDmMWq7WLkKIo23hsrpR3x084U= +github.com/moby/patternmatcher v0.6.1/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= github.com/moby/spdystream v0.5.1 h1:9sNYeYZUcci9R6/w7KDaFWEWeV4LStVG78Mpyq/Zm/Y= github.com/moby/spdystream v0.5.1/go.mod h1:xBAYlnt/ay+11ShkdFKNAG7LsyK/tmNBVvVOwrfMgdI= -github.com/moby/sys/sequential v0.6.0 h1:qrx7XFUd/5DxtqcoH1h438hF5TmOvzC/lspjy7zgvCU= -github.com/moby/sys/sequential v0.6.0/go.mod h1:uyv8EUTrca5PnDsdMGXhZe6CCe8U/UiTWd+lL+7b/Ko= +github.com/moby/sys/sequential v0.7.0 h1:ASQNGNROJSuOO6LL6bPHbKvuZu6NU8P4ldPWk31zj/8= +github.com/moby/sys/sequential v0.7.0/go.mod h1:NfSTAp6V3fw4tmkD62PEcOKeZKquXT8VKCkf7aVR79o= github.com/moby/sys/user v0.4.0 h1:jhcMKit7SA80hivmFJcbB1vqmw//wU61Zdui2eQXuMs= github.com/moby/sys/user v0.4.0/go.mod h1:bG+tYYYJgaMtRKgEmuueC0hJEAZWwtIbZTB+85uoHjs= github.com/moby/sys/userns v0.1.0 h1:tVLXkFOxVu9A64/yh59slHVv9ahO9UIev4JZusOLG/g= @@ -589,8 +591,8 @@ github.com/segmentio/ksuid v1.0.4 h1:sBo2BdShXjmcugAMwjugoGUdUV0pcxY5mW4xKRn3v4c github.com/segmentio/ksuid v1.0.4/go.mod h1:/XUiZBD3kVx5SmUOl55voK5yeAbBNNIed+2O73XgrPE= github.com/sergi/go-diff v1.4.0 h1:n/SP9D5ad1fORl+llWyN+D6qoUETXNZARKjyY2/KVCw= github.com/sergi/go-diff v1.4.0/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= -github.com/shirou/gopsutil/v4 v4.26.4 h1:B4SXVbcwTyrocPHEmWBC4uCYr4Xcu3MK1TXqbprAOWY= -github.com/shirou/gopsutil/v4 v4.26.4/go.mod h1:LZ6ewCSkBqUpvSOf+LsTGnRinC6iaNUNMGBtDkJBaLQ= +github.com/shirou/gopsutil/v4 v4.26.6 h1:Mzr/npDtQC/xpeEuQKHZt8Zo9CmPvhTj8nkR8w5TLDs= +github.com/shirou/gopsutil/v4 v4.26.6/go.mod h1:LZ6ewCSkBqUpvSOf+LsTGnRinC6iaNUNMGBtDkJBaLQ= github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k= github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME= github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= @@ -642,10 +644,10 @@ github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= -github.com/tklauser/go-sysconf v0.3.16 h1:frioLaCQSsF5Cy1jgRBrzr6t502KIIwQ0MArYICU0nA= -github.com/tklauser/go-sysconf v0.3.16/go.mod h1:/qNL9xxDhc7tx3HSRsLWNnuzbVfh3e7gh/BmM179nYI= -github.com/tklauser/numcpus v0.11.0 h1:nSTwhKH5e1dMNsCdVBukSZrURJRoHbSEQjdEbY+9RXw= -github.com/tklauser/numcpus v0.11.0/go.mod h1:z+LwcLq54uWZTX0u/bGobaV34u6V7KNlTZejzM6/3MQ= +github.com/tklauser/go-sysconf v0.4.0 h1:7H0uAN+7RkwWRaxhYXDLqa5V3LPrJeV8wmD9dRUgPQU= +github.com/tklauser/go-sysconf v0.4.0/go.mod h1:8mTNWyog7H+MpKijp4VmKJAd2bbYQ2zuUwkYRbUArPI= +github.com/tklauser/numcpus v0.12.0 h1:NR85qdvHA9pFse3x3weVZ0r0ST8R6l5RHbZrlRaqob4= +github.com/tklauser/numcpus v0.12.0/go.mod h1:ABHeXzJnr/qqwguhClkZKT1/8VABcYrsyUiUGobwWJg= github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 h1:6fotK7otjonDflCTK0BCfls4SPy3NcCVb5dqqmbRknE= github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75/go.mod h1:KO6IkyS8Y3j8OdNO85qEYBsRPuteD+YciPomcXdrMnk= github.com/vishvananda/netlink v1.3.1 h1:3AEMt62VKqz90r0tmNhog0r/PpWKmrEShJU0wJW6bV0= @@ -692,8 +694,8 @@ go.etcd.io/raft/v3 v3.6.0 h1:5NtvbDVYpnfZWcIHgGRk9DyzkBIXOi8j+DDp1IcnUWQ= go.etcd.io/raft/v3 v3.6.0/go.mod h1:nLvLevg6+xrVtHUmVaTcTz603gQPHfh7kUAwV6YpfGo= go.miloapis.com/dns-operator v0.5.1 h1:3HmturI/Opl+l4/KurN6B0/XEeVefoSvYU0ewcZ8BZ8= go.miloapis.com/dns-operator v0.5.1/go.mod h1:yQSBOx1ZhSzZgg5RyNKyNHfZgjXRKhxkcGD+jvbebQE= -go.miloapis.com/milo v0.28.1 h1:30bQS4EwadbOBsn3UC8qSBENN1koQgsQpcvqPjXfmlI= -go.miloapis.com/milo v0.28.1/go.mod h1:p9O2kk194mvoL8rhqjwb+LWB+GIyY4vQqiTowwibVWo= +go.miloapis.com/milo v0.28.4-0.20260629130346-79689376fe11 h1:gWqYe5aYpRn7jFUo2hxSsvj0KHC2xFzK2RGCcd3ClHg= +go.miloapis.com/milo v0.28.4-0.20260629130346-79689376fe11/go.mod h1:p9O2kk194mvoL8rhqjwb+LWB+GIyY4vQqiTowwibVWo= go.mongodb.org/mongo-driver v1.11.4/go.mod h1:PTSz5yu21bkT/wXpkS7WR5f0ddqw5quethTUn9WM+2g= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= @@ -703,8 +705,8 @@ go.opentelemetry.io/contrib/exporters/autoexport v0.67.0 h1:4fnRcNpc6YFtG3zsFw9a go.opentelemetry.io/contrib/exporters/autoexport v0.67.0/go.mod h1:qTvIHMFKoxW7HXg02gm6/Wofhq5p3Ib/A/NNt1EoBSQ= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0 h1:yI1/OhfEPy7J9eoa6Sj051C7n5dvpj0QX8g4sRchg04= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0/go.mod h1:NoUCKYWK+3ecatC4HjkRktREheMeEtrXoQxrqYFeHSc= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.68.0 h1:CqXxU8VOmDefoh0+ztfGaymYbhdB/tT3zs79QaZTNGY= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.68.0/go.mod h1:BuhAPThV8PBHBvg8ZzZ/Ok3idOdhWIodywz2xEcRbJo= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0 h1:8tvICD4vSTOOsNrsI4Ljf6C+6UKvpTEH5XY3JMoyPoo= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0/go.mod h1:z9+yiacE0IHRqM4qFfkbt/JYlmYXgss8GY/jXoNuPJI= go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU= go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc= go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.19.0 h1:Dn8rkudDzY6KV9dr/D/bTUuWgqDf9xe0rr4G2elrn0Y= @@ -762,8 +764,8 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.52.0 h1:RMs7fP2rXdep0CftQlK8Uf+kibLm7qkCcradZWYz988= -golang.org/x/crypto v0.52.0/go.mod h1:1QgfPxDqh0T2M/elOJtp9RvuR95kVjir0e6/BvEmGbc= +golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw= +golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk= golang.org/x/crypto/x509roots/fallback v0.0.0-20250406160420-959f8f3db0fb h1:Iu0p/klM0SM7atONioa/bPhLS7cjhnip99x1OIGibwg= golang.org/x/crypto/x509roots/fallback v0.0.0-20250406160420-959f8f3db0fb/go.mod h1:lxN5T34bK4Z/i6cMaU7frUU57VkDXFD4Kamfl/cp9oU= golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f h1:W3F4c+6OLc6H2lb//N1q4WpJkhzJCK5J6kUi1NTVXfM= @@ -771,8 +773,8 @@ golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f/go.mod h1:J1xhfL/vlindoeF/aI golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.36.0 h1:JJjpVx6myfUsUdAzZuOSTTmRE0PfZeNWzzvKrP7amb4= -golang.org/x/mod v0.36.0/go.mod h1:moc6ELqsWcOw5Ef3xVprK5ul/MvtVvkIXLziUOICjUQ= +golang.org/x/mod v0.37.0 h1:vF1DjpVEshcIqoEaauuHebaLk1O1forxjxBaVn884JQ= +golang.org/x/mod v0.37.0/go.mod h1:m8S8VeM9r4dzDwjrKO0a1sZP3YjeMamRRlD+fmR2Q/0= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -783,8 +785,8 @@ golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96b golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= -golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= +golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o= +golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec= golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -792,8 +794,8 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= -golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= +golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -815,18 +817,18 @@ golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= -golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.43.0 h1:S4RLU2sB31O/NCl+zFN9Aru9A/Cq2aqKpTZJ6B+DwT4= -golang.org/x/term v0.43.0/go.mod h1:lrhlHNdQJHO+1qVYiHfFKVuVioJIheAc3fBSMFYEIsk= +golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0= +golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= -golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= +golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs= +golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -834,8 +836,8 @@ golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.45.0 h1:18qN3FAooORvApf5XjCXgsuayZOEtXf6JK18I3+ONa8= -golang.org/x/tools v0.45.0/go.mod h1:LuUGqqaXcXMEFEruIVJVm5mgDD8vww/z/SR1gQ4uE/0= +golang.org/x/tools v0.47.0 h1:7Kn5x/d1svx/PzryTsqeoZN4TZwqeH5pGWjefhLi/1Q= +golang.org/x/tools v0.47.0/go.mod h1:dFHnyTvFWY212G+h7ZY4Vsp/K3U4/7W9TyVaAul8uCA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -898,16 +900,16 @@ k8s.io/klog/v2 v2.140.0 h1:Tf+J3AH7xnUzZyVVXhTgGhEKnFqye14aadWv7bzXdzc= k8s.io/klog/v2 v2.140.0/go.mod h1:o+/RWfJ6PwpnFn7OyAG3QnO47BFsymfEfrz6XyYSSp0= k8s.io/kms v0.36.1 h1:XdvKpywoW4k7YUHDh5uYP4mahJXECswHGfCddBBYLZs= k8s.io/kms v0.36.1/go.mod h1:g91diTD9h0oJCCHkTb00krlF+Qm5HTnkWLi9Q/TpRoc= -k8s.io/kube-openapi v0.0.0-20260317180543-43fb72c5454a h1:xCeOEAOoGYl2jnJoHkC3hkbPJgdATINPMAxaynU2Ovg= -k8s.io/kube-openapi v0.0.0-20260317180543-43fb72c5454a/go.mod h1:uGBT7iTA6c6MvqUvSXIaYZo9ukscABYi2btjhvgKGZ0= +k8s.io/kube-openapi v0.0.0-20260330154417-16be699c7b31 h1:V+sn9a/1fEYDGwnllCmqXBk8x7obZ+hl869Q3Abumkg= +k8s.io/kube-openapi v0.0.0-20260330154417-16be699c7b31/go.mod h1:uGBT7iTA6c6MvqUvSXIaYZo9ukscABYi2btjhvgKGZ0= k8s.io/kubectl v0.36.1 h1:96HqS9twIdHM0MlJLTwbo14b9kUKPkOzZ4tlRDLv4qI= k8s.io/kubectl v0.36.1/go.mod h1:/DGPAIewKsFWF9VFgGvkPhao2Ev4SNuE3BioZo8yPbk= k8s.io/metrics v0.36.1 h1:MQPb+G4RhrKEpt8NETPssbW8QgGUc4Jbqu1jx+kPqGk= k8s.io/metrics v0.36.1/go.mod h1:xqS8XcWLjDzo6E7DJm/GfjKpRKdN5/MtJAQFuV6nLUc= k8s.io/streaming v0.36.1 h1:L+K68n4Gg940BGNNYtUBvL1WTLL0YnKT3s+P1MNAmR4= k8s.io/streaming v0.36.1/go.mod h1:z6fV3D+NVkoeqRMtWwlUZK6U17SY/LqNzOxWL6GyR/s= -k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2 h1:AZYQSJemyQB5eRxqcPky+/7EdBj0xi3g0ZcxxJ7vbWU= -k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2/go.mod h1:xDxuJ0whA3d0I4mf/C4ppKHxXynQ+fxnkmQH0vTHnuk= +k8s.io/utils v0.0.0-20260319190234-28399d86e0b5 h1:kBawHLSnx/mYHmRnNUf9d4CpjREbeZuxoSGOX/J+aYM= +k8s.io/utils v0.0.0-20260319190234-28399d86e0b5/go.mod h1:xDxuJ0whA3d0I4mf/C4ppKHxXynQ+fxnkmQH0vTHnuk= oras.land/oras-go/v2 v2.6.0 h1:X4ELRsiGkrbeox69+9tzTu492FMUu7zJQW6eJU+I2oc= oras.land/oras-go/v2 v2.6.0/go.mod h1:magiQDfG6H1O9APp+rOsvCPcW1GD2MM7vgnKY0Y+u1o= periph.io/x/host/v3 v3.8.5 h1:g4g5xE1XZtDiGl1UAJaUur1aT7uNiFLMkyMEiZ7IHII= diff --git a/hack/fetch-oci-bundle.sh b/hack/fetch-oci-bundle.sh new file mode 100755 index 00000000..a5da7bad --- /dev/null +++ b/hack/fetch-oci-bundle.sh @@ -0,0 +1,105 @@ +#!/usr/bin/env bash +# Fetch a published manifest bundle from an OCI registry and extract it. +# +# Used for both the IPAM bundle and the dns-operator bundle. Sourcing manifests +# this way rather than from a Go module directory also removes a failure mode: +# `go list -m -f {{.Dir}}` returns an empty string on a cold module cache and +# exits 0, which silently produced paths with a leading slash. +# +# A bundle is a flux OCI artifact: a single layer of media type +# application/vnd.cncf.flux.content.v1.tar+gzip carrying the repo's config/ +# tree. It has no org.opencontainers.image.title annotation, so `oras pull` +# extracts nothing; reading the layer blob directly works with any OCI client +# and needs no flux-specific tooling. We use crane because it installs with +# `go install` like every other pinned tool here, where the flux CLI's own +# go.mod replace directives make that fail. +# +# Pull is anonymous — the repository is public, so CI needs no registry +# credentials. +# +# The revision check takes whatever substring identifies the pinned version — +# a commit sha for a bundle pinned to a commit, a tag for one pinned to a +# release — and fails if the artifact does not carry it. +# +# Usage: +# fetch-oci-bundle.sh + +set -euo pipefail + +CRANE="${1:?crane binary path required}" +CRANE_VERSION="${2:?crane version required}" +REPO="${3:?bundle repository required}" +DIGEST="${4:?bundle digest required}" +REVISION="${5:?expected revision required}" +OUT="${6:?output directory required}" + +# Install crane on demand rather than making the caller remember a prep step. +# `up` does not depend on the tools task, and the workflow that runs the e2e +# suite installs its binaries from an explicit list, so requiring crane to be +# there already means it works on whichever machine happened to install it and +# nowhere else. Same shape as the tools task's own per-binary guard. +if [ ! -x "$CRANE" ]; then + echo "🔧 installing crane ${CRANE_VERSION} into $(dirname "$CRANE")" >&2 + if ! GOBIN="$(cd "$(dirname "$CRANE")" && pwd)" \ + go install "github.com/google/go-containerregistry/cmd/crane@${CRANE_VERSION}" >&2; then + echo "❌ could not install crane ${CRANE_VERSION}; needs Go and network access" >&2 + exit 1 + fi +fi + +if [ ! -x "$CRANE" ]; then + echo "❌ no crane binary at ${CRANE} after installing" >&2 + exit 1 +fi + +# ghcr.io occasionally times out on the first dial. A registry read is now on +# the critical path of every `up`, so retry rather than fail the whole bring-up +# on one flaky connection. +retry() { + local attempt=1 + until "$@"; do + if [ "$attempt" -ge 3 ]; then + echo "❌ '$*' failed after ${attempt} attempts" >&2 + return 1 + fi + echo " retrying (${attempt}/3) after registry error" >&2 + attempt=$((attempt + 1)) + sleep $((attempt * 3)) + done +} + +manifest="$(retry "$CRANE" manifest "${REPO}@${DIGEST}")" + +field() { + printf '%s' "$manifest" | python3 -c "import json,sys; print($1)" +} + +# The artifact must carry the revision go.mod pins, or the manifests and the Go +# types NSO compiles against have drifted apart — exactly the failure the digest +# pin exists to prevent, so it is checked rather than assumed. +revision="$(field "json.load(sys.stdin).get('annotations',{}).get('org.opencontainers.image.revision','')")" +case "$revision" in + *"$REVISION"*) ;; + *) + echo "❌ bundle revision '${revision}' does not carry the pinned revision ${REVISION}" >&2 + exit 1 + ;; +esac + +layer="$(field "json.load(sys.stdin)['layers'][0]['digest']")" + +rm -rf "$OUT" +mkdir -p "$OUT" +# Buffer the blob before extracting: piping crane straight into tar hides a +# mid-transfer registry failure behind tar's exit status. +blob="$(mktemp)" +trap 'rm -f "$blob"' EXIT +retry "$CRANE" blob "${REPO}@${layer}" > "$blob" +tar xz -C "$OUT" < "$blob" + +if [ -z "$(ls -A "$OUT" 2>/dev/null)" ]; then + echo "❌ extracted bundle at ${OUT} is empty" >&2 + exit 1 +fi + +echo "$revision" diff --git a/hack/gen-ipam-impersonation-kubeconfig.sh b/hack/gen-ipam-impersonation-kubeconfig.sh new file mode 100755 index 00000000..7ede8ac9 --- /dev/null +++ b/hack/gen-ipam-impersonation-kubeconfig.sh @@ -0,0 +1,99 @@ +#!/usr/bin/env bash +# Write a kubeconfig whose contexts drive the IPAM aggregated apiserver as a +# specific project. +# +# IPAM scopes storage per tenant from three UserInfo.Extra keys +# (iam.miloapis.com/parent-api-group, parent-type, parent-name). Milo's front +# gate normally supplies them as X-Remote-Extra-* requestheader extras. A +# kubeconfig's as-user-extra becomes Impersonate-Extra-*, which the front proxy +# re-emits as X-Remote-Extra-* — the same place IPAM reads from. A request +# carrying none of the three reads nothing and its writes are refused, so every +# fixture and every assertion has to go through one of these contexts. +# +# Adapted from github.com/milo-os/ipam test/e2e/lib/gen-impersonation-kubeconfig.sh +# at ref 20865afe018c. The cross-project group is dropped: this env seeds no +# shared pools. +# +# Usage: gen-ipam-impersonation-kubeconfig.sh [project...] + +set -euo pipefail + +OUT="${1:?usage: gen-ipam-impersonation-kubeconfig.sh [project...]}" +SRC_CTX="${2:?source context required}" +shift 2 +PROJECTS=("$@") +if [ ${#PROJECTS[@]} -eq 0 ]; then + PROJECTS=(project-alpha project-beta) +fi + +TENANT_USER="e2e-tenant-tester" +PARENT_API_GROUP="resourcemanager.miloapis.com" +PARENT_TYPE="Project" + +# The output is assembled by hand rather than round-tripped through +# `kubectl config view`: kubectl v1.36 drops as / as-user-extra when it +# re-serialises a kubeconfig, which would silently remove the impersonation. +base="$(mktemp)" +trap 'rm -f "$base"' EXIT +kubectl --context "$SRC_CTX" config view --minify --flatten >"$base" + +BASE_CLUSTER="$(KUBECONFIG="$base" kubectl config view -o jsonpath="{.contexts[?(@.name=='${SRC_CTX}')].context.cluster}")" +BASE_USER="$(KUBECONFIG="$base" kubectl config view -o jsonpath="{.contexts[?(@.name=='${SRC_CTX}')].context.user}")" + +C_SERVER="$(KUBECONFIG="$base" kubectl config view --raw -o jsonpath="{.clusters[?(@.name=='${BASE_CLUSTER}')].cluster.server}")" +C_CA="$(KUBECONFIG="$base" kubectl config view --raw -o jsonpath="{.clusters[?(@.name=='${BASE_CLUSTER}')].cluster.certificate-authority-data}")" +C_INSECURE="$(KUBECONFIG="$base" kubectl config view --raw -o jsonpath="{.clusters[?(@.name=='${BASE_CLUSTER}')].cluster.insecure-skip-tls-verify}")" + +B_CERT="$(KUBECONFIG="$base" kubectl config view --raw -o jsonpath="{.users[?(@.name=='${BASE_USER}')].user.client-certificate-data}")" +B_KEY="$(KUBECONFIG="$base" kubectl config view --raw -o jsonpath="{.users[?(@.name=='${BASE_USER}')].user.client-key-data}")" +B_TOKEN="$(KUBECONFIG="$base" kubectl config view --raw -o jsonpath="{.users[?(@.name=='${BASE_USER}')].user.token}")" + +emit_base_credentials() { + [ -n "$B_CERT" ] && printf '%s\n' " client-certificate-data: ${B_CERT}" + [ -n "$B_KEY" ] && printf '%s\n' " client-key-data: ${B_KEY}" + [ -n "$B_TOKEN" ] && printf '%s\n' " token: ${B_TOKEN}" + return 0 +} + +{ + printf '%s\n' "apiVersion: v1" + printf '%s\n' "kind: Config" + printf '%s\n' "current-context: tenant-platform" + printf '%s\n' "clusters:" + printf '%s\n' "- name: ${BASE_CLUSTER}" + printf '%s\n' " cluster:" + printf '%s\n' " server: ${C_SERVER}" + [ -n "$C_CA" ] && printf '%s\n' " certificate-authority-data: ${C_CA}" + [ "$C_INSECURE" = "true" ] && printf '%s\n' " insecure-skip-tls-verify: true" + printf '%s\n' "users:" + printf '%s\n' "- name: ${BASE_USER}" + printf '%s\n' " user:" + emit_base_credentials + for proj in "${PROJECTS[@]}"; do + printf '%s\n' "- name: tenant-${proj}-as" + printf '%s\n' " user:" + emit_base_credentials + printf '%s\n' " as: ${TENANT_USER}" + printf '%s\n' " as-user-extra:" + printf '%s\n' " iam.miloapis.com/parent-api-group:" + printf '%s\n' " - ${PARENT_API_GROUP}" + printf '%s\n' " iam.miloapis.com/parent-type:" + printf '%s\n' " - ${PARENT_TYPE}" + printf '%s\n' " iam.miloapis.com/parent-name:" + printf '%s\n' " - ${proj}" + done + printf '%s\n' "contexts:" + for proj in "${PROJECTS[@]}"; do + printf '%s\n' "- name: tenant-${proj}" + printf '%s\n' " context:" + printf '%s\n' " cluster: ${BASE_CLUSTER}" + printf '%s\n' " user: tenant-${proj}-as" + done + printf '%s\n' "- name: tenant-platform" + printf '%s\n' " context:" + printf '%s\n' " cluster: ${BASE_CLUSTER}" + printf '%s\n' " user: ${BASE_USER}" +} >"$OUT" + +echo "wrote ${OUT} (source context ${SRC_CTX})" +echo " contexts: tenant-platform, $(printf 'tenant-%s ' "${PROJECTS[@]}")" diff --git a/hack/ipam-clear-fixtures.sh b/hack/ipam-clear-fixtures.sh new file mode 100755 index 00000000..986002ef --- /dev/null +++ b/hack/ipam-clear-fixtures.sh @@ -0,0 +1,112 @@ +#!/usr/bin/env bash +# Delete every IPAM object in the fixture projects. +# +# IPPool and IPClass are cluster-scoped, so chainsaw's namespace teardown never +# reaches them. A suite that dies partway leaves its pools and classes behind, +# and the next run fails on "already exists" for a reason unrelated to what it +# was testing. +# +# Order matters: a bound allocation refuses deletion of the pool it came from, +# and a class with pools offering it refuses deletion too. Claims first, +# allocations next, then pools, then classes. +# +# This script exists to guarantee a clean slate, so it must never report one it +# did not produce. It proves it can reach each tenant before deleting anything, +# and it verifies all four kinds are empty afterwards. +# +# Adapted from github.com/milo-os/ipam test/e2e/lib/clear-tenant-fixtures.sh at +# ref 20865afe018c. +# +# Usage: ipam-clear-fixtures.sh [project...] + +set -euo pipefail + +CONTEXT="${1:?usage: ipam-clear-fixtures.sh [project...]}" +shift +PROJECTS=("$@") +if [ ${#PROJECTS[@]} -eq 0 ]; then + PROJECTS=(project-alpha project-beta) +fi + +DELETE_TIMEOUT="${IPAM_CLEAR_TIMEOUT:-60s}" +KINDS="ipclaims ipallocations ippools ipclasses" + +here="$(cd "$(dirname "$0")" && pwd)" + +# Every call reaches IPAM as a project; a request without the project extras +# reads nothing, which would make an empty result look like a clean tenant. +k() { + IPAM_KUBE_CONTEXT="$CONTEXT" "${here}/ipam-tenant-kubectl.sh" "$@" +} + +# Positive control. Reading a type that always resolves proves the aggregated +# API is reachable AND that this context's impersonation is accepted. Without +# it, an unreachable cluster or a wrong context name would make every delete +# below a no-op and this script would report a clean tenant it never touched. +verify_reachable() { + local proj="$1" out + if ! out="$(k "$proj" get ipclasses 2>&1)"; then + echo "❌ cannot reach IPAM as ${proj}; refusing to report a clean tenant" >&2 + echo " ${out}" >&2 + exit 1 + fi +} + +# A delete that fails because the object is already gone is success. A pool +# pinned by a child that has not been deleted yet is expected mid-loop and is +# resolved by the next pass — the residue check at the end is what decides +# whether clearing actually worked. Anything else is surfaced rather than +# swallowed. +attempt_delete() { + local proj="$1" kind="$2" out rc + set +e + out="$(k "$proj" delete "$kind" --all --all-namespaces --ignore-not-found \ + --timeout="$DELETE_TIMEOUT" 2>&1)" + rc=$? + set -e + if [ "$rc" -ne 0 ] && ! printf '%s' "$out" | grep -qiE 'not found|no matches for|cannot delete IPPool'; then + echo " ⚠️ deleting ${kind} in ${proj}: ${out}" >&2 + fi +} + +residue() { + local proj="$1" kind="$2" + k "$proj" get "$kind" --all-namespaces \ + -o jsonpath='{range .items[*]}{.metadata.namespace}/{.metadata.name} {end}' 2>/dev/null || true +} + +for proj in "${PROJECTS[@]}"; do + verify_reachable "$proj" + + attempt_delete "$proj" ipclaims + attempt_delete "$proj" ipallocations + + # Pools need more than one pass. A cascade leaves child pools carved out of + # their parent, and the parent refuses to go while a carve is outstanding — + # but `delete --all` works through the list in name order, so a parent is + # usually attempted before the children that pin it. Each pass frees one + # level of the chain; the chain is at most a few deep. + for _ in 1 2 3 4; do + [ -z "$(residue "$proj" ippools)" ] && break + attempt_delete "$proj" ippools + done + + attempt_delete "$proj" ipclasses + + # The authoritative check, over every kind. Pools and classes block the next + # run with "already exists"; a stranded claim or allocation is worse, because + # a bound allocation refuses deletion of the pool it came from and wedges the + # run after that one. + left="" + for kind in $KINDS; do + found="$(residue "$proj" "$kind")" + [ -n "$found" ] && left="${left}${kind}: ${found}"$'\n' + done + if [ -n "$left" ]; then + echo "❌ ${proj} still holds IPAM fixtures after clearing:" >&2 + printf '%s' "$left" >&2 + exit 1 + fi + + echo "cleared IPAM fixtures in ${proj}" +done diff --git a/hack/ipam-tenant-kubectl.sh b/hack/ipam-tenant-kubectl.sh new file mode 100755 index 00000000..379d0f2a --- /dev/null +++ b/hack/ipam-tenant-kubectl.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +# Run kubectl against IPAM as a project tenant. +# +# IPAM scopes storage from three UserInfo.Extra keys. Milo's front gate supplies +# them as X-Remote-Extra-* requestheader extras; kubectl's --as-user-extra +# becomes Impersonate-Extra-*, which the front proxy re-emits as the same +# X-Remote-Extra-*, so it lands in UserInfo.Extra identically. A request +# carrying none of the three reads nothing and its writes are refused. +# +# This replaced a generated kubeconfig holding a flattened copy of the caller's +# credentials. kubectl takes the extras as flags directly, so there is no +# generated artefact to create before anything can talk to IPAM, and no token +# written to disk. +# +# Usage: ipam-tenant-kubectl.sh [kubectl args...] +# e.g. ipam-tenant-kubectl.sh project-alpha -n default get ipclaims + +set -euo pipefail + +PROJECT="${1:?usage: ipam-tenant-kubectl.sh [kubectl args...]}" +shift + +# The identity impersonated. Deliberately not cluster-admin: its access comes +# from the nso-ipam-tenant binding in test/e2e/fixtures/ipam/rbac.yaml, so the +# cross-project deny assertions cannot pass by privilege. +: "${IPAM_TENANT_USER:=e2e-tenant-tester}" +# Set by callers that already know which cluster; otherwise kubectl's current +# context applies. +CONTEXT="${IPAM_KUBE_CONTEXT:-}" + +set -- \ + --as="${IPAM_TENANT_USER}" \ + --as-user-extra="iam.miloapis.com/parent-api-group=resourcemanager.miloapis.com" \ + --as-user-extra="iam.miloapis.com/parent-type=Project" \ + --as-user-extra="iam.miloapis.com/parent-name=${PROJECT}" \ + "$@" + +if [ -n "$CONTEXT" ]; then + set -- --context "$CONTEXT" "$@" +fi + +exec kubectl "$@" diff --git a/internal/cmd/manager/manager.go b/internal/cmd/manager/manager.go index 11dd37b3..397c2d19 100644 --- a/internal/cmd/manager/manager.go +++ b/internal/cmd/manager/manager.go @@ -50,6 +50,7 @@ import ( networkingv1alphawebhooks "go.datum.net/network-services-operator/internal/webhook/v1alpha" webhookgatewayv1alpha1 "go.datum.net/network-services-operator/internal/webhook/v1alpha1" dnsv1alpha1 "go.miloapis.com/dns-operator/api/v1alpha1" + ipamv1alpha1 "go.miloapis.com/ipam/pkg/apis/ipam/v1alpha1" // +kubebuilder:scaffold:imports ) @@ -73,6 +74,7 @@ func init() { utilruntime.Must(cmacmev1.AddToScheme(scheme)) utilruntime.Must(cmv1.AddToScheme(scheme)) utilruntime.Must(dnsv1alpha1.AddToScheme(scheme)) + utilruntime.Must(ipamv1alpha1.AddToScheme(scheme)) // +kubebuilder:scaffold:scheme } @@ -370,6 +372,13 @@ func NewCommand(build BuildInfo) *cobra.Command { os.Exit(1) } + if serverConfig.NetworkInterface.Enabled { + if err := setupNetworkInterfaceClaimController(serverConfig, mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "NetworkInterfaceClaim") + os.Exit(1) + } + } + if err := (&controller.HTTPProxyReconciler{ Config: serverConfig, DownstreamCluster: downstreamCluster, @@ -492,48 +501,8 @@ func NewCommand(build BuildInfo) *cobra.Command { } } - if err := networkinggatewayv1webhooks.SetupGatewayWebhookWithManager(mgr, serverConfig); err != nil { - setupLog.Error(err, "unable to create webhook", "webhook", "Gateway") - os.Exit(1) - } - - if err := networkinggatewayv1webhooks.SetupHTTPRouteWebhookWithManager(mgr, serverConfig); err != nil { - setupLog.Error(err, "unable to create webhook", "webhook", "HTTPRoute") - os.Exit(1) - } - - if err := networkinggatewayv1webhooks.SetupBackendTLSPolicyWebhookWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create webhook", "webhook", "HTTPRoute") - os.Exit(1) - } - - if err := networkingv1alphawebhooks.SetupHTTPProxyWebhookWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create webhook", "webhook", "HTTPProxy") - os.Exit(1) - } - - if err := networkingv1alphawebhooks.SetupDomainWebhookWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create webhook", "webhook", "Domain") - os.Exit(1) - } - - if err = webhookgatewayv1alpha1.SetupBackendTrafficPolicyWebhookWithManager(mgr, serverConfig); err != nil { - setupLog.Error(err, "unable to create webhook", "webhook", "BackendTrafficPolicy") - os.Exit(1) - } - - if err = webhookgatewayv1alpha1.SetupSecurityPolicyWebhookWithManager(mgr, serverConfig); err != nil { - setupLog.Error(err, "unable to create webhook", "webhook", "SecurityPolicy") - os.Exit(1) - } - - if err = webhookgatewayv1alpha1.SetupHTTPRouteFilterWebhookWithManager(mgr, serverConfig); err != nil { - setupLog.Error(err, "unable to create webhook", "webhook", "HTTPRouteFilter") - os.Exit(1) - } - - if err = webhookgatewayv1alpha1.SetupBackendWebhookWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create webhook", "webhook", "Backend") + if webhook, err := setupWebhooks(mgr, serverConfig); err != nil { + setupLog.Error(err, "unable to create webhook", "webhook", webhook) os.Exit(1) } @@ -611,6 +580,88 @@ type legacyRunnableProvider interface { Run(context.Context, mcmanager.Manager) error } +// setupWebhooks registers every admission webhook. It returns the name of the +// webhook that failed. +func setupWebhooks(mgr mcmanager.Manager, serverConfig config.NetworkServicesOperator) (string, error) { + registrations := []struct { + name string + setup func() error + }{ + {"Gateway", func() error { + return networkinggatewayv1webhooks.SetupGatewayWebhookWithManager(mgr, serverConfig) + }}, + {"HTTPRoute", func() error { + return networkinggatewayv1webhooks.SetupHTTPRouteWebhookWithManager(mgr, serverConfig) + }}, + {"BackendTLSPolicy", func() error { + return networkinggatewayv1webhooks.SetupBackendTLSPolicyWebhookWithManager(mgr) + }}, + {"HTTPProxy", func() error { + return networkingv1alphawebhooks.SetupHTTPProxyWebhookWithManager(mgr) + }}, + {"Domain", func() error { + return networkingv1alphawebhooks.SetupDomainWebhookWithManager(mgr) + }}, + {"BackendTrafficPolicy", func() error { + return webhookgatewayv1alpha1.SetupBackendTrafficPolicyWebhookWithManager(mgr, serverConfig) + }}, + {"SecurityPolicy", func() error { + return webhookgatewayv1alpha1.SetupSecurityPolicyWebhookWithManager(mgr, serverConfig) + }}, + {"HTTPRouteFilter", func() error { + return webhookgatewayv1alpha1.SetupHTTPRouteFilterWebhookWithManager(mgr, serverConfig) + }}, + {"Backend", func() error { + return webhookgatewayv1alpha1.SetupBackendWebhookWithManager(mgr) + }}, + } + + for _, registration := range registrations { + if err := registration.setup(); err != nil { + return registration.name, err + } + } + return "", nil +} + +// setupNetworkInterfaceClaimController wires the controllers to one IPAM +// connection. Impersonation names the project on each request. +func setupNetworkInterfaceClaimController( + serverConfig config.NetworkServicesOperator, + mgr mcmanager.Manager, +) error { + ipamRestConfig, err := serverConfig.IPAM.RestConfig() + if err != nil { + return fmt.Errorf("unable to load IPAM kubeconfig: %w", err) + } + + ipamScheme, err := controller.IPAMScheme() + if err != nil { + return fmt.Errorf("unable to build IPAM scheme: %w", err) + } + + ipamClients, err := controller.NewIPAMClientFactory( + ipamRestConfig, + ipamScheme, + serverConfig.IPAM.ImpersonateUsername, + ) + if err != nil { + return fmt.Errorf("unable to build IPAM client factory: %w", err) + } + + if err := (&controller.NetworkInterfaceClaimReconciler{ + Config: serverConfig, + IPAM: ipamClients, + }).SetupWithManager(mgr); err != nil { + return err + } + + return (&controller.NetworkInterfaceReconciler{ + Config: serverConfig, + IPAM: ipamClients, + }).SetupWithManager(mgr) +} + func initializeClusterDiscovery( serverConfig config.NetworkServicesOperator, deploymentCluster cluster.Cluster, diff --git a/internal/config/config.go b/internal/config/config.go index 575e7f8f..0c6c852d 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -86,6 +86,95 @@ type NetworkServicesOperator struct { // ProjectClient configures the Kubernetes client connection used for both // project discovery and per-project cluster connections. ProjectClient ClientConnectionConfig `json:"projectClient,omitempty"` + + // IPAM configures the connection to the IPAM aggregated API server that + // network interface addresses are claimed from. + IPAM IPAMConfig `json:"ipam,omitempty"` + + // NetworkInterface configures the controller that fulfils + // NetworkInterfaceClaims. + NetworkInterface NetworkInterfaceConfig `json:"networkInterface,omitempty"` +} + +// +k8s:deepcopy-gen=true + +// IPAMConfig describes how the operator reaches the IPAM API server. One +// connection serves every project. Each request names its project through +// impersonation. +type IPAMConfig struct { + // KubeconfigPath is the path to a kubeconfig file pointing at the cluster + // serving the IPAM API. When empty, the operator's own in-cluster config is + // used. + KubeconfigPath string `json:"kubeconfigPath,omitempty"` + + // ImpersonateUsername is the user the operator impersonates when claiming + // addresses. IPAM checks this user's permissions, not the operator's. + // + // +default="nso-ipam-agent" + ImpersonateUsername string `json:"impersonateUsername,omitempty"` + + // Client configures the Kubernetes client connection to the IPAM API + // server. + Client ClientConnectionConfig `json:"client,omitempty"` +} + +func SetDefaults_IPAMConfig(obj *IPAMConfig) { + if obj.ImpersonateUsername == "" { + obj.ImpersonateUsername = "nso-ipam-agent" + } +} + +func (c *IPAMConfig) RestConfig() (*rest.Config, error) { + var ( + cfg *rest.Config + err error + ) + if c.KubeconfigPath == "" { + cfg, err = ctrl.GetConfig() + } else { + cfg, err = clientcmd.BuildConfigFromFlags("", c.KubeconfigPath) + } + if err != nil { + return nil, err + } + + c.Client.ApplyTo(cfg) + return cfg, nil +} + +// +k8s:deepcopy-gen=true + +// NetworkInterfaceConfig configures the NetworkInterfaceClaim controller. +type NetworkInterfaceConfig struct { + // Enabled registers the NetworkInterfaceClaim controller. Leave it off on a + // control plane that serves no location. + Enabled bool `json:"enabled,omitempty"` + + // Location is the location this control plane serves. Claims carry no + // location of their own, so every claim allocates against this one. + Location LocationConfig `json:"location,omitempty"` +} + +// +k8s:deepcopy-gen=true + +type LocationConfig struct { + Name string `json:"name,omitempty"` + + Namespace string `json:"namespace,omitempty"` +} + +func (c *NetworkInterfaceConfig) validate() error { + if !c.Enabled { + return nil + } + var errs []error + if c.Location.Name == "" { + errs = append(errs, errors.New("location.name is required when enabled is true")) + } + if c.Location.Namespace == "" { + errs = append(errs, errors.New("location.namespace is required when enabled is true")) + } + return errors.Join(errs...) } // +k8s:deepcopy-gen=true @@ -1273,6 +1362,9 @@ func (c *NetworkServicesOperator) Validate() error { if err := c.Gateway.validate(); err != nil { return fmt.Errorf("gateway: %w", err) } + if err := c.NetworkInterface.validate(); err != nil { + return fmt.Errorf("networkInterface: %w", err) + } return nil } diff --git a/internal/config/zz_generated.deepcopy.go b/internal/config/zz_generated.deepcopy.go index 131f416d..bee685a2 100644 --- a/internal/config/zz_generated.deepcopy.go +++ b/internal/config/zz_generated.deepcopy.go @@ -447,6 +447,22 @@ func (in *HTTPRouteValidationOptions) DeepCopy() *HTTPRouteValidationOptions { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *IPAMConfig) DeepCopyInto(out *IPAMConfig) { + *out = *in + out.Client = in.Client +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new IPAMConfig. +func (in *IPAMConfig) DeepCopy() *IPAMConfig { + if in == nil { + return nil + } + out := new(IPAMConfig) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *IrohConnectorConfig) DeepCopyInto(out *IrohConnectorConfig) { *out = *in @@ -496,6 +512,21 @@ func (in *LeaderElectionConfig) DeepCopy() *LeaderElectionConfig { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LocationConfig) DeepCopyInto(out *LocationConfig) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LocationConfig. +func (in *LocationConfig) DeepCopy() *LocationConfig { + if in == nil { + return nil + } + out := new(LocationConfig) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *MetricsServerConfig) DeepCopyInto(out *MetricsServerConfig) { *out = *in @@ -517,6 +548,22 @@ func (in *MetricsServerConfig) DeepCopy() *MetricsServerConfig { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetworkInterfaceConfig) DeepCopyInto(out *NetworkInterfaceConfig) { + *out = *in + out.Location = in.Location +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkInterfaceConfig. +func (in *NetworkInterfaceConfig) DeepCopy() *NetworkInterfaceConfig { + if in == nil { + return nil + } + out := new(NetworkInterfaceConfig) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *NetworkServicesOperator) DeepCopyInto(out *NetworkServicesOperator) { *out = *in @@ -535,6 +582,8 @@ func (in *NetworkServicesOperator) DeepCopyInto(out *NetworkServicesOperator) { out.ControlPlaneClient = in.ControlPlaneClient out.DownstreamClient = in.DownstreamClient out.ProjectClient = in.ProjectClient + out.IPAM = in.IPAM + out.NetworkInterface = in.NetworkInterface } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkServicesOperator. diff --git a/internal/config/zz_generated.defaults.go b/internal/config/zz_generated.defaults.go index 87f5179c..acc4d62d 100644 --- a/internal/config/zz_generated.defaults.go +++ b/internal/config/zz_generated.defaults.go @@ -359,4 +359,15 @@ func SetObjectDefaults_NetworkServicesOperator(in *NetworkServicesOperator) { if in.ProjectClient.Burst == 0 { in.ProjectClient.Burst = 100 } + SetDefaults_IPAMConfig(&in.IPAM) + if in.IPAM.ImpersonateUsername == "" { + in.IPAM.ImpersonateUsername = "nso-ipam-agent" + } + SetDefaults_ClientConnectionConfig(&in.IPAM.Client) + if in.IPAM.Client.QPS == 0 { + in.IPAM.Client.QPS = 50 + } + if in.IPAM.Client.Burst == 0 { + in.IPAM.Client.Burst = 100 + } } diff --git a/internal/controller/ipam_errors.go b/internal/controller/ipam_errors.go new file mode 100644 index 00000000..805db6c5 --- /dev/null +++ b/internal/controller/ipam_errors.go @@ -0,0 +1,57 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package controller + +import ( + "errors" + "net/http" + + apierrors "k8s.io/apimachinery/pkg/api/errors" +) + +// allocationFailureReason classifies why IPAM refused an allocation, by status +// code only. IPAM's own message is passed through unread. +type allocationFailureReason string + +const ( + allocationFailureExhausted allocationFailureReason = "AddressPoolExhausted" + allocationFailureConflict allocationFailureReason = "RetainedAddressConflict" + // Usually an address class or pool that is not configured, rather than a + // bad request, and the status code does not tell the two apart. + allocationFailureRejected allocationFailureReason = "AddressAllocationRejected" + allocationFailureUnknown allocationFailureReason = "AllocationFailed" +) + +const httpStatusInsufficientStorage = 507 + +func classifyAllocationFailure(err error) allocationFailureReason { + var status apierrors.APIStatus + if !errors.As(err, &status) { + return allocationFailureUnknown + } + + switch int(status.Status().Code) { + case httpStatusInsufficientStorage: + return allocationFailureExhausted + case http.StatusConflict: + return allocationFailureConflict + case http.StatusBadRequest, http.StatusUnprocessableEntity: + return allocationFailureRejected + default: + return allocationFailureUnknown + } +} + +func allocationFailureMessage(reason allocationFailureReason, request allocationRequest, err error) string { + switch reason { + case allocationFailureExhausted: + return "No address is left for " + request.describe() + ": " + err.Error() + case allocationFailureConflict: + return "A retained allocation still holds the name for " + request.describe() + ": " + err.Error() + case allocationFailureRejected: + return "IPAM would not allocate " + request.describe() + + ". This usually means the address class, or a pool backing it, is not configured in this project: " + err.Error() + default: + return "Allocating " + request.describe() + " failed: " + err.Error() + } +} diff --git a/internal/controller/ipam_project_client.go b/internal/controller/ipam_project_client.go new file mode 100644 index 00000000..22a9957f --- /dev/null +++ b/internal/controller/ipam_project_client.go @@ -0,0 +1,144 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package controller + +import ( + "context" + "fmt" + "sync" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" + + ipamv1alpha1 "go.miloapis.com/ipam/pkg/apis/ipam/v1alpha1" + iamv1alpha1 "go.miloapis.com/milo/pkg/apis/iam/v1alpha1" + resourcemanagerv1alpha1 "go.miloapis.com/milo/pkg/apis/resourcemanager/v1alpha1" + + "go.datum.net/network-services-operator/internal/downstreamclient" +) + +// Milo exports no constant for a kind name, so the project kind is spelled +// here. +const ipamParentType = "Project" + +// IPAMClientFactory returns a client bound to one project. Every IPAM request +// goes through one, so no request can reach IPAM without naming a project. +type IPAMClientFactory interface { + ClientForProject(project string) (client.Client, error) +} + +// NewIPAMClientFactory builds project-scoped clients from one connection. The +// clients are uncached, because a cache would watch every project served. +func NewIPAMClientFactory(base *rest.Config, scheme *runtime.Scheme, actAsUsername string) (IPAMClientFactory, error) { + if base == nil { + return nil, fmt.Errorf("a rest config is required") + } + if actAsUsername == "" { + return nil, fmt.Errorf("an impersonation username is required") + } + return &impersonatingIPAMClientFactory{ + base: base, + scheme: scheme, + actAsUsername: actAsUsername, + clients: map[string]client.Client{}, + }, nil +} + +type impersonatingIPAMClientFactory struct { + base *rest.Config + scheme *runtime.Scheme + actAsUsername string + + mu sync.Mutex + clients map[string]client.Client +} + +func (f *impersonatingIPAMClientFactory) ClientForProject(project string) (client.Client, error) { + if project == "" { + return nil, errNoProject + } + + f.mu.Lock() + defer f.mu.Unlock() + + if existing, ok := f.clients[project]; ok { + return existing, nil + } + + cfg := rest.CopyConfig(f.base) + cfg.Impersonate = rest.ImpersonationConfig{ + UserName: f.actAsUsername, + Extra: map[string][]string{ + iamv1alpha1.ParentAPIGroupExtraKey: {resourcemanagerv1alpha1.GroupVersion.Group}, + iamv1alpha1.ParentKindExtraKey: {ipamParentType}, + iamv1alpha1.ParentNameExtraKey: {project}, + }, + } + + cl, err := client.New(cfg, client.Options{Scheme: f.scheme}) + if err != nil { + return nil, fmt.Errorf("failed building IPAM client for project %q: %w", project, err) + } + + f.clients[project] = cl + return cl, nil +} + +// IPAMScheme is the scheme a project-scoped IPAM client is built with. +func IPAMScheme() (*runtime.Scheme, error) { + scheme := runtime.NewScheme() + if err := corev1.AddToScheme(scheme); err != nil { + return nil, err + } + if err := ipamv1alpha1.AddToScheme(scheme); err != nil { + return nil, err + } + return scheme, nil +} + +var errNoProject = fmt.Errorf("no project") + +// projectFromNamespace reads the project a namespace belongs to. A namespace +// that names no project resolves to nothing, never to a default. +func projectFromNamespace(ns *corev1.Namespace) (string, error) { + value, ok := ns.Labels[downstreamclient.UpstreamOwnerClusterNameLabel] + if !ok || value == "" { + return "", fmt.Errorf("namespace %q carries no %s label", ns.Name, downstreamclient.UpstreamOwnerClusterNameLabel) + } + + project := downstreamclient.UpstreamClusterNameFromLabel(value) + if project == "" { + return "", fmt.Errorf("namespace %q has %s=%q, which names no project", ns.Name, downstreamclient.UpstreamOwnerClusterNameLabel, value) + } + + return project, nil +} + +func projectNamespaceFromNamespace(ns *corev1.Namespace) (string, error) { + value, ok := ns.Labels[downstreamclient.UpstreamOwnerNamespaceLabel] + if !ok || value == "" { + return "", fmt.Errorf("namespace %q carries no %s label", ns.Name, downstreamclient.UpstreamOwnerNamespaceLabel) + } + return value, nil +} + +func ensureProjectNamespace(ctx context.Context, cl client.Client, name string) error { + var existing corev1.Namespace + err := cl.Get(ctx, client.ObjectKey{Name: name}, &existing) + if err == nil { + return nil + } + if !apierrors.IsNotFound(err) { + return err + } + + namespace := &corev1.Namespace{} + namespace.Name = name + if err := cl.Create(ctx, namespace); err != nil && !apierrors.IsAlreadyExists(err) { + return err + } + return nil +} diff --git a/internal/controller/metrics.go b/internal/controller/metrics.go index 78784ca8..71cf0c16 100644 --- a/internal/controller/metrics.go +++ b/internal/controller/metrics.go @@ -17,9 +17,20 @@ const ( metricLabelHostname = "hostname" metricLabelSecret = "secret" metricLabelReason = "reason" + metricLabelProject = "project" ) var ( + // missingAllocationsTotal counts addresses no IPClaim holds. IPAM may give + // the same address to another claim. + missingAllocationsTotal = promauto.NewCounterVec( + prometheus.CounterOpts{ + Name: "nso_network_interface_missing_allocations_total", + Help: "Total addresses advertised by a NetworkInterface with no IPClaim holding them, by project.", + }, + []string{metricLabelProject}, + ) + // replicatorConflictsTotal counts resource-version conflicts observed by the // gateway-resource-replicator controller. Conflicts arise when the upstream or // downstream API server rejects an update because the local object's diff --git a/internal/controller/networkinterface_controller.go b/internal/controller/networkinterface_controller.go new file mode 100644 index 00000000..73bf047a --- /dev/null +++ b/internal/controller/networkinterface_controller.go @@ -0,0 +1,132 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package controller + +import ( + "context" + "errors" + "fmt" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/log" + mcbuilder "sigs.k8s.io/multicluster-runtime/pkg/builder" + mcmanager "sigs.k8s.io/multicluster-runtime/pkg/manager" + mcreconcile "sigs.k8s.io/multicluster-runtime/pkg/reconcile" + + networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" + "go.datum.net/network-services-operator/internal/config" +) + +// NetworkInterfaceReconciler releases an interface's addresses when the +// interface is deleted. A retained interface has no claim to do this for it. +type NetworkInterfaceReconciler struct { + Config config.NetworkServicesOperator + IPAM IPAMClientFactory + + claims *NetworkInterfaceClaimReconciler + mgr mcmanager.Manager +} + +// +kubebuilder:rbac:groups=networking.datumapis.com,resources=networkinterfaces/finalizers,verbs=update + +func (r *NetworkInterfaceReconciler) Reconcile(ctx context.Context, req mcreconcile.Request) (ctrl.Result, error) { + logger := log.FromContext(ctx, "cluster", req.ClusterName) + + cl, err := r.mgr.GetCluster(ctx, req.ClusterName) + if err != nil { + return ctrl.Result{}, err + } + + logger.Info("reconciling network interface") + defer logger.Info("reconcile complete") + + return ctrl.Result{}, r.reconcileInterface(ctx, cl.GetClient(), req.NamespacedName) +} + +func (r *NetworkInterfaceReconciler) reconcileInterface( + ctx context.Context, + cl client.Client, + key client.ObjectKey, +) error { + var iface networkingv1alpha.NetworkInterface + if err := cl.Get(ctx, key, &iface); err != nil { + return client.IgnoreNotFound(err) + } + + if iface.DeletionTimestamp.IsZero() { + if controllerutil.AddFinalizer(&iface, networkInterfaceFinalizer) { + if err := cl.Update(ctx, &iface); err != nil { + return fmt.Errorf("failed adding finalizer: %w", err) + } + } + return nil + } + + if !controllerutil.ContainsFinalizer(&iface, networkInterfaceFinalizer) { + return nil + } + + held, err := r.heldByLiveClaim(ctx, cl, &iface) + if err != nil { + return err + } + + // The claim rebuilds the interface and finds the same addresses by name. + // Releasing here would give a running workload new ones. + if !held { + if err := r.claims.releaseAddresses(ctx, cl, iface.Namespace, &iface); err != nil { + return err + } + } + + controllerutil.RemoveFinalizer(&iface, networkInterfaceFinalizer) + if err := cl.Update(ctx, &iface); err != nil { + return fmt.Errorf("failed removing finalizer: %w", err) + } + + return nil +} + +// heldByLiveClaim reports whether a claim named in claimRef still exists and is +// not being deleted. +func (r *NetworkInterfaceReconciler) heldByLiveClaim( + ctx context.Context, + cl client.Client, + iface *networkingv1alpha.NetworkInterface, +) (bool, error) { + ref := iface.Spec.ClaimRef + if ref == nil { + return false, nil + } + + var claim networkingv1alpha.NetworkInterfaceClaim + key := client.ObjectKey{Namespace: iface.Namespace, Name: ref.Name} + if err := cl.Get(ctx, key, &claim); err != nil { + if apierrors.IsNotFound(err) { + return false, nil + } + return false, fmt.Errorf("failed reading claim %q: %w", ref.Name, err) + } + + // A claim being deleted holds nothing. Its own release cannot free these + // addresses once the interface is gone. + return claim.DeletionTimestamp.IsZero(), nil +} + +// SetupWithManager sets up the controller with the Manager. +func (r *NetworkInterfaceReconciler) SetupWithManager(mgr mcmanager.Manager) error { + if r.IPAM == nil { + return errors.New("an IPAM client factory is required") + } + + r.mgr = mgr + r.claims = &NetworkInterfaceClaimReconciler{Config: r.Config, IPAM: r.IPAM, mgr: mgr} + + return mcbuilder.ControllerManagedBy(mgr). + For(&networkingv1alpha.NetworkInterface{}, mcbuilder.WithEngageWithLocalCluster(false)). + Named("networkinterface"). + Complete(r) +} diff --git a/internal/controller/networkinterfaceclaim_controller.go b/internal/controller/networkinterfaceclaim_controller.go new file mode 100644 index 00000000..c48827d1 --- /dev/null +++ b/internal/controller/networkinterfaceclaim_controller.go @@ -0,0 +1,1257 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package controller + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "net/netip" + "slices" + "strings" + "time" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/validation" + "k8s.io/client-go/tools/events" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/cluster" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + mcbuilder "sigs.k8s.io/multicluster-runtime/pkg/builder" + mchandler "sigs.k8s.io/multicluster-runtime/pkg/handler" + mcmanager "sigs.k8s.io/multicluster-runtime/pkg/manager" + "sigs.k8s.io/multicluster-runtime/pkg/multicluster" + mcreconcile "sigs.k8s.io/multicluster-runtime/pkg/reconcile" + + ipamv1alpha1 "go.miloapis.com/ipam/pkg/apis/ipam/v1alpha1" + + networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" + "go.datum.net/network-services-operator/internal/config" +) + +const ( + networkInterfaceClaimFinalizer = "networking.datumapis.com/networkinterfaceclaim-release" + networkInterfaceFinalizer = "networking.datumapis.com/networkinterface-release" + + allocationClaimAnnotation = "networking.datumapis.com/allocation-claim" + + ipamScopeRoleNetwork = "network" + ipamScopeRoleLocation = "location" + + datumNetworkingAPIGroup = "networking.datumapis.com" + + maxObjectNameLength = 253 + ipClaimNameHashLen = 12 + + // Nothing watches the network, the namespace or IPAM, so a rejected claim + // needs its own way back. + rejectedClaimRetryInterval = time.Minute +) + +// NetworkInterfaceClaimReconciler binds a NetworkInterfaceClaim to a +// NetworkInterface. +type NetworkInterfaceClaimReconciler struct { + Config config.NetworkServicesOperator + IPAM IPAMClientFactory + + mgr mcmanager.Manager +} + +// +kubebuilder:rbac:groups=networking.datumapis.com,resources=networkinterfaceclaims,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=networking.datumapis.com,resources=networkinterfaceclaims/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=networking.datumapis.com,resources=networkinterfaceclaims/finalizers,verbs=update +// +kubebuilder:rbac:groups=networking.datumapis.com,resources=networkinterfaces,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=networking.datumapis.com,resources=networkinterfaces/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=networking.datumapis.com,resources=networkbindings,verbs=get;list;watch;create;update;patch +// +kubebuilder:rbac:groups=networking.datumapis.com,resources=networks,verbs=get;list;watch +// +kubebuilder:rbac:groups=networking.datumapis.com,resources=subnets,verbs=get;list;watch +// +kubebuilder:rbac:groups=events.k8s.io,resources=events,verbs=create;patch + +func (r *NetworkInterfaceClaimReconciler) Reconcile(ctx context.Context, req mcreconcile.Request) (ctrl.Result, error) { + logger := log.FromContext(ctx, "cluster", req.ClusterName) + + cl, err := r.mgr.GetCluster(ctx, req.ClusterName) + if err != nil { + return ctrl.Result{}, err + } + + logger.Info("reconciling network interface claim") + defer logger.Info("reconcile complete") + + return r.reconcileClaim(ctx, cl.GetClient(), cl.GetEventRecorder("networkinterfaceclaim-controller"), req.NamespacedName) +} + +func (r *NetworkInterfaceClaimReconciler) reconcileClaim( + ctx context.Context, + cl client.Client, + recorder events.EventRecorder, + key client.ObjectKey, +) (ctrl.Result, error) { + var claim networkingv1alpha.NetworkInterfaceClaim + if err := cl.Get(ctx, key, &claim); err != nil { + return ctrl.Result{}, client.IgnoreNotFound(err) + } + + if !claim.DeletionTimestamp.IsZero() { + return ctrl.Result{}, r.release(ctx, cl, &claim) + } + + if controllerutil.AddFinalizer(&claim, networkInterfaceClaimFinalizer) { + if err := cl.Update(ctx, &claim); err != nil { + return ctrl.Result{}, fmt.Errorf("failed adding finalizer: %w", err) + } + } + + return r.fulfill(ctx, cl, recorder, &claim) +} + +func (r *NetworkInterfaceClaimReconciler) fulfill( + ctx context.Context, + cl client.Client, + recorder events.EventRecorder, + claim *networkingv1alpha.NetworkInterfaceClaim, +) (ctrl.Result, error) { + routing, err := r.resolveProject(ctx, cl, claim.Namespace) + if err != nil { + var unresolvable *projectUnresolvable + if errors.As(err, &unresolvable) { + return r.reject(ctx, cl, claim, "ProjectUnresolved", unresolvable.Error()) + } + return ctrl.Result{}, err + } + + var network networkingv1alpha.Network + networkKey := client.ObjectKey{Namespace: claim.Namespace, Name: claim.Spec.Network.Name} + if err := cl.Get(ctx, networkKey, &network); err != nil { + if apierrors.IsNotFound(err) { + return r.reject(ctx, cl, claim, "NetworkNotFound", + fmt.Sprintf("Network %q was not found in namespace %q", networkKey.Name, networkKey.Namespace)) + } + return ctrl.Result{}, fmt.Errorf("failed fetching network: %w", err) + } + + for _, family := range claim.Spec.IPFamilies { + if !slices.Contains(network.Spec.IPFamilies, family) { + return r.reject(ctx, cl, claim, "AddressFamilyNotCarried", + fmt.Sprintf("Network %q in namespace %q does not carry address family %s", + network.Name, network.Namespace, family)) + } + } + + networkContextName := r.resolveNetworkContext(ctx, cl, claim, &network) + + ipamClient, err := r.IPAM.ClientForProject(routing.project) + if err != nil { + return ctrl.Result{}, fmt.Errorf("failed building IPAM client: %w", err) + } + + iface, err := r.bindInterface(ctx, cl, ipamClient, routing, claim, &network) + if err != nil { + var failure *allocationFailure + if errors.As(err, &failure) { + return r.reject(ctx, cl, claim, string(failure.reason), failure.message) + } + var refused *bindingRefused + if errors.As(err, &refused) { + return r.reject(ctx, cl, claim, refused.reason, refused.message) + } + return ctrl.Result{}, err + } + + allocations, err := r.checkAllocations(ctx, ipamClient, recorder, routing, claim, iface) + if err != nil { + return ctrl.Result{}, err + } + + if err := r.syncInterface(ctx, cl, iface, claim, &network, networkContextName); err != nil { + return ctrl.Result{}, err + } + + if err := r.publishInterfaceStatus(ctx, cl, iface, networkContextName, allocations); err != nil { + return ctrl.Result{}, err + } + + return ctrl.Result{}, r.publishClaimStatus(ctx, cl, claim, iface, allocations) +} + +type projectRouting struct { + project string + projectNamespace string +} + +func (r *NetworkInterfaceClaimReconciler) resolveProject( + ctx context.Context, + cl client.Client, + namespaceName string, +) (projectRouting, error) { + var namespace corev1.Namespace + if err := cl.Get(ctx, client.ObjectKey{Name: namespaceName}, &namespace); err != nil { + return projectRouting{}, fmt.Errorf("failed reading namespace %q: %w", namespaceName, err) + } + + project, err := projectFromNamespace(&namespace) + if err != nil { + return projectRouting{}, &projectUnresolvable{message: err.Error()} + } + + projectNamespace, err := projectNamespaceFromNamespace(&namespace) + if err != nil { + return projectRouting{}, &projectUnresolvable{message: err.Error()} + } + + return projectRouting{project: project, projectNamespace: projectNamespace}, nil +} + +// projectUnresolvable means the namespace does not name a project. A failure to +// read the namespace is an ordinary error, and retrying fixes it. +type projectUnresolvable struct { + message string +} + +func (e *projectUnresolvable) Error() string { return e.message } + +// resolveNetworkContext records where the network lives. An unresolved context +// does not hold up allocation. +func (r *NetworkInterfaceClaimReconciler) resolveNetworkContext( + ctx context.Context, + cl client.Client, + claim *networkingv1alpha.NetworkInterfaceClaim, + network *networkingv1alpha.Network, +) string { + logger := log.FromContext(ctx) + location := r.location() + + binding := &networkingv1alpha.NetworkBinding{} + binding.Namespace = claim.Namespace + binding.Name = fmt.Sprintf("%s-%s-%s", network.Name, location.Namespace, location.Name) + + _, err := controllerutil.CreateOrUpdate(ctx, cl, binding, func() error { + binding.Spec.Network = networkingv1alpha.NetworkRef{ + Namespace: network.Namespace, + Name: network.Name, + } + binding.Spec.Location = location + return controllerutil.SetControllerReference(network, binding, cl.Scheme()) + }) + if err != nil { + logger.Error(err, "failed ensuring network binding", "binding", binding.Name) + return "" + } + + if binding.Status.NetworkContextRef == nil { + return "" + } + return binding.Status.NetworkContextRef.Name +} + +func (r *NetworkInterfaceClaimReconciler) location() networkingv1alpha.LocationReference { + return networkingv1alpha.LocationReference{ + Name: r.Config.NetworkInterface.Location.Name, + Namespace: r.Config.NetworkInterface.Location.Namespace, + } +} + +type bindingRefused struct { + reason string + message string +} + +func (e *bindingRefused) Error() string { return e.message } + +func interfaceSatisfies( + iface *networkingv1alpha.NetworkInterface, + claim *networkingv1alpha.NetworkInterfaceClaim, +) error { + // The addresses belong to the network they were allocated from. Publishing + // them under another network would hand one network's addresses to another. + if iface.Spec.Network.Name != claim.Spec.Network.Name { + return &bindingRefused{ + reason: "NetworkMismatch", + message: fmt.Sprintf( + "Network interface %q holds addresses on network %q and cannot be bound by a claim naming network %q", + iface.Name, iface.Spec.Network.Name, claim.Spec.Network.Name), + } + } + + if iface.Spec.InterfaceName != claim.Spec.InterfaceName { + return &bindingRefused{ + reason: "InterfaceNameMismatch", + message: fmt.Sprintf( + "Network interface %q presents as %q to the guest and cannot be bound by a claim asking for %q", + iface.Name, iface.Spec.InterfaceName, claim.Spec.InterfaceName), + } + } + + for _, family := range claim.Spec.IPFamilies { + if !slices.ContainsFunc(iface.Spec.Addresses, func(a networkingv1alpha.NetworkInterfaceAddress) bool { + return a.Family == family + }) { + return &bindingRefused{ + reason: "AddressFamilyMissing", + message: fmt.Sprintf( + "Network interface %q holds no %s address, which this claim requires", + iface.Name, family), + } + } + } + + for _, request := range claim.Spec.Addresses { + if !slices.ContainsFunc(iface.Spec.ExternalAddresses, func(a networkingv1alpha.NetworkInterfaceExternalAddress) bool { + return a.Class == request.Class + }) { + return &bindingRefused{ + reason: "AddressClassMissing", + message: fmt.Sprintf( + "Network interface %q holds no address of class %q, which this claim requires", + iface.Name, request.Class), + } + } + } + + // An address keeps the reclaim policy it was allocated under, so a claim + // asking for a different one cannot be honoured. + if iface.Spec.ReclaimPolicy != claim.Spec.ReclaimPolicy { + return &bindingRefused{ + reason: "ReclaimPolicyMismatch", + message: fmt.Sprintf( + "Network interface %q holds addresses allocated with reclaimPolicy %s and cannot be bound by a claim requesting %s", + iface.Name, iface.Spec.ReclaimPolicy, claim.Spec.ReclaimPolicy), + } + } + + return nil +} + +func (r *NetworkInterfaceClaimReconciler) bindInterface( + ctx context.Context, + cl client.Client, + ipamClient client.Client, + routing projectRouting, + claim *networkingv1alpha.NetworkInterfaceClaim, + network *networkingv1alpha.Network, +) (*networkingv1alpha.NetworkInterface, error) { + interfaceKey := client.ObjectKey{Namespace: claim.Namespace, Name: interfaceNameForClaim(claim)} + + var existing networkingv1alpha.NetworkInterface + err := cl.Get(ctx, interfaceKey, &existing) + if err != nil && !apierrors.IsNotFound(err) { + return nil, fmt.Errorf("failed fetching network interface: %w", err) + } + + if err == nil { + if ref := existing.Spec.ClaimRef; ref != nil && ref.Name != claim.Name { + return nil, &bindingRefused{ + reason: "InterfaceHeldByAnotherClaim", + message: fmt.Sprintf( + "Network interface %q is held by claim %q", existing.Name, ref.Name), + } + } + + if err := interfaceSatisfies(&existing, claim); err != nil { + return nil, err + } + + existing.Spec.ClaimRef = &networkingv1alpha.NetworkInterfaceClaimRef{Name: claim.Name} + controllerutil.AddFinalizer(&existing, networkInterfaceFinalizer) + if err := cl.Update(ctx, &existing); err != nil { + return nil, fmt.Errorf("failed binding network interface: %w", err) + } + return &existing, nil + } + + requests, err := r.allocationRequests(ctx, ipamClient, claim) + if err != nil { + return nil, err + } + + allocated, err := r.allocate(ctx, ipamClient, routing, claim, network, requests) + if err != nil { + return nil, err + } + + iface := &networkingv1alpha.NetworkInterface{} + iface.Namespace = interfaceKey.Namespace + iface.Name = interfaceKey.Name + iface.Annotations = map[string]string{allocationClaimAnnotation: claim.Name} + iface.Finalizers = []string{networkInterfaceFinalizer} + iface.Spec = networkingv1alpha.NetworkInterfaceSpec{ + Network: networkingv1alpha.LocalNetworkRef{Name: network.Name}, + ClaimRef: &networkingv1alpha.NetworkInterfaceClaimRef{Name: claim.Name}, + InterfaceName: claim.Spec.InterfaceName, + MTU: network.Spec.MTU, + ReclaimPolicy: claim.Spec.ReclaimPolicy, + } + + for _, entry := range allocated { + if entry.request.external { + iface.Spec.ExternalAddresses = append(iface.Spec.ExternalAddresses, networkingv1alpha.NetworkInterfaceExternalAddress{ + Family: entry.request.family, + Address: entry.bareAddress(), + Class: entry.request.className, + }) + continue + } + + iface.Spec.Addresses = append(iface.Spec.Addresses, networkingv1alpha.NetworkInterfaceAddress{ + Family: entry.request.family, + Address: entry.cidr, + Primary: entry.request.family == claim.Spec.IPFamilies[0], + Class: entry.request.className, + }) + } + + if err := cl.Create(ctx, iface); err != nil { + return nil, fmt.Errorf("failed creating network interface: %w", err) + } + + return iface, nil +} + +type allocationRequest struct { + discriminator string + family networkingv1alpha.IPFamily + className string + external bool +} + +func (a allocationRequest) describe() string { + if a.className != "" { + return fmt.Sprintf("an address of class %q", a.className) + } + return fmt.Sprintf("an %s address", a.family) +} + +type allocatedAddress struct { + request allocationRequest + cidr string +} + +func (a allocatedAddress) bareAddress() string { + prefix, err := netip.ParsePrefix(a.cidr) + if err != nil { + return a.cidr + } + if prefix.Bits() != prefix.Addr().BitLen() { + return a.cidr + } + return prefix.Addr().String() +} + +func (r *NetworkInterfaceClaimReconciler) allocationRequests( + ctx context.Context, + ipamClient client.Client, + claim *networkingv1alpha.NetworkInterfaceClaim, +) ([]allocationRequest, error) { + requests := make([]allocationRequest, 0, len(claim.Spec.IPFamilies)+len(claim.Spec.Addresses)) + + for _, family := range claim.Spec.IPFamilies { + requests = append(requests, allocationRequest{ + discriminator: familyDiscriminator(family), + family: family, + }) + } + + for _, address := range claim.Spec.Addresses { + var class ipamv1alpha1.IPClass + if err := ipamClient.Get(ctx, client.ObjectKey{Name: address.Class}, &class); err != nil { + return nil, &allocationFailure{ + reason: allocationFailureRejected, + message: fmt.Sprintf("Address class %q could not be read: %v", + address.Class, err), + } + } + + family := networkingv1alpha.IPFamily(class.Spec.IPFamily) + if !slices.Contains(claim.Spec.IPFamilies, family) { + return nil, &allocationFailure{ + reason: allocationFailureRejected, + message: fmt.Sprintf( + "Address class %q hands out %s addresses, which map onto an %s address this interface does not carry", + address.Class, family, family), + } + } + + requests = append(requests, allocationRequest{ + discriminator: classDiscriminator(address.Class), + family: family, + className: address.Class, + external: true, + }) + } + + return requests, nil +} + +type allocationFailure struct { + reason allocationFailureReason + message string +} + +func (e *allocationFailure) Error() string { return e.message } + +// allocate claims every requested address, or none. Names are derived from the +// claim, so a failed rollback leaves addresses the next attempt finds again. +func (r *NetworkInterfaceClaimReconciler) allocate( + ctx context.Context, + ipamClient client.Client, + routing projectRouting, + claim *networkingv1alpha.NetworkInterfaceClaim, + network *networkingv1alpha.Network, + requests []allocationRequest, +) ([]allocatedAddress, error) { + logger := log.FromContext(ctx) + + if err := ensureProjectNamespace(ctx, ipamClient, routing.projectNamespace); err != nil { + return nil, fmt.Errorf("failed ensuring project namespace %q: %w", routing.projectNamespace, err) + } + + location := r.location() + allocated := make([]allocatedAddress, 0, len(requests)) + created := make([]*ipamv1alpha1.IPClaim, 0, len(requests)) + + rollback := func() { + var leaked []string + for _, ipClaim := range created { + if err := ipamClient.Delete(ctx, ipClaim); err != nil && !apierrors.IsNotFound(err) { + leaked = append(leaked, ipClaim.Name) + logger.Error(err, "failed releasing address after a partial allocation", + "ipclaim", ipClaim.Name, "project", routing.project) + } + } + if len(leaked) > 0 { + logger.Info("addresses remain claimed after a failed rollback and will be reused on retry", + "ipclaims", strings.Join(leaked, ","), "project", routing.project) + } + } + + for _, request := range requests { + ipClaim := &ipamv1alpha1.IPClaim{} + ipClaim.Namespace = routing.projectNamespace + ipClaim.Name = ipClaimName(claim.Name, request.discriminator) + ipClaim.Spec = ipamv1alpha1.IPClaimSpec{ + ClassName: request.className, + ReclaimPolicy: ipamReclaimPolicy(claim.Spec.ReclaimPolicy), + Scope: map[string]ipamv1alpha1.ScopeRef{ + ipamScopeRoleNetwork: { + APIGroup: datumNetworkingAPIGroup, + Kind: "Network", + Name: network.Name, + }, + ipamScopeRoleLocation: { + APIGroup: datumNetworkingAPIGroup, + Kind: "Location", + Name: location.Name, + }, + }, + } + if request.className == "" { + ipClaim.Spec.IPFamily = ipamv1alpha1.IPFamily(request.family) + } + + // IPAM reports a duplicate name inconsistently, so ask whether the + // address exists instead of reading the refusal. + existing := &ipamv1alpha1.IPClaim{} + getErr := ipamClient.Get(ctx, client.ObjectKeyFromObject(ipClaim), existing) + if getErr != nil && !apierrors.IsNotFound(getErr) { + rollback() + return nil, fmt.Errorf("failed reading IPClaim %q: %w", ipClaim.Name, getErr) + } + + if getErr == nil { + ipClaim = existing + } else if createErr := ipamClient.Create(ctx, ipClaim); createErr != nil { + raced := &ipamv1alpha1.IPClaim{} + if err := ipamClient.Get(ctx, client.ObjectKeyFromObject(ipClaim), raced); err != nil { + rollback() + reason := classifyAllocationFailure(createErr) + return nil, &allocationFailure{ + reason: reason, + message: allocationFailureMessage(reason, request, createErr), + } + } + ipClaim = raced + } else { + created = append(created, ipClaim) + } + + if ipClaim.Status.AllocatedCIDR == "" { + rollback() + return nil, &allocationFailure{ + reason: allocationFailureUnknown, + message: fmt.Sprintf("IPAM reported no address for %s (phase %q)", + request.describe(), ipClaim.Status.Phase), + } + } + + allocated = append(allocated, allocatedAddress{ + request: request, + cidr: ipClaim.Status.AllocatedCIDR, + }) + } + + return allocated, nil +} + +func (r *NetworkInterfaceClaimReconciler) checkAllocations( + ctx context.Context, + ipamClient client.Client, + recorder events.EventRecorder, + routing projectRouting, + claim *networkingv1alpha.NetworkInterfaceClaim, + iface *networkingv1alpha.NetworkInterface, +) (allocationHealth, error) { + owner := allocationClaimName(iface) + health := allocationHealth{intact: true} + + var errs []error + for _, entry := range allocationEntries(iface) { + var ipClaim ipamv1alpha1.IPClaim + key := client.ObjectKey{ + Namespace: routing.projectNamespace, + Name: ipClaimName(owner, entry.discriminator), + } + if err := ipamClient.Get(ctx, key, &ipClaim); err != nil { + if apierrors.IsNotFound(err) { + health = allocationHealth{ + message: fmt.Sprintf("Address %s is held by no allocation in project %q", + entry.address, routing.project), + } + r.reportMissingAllocation(ctx, recorder, routing, claim, iface, entry, key.Name, "") + continue + } + errs = append(errs, fmt.Errorf("failed reading IPClaim %q: %w", key.Name, err)) + continue + } + + if !entry.holds(&ipClaim) { + health = allocationHealth{ + message: fmt.Sprintf("Address %s is published, but allocation %q holds %s", + entry.address, key.Name, ipClaim.Status.AllocatedCIDR), + } + r.reportMissingAllocation(ctx, recorder, routing, claim, iface, entry, key.Name, + ipClaim.Status.AllocatedCIDR) + } + } + + return health, errors.Join(errs...) +} + +// allocationClaimName is the claim whose name the addresses were allocated +// under, which is not always the claim holding the interface now. +func allocationClaimName(iface *networkingv1alpha.NetworkInterface) string { + if recorded := iface.Annotations[allocationClaimAnnotation]; recorded != "" { + return recorded + } + if ref := iface.Spec.ClaimRef; ref != nil { + return ref.Name + } + return iface.Name +} + +// reportMissingAllocation warns that an address may be handed to another claim. +// Reallocating instead would change the address of a running workload. +func (r *NetworkInterfaceClaimReconciler) reportMissingAllocation( + ctx context.Context, + recorder events.EventRecorder, + routing projectRouting, + claim *networkingv1alpha.NetworkInterfaceClaim, + iface *networkingv1alpha.NetworkInterface, + entry allocationEntry, + ipClaimName string, + heldInstead string, +) { + missingAllocationsTotal.WithLabelValues(routing.project).Inc() + + log.FromContext(ctx).Error(errNoAllocationBehindAddress, + "network interface advertises an address with no allocation behind it", + "interface", iface.Name, "address", entry.address, "heldInstead", heldInstead, + "ipclaim", ipClaimName, "project", routing.project) + + if recorder == nil { + return + } + + if heldInstead != "" { + recorder.Eventf(claim, iface, corev1.EventTypeWarning, "AddressAllocationMissing", "VerifyAllocation", + "Address %s on network interface %q is published, but allocation %q in project %q holds %s. "+ + "The published address belongs to no one and may be given to another claim.", + entry.address, iface.Name, ipClaimName, routing.project, heldInstead) + return + } + + recorder.Eventf(claim, iface, corev1.EventTypeWarning, "AddressAllocationMissing", "VerifyAllocation", + "Address %s on network interface %q has no allocation in project %q (IPClaim %q is gone). "+ + "IPAM considers the address free and may hand it to another claim.", + entry.address, iface.Name, routing.project, ipClaimName) +} + +var errNoAllocationBehindAddress = errors.New("no IPClaim holds this address") + +type allocationEntry struct { + discriminator string + address string + external bool +} + +// holds reports whether an allocation still carries the address the interface +// publishes. An IPClaim recreated under the same name may hold a different one. +func (e allocationEntry) holds(ipClaim *ipamv1alpha1.IPClaim) bool { + allocated := allocatedAddress{cidr: ipClaim.Status.AllocatedCIDR} + if e.external { + return allocated.bareAddress() == e.address + } + return ipClaim.Status.AllocatedCIDR == e.address +} + +func allocationEntries(iface *networkingv1alpha.NetworkInterface) []allocationEntry { + entries := make([]allocationEntry, 0, len(iface.Spec.Addresses)+len(iface.Spec.ExternalAddresses)) + for _, address := range iface.Spec.Addresses { + entries = append(entries, allocationEntry{ + discriminator: familyDiscriminator(address.Family), + address: address.Address, + }) + } + for _, address := range iface.Spec.ExternalAddresses { + entries = append(entries, allocationEntry{ + discriminator: classDiscriminator(address.Class), + address: address.Address, + external: true, + }) + } + return entries +} + +func allocationDiscriminators(iface *networkingv1alpha.NetworkInterface) []string { + entries := allocationEntries(iface) + discriminators := make([]string, 0, len(entries)) + for _, entry := range entries { + discriminators = append(discriminators, entry.discriminator) + } + return discriminators +} + +func (r *NetworkInterfaceClaimReconciler) publishInterfaceStatus( + ctx context.Context, + cl client.Client, + iface *networkingv1alpha.NetworkInterface, + networkContextName string, + allocations allocationHealth, +) error { + iface.Status.Phase = networkingv1alpha.NetworkInterfacePhaseBound + if networkContextName != "" { + iface.Status.NetworkContextRef = &networkingv1alpha.LocalNetworkContextRef{Name: networkContextName} + } + + apimeta.SetStatusCondition(&iface.Status.Conditions, allocatedCondition( + networkingv1alpha.NetworkInterfaceAllocated, iface.Generation, allocations)) + + if apimeta.FindStatusCondition(iface.Status.Conditions, networkingv1alpha.NetworkInterfaceProgrammed) == nil { + apimeta.SetStatusCondition(&iface.Status.Conditions, metav1.Condition{ + Type: networkingv1alpha.NetworkInterfaceProgrammed, + Status: metav1.ConditionUnknown, + Reason: "Pending", + ObservedGeneration: iface.Generation, + Message: "Waiting for the data plane to report the attachment", + }) + } + + if err := cl.Status().Update(ctx, iface); err != nil { + return fmt.Errorf("failed updating network interface status: %w", err) + } + return nil +} + +func (r *NetworkInterfaceClaimReconciler) publishClaimStatus( + ctx context.Context, + cl client.Client, + claim *networkingv1alpha.NetworkInterfaceClaim, + iface *networkingv1alpha.NetworkInterface, + allocations allocationHealth, +) error { + claim.Status.Addresses = append([]networkingv1alpha.NetworkInterfaceAddress(nil), iface.Spec.Addresses...) + claim.Status.NetworkInterfaceRef = &networkingv1alpha.LocalNetworkInterfaceRef{Name: iface.Name} + claim.Status.ExternalAddresses = append([]networkingv1alpha.NetworkInterfaceExternalAddress(nil), iface.Spec.ExternalAddresses...) + + apimeta.SetStatusCondition(&claim.Status.Conditions, metav1.Condition{ + Type: networkingv1alpha.NetworkInterfaceClaimBound, + Status: metav1.ConditionTrue, + Reason: "Bound", + ObservedGeneration: claim.Generation, + Message: fmt.Sprintf("Bound to network interface %q", iface.Name), + }) + apimeta.SetStatusCondition(&claim.Status.Conditions, allocatedCondition( + networkingv1alpha.NetworkInterfaceClaimAllocated, claim.Generation, allocations)) + seedProgrammed(&claim.Status.Conditions, claim.Generation) + setReady(&claim.Status.Conditions, claim.Generation) + + if err := cl.Status().Update(ctx, claim); err != nil { + return fmt.Errorf("failed updating claim status: %w", err) + } + return nil +} + +// syncGateways writes each address's gateway onto the interface once the +// location has a subnet. A provider configures a NIC from the interface alone, +// so the gateway has to live there and not only on the claim's status copy. +func (r *NetworkInterfaceClaimReconciler) syncInterface( + ctx context.Context, + cl client.Client, + iface *networkingv1alpha.NetworkInterface, + claim *networkingv1alpha.NetworkInterfaceClaim, + network *networkingv1alpha.Network, + networkContextName string, +) error { + changed := false + + // MTU follows the network, and the primary address follows the claim's + // first family. Both are derived, and an adopted interface carries whatever + // its previous claim left behind. + if iface.Spec.MTU != network.Spec.MTU { + iface.Spec.MTU = network.Spec.MTU + changed = true + } + for i := range iface.Spec.Addresses { + primary := iface.Spec.Addresses[i].Family == claim.Spec.IPFamilies[0] + if iface.Spec.Addresses[i].Primary != primary { + iface.Spec.Addresses[i].Primary = primary + changed = true + } + } + + if err := r.applyGateways(ctx, cl, iface, networkContextName, &changed); err != nil { + return err + } + + if !changed { + return nil + } + + if err := cl.Update(ctx, iface); err != nil { + return fmt.Errorf("failed updating network interface: %w", err) + } + return nil +} + +func (r *NetworkInterfaceClaimReconciler) applyGateways( + ctx context.Context, + cl client.Client, + iface *networkingv1alpha.NetworkInterface, + networkContextName string, + changed *bool, +) error { + if networkContextName == "" { + return nil + } + + var subnets networkingv1alpha.SubnetList + if err := cl.List(ctx, &subnets, client.InNamespace(iface.Namespace)); err != nil { + return fmt.Errorf("failed listing subnets: %w", err) + } + + for i := range iface.Spec.Addresses { + gateway := subnetGatewayFor(&subnets, networkContextName, iface.Spec.Addresses[i].Family) + if gateway == "" || iface.Spec.Addresses[i].Gateway == gateway { + continue + } + iface.Spec.Addresses[i].Gateway = gateway + *changed = true + } + return nil +} + +func subnetGatewayFor( + subnets *networkingv1alpha.SubnetList, + networkContextName string, + family networkingv1alpha.IPFamily, +) string { + for i := range subnets.Items { + subnet := &subnets.Items[i] + if subnet.Spec.NetworkContext.Name == networkContextName && subnet.Spec.IPFamily == family { + return subnetGateway(subnet) + } + } + return "" +} + +func subnetGateway(subnet *networkingv1alpha.Subnet) string { + start := subnet.Spec.StartAddress + if subnet.Status.StartAddress != nil { + start = *subnet.Status.StartAddress + } + + addr, err := netip.ParseAddr(start) + if err != nil { + return "" + } + return addr.Next().String() +} + +func (r *NetworkInterfaceClaimReconciler) reject( + ctx context.Context, + cl client.Client, + claim *networkingv1alpha.NetworkInterfaceClaim, + reason string, + message string, +) (ctrl.Result, error) { + log.FromContext(ctx).Info("claim cannot be fulfilled", "reason", reason, "message", message) + + demoted := []string{networkingv1alpha.NetworkInterfaceClaimReady} + + // A claim that already holds an interface still holds it, and still holds + // the addresses IPAM allocated. Only Ready is false. + if claim.Status.NetworkInterfaceRef == nil { + demoted = append(demoted, + networkingv1alpha.NetworkInterfaceClaimBound, + networkingv1alpha.NetworkInterfaceClaimAllocated) + } + + for _, conditionType := range demoted { + apimeta.SetStatusCondition(&claim.Status.Conditions, metav1.Condition{ + Type: conditionType, + Status: metav1.ConditionFalse, + Reason: reason, + ObservedGeneration: claim.Generation, + Message: message, + }) + } + seedProgrammed(&claim.Status.Conditions, claim.Generation) + + if err := cl.Status().Update(ctx, claim); err != nil { + return ctrl.Result{}, fmt.Errorf("failed updating claim status: %w", err) + } + + // Nothing watches the network, the namespace or IPAM, so a claim rejected + // for a condition that later clears has no other way back. + return ctrl.Result{RequeueAfter: rejectedClaimRetryInterval}, nil +} + +// allocationHealth reports whether every address the interface publishes is +// still backed by an allocation. +type allocationHealth struct { + intact bool + message string +} + +func allocatedCondition(conditionType string, generation int64, health allocationHealth) metav1.Condition { + if health.intact { + return metav1.Condition{ + Type: conditionType, + Status: metav1.ConditionTrue, + Reason: "Allocated", + ObservedGeneration: generation, + Message: "Every requested address is held", + } + } + return metav1.Condition{ + Type: conditionType, + Status: metav1.ConditionFalse, + Reason: "AddressAllocationMissing", + ObservedGeneration: generation, + Message: health.message, + } +} + +// setReady derives Ready from the conditions it depends on, so it becomes true +// on its own once the data plane reports the attachment. +func setReady(conditions *[]metav1.Condition, generation int64) { + unmet := "" + for _, conditionType := range []string{ + networkingv1alpha.NetworkInterfaceClaimBound, + networkingv1alpha.NetworkInterfaceClaimAllocated, + networkingv1alpha.NetworkInterfaceClaimProgrammed, + } { + if !apimeta.IsStatusConditionTrue(*conditions, conditionType) { + unmet = conditionType + break + } + } + + if unmet == "" { + apimeta.SetStatusCondition(conditions, metav1.Condition{ + Type: networkingv1alpha.NetworkInterfaceClaimReady, + Status: metav1.ConditionTrue, + Reason: "Ready", + ObservedGeneration: generation, + Message: "The interface is bound, addressed, and programmed", + }) + return + } + + status := metav1.ConditionFalse + if apimeta.IsStatusConditionPresentAndEqual(*conditions, unmet, metav1.ConditionUnknown) { + status = metav1.ConditionUnknown + } + apimeta.SetStatusCondition(conditions, metav1.Condition{ + Type: networkingv1alpha.NetworkInterfaceClaimReady, + Status: status, + Reason: "Not" + unmet, + ObservedGeneration: generation, + Message: fmt.Sprintf("Waiting for %s", unmet), + }) +} + +// seedProgrammed sets Programmed only when it is absent. The data plane owns +// this condition; overwriting it would revert whoever reported the attachment. +func seedProgrammed(conditions *[]metav1.Condition, generation int64) { + if apimeta.FindStatusCondition(*conditions, networkingv1alpha.NetworkInterfaceClaimProgrammed) != nil { + return + } + apimeta.SetStatusCondition(conditions, metav1.Condition{ + Type: networkingv1alpha.NetworkInterfaceClaimProgrammed, + Status: metav1.ConditionUnknown, + Reason: "Pending", + ObservedGeneration: generation, + Message: "Waiting for the data plane to report the attachment", + }) +} + +// release returns the addresses to their pools, or under Retain leaves the +// interface holding them for the next claim of this name. +func (r *NetworkInterfaceClaimReconciler) release( + ctx context.Context, + cl client.Client, + claim *networkingv1alpha.NetworkInterfaceClaim, +) error { + if !controllerutil.ContainsFinalizer(claim, networkInterfaceClaimFinalizer) { + return nil + } + + var iface networkingv1alpha.NetworkInterface + interfaceKey := client.ObjectKey{Namespace: claim.Namespace, Name: interfaceNameForClaim(claim)} + err := cl.Get(ctx, interfaceKey, &iface) + if err != nil && !apierrors.IsNotFound(err) { + return fmt.Errorf("failed fetching network interface: %w", err) + } + + interfaceExists := err == nil + held := interfaceExists && iface.Spec.ClaimRef != nil && + iface.Spec.ClaimRef.Name == claim.Name + retain := claim.Spec.ReclaimPolicy == networkingv1alpha.NetworkInterfaceReclaimPolicyRetain + + switch { + case held && retain: + iface.Spec.ClaimRef = nil + if err := cl.Update(ctx, &iface); err != nil { + return fmt.Errorf("failed unbinding network interface: %w", err) + } + if err := markInterfaceAvailable(ctx, cl, &iface); err != nil { + return err + } + + case held: + if err := r.releaseAddresses(ctx, cl, claim.Namespace, &iface); err != nil { + return err + } + if controllerutil.RemoveFinalizer(&iface, networkInterfaceFinalizer) { + if err := cl.Update(ctx, &iface); err != nil { + return fmt.Errorf("failed clearing network interface finalizer: %w", err) + } + } + if err := cl.Delete(ctx, &iface); err != nil && !apierrors.IsNotFound(err) { + return fmt.Errorf("failed deleting network interface: %w", err) + } + + case interfaceExists && iface.Spec.ClaimRef == nil && retain: + // A previous attempt unbound the interface and failed before recording + // it. Nothing else writes phase for an unbound interface. + if err := markInterfaceAvailable(ctx, cl, &iface); err != nil { + return err + } + + case !retain: + // The interface is gone, or never named this claim. Either way the + // addresses this claim allocated are named after it. + if err := r.releaseClaimAllocations(ctx, cl, claim); err != nil { + return err + } + } + + controllerutil.RemoveFinalizer(claim, networkInterfaceClaimFinalizer) + if err := cl.Update(ctx, claim); err != nil { + return fmt.Errorf("failed removing finalizer: %w", err) + } + return nil +} + +func (r *NetworkInterfaceClaimReconciler) releaseAddresses( + ctx context.Context, + cl client.Client, + namespace string, + iface *networkingv1alpha.NetworkInterface, +) error { + return r.releaseIPClaims(ctx, cl, namespace, + allocationClaimName(iface), allocationDiscriminators(iface)) +} + +// releaseClaimAllocations releases the addresses this claim allocated. A claim +// that only adopted an existing interface allocated none and releases none. +func (r *NetworkInterfaceClaimReconciler) releaseClaimAllocations( + ctx context.Context, + cl client.Client, + claim *networkingv1alpha.NetworkInterfaceClaim, +) error { + // A claim that never bound allocated nothing, so there is nothing to + // release and no reason to block its deletion on a project it never named. + if claim.Status.NetworkInterfaceRef == nil { + return nil + } + + return r.releaseIPClaims(ctx, cl, claim.Namespace, + claim.Name, claimDiscriminators(claim)) +} + +func markInterfaceAvailable( + ctx context.Context, + cl client.Client, + iface *networkingv1alpha.NetworkInterface, +) error { + if iface.Status.Phase == networkingv1alpha.NetworkInterfacePhaseAvailable { + return nil + } + iface.Status.Phase = networkingv1alpha.NetworkInterfacePhaseAvailable + if err := cl.Status().Update(ctx, iface); err != nil { + return fmt.Errorf("failed updating network interface status: %w", err) + } + return nil +} + +func (r *NetworkInterfaceClaimReconciler) releaseIPClaims( + ctx context.Context, + cl client.Client, + namespace string, + owner string, + discriminators []string, +) error { + if len(discriminators) == 0 { + return nil + } + + routing, err := r.resolveProject(ctx, cl, namespace) + if err != nil { + return fmt.Errorf("cannot release addresses claimed by %q: %w", owner, err) + } + + ipamClient, err := r.IPAM.ClientForProject(routing.project) + if err != nil { + return fmt.Errorf("failed building IPAM client: %w", err) + } + + var errs []error + for _, discriminator := range discriminators { + ipClaim := &ipamv1alpha1.IPClaim{} + ipClaim.Namespace = routing.projectNamespace + ipClaim.Name = ipClaimName(owner, discriminator) + if err := ipamClient.Delete(ctx, ipClaim); err != nil && !apierrors.IsNotFound(err) { + errs = append(errs, fmt.Errorf("failed releasing %q: %w", ipClaim.Name, err)) + } + } + return errors.Join(errs...) +} + +func claimDiscriminators(claim *networkingv1alpha.NetworkInterfaceClaim) []string { + discriminators := make([]string, 0, len(claim.Spec.IPFamilies)+len(claim.Spec.Addresses)) + for _, family := range claim.Spec.IPFamilies { + discriminators = append(discriminators, familyDiscriminator(family)) + } + for _, address := range claim.Spec.Addresses { + discriminators = append(discriminators, classDiscriminator(address.Class)) + } + return discriminators +} + +func interfaceNameForClaim(claim *networkingv1alpha.NetworkInterfaceClaim) string { + if claim.Spec.NetworkInterfaceName != "" { + return claim.Spec.NetworkInterfaceName + } + return claim.Name +} + +func familyDiscriminator(family networkingv1alpha.IPFamily) string { + return "f-" + strings.ToLower(string(family)) +} + +func classDiscriminator(class string) string { + return "c-" + class +} + +// ipClaimName derives a stable name from the claim and what it asks for, so a +// replacement instance finds the addresses it already has. +func ipClaimName(claimName, discriminator string) string { + candidate := claimName + "-" + discriminator + if len(candidate) <= maxObjectNameLength && len(validation.IsDNS1123Subdomain(candidate)) == 0 { + return candidate + } + + sum := sha256.Sum256([]byte(claimName + "\x00" + discriminator)) + suffix := hex.EncodeToString(sum[:])[:ipClaimNameHashLen] + + prefix := claimName + if max := maxObjectNameLength - 1 - ipClaimNameHashLen; len(prefix) > max { + prefix = prefix[:max] + } + return strings.TrimRight(prefix, "-.") + "-" + suffix +} + +func ipamReclaimPolicy(policy networkingv1alpha.NetworkInterfaceReclaimPolicy) ipamv1alpha1.ReclaimPolicy { + if policy == networkingv1alpha.NetworkInterfaceReclaimPolicyRetain { + return ipamv1alpha1.ReclaimRetain + } + return ipamv1alpha1.ReclaimDelete +} + +// SetupWithManager sets up the controller with the Manager. +func (r *NetworkInterfaceClaimReconciler) SetupWithManager(mgr mcmanager.Manager) error { + if r.IPAM == nil { + return errors.New("an IPAM client factory is required") + } + if r.Config.NetworkInterface.Location.Name == "" || r.Config.NetworkInterface.Location.Namespace == "" { + return errors.New("a location is required to fulfil network interface claims") + } + + r.mgr = mgr + return mcbuilder.ControllerManagedBy(mgr). + For(&networkingv1alpha.NetworkInterfaceClaim{}, mcbuilder.WithEngageWithLocalCluster(false)). + // A deleted interface is rebuilt by the claim that holds it. + Watches(&networkingv1alpha.NetworkInterface{}, mchandler.EnqueueRequestsFromMapFunc( + func(_ context.Context, obj client.Object) []ctrl.Request { + iface, ok := obj.(*networkingv1alpha.NetworkInterface) + if !ok || iface.Spec.ClaimRef == nil { + return nil + } + return []ctrl.Request{{NamespacedName: client.ObjectKey{ + Namespace: iface.Namespace, + Name: iface.Spec.ClaimRef.Name, + }}} + })). + // A subnet appearing is what makes the gateway resolvable, and no claim + // event follows it. + Watches(&networkingv1alpha.Subnet{}, func(clusterName multicluster.ClusterName, cl cluster.Cluster) mchandler.EventHandler { + return mchandler.ForCluster(handler.EnqueueRequestsFromMapFunc( + func(ctx context.Context, obj client.Object) []reconcile.Request { + var claims networkingv1alpha.NetworkInterfaceClaimList + if err := cl.GetClient().List(ctx, &claims, client.InNamespace(obj.GetNamespace())); err != nil { + log.FromContext(ctx).Error(err, "failed listing claims for a subnet event") + return nil + } + + requests := make([]reconcile.Request, 0, len(claims.Items)) + for i := range claims.Items { + requests = append(requests, reconcile.Request{ + NamespacedName: client.ObjectKeyFromObject(&claims.Items[i]), + }) + } + return requests + }), clusterName) + }). + Named("networkinterfaceclaim"). + Complete(r) +} diff --git a/internal/controller/networkinterfaceclaim_controller_test.go b/internal/controller/networkinterfaceclaim_controller_test.go new file mode 100644 index 00000000..e62f1dfe --- /dev/null +++ b/internal/controller/networkinterfaceclaim_controller_test.go @@ -0,0 +1,1301 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package controller + +import ( + "context" + "fmt" + "maps" + "os" + "path/filepath" + "slices" + "strings" + "sync" + "testing" + "time" + + "github.com/prometheus/client_golang/prometheus/testutil" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + eventsv1 "k8s.io/api/events/v1" + rbacv1 "k8s.io/api/rbac/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/validation" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + eventsv1client "k8s.io/client-go/kubernetes/typed/events/v1" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/events" + "sigs.k8s.io/controller-runtime/pkg/client" + fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/envtest" + "sigs.k8s.io/yaml" + + ipamv1alpha1 "go.miloapis.com/ipam/pkg/apis/ipam/v1alpha1" + + networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" + "go.datum.net/network-services-operator/internal/config" + "go.datum.net/network-services-operator/internal/downstreamclient" +) + +const ( + testProject = "project-alpha" + testProjectNS = "default" + testLocationName = "us-central-1" + testLocationNS = "datum-locations" + testPublicV4Class = "datum-public-v4" +) + +// fakeIPAM stands in for the IPAM API server. Allocation is synchronous there, +// so the create response already carries the address. +type fakeIPAM struct { + mu sync.Mutex + + scheme *runtime.Scheme + clients map[string]client.Client + + classes []*ipamv1alpha1.IPClass + + // createdIn records, per project, the IPClaim names Create was called with. + createdIn map[string][]string + // deletedIn records, per project, the IPClaim names Delete was called with. + deletedIn map[string][]string + // allocationPolicy is the policy IPAM froze onto the allocation. The server + // releases on this value alone and never re-reads the IPClaim. + allocationPolicy map[string]ipamv1alpha1.ReclaimPolicy + // orphans maps an allocation to the CIDR it holds after its IPClaim was + // deleted under a frozen Retain. + orphans map[string]string + + // failOn refuses the named IPClaim with the given error. + failOn map[string]error + + nextV4 int + nextV6 int +} + +func newFakeIPAM(t *testing.T, classes ...*ipamv1alpha1.IPClass) *fakeIPAM { + t.Helper() + scheme, err := IPAMScheme() + require.NoError(t, err) + + return &fakeIPAM{ + scheme: scheme, + clients: map[string]client.Client{}, + classes: classes, + createdIn: map[string][]string{}, + deletedIn: map[string][]string{}, + allocationPolicy: map[string]ipamv1alpha1.ReclaimPolicy{}, + orphans: map[string]string{}, + failOn: map[string]error{}, + } +} + +func (f *fakeIPAM) ClientForProject(project string) (client.Client, error) { + f.mu.Lock() + defer f.mu.Unlock() + + if existing, ok := f.clients[project]; ok { + return existing, nil + } + + builder := fakeclient.NewClientBuilder().WithScheme(f.scheme) + for _, class := range f.classes { + builder = builder.WithObjects(class.DeepCopy()) + } + + cl := builder.WithInterceptorFuncs(interceptor.Funcs{ + Create: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.CreateOption) error { + ipClaim, ok := obj.(*ipamv1alpha1.IPClaim) + if !ok { + return c.Create(ctx, obj, opts...) + } + + f.mu.Lock() + f.createdIn[project] = append(f.createdIn[project], ipClaim.Name) + failure := f.failOn[ipClaim.Name] + if failure == nil { + failure = f.retainedConflictLocked(ipClaim) + } + if failure == nil { + f.allocateLocked(project, ipClaim) + } + f.mu.Unlock() + + if failure != nil { + return failure + } + + if err := c.Create(ctx, obj, opts...); err != nil { + // IPAM lets this collision escape as a raw 500, which no status + // code tells apart from a genuine internal error. + if apierrors.IsAlreadyExists(err) { + return apierrors.NewInternalError(fmt.Errorf( + `insert allocation: ERROR: duplicate key value violates unique constraint "ipam_cidr_allocations_claim_key_key" (SQLSTATE 23505)`)) + } + return err + } + return nil + }, + Delete: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.DeleteOption) error { + if ipClaim, ok := obj.(*ipamv1alpha1.IPClaim); ok { + var stored ipamv1alpha1.IPClaim + _ = c.Get(ctx, client.ObjectKeyFromObject(ipClaim), &stored) + + f.mu.Lock() + f.deletedIn[project] = append(f.deletedIn[project], ipClaim.Name) + if f.allocationPolicy[ipClaim.Name] == ipamv1alpha1.ReclaimRetain { + f.orphans[allocationNameFor(ipClaim.Name)] = stored.Status.AllocatedCIDR + } + f.mu.Unlock() + } + return c.Delete(ctx, obj, opts...) + }, + }).Build() + + f.clients[project] = cl + return cl, nil +} + +func allocationNameFor(ipClaimName string) string { + return "alloc-" + ipClaimName +} + +// retainedConflictLocked models the other duplicate, which IPAM maps to a 409. +// An allocation left behind by a deleted claim blocks the name it used. +func (f *fakeIPAM) retainedConflictLocked(ipClaim *ipamv1alpha1.IPClaim) error { + allocationName := allocationNameFor(ipClaim.Name) + if _, orphaned := f.orphans[allocationName]; !orphaned { + return nil + } + return apierrors.NewConflict( + ipamv1alpha1.SchemeGroupVersion.WithResource("ipclaims").GroupResource(), + ipClaim.Name, + fmt.Errorf("an allocation under this identity already exists: IPAllocation %q, retained by an earlier claim of the same name; delete it to reuse the name", allocationName), + ) +} + +func (f *fakeIPAM) allocateLocked(project string, ipClaim *ipamv1alpha1.IPClaim) { + family := ipClaim.Spec.IPFamily + if ipClaim.Spec.ClassName != "" { + for _, class := range f.classes { + if class.Name == ipClaim.Spec.ClassName { + family = class.Spec.IPFamily + } + } + } + + // Frozen once, here. Later edits to the IPClaim never reach it. + f.allocationPolicy[ipClaim.Name] = ipClaim.Spec.ReclaimPolicy + + // The real server writes only allocatedCIDR, host prefixes included, and + // never status.address. + ipClaim.Status.Phase = ipamv1alpha1.ClaimBound + if family == ipamv1alpha1.IPv6 { + f.nextV6++ + ipClaim.Status.AllocatedCIDR = fmt.Sprintf("2001:db8:a000:%d::/96", f.nextV6) + } else { + f.nextV4++ + ipClaim.Status.AllocatedCIDR = fmt.Sprintf("10.128.0.%d/32", f.nextV4) + } + ipClaim.Status.PoolRef = &ipamv1alpha1.LocalRef{Name: "pool-" + project} +} + +// created and deleted are keyed by project, so a test can assert which project +// an allocation reached. +func (f *fakeIPAM) created() map[string][]string { + f.mu.Lock() + defer f.mu.Unlock() + return maps.Clone(f.createdIn) +} + +func (f *fakeIPAM) deleted() map[string][]string { + f.mu.Lock() + defer f.mu.Unlock() + return maps.Clone(f.deletedIn) +} + +func (f *fakeIPAM) createdAnywhere() int { + f.mu.Lock() + defer f.mu.Unlock() + total := 0 + for _, names := range f.createdIn { + total += len(names) + } + return total +} + +// orphanedAllocations reports allocations no IPClaim references. +func (f *fakeIPAM) orphanedAllocations() map[string]string { + f.mu.Lock() + defer f.mu.Unlock() + return maps.Clone(f.orphans) +} + +func (f *fakeIPAM) refuse(name string, err error) { + f.mu.Lock() + defer f.mu.Unlock() + f.failOn[name] = err +} + +func publicV4Class() *ipamv1alpha1.IPClass { + class := &ipamv1alpha1.IPClass{} + class.Name = testPublicV4Class + class.Spec.IPFamily = ipamv1alpha1.IPv4 + return class +} + +func startNetworkInterfaceEnv(t *testing.T) (client.Client, *rest.Config) { + t.Helper() + if os.Getenv("KUBEBUILDER_ASSETS") == "" { + t.Skip("KUBEBUILDER_ASSETS unset; run via `make test` to exercise envtest") + } + + testScheme := runtime.NewScheme() + require.NoError(t, clientgoscheme.AddToScheme(testScheme)) + require.NoError(t, networkingv1alpha.AddToScheme(testScheme)) + + env := &envtest.Environment{ + CRDDirectoryPaths: []string{filepath.Join("..", "..", "config", "crd", "bases")}, + ErrorIfCRDPathMissing: true, + } + cfg, err := env.Start() + require.NoError(t, err) + t.Cleanup(func() { _ = env.Stop() }) + + cl, err := client.New(cfg, client.Options{Scheme: testScheme}) + require.NoError(t, err) + return cl, cfg +} + +type scenario struct { + restConfig *rest.Config + events *events.FakeRecorder + t *testing.T + ctx context.Context + client client.Client + ipam *fakeIPAM + reconciler *NetworkInterfaceClaimReconciler + namespace string +} + +// newScenario builds one namespace of objects. Set labelled to false for a +// namespace that names no project. +func newScenario(t *testing.T, labelled bool, networkFamilies []networkingv1alpha.IPFamily, classes ...*ipamv1alpha1.IPClass) *scenario { + t.Helper() + cl, restConfig := startNetworkInterfaceEnv(t) + ctx := context.Background() + + namespaceName := "ns-" + sanitizeName(strings.ToLower(t.Name())) + + namespace := &corev1.Namespace{} + namespace.Name = namespaceName + namespace.Labels = map[string]string{ + downstreamclient.UpstreamOwnerNamespaceLabel: testProjectNS, + } + if labelled { + namespace.Labels[downstreamclient.UpstreamOwnerClusterNameLabel] = "cluster-" + testProject + } + require.NoError(t, cl.Create(ctx, namespace)) + + network := &networkingv1alpha.Network{} + network.Namespace = namespaceName + network.Name = "default" + network.Spec = networkingv1alpha.NetworkSpec{ + IPAM: networkingv1alpha.NetworkIPAM{Mode: networkingv1alpha.NetworkIPAMModeAuto}, + IPFamilies: networkFamilies, + MTU: 1460, + } + require.NoError(t, cl.Create(ctx, network)) + + ipam := newFakeIPAM(t, classes...) + + operatorConfig := config.NetworkServicesOperator{} + operatorConfig.NetworkInterface.Enabled = true + operatorConfig.NetworkInterface.Location = config.LocationConfig{ + Name: testLocationName, + Namespace: testLocationNS, + } + + return &scenario{ + restConfig: restConfig, + events: events.NewFakeRecorder(64), + t: t, + ctx: ctx, + client: cl, + ipam: ipam, + reconciler: &NetworkInterfaceClaimReconciler{Config: operatorConfig, IPAM: ipam}, + namespace: namespaceName, + } +} + +// transientNamespaceClient fails namespace reads the way a flaky API server +// does, without the namespace itself changing. +type transientNamespaceClient struct { + client.Client +} + +func (c *transientNamespaceClient) Get( + ctx context.Context, + key client.ObjectKey, + obj client.Object, + opts ...client.GetOption, +) error { + if _, ok := obj.(*corev1.Namespace); ok { + return apierrors.NewServiceUnavailable("the server is currently unable to handle the request") + } + return c.Client.Get(ctx, key, obj, opts...) +} + +func sanitizeName(in string) string { + var b strings.Builder + for _, r := range in { + switch { + case r >= 'a' && r <= 'z', r >= '0' && r <= '9', r == '-': + b.WriteRune(r) + default: + b.WriteRune('-') + } + } + return strings.Trim(b.String(), "-") +} + +func (s *scenario) createClaim(name string, spec networkingv1alpha.NetworkInterfaceClaimSpec) *networkingv1alpha.NetworkInterfaceClaim { + s.t.Helper() + claim := &networkingv1alpha.NetworkInterfaceClaim{} + claim.Namespace = s.namespace + claim.Name = name + claim.Spec = spec + claim.Spec.Network = networkingv1alpha.LocalNetworkRef{Name: "default"} + require.NoError(s.t, s.client.Create(s.ctx, claim)) + return claim +} + +func (s *scenario) reconcile(claim *networkingv1alpha.NetworkInterfaceClaim) { + s.t.Helper() + _, err := s.reconciler.reconcileClaim(s.ctx, s.client, s.events, client.ObjectKeyFromObject(claim)) + require.NoError(s.t, err) +} + +func (s *scenario) reconcileInterface(name string) { + s.t.Helper() + interfaces := &NetworkInterfaceReconciler{Config: s.reconciler.Config, IPAM: s.ipam, claims: s.reconciler} + require.NoError(s.t, interfaces.reconcileInterface(s.ctx, s.client, + client.ObjectKey{Namespace: s.namespace, Name: name})) +} + +// programNetworkContext marks the binding's context ready, which is what makes +// the interface record a context and the gateway resolvable. +func (s *scenario) programNetworkContext() string { + s.t.Helper() + + bindingName := fmt.Sprintf("default-%s-%s", testLocationNS, testLocationName) + var binding networkingv1alpha.NetworkBinding + require.NoError(s.t, s.client.Get(s.ctx, + client.ObjectKey{Namespace: s.namespace, Name: bindingName}, &binding)) + + contextName := bindingName + binding.Status.NetworkContextRef = &networkingv1alpha.NetworkContextRef{ + Namespace: s.namespace, + Name: contextName, + } + require.NoError(s.t, s.client.Status().Update(s.ctx, &binding)) + return contextName +} + +func (s *scenario) createSubnet( + name, contextName string, + family networkingv1alpha.IPFamily, + startAddress string, + prefixLength int32, +) { + s.t.Helper() + + subnet := &networkingv1alpha.Subnet{} + subnet.Namespace = s.namespace + subnet.Name = name + subnet.Spec = networkingv1alpha.SubnetSpec{ + SubnetClass: "private", + NetworkContext: networkingv1alpha.LocalNetworkContextRef{Name: contextName}, + Location: networkingv1alpha.LocationReference{ + Name: testLocationName, + Namespace: testLocationNS, + }, + IPFamily: family, + StartAddress: startAddress, + PrefixLength: prefixLength, + } + require.NoError(s.t, s.client.Create(s.ctx, subnet)) +} + +func (s *scenario) getClaim(name string) *networkingv1alpha.NetworkInterfaceClaim { + s.t.Helper() + var claim networkingv1alpha.NetworkInterfaceClaim + require.NoError(s.t, s.client.Get(s.ctx, client.ObjectKey{Namespace: s.namespace, Name: name}, &claim)) + return &claim +} + +func (s *scenario) getInterface(name string) (*networkingv1alpha.NetworkInterface, error) { + var iface networkingv1alpha.NetworkInterface + err := s.client.Get(s.ctx, client.ObjectKey{Namespace: s.namespace, Name: name}, &iface) + return &iface, err +} + +func (s *scenario) deleteClaim(claim *networkingv1alpha.NetworkInterfaceClaim) { + s.t.Helper() + require.NoError(s.t, s.client.Delete(s.ctx, claim)) + s.reconcile(claim) +} + +func (s *scenario) ipClaim(name string) *ipamv1alpha1.IPClaim { + s.t.Helper() + cl, err := s.ipam.ClientForProject(testProject) + require.NoError(s.t, err) + + var ipClaim ipamv1alpha1.IPClaim + require.NoError(s.t, cl.Get(s.ctx, client.ObjectKey{Namespace: testProjectNS, Name: name}, &ipClaim)) + return &ipClaim +} + +func conditionOf(claim *networkingv1alpha.NetworkInterfaceClaim, conditionType string) *metav1.Condition { + return apimeta.FindStatusCondition(claim.Status.Conditions, conditionType) +} + +func TestNetworkInterfaceClaimBindsDualStack(t *testing.T) { + s := newScenario(t, true, + []networkingv1alpha.IPFamily{networkingv1alpha.IPv4Protocol, networkingv1alpha.IPv6Protocol}, + publicV4Class()) + + claim := s.createClaim("web-0-eth0", networkingv1alpha.NetworkInterfaceClaimSpec{ + InterfaceName: "eth0", + IPFamilies: []networkingv1alpha.IPFamily{ + networkingv1alpha.IPv6Protocol, + networkingv1alpha.IPv4Protocol, + }, + Addresses: []networkingv1alpha.NetworkInterfaceAddressRequest{{Class: testPublicV4Class}}, + ReclaimPolicy: networkingv1alpha.NetworkInterfaceReclaimPolicyDelete, + }) + s.reconcile(claim) + + iface, err := s.getInterface("web-0-eth0") + require.NoError(t, err) + require.Equal(t, int32(1460), iface.Spec.MTU) + require.Equal(t, "eth0", iface.Spec.InterfaceName) + require.Equal(t, networkingv1alpha.NetworkInterfacePhaseBound, iface.Status.Phase) + require.NotNil(t, iface.Spec.ClaimRef) + require.Equal(t, claim.Name, iface.Spec.ClaimRef.Name) + + require.Len(t, iface.Spec.Addresses, 2) + require.Equal(t, networkingv1alpha.IPv6Protocol, iface.Spec.Addresses[0].Family) + require.True(t, iface.Spec.Addresses[0].Primary, "the first family listed holds the primary address") + require.Equal(t, "2001:db8:a000:1::/96", iface.Spec.Addresses[0].Address) + require.Equal(t, networkingv1alpha.IPv4Protocol, iface.Spec.Addresses[1].Family) + require.False(t, iface.Spec.Addresses[1].Primary) + require.Equal(t, "10.128.0.1/32", iface.Spec.Addresses[1].Address, + "an address inside the network keeps its prefix, host prefixes included") + + require.Len(t, iface.Spec.ExternalAddresses, 1) + require.Equal(t, testPublicV4Class, iface.Spec.ExternalAddresses[0].Class) + require.Equal(t, "10.128.0.2", iface.Spec.ExternalAddresses[0].Address, + "an externally reachable address is bare, with no prefix") + + require.ElementsMatch(t, []string{ + "web-0-eth0-f-ipv6", + "web-0-eth0-f-ipv4", + "web-0-eth0-c-" + testPublicV4Class, + }, s.ipam.created()[testProject]) + + bound := s.getClaim("web-0-eth0") + require.Equal(t, "web-0-eth0", bound.Status.NetworkInterfaceRef.Name) + require.Equal(t, iface.Spec.Addresses, bound.Status.Addresses) + require.Equal(t, iface.Spec.ExternalAddresses, bound.Status.ExternalAddresses) + + require.Equal(t, metav1.ConditionTrue, conditionOf(bound, networkingv1alpha.NetworkInterfaceClaimBound).Status) + require.Equal(t, metav1.ConditionTrue, conditionOf(bound, networkingv1alpha.NetworkInterfaceClaimAllocated).Status) + require.Equal(t, metav1.ConditionUnknown, conditionOf(bound, networkingv1alpha.NetworkInterfaceClaimProgrammed).Status, + "programming is out of scope and must stay unknown") + require.NotEqual(t, metav1.ConditionTrue, conditionOf(bound, networkingv1alpha.NetworkInterfaceClaimReady).Status, + "Ready requires Programmed, which nothing reports yet") +} + +func TestNetworkInterfaceClaimFailsClosedWithoutProject(t *testing.T) { + s := newScenario(t, false, []networkingv1alpha.IPFamily{networkingv1alpha.IPv6Protocol}) + + claim := s.createClaim("orphan-eth0", networkingv1alpha.NetworkInterfaceClaimSpec{ + InterfaceName: "eth0", + IPFamilies: []networkingv1alpha.IPFamily{networkingv1alpha.IPv6Protocol}, + ReclaimPolicy: networkingv1alpha.NetworkInterfaceReclaimPolicyDelete, + }) + s.reconcile(claim) + + require.Zero(t, s.ipam.createdAnywhere(), + "a namespace naming no project must not reach IPAM at all") + + _, err := s.getInterface("orphan-eth0") + require.True(t, apierrors.IsNotFound(err), "nothing may be published without a project") + + rejected := s.getClaim("orphan-eth0") + for _, conditionType := range []string{ + networkingv1alpha.NetworkInterfaceClaimBound, + networkingv1alpha.NetworkInterfaceClaimAllocated, + networkingv1alpha.NetworkInterfaceClaimReady, + } { + condition := conditionOf(rejected, conditionType) + require.Equal(t, metav1.ConditionFalse, condition.Status, conditionType) + require.Equal(t, "ProjectUnresolved", condition.Reason, conditionType) + require.Contains(t, condition.Message, downstreamclient.UpstreamOwnerClusterNameLabel, + "the failure must name the label that is missing") + } + + // A claim that allocated nothing must still be deletable. + s.deleteClaim(rejected) + var gone networkingv1alpha.NetworkInterfaceClaim + err = s.client.Get(s.ctx, client.ObjectKeyFromObject(rejected), &gone) + require.True(t, apierrors.IsNotFound(err)) +} + +func TestNetworkInterfaceClaimRejectsFamilyTheNetworkDoesNotCarry(t *testing.T) { + s := newScenario(t, true, []networkingv1alpha.IPFamily{networkingv1alpha.IPv4Protocol}) + + claim := s.createClaim("v6-on-v4-eth0", networkingv1alpha.NetworkInterfaceClaimSpec{ + InterfaceName: "eth0", + IPFamilies: []networkingv1alpha.IPFamily{networkingv1alpha.IPv6Protocol}, + ReclaimPolicy: networkingv1alpha.NetworkInterfaceReclaimPolicyDelete, + }) + s.reconcile(claim) + + require.Zero(t, s.ipam.createdAnywhere(), + "IPAM binds the class's family and would never report this, so NSO must catch it first") + + rejected := s.getClaim("v6-on-v4-eth0") + condition := conditionOf(rejected, networkingv1alpha.NetworkInterfaceClaimAllocated) + require.Equal(t, metav1.ConditionFalse, condition.Status) + require.Equal(t, "AddressFamilyNotCarried", condition.Reason) + require.Contains(t, condition.Message, "default") + require.Contains(t, condition.Message, "IPv6") +} + +func TestNetworkInterfaceClaimRollsBackPartialAllocation(t *testing.T) { + s := newScenario(t, true, + []networkingv1alpha.IPFamily{networkingv1alpha.IPv4Protocol, networkingv1alpha.IPv6Protocol}) + + s.ipam.refuse("half-eth0-f-ipv4", apierrors.NewGenericServerResponse( + httpStatusInsufficientStorage, "POST", + schema.GroupResource{Group: "ipam.miloapis.com", Resource: "ipclaims"}, + "", "IPPool exhausted", 0, false)) + + claim := s.createClaim("half-eth0", networkingv1alpha.NetworkInterfaceClaimSpec{ + InterfaceName: "eth0", + IPFamilies: []networkingv1alpha.IPFamily{ + networkingv1alpha.IPv6Protocol, + networkingv1alpha.IPv4Protocol, + }, + ReclaimPolicy: networkingv1alpha.NetworkInterfaceReclaimPolicyDelete, + }) + s.reconcile(claim) + + _, err := s.getInterface("half-eth0") + require.True(t, apierrors.IsNotFound(err), "a partly addressed interface is never published") + + require.Equal(t, []string{"half-eth0-f-ipv6"}, s.ipam.deleted()[testProject], + "the address that did allocate is released rather than leaked") + + rejected := s.getClaim("half-eth0") + condition := conditionOf(rejected, networkingv1alpha.NetworkInterfaceClaimAllocated) + require.Equal(t, metav1.ConditionFalse, condition.Status) + require.Equal(t, string(allocationFailureExhausted), condition.Reason) + require.Contains(t, condition.Message, "IPPool exhausted", + "IPAM's own message is carried through verbatim") +} + +func TestNetworkInterfaceRetainRebindsSameAddresses(t *testing.T) { + s := newScenario(t, true, []networkingv1alpha.IPFamily{networkingv1alpha.IPv6Protocol}) + + spec := networkingv1alpha.NetworkInterfaceClaimSpec{ + InterfaceName: "eth0", + IPFamilies: []networkingv1alpha.IPFamily{networkingv1alpha.IPv6Protocol}, + ReclaimPolicy: networkingv1alpha.NetworkInterfaceReclaimPolicyRetain, + } + + claim := s.createClaim("slot-0-eth0", spec) + s.reconcile(claim) + + first, err := s.getInterface("slot-0-eth0") + require.NoError(t, err) + originalAddresses := first.Spec.Addresses + originalClaimUID := claim.UID + + s.deleteClaim(s.getClaim("slot-0-eth0")) + + retained, err := s.getInterface("slot-0-eth0") + require.NoError(t, err, "Retain keeps the interface") + require.Nil(t, retained.Spec.ClaimRef) + require.Equal(t, networkingv1alpha.NetworkInterfacePhaseAvailable, retained.Status.Phase) + require.Equal(t, originalAddresses, retained.Spec.Addresses, "a retained interface keeps its addresses") + require.Empty(t, s.ipam.deleted()[testProject], "Retain releases no addresses") + require.Empty(t, s.ipam.orphanedAllocations(), + "retention comes from keeping the IPClaim alive, so nothing is ever orphaned") + + allocationsBefore := len(s.ipam.created()[testProject]) + + replacement := s.createClaim("slot-0-eth0", spec) + require.NotEqual(t, originalClaimUID, replacement.UID, + "the replacement is a genuinely different object, not the same claim resurrected") + s.reconcile(replacement) + + rebound, err := s.getInterface("slot-0-eth0") + require.NoError(t, err) + require.Equal(t, originalAddresses, rebound.Spec.Addresses, + "the replacement comes back to the addresses the predecessor held") + require.Equal(t, "slot-0-eth0", rebound.Spec.ClaimRef.Name, + "the interface records the claim now holding it") + require.Equal(t, networkingv1alpha.NetworkInterfacePhaseBound, rebound.Status.Phase) + require.Len(t, s.ipam.created()[testProject], allocationsBefore, + "rebinding allocates nothing new") + + bound := s.getClaim("slot-0-eth0") + require.Equal(t, originalAddresses, bound.Status.Addresses) +} + +func TestNetworkInterfaceDeleteReleasesAddresses(t *testing.T) { + s := newScenario(t, true, []networkingv1alpha.IPFamily{networkingv1alpha.IPv6Protocol}) + + claim := s.createClaim("ephemeral-eth0", networkingv1alpha.NetworkInterfaceClaimSpec{ + InterfaceName: "eth0", + IPFamilies: []networkingv1alpha.IPFamily{networkingv1alpha.IPv6Protocol}, + ReclaimPolicy: networkingv1alpha.NetworkInterfaceReclaimPolicyDelete, + }) + s.reconcile(claim) + + _, err := s.getInterface("ephemeral-eth0") + require.NoError(t, err) + + s.deleteClaim(s.getClaim("ephemeral-eth0")) + + _, err = s.getInterface("ephemeral-eth0") + require.True(t, apierrors.IsNotFound(err), "Delete removes the interface") + require.Equal(t, []string{"ephemeral-eth0-f-ipv6"}, s.ipam.deleted()[testProject]) + require.Empty(t, s.ipam.orphanedAllocations(), + "an address claimed under Delete goes back to its pool") +} + +// IPAM reports a duplicate name as a 500. Treating that as a hard failure +// rejects the claim forever, because every retry asks for the same name. +func TestAllocationResumesAfterTheInterfaceFailsToLand(t *testing.T) { + s := newScenario(t, true, + []networkingv1alpha.IPFamily{networkingv1alpha.IPv4Protocol, networkingv1alpha.IPv6Protocol}) + + spec := networkingv1alpha.NetworkInterfaceClaimSpec{ + InterfaceName: "eth0", + IPFamilies: []networkingv1alpha.IPFamily{ + networkingv1alpha.IPv6Protocol, + networkingv1alpha.IPv4Protocol, + }, + ReclaimPolicy: networkingv1alpha.NetworkInterfaceReclaimPolicyDelete, + } + + claim := s.createClaim("resumed", spec) + s.reconcile(claim) + + original, err := s.getInterface("resumed") + require.NoError(t, err) + originalAddresses := original.Spec.Addresses + + // Stand in for the interface write never landing. + controllerutil.RemoveFinalizer(original, networkInterfaceFinalizer) + require.NoError(t, s.client.Update(s.ctx, original)) + require.NoError(t, s.client.Delete(s.ctx, original)) + + allocationsBefore := len(s.ipam.created()[testProject]) + + s.reconcile(s.getClaim("resumed")) + + recovered, err := s.getInterface("resumed") + require.NoError(t, err, "the next reconcile must rebuild the interface, not reject the claim") + require.Equal(t, originalAddresses, recovered.Spec.Addresses, + "it comes back on the addresses already allocated under those names") + + require.Len(t, s.ipam.created()[testProject], allocationsBefore, + "the addresses are found by name, so nothing is re-allocated") + + bound := s.getClaim("resumed") + require.Equal(t, metav1.ConditionTrue, conditionOf(bound, networkingv1alpha.NetworkInterfaceClaimAllocated).Status, + "a 409 from a name we already own is not an allocation failure") +} + +// A retained interface sits Available and bound to nothing, which is when an +// operator is most likely to delete it. Its addresses must still be released. +func TestDeletingAnInterfaceReleasesItsAddresses(t *testing.T) { + s := newScenario(t, true, []networkingv1alpha.IPFamily{networkingv1alpha.IPv6Protocol}) + + claim := s.createClaim("stranded", networkingv1alpha.NetworkInterfaceClaimSpec{ + InterfaceName: "eth0", + IPFamilies: []networkingv1alpha.IPFamily{networkingv1alpha.IPv6Protocol}, + ReclaimPolicy: networkingv1alpha.NetworkInterfaceReclaimPolicyRetain, + }) + s.reconcile(claim) + s.deleteClaim(s.getClaim("stranded")) + + available, err := s.getInterface("stranded") + require.NoError(t, err) + require.Equal(t, networkingv1alpha.NetworkInterfacePhaseAvailable, available.Status.Phase) + require.Contains(t, available.Finalizers, networkInterfaceFinalizer, + "a retained interface carries its own finalizer, having no claim to carry one for it") + + require.NoError(t, s.client.Delete(s.ctx, available)) + s.reconcileInterface("stranded") + + _, err = s.getInterface("stranded") + require.True(t, apierrors.IsNotFound(err), "the finalizer must not hold the delete open once released") + require.Equal(t, []string{"stranded-f-ipv6"}, s.ipam.deleted()[testProject], + "the addresses go back to the pool rather than outliving every object naming them") +} + +// A live claim rebuilds the interface, so releasing its addresses here would +// renumber a running workload. +func TestDeletingABoundInterfaceKeepsItsAddresses(t *testing.T) { + s := newScenario(t, true, []networkingv1alpha.IPFamily{networkingv1alpha.IPv6Protocol}) + + claim := s.createClaim("still-held", networkingv1alpha.NetworkInterfaceClaimSpec{ + InterfaceName: "eth0", + IPFamilies: []networkingv1alpha.IPFamily{networkingv1alpha.IPv6Protocol}, + ReclaimPolicy: networkingv1alpha.NetworkInterfaceReclaimPolicyDelete, + }) + s.reconcile(claim) + + bound, err := s.getInterface("still-held") + require.NoError(t, err) + originalAddresses := bound.Spec.Addresses + + require.NoError(t, s.client.Delete(s.ctx, bound)) + s.reconcileInterface("still-held") + + require.Empty(t, s.ipam.deleted()[testProject], + "a claim still holds this interface, so its addresses stay allocated") + + s.reconcile(s.getClaim("still-held")) + + rebuilt, err := s.getInterface("still-held") + require.NoError(t, err, "the claim rebuilds the interface it still holds") + require.Equal(t, originalAddresses, rebuilt.Spec.Addresses, + "and it comes back on the same addresses rather than renumbering") +} + +// A fake recorder cannot be forbidden, so no other test catches an event the +// controller has no grant to write. The generated role is the only signal. +func TestManagerRoleGrantsEventCreation(t *testing.T) { + manifest, err := os.ReadFile(filepath.Join("..", "..", "config", "rbac", "role.yaml")) + require.NoError(t, err) + + var role rbacv1.ClusterRole + require.NoError(t, yaml.Unmarshal(manifest, &role)) + + granted := false + for _, rule := range role.Rules { + if slices.Contains(rule.APIGroups, "events.k8s.io") && + slices.Contains(rule.Resources, "events") && + slices.Contains(rule.Verbs, "create") { + granted = true + break + } + } + + require.True(t, granted, + "the controller records events through the events.k8s.io API; without that group granted they are "+ + "rejected and the warning never reaches an operator, whatever the core-group grant says") +} + +// events.k8s.io/v1 requires an action and a resolvable regarding reference, +// and a fake recorder checks neither. This posts through a real broadcaster so +// the event has to survive validation. Authorization is covered separately by +// TestManagerRoleGrantsEventCreation, because envtest does not enforce RBAC. +func TestMissingAllocationEventReachesTheAPIServer(t *testing.T) { + s := newScenario(t, true, []networkingv1alpha.IPFamily{networkingv1alpha.IPv6Protocol}) + + claim := s.createClaim("delivered", networkingv1alpha.NetworkInterfaceClaimSpec{ + InterfaceName: "eth0", + IPFamilies: []networkingv1alpha.IPFamily{networkingv1alpha.IPv6Protocol}, + ReclaimPolicy: networkingv1alpha.NetworkInterfaceReclaimPolicyDelete, + }) + s.reconcile(claim) + + iface, err := s.getInterface("delivered") + require.NoError(t, err) + + eventClient, err := eventsv1client.NewForConfig(s.restConfig) + require.NoError(t, err) + + broadcaster := events.NewBroadcaster(&events.EventSinkImpl{Interface: eventClient}) + require.NoError(t, broadcaster.StartRecordingToSinkWithContext(s.ctx)) + t.Cleanup(broadcaster.Shutdown) + + recorder := broadcaster.NewRecorder(s.client.Scheme(), "networkinterfaceclaim-controller") + + s.reconciler.reportMissingAllocation(s.ctx, recorder, + projectRouting{project: testProject, projectNamespace: testProjectNS}, + s.getClaim("delivered"), iface, + allocationEntry{discriminator: "f-ipv6", address: iface.Spec.Addresses[0].Address}, + "delivered-f-ipv6", "") + + var delivered *eventsv1.Event + require.Eventually(t, func() bool { + list, err := eventClient.Events(s.namespace).List(s.ctx, metav1.ListOptions{}) + if err != nil { + return false + } + for i := range list.Items { + if list.Items[i].Reason == "AddressAllocationMissing" { + delivered = &list.Items[i] + return true + } + } + return false + }, 30*time.Second, 250*time.Millisecond, + "the event never landed; the API server rejected it or the recorder never posted it") + + require.Equal(t, corev1.EventTypeWarning, delivered.Type) + require.NotEmpty(t, delivered.Action, "events.k8s.io/v1 requires an action") + require.Equal(t, "NetworkInterfaceClaim", delivered.Regarding.Kind) + require.Equal(t, "delivered", delivered.Regarding.Name) + require.NotNil(t, delivered.Related, "the interface is carried as the related object") + require.Equal(t, "NetworkInterface", delivered.Related.Kind) + require.Contains(t, delivered.Note, iface.Spec.Addresses[0].Address) + require.Contains(t, delivered.Note, "delivered-f-ipv6") + require.Contains(t, delivered.Note, testProject) +} + +// Addresses belong to the network they were allocated from. Publishing them +// under another network hands one network's addresses to another. +func TestAdoptionRefusesAnInterfaceOnAnotherNetwork(t *testing.T) { + s := newScenario(t, true, []networkingv1alpha.IPFamily{networkingv1alpha.IPv6Protocol}) + + other := &networkingv1alpha.Network{} + other.Namespace = s.namespace + other.Name = "other" + other.Spec = networkingv1alpha.NetworkSpec{ + IPAM: networkingv1alpha.NetworkIPAM{Mode: networkingv1alpha.NetworkIPAMModeAuto}, + IPFamilies: []networkingv1alpha.IPFamily{networkingv1alpha.IPv6Protocol}, + MTU: 1460, + } + require.NoError(t, s.client.Create(s.ctx, other)) + + spec := networkingv1alpha.NetworkInterfaceClaimSpec{ + InterfaceName: "eth0", + IPFamilies: []networkingv1alpha.IPFamily{networkingv1alpha.IPv6Protocol}, + ReclaimPolicy: networkingv1alpha.NetworkInterfaceReclaimPolicyRetain, + } + + first := s.createClaim("crossnet", spec) + s.reconcile(first) + s.deleteClaim(s.getClaim("crossnet")) + + retained, err := s.getInterface("crossnet") + require.NoError(t, err) + heldAddress := retained.Spec.Addresses[0].Address + + onOtherNetwork := spec + onOtherNetwork.Network = networkingv1alpha.LocalNetworkRef{Name: "other"} + claim := &networkingv1alpha.NetworkInterfaceClaim{} + claim.Namespace = s.namespace + claim.Name = "crossnet" + claim.Spec = onOtherNetwork + require.NoError(t, s.client.Create(s.ctx, claim)) + s.reconcile(claim) + + rejected := s.getClaim("crossnet") + condition := conditionOf(rejected, networkingv1alpha.NetworkInterfaceClaimReady) + require.Equal(t, metav1.ConditionFalse, condition.Status) + require.Equal(t, "NetworkMismatch", condition.Reason) + require.Contains(t, condition.Message, "other") + + require.Empty(t, rejected.Status.Addresses, + "a claim on another network must not publish addresses allocated for this one") + + unchanged, err := s.getInterface("crossnet") + require.NoError(t, err) + require.Equal(t, heldAddress, unchanged.Spec.Addresses[0].Address) + require.Nil(t, unchanged.Spec.ClaimRef, "the interface stays unbound") +} + +// The data plane owns Programmed. Overwriting it puts Ready permanently out of +// reach, and Ready is what consumers gate on. +func TestProgrammedIsSeededThenLeftAlone(t *testing.T) { + s := newScenario(t, true, []networkingv1alpha.IPFamily{networkingv1alpha.IPv6Protocol}) + + claim := s.createClaim("gated", networkingv1alpha.NetworkInterfaceClaimSpec{ + InterfaceName: "eth0", + IPFamilies: []networkingv1alpha.IPFamily{networkingv1alpha.IPv6Protocol}, + ReclaimPolicy: networkingv1alpha.NetworkInterfaceReclaimPolicyDelete, + }) + s.reconcile(claim) + + seeded := s.getClaim("gated") + require.Equal(t, metav1.ConditionUnknown, + conditionOf(seeded, networkingv1alpha.NetworkInterfaceClaimProgrammed).Status) + require.NotEqual(t, metav1.ConditionTrue, + conditionOf(seeded, networkingv1alpha.NetworkInterfaceClaimReady).Status) + + // Stand in for the data plane reporting the attachment. + apimeta.SetStatusCondition(&seeded.Status.Conditions, metav1.Condition{ + Type: networkingv1alpha.NetworkInterfaceClaimProgrammed, + Status: metav1.ConditionTrue, + Reason: "Programmed", + Message: "The attachment is ready", + }) + require.NoError(t, s.client.Status().Update(s.ctx, seeded)) + + s.reconcile(s.getClaim("gated")) + + settled := s.getClaim("gated") + require.Equal(t, metav1.ConditionTrue, + conditionOf(settled, networkingv1alpha.NetworkInterfaceClaimProgrammed).Status, + "NSO must not revert the condition the data plane owns") + require.Equal(t, metav1.ConditionTrue, + conditionOf(settled, networkingv1alpha.NetworkInterfaceClaimReady).Status, + "Ready follows from the other three rather than being hardcoded") +} + +// A blip reading the namespace must not demote a healthy claim and then leave +// nothing to bring it back. +func TestTransientProjectFailureDoesNotWedgeTheClaim(t *testing.T) { + s := newScenario(t, true, []networkingv1alpha.IPFamily{networkingv1alpha.IPv6Protocol}) + + claim := s.createClaim("blip", networkingv1alpha.NetworkInterfaceClaimSpec{ + InterfaceName: "eth0", + IPFamilies: []networkingv1alpha.IPFamily{networkingv1alpha.IPv6Protocol}, + ReclaimPolicy: networkingv1alpha.NetworkInterfaceReclaimPolicyDelete, + }) + s.reconcile(claim) + + failing := &transientNamespaceClient{Client: s.client} + _, err := s.reconciler.reconcileClaim(s.ctx, failing, s.events, + client.ObjectKey{Namespace: s.namespace, Name: "blip"}) + require.Error(t, err, "a failed read must be retried, not turned into a rejection") + + bound := s.getClaim("blip") + require.Equal(t, metav1.ConditionTrue, + conditionOf(bound, networkingv1alpha.NetworkInterfaceClaimBound).Status, + "a read failure says nothing about whether the claim is bound") +} + +// A rejection must carry its own way back, because nothing watches the network. +func TestRejectionRequeues(t *testing.T) { + s := newScenario(t, true, []networkingv1alpha.IPFamily{networkingv1alpha.IPv6Protocol}) + + claim := &networkingv1alpha.NetworkInterfaceClaim{} + claim.Namespace = s.namespace + claim.Name = "nonetwork" + claim.Spec = networkingv1alpha.NetworkInterfaceClaimSpec{ + Network: networkingv1alpha.LocalNetworkRef{Name: "absent"}, + InterfaceName: "eth0", + IPFamilies: []networkingv1alpha.IPFamily{networkingv1alpha.IPv6Protocol}, + ReclaimPolicy: networkingv1alpha.NetworkInterfaceReclaimPolicyDelete, + } + require.NoError(t, s.client.Create(s.ctx, claim)) + + result, err := s.reconciler.reconcileClaim(s.ctx, s.client, s.events, + client.ObjectKeyFromObject(claim)) + require.NoError(t, err) + require.NotZero(t, result.RequeueAfter, + "nothing else will bring this claim back once the network appears") +} + +// A provider configures a NIC from the interface alone, so the gateway has to +// reach NetworkInterface.spec. The subnet usually appears after the interface, +// so it has to be filled in later rather than only at creation. +func TestGatewayReachesTheInterfaceWhenTheSubnetAppears(t *testing.T) { + s := newScenario(t, true, []networkingv1alpha.IPFamily{networkingv1alpha.IPv4Protocol}) + + claim := s.createClaim("routed", networkingv1alpha.NetworkInterfaceClaimSpec{ + InterfaceName: "eth0", + IPFamilies: []networkingv1alpha.IPFamily{networkingv1alpha.IPv4Protocol}, + ReclaimPolicy: networkingv1alpha.NetworkInterfaceReclaimPolicyDelete, + }) + s.reconcile(claim) + + iface, err := s.getInterface("routed") + require.NoError(t, err) + require.Empty(t, iface.Spec.Addresses[0].Gateway, + "no subnet yet, so no gateway, which is a legitimate state") + + contextName := s.programNetworkContext() + s.createSubnet("v4", contextName, networkingv1alpha.IPv4Protocol, "10.128.0.0", 24) + + s.reconcile(s.getClaim("routed")) + + routed, err := s.getInterface("routed") + require.NoError(t, err) + require.Equal(t, "10.128.0.1", routed.Spec.Addresses[0].Gateway, + "the gateway must be on the interface, which is all a provider reads") + + bound := s.getClaim("routed") + require.Equal(t, "10.128.0.1", bound.Status.Addresses[0].Gateway, + "the claim's copy comes from the interface rather than being resolved twice") + + // Reconciling again must not keep rewriting the same value. + before := routed.ResourceVersion + s.reconcile(s.getClaim("routed")) + settled, err := s.getInterface("routed") + require.NoError(t, err) + require.Equal(t, before, settled.ResourceVersion, + "an unchanged gateway must not write to the API server") +} + +// IPAM may give a missing address to another claim, so an operator has to see +// it. Reallocating instead would renumber a running workload. +func TestMissingAllocationIsReportedOnTheClaim(t *testing.T) { + s := newScenario(t, true, []networkingv1alpha.IPFamily{networkingv1alpha.IPv6Protocol}) + + claim := s.createClaim("unbacked", networkingv1alpha.NetworkInterfaceClaimSpec{ + InterfaceName: "eth0", + IPFamilies: []networkingv1alpha.IPFamily{networkingv1alpha.IPv6Protocol}, + ReclaimPolicy: networkingv1alpha.NetworkInterfaceReclaimPolicyDelete, + }) + s.reconcile(claim) + + iface, err := s.getInterface("unbacked") + require.NoError(t, err) + advertised := iface.Spec.Addresses[0].Address + + ipamClient, err := s.ipam.ClientForProject(testProject) + require.NoError(t, err) + require.NoError(t, ipamClient.Delete(s.ctx, s.ipClaim("unbacked-f-ipv6"))) + + before := testutil.ToFloat64(missingAllocationsTotal.WithLabelValues(testProject)) + s.reconcile(s.getClaim("unbacked")) + + require.Equal(t, before+1, testutil.ToFloat64(missingAllocationsTotal.WithLabelValues(testProject)), + "the metric is what makes this alertable") + + var warning string + select { + case warning = <-s.events.Events: + default: + t.Fatal("no event was recorded for an address nothing holds") + } + + require.Contains(t, warning, corev1.EventTypeWarning) + require.Contains(t, warning, "AddressAllocationMissing") + require.Contains(t, warning, advertised, "the event names the address at risk") + require.Contains(t, warning, "unbacked-f-ipv6", "and the allocation that went missing") + require.Contains(t, warning, testProject) + + // Allocated must not claim every address is held once one is not. + reported := s.getClaim("unbacked") + allocated := conditionOf(reported, networkingv1alpha.NetworkInterfaceClaimAllocated) + require.Equal(t, metav1.ConditionFalse, allocated.Status) + require.Equal(t, "AddressAllocationMissing", allocated.Reason) + require.Contains(t, allocated.Message, advertised) + + require.NotEqual(t, metav1.ConditionTrue, + conditionOf(reported, networkingv1alpha.NetworkInterfaceClaimReady).Status) + + still, err := s.getInterface("unbacked") + require.NoError(t, err) + require.Equal(t, advertised, still.Spec.Addresses[0].Address, + "the address is never silently renumbered") +} + +// Release reads addresses off the interface. Deleting the interface first +// leaves nothing to read, so it must fall back to what the claim asked for. +func TestDeletingAClaimWhoseInterfaceIsGoneReleasesItsAddresses(t *testing.T) { + s := newScenario(t, true, + []networkingv1alpha.IPFamily{networkingv1alpha.IPv4Protocol, networkingv1alpha.IPv6Protocol}, + publicV4Class()) + + claim := s.createClaim("outlived", networkingv1alpha.NetworkInterfaceClaimSpec{ + InterfaceName: "eth0", + IPFamilies: []networkingv1alpha.IPFamily{ + networkingv1alpha.IPv6Protocol, + networkingv1alpha.IPv4Protocol, + }, + Addresses: []networkingv1alpha.NetworkInterfaceAddressRequest{{Class: testPublicV4Class}}, + ReclaimPolicy: networkingv1alpha.NetworkInterfaceReclaimPolicyDelete, + }) + s.reconcile(claim) + + iface, err := s.getInterface("outlived") + require.NoError(t, err) + controllerutil.RemoveFinalizer(iface, networkInterfaceFinalizer) + require.NoError(t, s.client.Update(s.ctx, iface)) + require.NoError(t, s.client.Delete(s.ctx, iface)) + + s.deleteClaim(s.getClaim("outlived")) + + require.ElementsMatch(t, []string{ + "outlived-f-ipv6", + "outlived-f-ipv4", + "outlived-c-" + testPublicV4Class, + }, s.ipam.deleted()[testProject], + "every address the claim minted goes back, interface or no interface") +} + +// An interface outlives the claim that allocated its addresses. A lookup keyed +// on the current holder finds nothing, which reads as nothing to release. +func TestAllocationsFollowTheMintingClaimNotTheHolder(t *testing.T) { + s := newScenario(t, true, []networkingv1alpha.IPFamily{networkingv1alpha.IPv6Protocol}) + + spec := func() networkingv1alpha.NetworkInterfaceClaimSpec { + return networkingv1alpha.NetworkInterfaceClaimSpec{ + InterfaceName: "eth0", + IPFamilies: []networkingv1alpha.IPFamily{networkingv1alpha.IPv6Protocol}, + ReclaimPolicy: networkingv1alpha.NetworkInterfaceReclaimPolicyRetain, + NetworkInterfaceName: "shared", + } + } + + minter := s.createClaim("minter", spec()) + s.reconcile(minter) + require.Equal(t, []string{"minter-f-ipv6"}, s.ipam.created()[testProject]) + + s.deleteClaim(s.getClaim("minter")) + + adopter := s.createClaim("adopter", spec()) + s.reconcile(adopter) + + iface, err := s.getInterface("shared") + require.NoError(t, err) + require.Equal(t, "minter", iface.Annotations[allocationClaimAnnotation], + "the interface keeps naming the claim its addresses were minted under") + require.Equal(t, "adopter", iface.Spec.ClaimRef.Name) + require.Len(t, s.ipam.created()[testProject], 1, "adoption allocates nothing new") + + s.deleteClaim(s.getClaim("adopter")) + require.NoError(t, s.client.Delete(s.ctx, iface)) + s.reconcileInterface("shared") + + require.Equal(t, []string{"minter-f-ipv6"}, s.ipam.deleted()[testProject], + "release must find the address under the name it was minted with") +} + +func TestAdoptedInterfaceMustSatisfyTheClaim(t *testing.T) { + s := newScenario(t, true, + []networkingv1alpha.IPFamily{networkingv1alpha.IPv4Protocol, networkingv1alpha.IPv6Protocol}) + + single := s.createClaim("grower", networkingv1alpha.NetworkInterfaceClaimSpec{ + InterfaceName: "eth0", + IPFamilies: []networkingv1alpha.IPFamily{networkingv1alpha.IPv6Protocol}, + ReclaimPolicy: networkingv1alpha.NetworkInterfaceReclaimPolicyRetain, + }) + s.reconcile(single) + s.deleteClaim(s.getClaim("grower")) + + dualStack := s.createClaim("grower", networkingv1alpha.NetworkInterfaceClaimSpec{ + InterfaceName: "eth0", + IPFamilies: []networkingv1alpha.IPFamily{ + networkingv1alpha.IPv6Protocol, + networkingv1alpha.IPv4Protocol, + }, + ReclaimPolicy: networkingv1alpha.NetworkInterfaceReclaimPolicyRetain, + }) + s.reconcile(dualStack) + + rejected := s.getClaim("grower") + condition := conditionOf(rejected, networkingv1alpha.NetworkInterfaceClaimAllocated) + require.Equal(t, metav1.ConditionFalse, condition.Status, + "a retained interface holding one family cannot satisfy a dual-stack claim") + require.Contains(t, condition.Message, "IPv4") +} + +// IPAM freezes the reclaim policy onto the allocation. A claim asking for a +// different one cannot be honoured, so binding it would strand the address. +func TestAdoptionRefusesADifferentReclaimPolicy(t *testing.T) { + s := newScenario(t, true, []networkingv1alpha.IPFamily{networkingv1alpha.IPv6Protocol}) + + retained := s.createClaim("switcher", networkingv1alpha.NetworkInterfaceClaimSpec{ + InterfaceName: "eth0", + IPFamilies: []networkingv1alpha.IPFamily{networkingv1alpha.IPv6Protocol}, + ReclaimPolicy: networkingv1alpha.NetworkInterfaceReclaimPolicyRetain, + }) + s.reconcile(retained) + s.deleteClaim(s.getClaim("switcher")) + + replacement := s.createClaim("switcher", networkingv1alpha.NetworkInterfaceClaimSpec{ + InterfaceName: "eth0", + IPFamilies: []networkingv1alpha.IPFamily{networkingv1alpha.IPv6Protocol}, + ReclaimPolicy: networkingv1alpha.NetworkInterfaceReclaimPolicyDelete, + }) + s.reconcile(replacement) + + condition := conditionOf(s.getClaim("switcher"), networkingv1alpha.NetworkInterfaceClaimAllocated) + require.Equal(t, metav1.ConditionFalse, condition.Status) + require.Contains(t, condition.Message, "Retain") + require.Empty(t, s.ipam.orphanedAllocations(), + "refusing at bind time is what keeps the allocation from being stranded later") +} + +// An immutable reclaim policy is what stops an address being stranded. +func TestReclaimPolicyIsImmutable(t *testing.T) { + s := newScenario(t, true, []networkingv1alpha.IPFamily{networkingv1alpha.IPv6Protocol}) + + claim := s.createClaim("frozen", networkingv1alpha.NetworkInterfaceClaimSpec{ + InterfaceName: "eth0", + IPFamilies: []networkingv1alpha.IPFamily{networkingv1alpha.IPv6Protocol}, + ReclaimPolicy: networkingv1alpha.NetworkInterfaceReclaimPolicyRetain, + }) + s.reconcile(claim) + + switched := s.getClaim("frozen") + switched.Spec.ReclaimPolicy = networkingv1alpha.NetworkInterfaceReclaimPolicyDelete + err := s.client.Update(s.ctx, switched) + require.Error(t, err, "the API must reject the switch that would strand the allocation") + require.Contains(t, err.Error(), "immutable") +} + +func TestExternalAddressesDropTheHostPrefix(t *testing.T) { + for _, tc := range []struct { + allocated string + want string + }{ + {"198.51.100.11/32", "198.51.100.11"}, + {"2001:db8::1/128", "2001:db8::1"}, + {"10.128.0.0/24", "10.128.0.0/24"}, + {"2001:db8:a000:1::/96", "2001:db8:a000:1::/96"}, + {"not-an-address", "not-an-address"}, + } { + require.Equal(t, tc.want, allocatedAddress{cidr: tc.allocated}.bareAddress(), tc.allocated) + } +} + +func TestIPClaimNamesSurviveInstanceReplacement(t *testing.T) { + const claimName = "workload-default-us-central-1-0-eth0" + + require.Equal(t, "workload-default-us-central-1-0-eth0-f-ipv6", + ipClaimName(claimName, familyDiscriminator(networkingv1alpha.IPv6Protocol))) + + require.Equal(t, + ipClaimName(claimName, familyDiscriminator(networkingv1alpha.IPv6Protocol)), + ipClaimName(claimName, familyDiscriminator(networkingv1alpha.IPv6Protocol)), + "the name depends on the claim and the request, and on nothing that changes") + + require.NotEqual(t, + ipClaimName(claimName, familyDiscriminator(networkingv1alpha.IPv4Protocol)), + ipClaimName(claimName, familyDiscriminator(networkingv1alpha.IPv6Protocol))) + + require.NotEqual(t, + ipClaimName(claimName, classDiscriminator("f-ipv6")), + ipClaimName(claimName, familyDiscriminator(networkingv1alpha.IPv6Protocol)), + "a class named like a family must not collide with the family") + + longName := strings.Repeat("a", 253) + for _, discriminator := range []string{ + familyDiscriminator(networkingv1alpha.IPv4Protocol), + familyDiscriminator(networkingv1alpha.IPv6Protocol), + classDiscriminator(strings.Repeat("c", 63)), + } { + name := ipClaimName(longName, discriminator) + require.LessOrEqual(t, len(name), 253, "names must stay valid at the maximum claim length") + require.Empty(t, validation.IsDNS1123Subdomain(name)) + require.Equal(t, name, ipClaimName(longName, discriminator), "the hashed form is deterministic too") + } + + require.NotEqual(t, + ipClaimName(longName, familyDiscriminator(networkingv1alpha.IPv4Protocol)), + ipClaimName(longName, familyDiscriminator(networkingv1alpha.IPv6Protocol)), + "truncation must not merge two requests into one name") +} diff --git a/test/e2e/fixtures/ipam/RANGES.md b/test/e2e/fixtures/ipam/RANGES.md new file mode 100644 index 00000000..6cb38427 --- /dev/null +++ b/test/e2e/fixtures/ipam/RANGES.md @@ -0,0 +1,83 @@ +# Fixture range allocation + +`IPPool` and `IPClass` are cluster-scoped and every suite in a project shares +them, so root pool CIDRs must be disjoint — a root pool that overlaps another +in the same project is refused with 409, and chainsaw runs suites concurrently +within one project. The enforcing script upstream (`hack/verify-fixture-ranges.sh`) +does not exist at the pinned ref, so the discipline is this table. Add a range +here before you add a pool. + +| project | pool | CIDR | class it backs | +|---|---|---|---| +| project-alpha | `datum-network-v6-root` | `2001:db8:a000::/36` | `datum-network-v6` | +| project-alpha | `datum-endpoint-v4-root` | `10.128.0.0/16` | `datum-endpoint-v4` | +| project-alpha | `datum-public-v4-root` | `198.51.100.0/24` | `datum-public-v4` | +| project-beta | `datum-network-v6-root` | `2001:db8:b000::/36` | `datum-network-v6` | +| project-beta | `datum-endpoint-v4-root` | `10.129.0.0/16` | `datum-endpoint-v4` | +| project-beta | `datum-public-v4-root` | `203.0.113.0/24` | `datum-public-v4` | + +Class names are deliberately IDENTICAL across the two projects: a controller +routing by project must reach different address space through the same class +name, and identical names are what makes a routing bug visible. + +## Default classes + +A claim naming no class but setting `spec.ipFamily` resolves through +`ipam.miloapis.com/is-default-class`. Exactly one class per family per project +carries it: + +| project | IPv4 default | IPv6 default | +|---|---|---| +| project-alpha | `datum-endpoint-v4` | `datum-endpoint-v6` | +| project-beta | `datum-endpoint-v4` | `datum-endpoint-v6` | + +The annotation goes on the LEAF of the IPv6 chain, not its root. Resolution +returns a class and allocation proceeds from there, so annotating +`datum-network-v6` would hand a claim a `/48` where an endpoint block was +wanted — and report it as a successful allocation. + +`datum-public-v4` is deliberately NOT annotated: it is the named-class case and +must be reachable only through an explicit class name. Two defaults for one +family is not an error — IPAM lists the annotated classes `ORDER BY key` within +the project and takes the first whose family matches, silently ignoring the +rest — so a stray second annotation is a fixture that looks like it works. + +## The IPv6 chain + +`datum-network-v6` (`poolPer: [network]`, `/48`) + → `datum-subnet-v6` (`poolPer: [network, location]`, `/64`) + → `datum-endpoint-v6` (leaf, `/96`) + +Only the root class is backed by an operator-authored pool. The `/48` and `/64` +pools are provisioned by the allocator on first claim. A claim of +`datum-endpoint-v6` must therefore carry both scope roles, `network` and +`location`; one missing a role is rejected rather than widened. + +`datum-subnet-v6` repeats `network` in its `poolPer` on purpose. Pool identity +is keyed on (class name, scope digest) alone, with no reference to the parent +chain, so a subnet class scoped only by location would hand two networks in one +location the same pool. + +## Host addresses vs blocks + +A host-address class is one whose `allowedPrefixLengths` pins min == max at the +family's full width — 32 for IPv4, 128 for IPv6. The two IPv4 classes here are +host-address classes; every class in the IPv6 chain hands out blocks. + +**At the pinned IPAM ref, nothing ever writes `status.address`.** The field +exists on IPClaim and IPAllocation and round-trips through conversion, but no +code path populates it, so it reads empty for host-address classes too. A +host address arrives as a `/32` in `status.allocatedCIDR` and a consumer must +take it from there — reading `status.address` gets `""` and looks like a claim +that did not bind. + +## Scope roles + +Free strings, compared and never interpreted. The conventional pairs: + +| role | apiGroup | kind | +|---|---|---| +| `network` | `networking.datumapis.com` | `Network` | +| `location` | `networking.datumapis.com` | `Location` | + +Scope values are object NAMES, not UIDs. diff --git a/test/e2e/fixtures/ipam/README.md b/test/e2e/fixtures/ipam/README.md new file mode 100644 index 00000000..0ddb1125 --- /dev/null +++ b/test/e2e/fixtures/ipam/README.md @@ -0,0 +1,25 @@ +# IPAM test fixtures + +Test data for the suites that exercise NetworkInterfaceClaim allocation against +a real IPAM. None of it is needed to deploy IPAM — that lives in +`config/dependencies/ipam/`. + +This directory holds no `chainsaw-test.yaml`, so chainsaw walks past it: it +recurses looking for test files and ignores directories without one, the same +way it ignores the `networkbinding/` grouping directory. + +| file | what it is | +|---|---| +| `RANGES.md` | The range allocation per project, the class model, and the rules that keep concurrent suites from colliding. **Read this before adding a pool.** | +| `project-alpha/`, `project-beta/` | `IPClass` and `IPPool` seeds. Identical class names in both projects, disjoint address space, so a controller routing to the wrong project allocates from visibly wrong space instead of quietly succeeding. | +| `namespaces.yaml` | The `ipam-e2e-*` namespaces a claim resolves its project from, carrying the real encoded `meta.datumapis.com/upstream-cluster-name` label — plus one deliberately without it, for the fail-closed case. | +| `rbac.yaml` | Binds the identity the suites impersonate to the operator's own tenant role. | + +Applied by `task test-infra:ipam-fixtures` and `task test-infra:ipam-namespaces`, +both of which run as part of `test-infra:up` and again at the head of every +suite-running task. + +These are seeded, not asserted on directly: they are the address space the +suites allocate from. `IPPool` and `IPClass` are cluster-scoped, so chainsaw's +namespace teardown never reaches them — `task test-infra:ipam-fixtures-clear` +is what stops a run that died mid-suite from poisoning the next one. diff --git a/test/e2e/fixtures/ipam/namespaces.yaml b/test/e2e/fixtures/ipam/namespaces.yaml new file mode 100644 index 00000000..e48e9421 --- /dev/null +++ b/test/e2e/fixtures/ipam/namespaces.yaml @@ -0,0 +1,58 @@ +# Namespaces carrying the labels the mapped-namespace strategy really writes, +# so a scenario exercises the production decode path rather than a hand-written +# approximation of it. +# +# Applied to BOTH clusters. The label is written on downstream namespaces in +# production, but the consumer here is the NetworkInterfaceClaim reconciler: +# it runs on the upstream cell and resolveProject reads the namespace of the +# CLAIM through the cell client. Claims live upstream, so upstream is where +# these must exist for a project to resolve at all. +# +# meta.datumapis.com/upstream-cluster-name is ENCODED: a "cluster-" prefix with +# every "/" written as "_" (see internal/downstreamclient/mappednamespace.go, +# UpstreamClusterNameFromLabel). A fixture that writes the bare cluster name +# passes on its own and fails against the real path. +apiVersion: v1 +kind: Namespace +metadata: + name: ipam-e2e-alpha + labels: + meta.datumapis.com/upstream-cluster-name: cluster-project-alpha + meta.datumapis.com/upstream-namespace: default +--- +apiVersion: v1 +kind: Namespace +metadata: + name: ipam-e2e-beta + labels: + meta.datumapis.com/upstream-cluster-name: cluster-project-beta + meta.datumapis.com/upstream-namespace: default +--- +# What this env's single-cluster provider actually stamps: the provider engages +# one cluster named "single". +apiVersion: v1 +kind: Namespace +metadata: + name: ipam-e2e-single + labels: + meta.datumapis.com/upstream-cluster-name: cluster-single + meta.datumapis.com/upstream-namespace: default +--- +# Pre-#196 encoding: the cluster name carried a leading slash, so "/project-alpha" +# encodes to "cluster-_project-alpha" and must still decode to "project-alpha". +apiVersion: v1 +kind: Namespace +metadata: + name: ipam-e2e-legacy + labels: + meta.datumapis.com/upstream-cluster-name: cluster-_project-alpha + meta.datumapis.com/upstream-namespace: default +--- +# No cluster-name label at all. Anything resolving a project from a downstream +# namespace must fail closed here, not fall back to a default project. +apiVersion: v1 +kind: Namespace +metadata: + name: ipam-e2e-unlabeled + labels: + meta.datumapis.com/upstream-namespace: default diff --git a/test/e2e/fixtures/ipam/project-alpha/classes.yaml b/test/e2e/fixtures/ipam/project-alpha/classes.yaml new file mode 100644 index 00000000..9536a955 --- /dev/null +++ b/test/e2e/fixtures/ipam/project-alpha/classes.yaml @@ -0,0 +1,89 @@ +apiVersion: ipam.miloapis.com/v1alpha1 +kind: IPClass +metadata: + name: datum-network-v6 + labels: + nso-fixture: "true" +spec: + ipFamily: IPv6 + poolPer: + - network + defaultPrefixLength: 48 + allowedPrefixLengths: + min: 48 + max: 48 + routing: + external: Aggregate +--- +apiVersion: ipam.miloapis.com/v1alpha1 +kind: IPClass +metadata: + name: datum-subnet-v6 + labels: + nso-fixture: "true" +spec: + ipFamily: IPv6 + parentClassName: datum-network-v6 + poolPer: + - network + - location + defaultPrefixLength: 64 + allowedPrefixLengths: + min: 64 + max: 64 +--- +apiVersion: ipam.miloapis.com/v1alpha1 +kind: IPClass +metadata: + name: datum-endpoint-v6 + annotations: + # The default class for this family: a claim naming no class but + # setting ipFamily resolves to this one. + ipam.miloapis.com/is-default-class: "true" + labels: + nso-fixture: "true" +spec: + ipFamily: IPv6 + parentClassName: datum-subnet-v6 + defaultPrefixLength: 96 + allowedPrefixLengths: + min: 96 + max: 96 + reclaimPolicy: Delete + routing: + internal: Host +--- +apiVersion: ipam.miloapis.com/v1alpha1 +kind: IPClass +metadata: + name: datum-endpoint-v4 + annotations: + # The default class for this family: a claim naming no class but + # setting ipFamily resolves to this one. + ipam.miloapis.com/is-default-class: "true" + labels: + nso-fixture: "true" +spec: + ipFamily: IPv4 + allowedPrefixLengths: + min: 32 + max: 32 + reclaimPolicy: Delete + routing: + internal: Host +--- +apiVersion: ipam.miloapis.com/v1alpha1 +kind: IPClass +metadata: + name: datum-public-v4 + labels: + nso-fixture: "true" +spec: + ipFamily: IPv4 + allowedPrefixLengths: + min: 32 + max: 32 + reclaimPolicy: Delete + routing: + internal: Host + external: Aggregate diff --git a/test/e2e/fixtures/ipam/project-alpha/pools.yaml b/test/e2e/fixtures/ipam/project-alpha/pools.yaml new file mode 100644 index 00000000..5ea74a3c --- /dev/null +++ b/test/e2e/fixtures/ipam/project-alpha/pools.yaml @@ -0,0 +1,52 @@ +# Ranges are registered in ../RANGES.md. Root pools of one project may not +# overlap. +apiVersion: ipam.miloapis.com/v1alpha1 +kind: IPPool +metadata: + name: datum-network-v6-root + labels: + nso-fixture: "true" +spec: + cidr: 2001:db8:a000::/36 + ipFamily: IPv6 + visibility: consumer + classNames: + - datum-network-v6 + allocation: + minPrefixLength: 48 + maxPrefixLength: 48 + strategy: FirstFit +--- +apiVersion: ipam.miloapis.com/v1alpha1 +kind: IPPool +metadata: + name: datum-endpoint-v4-root + labels: + nso-fixture: "true" +spec: + cidr: 10.128.0.0/16 + ipFamily: IPv4 + visibility: consumer + classNames: + - datum-endpoint-v4 + allocation: + minPrefixLength: 32 + maxPrefixLength: 32 + strategy: FirstFit +--- +apiVersion: ipam.miloapis.com/v1alpha1 +kind: IPPool +metadata: + name: datum-public-v4-root + labels: + nso-fixture: "true" +spec: + cidr: 198.51.100.0/24 + ipFamily: IPv4 + visibility: consumer + classNames: + - datum-public-v4 + allocation: + minPrefixLength: 32 + maxPrefixLength: 32 + strategy: FirstFit diff --git a/test/e2e/fixtures/ipam/project-beta/classes.yaml b/test/e2e/fixtures/ipam/project-beta/classes.yaml new file mode 100644 index 00000000..9536a955 --- /dev/null +++ b/test/e2e/fixtures/ipam/project-beta/classes.yaml @@ -0,0 +1,89 @@ +apiVersion: ipam.miloapis.com/v1alpha1 +kind: IPClass +metadata: + name: datum-network-v6 + labels: + nso-fixture: "true" +spec: + ipFamily: IPv6 + poolPer: + - network + defaultPrefixLength: 48 + allowedPrefixLengths: + min: 48 + max: 48 + routing: + external: Aggregate +--- +apiVersion: ipam.miloapis.com/v1alpha1 +kind: IPClass +metadata: + name: datum-subnet-v6 + labels: + nso-fixture: "true" +spec: + ipFamily: IPv6 + parentClassName: datum-network-v6 + poolPer: + - network + - location + defaultPrefixLength: 64 + allowedPrefixLengths: + min: 64 + max: 64 +--- +apiVersion: ipam.miloapis.com/v1alpha1 +kind: IPClass +metadata: + name: datum-endpoint-v6 + annotations: + # The default class for this family: a claim naming no class but + # setting ipFamily resolves to this one. + ipam.miloapis.com/is-default-class: "true" + labels: + nso-fixture: "true" +spec: + ipFamily: IPv6 + parentClassName: datum-subnet-v6 + defaultPrefixLength: 96 + allowedPrefixLengths: + min: 96 + max: 96 + reclaimPolicy: Delete + routing: + internal: Host +--- +apiVersion: ipam.miloapis.com/v1alpha1 +kind: IPClass +metadata: + name: datum-endpoint-v4 + annotations: + # The default class for this family: a claim naming no class but + # setting ipFamily resolves to this one. + ipam.miloapis.com/is-default-class: "true" + labels: + nso-fixture: "true" +spec: + ipFamily: IPv4 + allowedPrefixLengths: + min: 32 + max: 32 + reclaimPolicy: Delete + routing: + internal: Host +--- +apiVersion: ipam.miloapis.com/v1alpha1 +kind: IPClass +metadata: + name: datum-public-v4 + labels: + nso-fixture: "true" +spec: + ipFamily: IPv4 + allowedPrefixLengths: + min: 32 + max: 32 + reclaimPolicy: Delete + routing: + internal: Host + external: Aggregate diff --git a/test/e2e/fixtures/ipam/project-beta/pools.yaml b/test/e2e/fixtures/ipam/project-beta/pools.yaml new file mode 100644 index 00000000..11d50b0f --- /dev/null +++ b/test/e2e/fixtures/ipam/project-beta/pools.yaml @@ -0,0 +1,52 @@ +# Ranges are registered in ../RANGES.md. Root pools of one project may not +# overlap. +apiVersion: ipam.miloapis.com/v1alpha1 +kind: IPPool +metadata: + name: datum-network-v6-root + labels: + nso-fixture: "true" +spec: + cidr: 2001:db8:b000::/36 + ipFamily: IPv6 + visibility: consumer + classNames: + - datum-network-v6 + allocation: + minPrefixLength: 48 + maxPrefixLength: 48 + strategy: FirstFit +--- +apiVersion: ipam.miloapis.com/v1alpha1 +kind: IPPool +metadata: + name: datum-endpoint-v4-root + labels: + nso-fixture: "true" +spec: + cidr: 10.129.0.0/16 + ipFamily: IPv4 + visibility: consumer + classNames: + - datum-endpoint-v4 + allocation: + minPrefixLength: 32 + maxPrefixLength: 32 + strategy: FirstFit +--- +apiVersion: ipam.miloapis.com/v1alpha1 +kind: IPPool +metadata: + name: datum-public-v4-root + labels: + nso-fixture: "true" +spec: + cidr: 203.0.113.0/24 + ipFamily: IPv4 + visibility: consumer + classNames: + - datum-public-v4 + allocation: + minPrefixLength: 32 + maxPrefixLength: 32 + strategy: FirstFit diff --git a/test/e2e/fixtures/ipam/rbac.yaml b/test/e2e/fixtures/ipam/rbac.yaml new file mode 100644 index 00000000..4f5e8b40 --- /dev/null +++ b/test/e2e/fixtures/ipam/rbac.yaml @@ -0,0 +1,28 @@ +# The test identity, kept out of the IPAM deployment. +# +# The suites and the fixture seeder reach IPAM by impersonating +# e2e-tenant-tester with the three iam.miloapis.com project extras. IPAM +# authorizes through a delegated SubjectAccessReview against the host +# apiserver, so that impersonated user needs ordinary RBAC of its own. +# +# It binds the nso-ipam-tenant ClusterRole from +# config/dependencies/ipam/overlay/rbac.yaml rather than restating the rules: +# the tester must hold exactly what the operator holds, and two copies of a +# permission set drift. +# +# Nothing grants the *right to impersonate* here. The seeder runs as the kind +# admin credential, which is in system:masters and already authorized for +# everything; only the NSO manager's ServiceAccount needs an explicit +# impersonation grant, and that ships with the deployment. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: nso-ipam-tenant-e2e +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: nso-ipam-tenant +subjects: + - kind: User + apiGroup: rbac.authorization.k8s.io + name: e2e-tenant-tester diff --git a/test/e2e/networkinterfaceclaim-bound-interface-delete/chainsaw-test.yaml b/test/e2e/networkinterfaceclaim-bound-interface-delete/chainsaw-test.yaml new file mode 100644 index 00000000..a39c11ce --- /dev/null +++ b/test/e2e/networkinterfaceclaim-bound-interface-delete/chainsaw-test.yaml @@ -0,0 +1,220 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/kyverno/chainsaw/main/.schemas/json/test-chainsaw-v1alpha1.json +# +# Deleting an interface a live claim still holds must not take the addresses +# with it. +# +# The delete is allowed — it is not refused and it does not cascade to the +# claim — but the release must NOT happen, because the addresses belong to a +# workload that is still running. Releasing here and re-allocating later is +# silent renumbering, which is worse than either refusing the delete or +# cascading it. +# +# The guarantee is asserted where it is real: the IPClaims in project-alpha +# still hold the same CIDRs after the interface is gone. NSO's own copy of the +# addresses proves nothing here — it is written once at bind and would look +# identical either way. +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: networkinterfaceclaim-bound-interface-delete +spec: + cluster: nso-standard + steps: + - name: A bound claim holds three addresses + try: + - create: + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: Network + metadata: + name: nic-boundifdel-net + namespace: ipam-e2e-alpha + spec: + ipam: + mode: Auto + ipFamilies: + - IPv4 + - IPv6 + - create: + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: NetworkInterfaceClaim + metadata: + name: nic-boundifdel + namespace: ipam-e2e-alpha + spec: + network: + name: nic-boundifdel-net + ipFamilies: + - IPv6 + - IPv4 + addresses: + - class: datum-public-v4 + - assert: + timeout: 120s + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: NetworkInterfaceClaim + metadata: + name: nic-boundifdel + namespace: ipam-e2e-alpha + status: + (conditions[?type == 'Allocated'] | [0].status): "True" + - script: + content: | + set -eu + IPAM=$(git rev-parse --show-toplevel)/hack/ipam-tenant-kubectl.sh + record="${TMPDIR:-/tmp}/nic-boundifdel-addresses" + "$IPAM" project-alpha -n default get ipclaim \ + -o 'jsonpath={range .items[*]}{.metadata.name}={.status.allocatedCIDR} {end}' > "$record" + kubectl -n ipam-e2e-alpha get networkinterface nic-boundifdel -o jsonpath='{.metadata.uid}' > "${record}.uid" + cat "$record" + grep -q 'nic-boundifdel-f-ipv6=' "$record" || { echo "the v6 address was never allocated in IPAM"; exit 1; } + catch: + - script: + content: | + kubectl -n ipam-e2e-alpha get networkinterfaceclaim,networkinterface -o yaml + kubectl -n network-services-operator-system logs -l app.kubernetes.io/name=network-services-operator --tail=100 + cleanup: + - description: > + Safety net. The count it prints is the signal: a claim now releases + its addresses from what it asked for rather than from an interface + it may no longer have, so anything above zero here means a claim + went away without handing its addresses back. + script: + content: | + set -u + IPAM=$(git rev-parse --show-toplevel)/hack/ipam-tenant-kubectl.sh + ipam() { "$IPAM" project-alpha -n default "$@"; } + + kubectl -n ipam-e2e-alpha delete networkinterfaceclaim nic-boundifdel --ignore-not-found --timeout=60s + kubectl -n ipam-e2e-alpha delete networkinterface nic-boundifdel --ignore-not-found --timeout=60s + stranded=$(ipam get ipclaim -o jsonpath='{range .items[*]}{.metadata.name} {end}' | tr ' ' '\n' | grep -E '^nic-boundifdel-' || true) + echo "IPClaims stranded by the deleted interface: $(echo "$stranded" | grep -c . || true)" + [ -z "$stranded" ] || ipam delete ipclaim $stranded --ignore-not-found --timeout=60s + orphans=$(ipam get ipallocation -o name) + [ -z "$orphans" ] || ipam delete $orphans --ignore-not-found + rm -f "${TMPDIR:-/tmp}/nic-boundifdel-addresses" "${TMPDIR:-/tmp}/nic-boundifdel-addresses.uid" + + - name: Deleting the interface under a live claim is allowed + description: | + Not refused and not cascaded: the interface's finalizer clears at once + because a live claim still holds it, and the claim itself is untouched. + try: + - script: + timeout: 120s + content: | + set -eu + # --wait=false on purpose: the claim controller rebuilds the + # interface within seconds, so a waiting delete can watch the + # replacement appear under the same name and never return. + kubectl -n ipam-e2e-alpha delete networkinterface nic-boundifdel --wait=false + - assert: + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: NetworkInterfaceClaim + metadata: + name: nic-boundifdel + namespace: ipam-e2e-alpha + + - name: The addresses were not released + description: | + The assertion that matters. Every IPClaim must still exist and still + hold the CIDR recorded before the delete — a release here would hand the + workload new addresses on the next reconcile. + try: + - script: + content: | + set -eu + IPAM=$(git rev-parse --show-toplevel)/hack/ipam-tenant-kubectl.sh + ipam() { "$IPAM" project-alpha -n default "$@"; } + + before=$(cat "${TMPDIR:-/tmp}/nic-boundifdel-addresses") + after=$(ipam get ipclaim -o 'jsonpath={range .items[*]}{.metadata.name}={.status.allocatedCIDR} {end}') + echo "before: ${before}" + echo "after: ${after}" + [ "$before" = "$after" ] || { echo "the addresses changed when the interface was deleted"; exit 1; } + + - name: The claim rebuilds the interface on the same addresses + description: | + The rebuild is driven by the claim controller watching NetworkInterface + and enqueuing the holder, so no nudge is needed here; a claim left + Bound against a missing interface would be the failure. + + The rebuilt interface must carry the addresses IPAM already holds, not + newly allocated ones — that reuse is the whole reason deleting a bound + interface is allowed to be survivable. + try: + - assert: + timeout: 120s + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: NetworkInterface + metadata: + name: nic-boundifdel + namespace: ipam-e2e-alpha + spec: + claimRef: + name: nic-boundifdel + status: + phase: Bound + - assert: + timeout: 60s + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: NetworkInterfaceClaim + metadata: + name: nic-boundifdel + namespace: ipam-e2e-alpha + status: + (conditions[?type == 'Bound'] | [0].status): "True" + (conditions[?type == 'Allocated'] | [0].status): "True" + - script: + content: | + set -eu + IPAM=$(git rev-parse --show-toplevel)/hack/ipam-tenant-kubectl.sh + ipam() { "$IPAM" project-alpha -n default "$@"; } + + before=$(cat "${TMPDIR:-/tmp}/nic-boundifdel-addresses") + after=$(ipam get ipclaim -o 'jsonpath={range .items[*]}{.metadata.name}={.status.allocatedCIDR} {end}') + [ "$before" = "$after" ] || { echo "the rebuild reallocated: before ${before}, after ${after}"; exit 1; } + + # A rebuild, not a delete that never took: the object carrying + # these addresses is a different one. + was=$(cat "${TMPDIR:-/tmp}/nic-boundifdel-addresses.uid") + now=$(kubectl -n ipam-e2e-alpha get networkinterface nic-boundifdel -o jsonpath='{.metadata.uid}') + [ "$was" != "$now" ] || { echo "the interface was never actually deleted (same uid ${now})"; exit 1; } + + for entry in $before; do + name=${entry%%=*} + cidr=${entry#*=} + case "$name" in + *-c-datum-public-v4) published=$(kubectl -n ipam-e2e-alpha get networkinterfaceclaim nic-boundifdel -o jsonpath='{.status.externalAddresses[0].address}')/32 ;; + *-f-ipv6) published=$(kubectl -n ipam-e2e-alpha get networkinterfaceclaim nic-boundifdel -o 'jsonpath={.status.addresses[?(@.family=="IPv6")].address}') ;; + *-f-ipv4) published=$(kubectl -n ipam-e2e-alpha get networkinterfaceclaim nic-boundifdel -o 'jsonpath={.status.addresses[?(@.family=="IPv4")].address}') ;; + esac + [ "$published" = "$cidr" ] || { echo "the rebuilt interface publishes ${published} where IPAM holds ${cidr}"; exit 1; } + done + + - name: The rebuilt claim releases everything on delete + try: + - delete: + timeout: 120s + ref: + apiVersion: networking.datumapis.com/v1alpha + kind: NetworkInterfaceClaim + name: nic-boundifdel + namespace: ipam-e2e-alpha + - script: + content: | + set -eu + IPAM=$(git rev-parse --show-toplevel)/hack/ipam-tenant-kubectl.sh + ipam() { "$IPAM" project-alpha -n default "$@"; } + + ipam get ipclass datum-public-v4 >/dev/null \ + || { echo "project-alpha's fixtures are not readable through this context, so an empty IPClaim list means nothing"; exit 1; } + + for found in $(ipam get ipclaim -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' | grep -E '^nic-boundifdel-' || true); do + echo "IPClaim ${found} survived the claim that held it" + exit 1 + done diff --git a/test/e2e/networkinterfaceclaim-dual-stack/chainsaw-test.yaml b/test/e2e/networkinterfaceclaim-dual-stack/chainsaw-test.yaml new file mode 100644 index 00000000..b0f5e898 --- /dev/null +++ b/test/e2e/networkinterfaceclaim-dual-stack/chainsaw-test.yaml @@ -0,0 +1,146 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/kyverno/chainsaw/main/.schemas/json/test-chainsaw-v1alpha1.json +# +# A dual-stack claim binds, and the addresses it publishes are real allocations +# in the upstream IPAM service rather than values NSO invented. +# +# The claim lives in ipam-e2e-alpha, whose upstream-cluster-name label routes to +# project-alpha; the IPClaims are therefore read back as project-alpha, in the +# project namespace named by the cell namespace's upstream-namespace label +# (default). +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: networkinterfaceclaim-dual-stack +spec: + cluster: nso-standard + steps: + - name: Create a dual-stack network + try: + - create: + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: Network + metadata: + name: nic-dualstack-net + namespace: ipam-e2e-alpha + spec: + ipam: + mode: Auto + ipFamilies: + - IPv4 + - IPv6 + + - name: A dual-stack claim binds and publishes one address per family + description: | + Programmed stays Unknown because no data plane reports the attachment in + this env, so Ready is never True. Waiting for Ready here would hang. + try: + - create: + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: NetworkInterfaceClaim + metadata: + name: nic-dualstack + namespace: ipam-e2e-alpha + spec: + network: + name: nic-dualstack-net + ipFamilies: + - IPv6 + - IPv4 + - assert: + timeout: 120s + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: NetworkInterfaceClaim + metadata: + name: nic-dualstack + namespace: ipam-e2e-alpha + status: + networkInterfaceRef: + name: nic-dualstack + addresses: + - family: IPv6 + primary: true + - family: IPv4 + (length(addresses || `[]`) == `2`): true + (conditions[?type == 'Bound'] | [0].status): "True" + (conditions[?type == 'Allocated'] | [0].status): "True" + (conditions[?type == 'Programmed'] | [0].status): Unknown + (length(conditions[?type == 'Ready' && status == 'True']) == `0`): true + catch: + - script: + content: | + kubectl -n ipam-e2e-alpha get networkinterfaceclaim,networkinterface -o yaml + kubectl -n network-services-operator-system logs -l app.kubernetes.io/name=network-services-operator --tail=100 + + - name: The interface the claim names exists and is bound to it + try: + - assert: + timeout: 60s + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: NetworkInterface + metadata: + name: nic-dualstack + namespace: ipam-e2e-alpha + spec: + claimRef: + name: nic-dualstack + status: + phase: Bound + + - name: Every published address is an IPClaim held in project-alpha + description: | + The assertion this environment exists to make. Each address is looked up + in IPAM under its deterministic name — -f-ipv6 / -f-ipv4 — + and must be Bound to exactly the CIDR NSO published. + try: + - script: + content: | + set -eu + IPAM=$(git rev-parse --show-toplevel)/hack/ipam-tenant-kubectl.sh + ipam() { proj=$1; shift; "$IPAM" "$proj" -n default "$@"; } + + for family in ipv6 ipv4; do + upper=$(echo "$family" | tr '[:lower:]' '[:upper:]' | sed 's/IPV/IPv/') + published=$(kubectl -n ipam-e2e-alpha get networkinterfaceclaim nic-dualstack \ + -o "jsonpath={.status.addresses[?(@.family=='${upper}')].address}") + allocated=$(ipam project-alpha get ipclaim "nic-dualstack-f-${family}" -o jsonpath='{.status.allocatedCIDR}') + phase=$(ipam project-alpha get ipclaim "nic-dualstack-f-${family}" -o jsonpath='{.status.phase}') + + echo "${upper}: claim=${published} ipam=${allocated} (${phase})" + [ -n "$published" ] || { echo "claim published no ${upper} address"; exit 1; } + [ "$phase" = "Bound" ] || { echo "IPClaim nic-dualstack-f-${family} is ${phase}, want Bound"; exit 1; } + [ "$published" = "$allocated" ] || { echo "claim published ${published}, IPAM holds ${allocated}"; exit 1; } + done + + # Addresses on the interface are always CIDRs, host prefix included. + # A bare address here is a regression. + v6=$(kubectl -n ipam-e2e-alpha get networkinterfaceclaim nic-dualstack -o "jsonpath={.status.addresses[?(@.family=='IPv6')].address}") + v4=$(kubectl -n ipam-e2e-alpha get networkinterfaceclaim nic-dualstack -o "jsonpath={.status.addresses[?(@.family=='IPv4')].address}") + echo "$v6" | grep -Eq '^2001:db8:a[0-9a-f]{3}:.*/96$' || { echo "IPv6 ${v6} is not a /96 out of project-alpha's 2001:db8:a000::/36"; exit 1; } + echo "$v4" | grep -Eq '^10\.128\.[0-9]+\.[0-9]+/32$' || { echo "IPv4 ${v4} is not a /32 out of project-alpha's 10.128.0.0/16"; exit 1; } + + - name: Deleting the claim releases both addresses + try: + - delete: + timeout: 120s + ref: + apiVersion: networking.datumapis.com/v1alpha + kind: NetworkInterfaceClaim + name: nic-dualstack + namespace: ipam-e2e-alpha + - script: + content: | + set -eu + IPAM=$(git rev-parse --show-toplevel)/hack/ipam-tenant-kubectl.sh + ipam() { "$IPAM" project-alpha -n default "$@"; } + + ipam get ipclass datum-endpoint-v4 >/dev/null \ + || { echo "project-alpha's fixtures are not readable through this context, so an empty IPClaim list means nothing"; exit 1; } + + for found in $(ipam get ipclaim -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' | grep -E '^nic-dualstack-' || true); do + echo "IPClaim ${found} still exists in project-alpha after the claim was deleted" + exit 1 + done diff --git a/test/e2e/networkinterfaceclaim-external-address/chainsaw-test.yaml b/test/e2e/networkinterfaceclaim-external-address/chainsaw-test.yaml new file mode 100644 index 00000000..06b1abb2 --- /dev/null +++ b/test/e2e/networkinterfaceclaim-external-address/chainsaw-test.yaml @@ -0,0 +1,86 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/kyverno/chainsaw/main/.schemas/json/test-chainsaw-v1alpha1.json +# +# A class named in spec.addresses[] yields an EXTERNAL address, and it arrives +# bare. +# +# datum-public-v4 carries no default-class annotation, so the only way to reach +# 198.51.100.0/24 is to name it. It is a /32 class: IPAM hands back +# "198.51.100.x/32" and the interface must publish "198.51.100.x". A prefix +# surviving into externalAddresses is a regression, not a fixture quirk. +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: networkinterfaceclaim-external-address +spec: + cluster: nso-standard + steps: + - name: A claim naming datum-public-v4 binds + try: + - create: + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: Network + metadata: + name: nic-external-net + namespace: ipam-e2e-alpha + spec: + ipam: + mode: Auto + ipFamilies: + - IPv4 + - create: + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: NetworkInterfaceClaim + metadata: + name: nic-external + namespace: ipam-e2e-alpha + spec: + network: + name: nic-external-net + ipFamilies: + - IPv4 + addresses: + - class: datum-public-v4 + - assert: + timeout: 120s + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: NetworkInterfaceClaim + metadata: + name: nic-external + namespace: ipam-e2e-alpha + status: + externalAddresses: + - family: IPv4 + class: datum-public-v4 + (length(externalAddresses || `[]`) == `1`): true + (length(addresses || `[]`) == `1`): true + (conditions[?type == 'Allocated'] | [0].status): "True" + catch: + - script: + content: | + kubectl -n ipam-e2e-alpha get networkinterfaceclaim,networkinterface -o yaml + kubectl -n network-services-operator-system logs -l app.kubernetes.io/name=network-services-operator --tail=100 + + - name: The external address is bare and comes from project-alpha's public range + try: + - script: + content: | + set -eu + IPAM=$(git rev-parse --show-toplevel)/hack/ipam-tenant-kubectl.sh + + external=$(kubectl -n ipam-e2e-alpha get networkinterfaceclaim nic-external -o jsonpath='{.status.externalAddresses[0].address}') + internal=$(kubectl -n ipam-e2e-alpha get networkinterfaceclaim nic-external -o jsonpath='{.status.addresses[0].address}') + held=$("$IPAM" project-alpha -n default \ + get ipclaim nic-external-c-datum-public-v4 -o jsonpath='{.status.allocatedCIDR}') + echo "external=${external} internal=${internal} ipam=${held}" + + echo "$external" | grep -Eq '^198\.51\.100\.[0-9]+$' || { echo "external address ${external} is not a bare address from 198.51.100.0/24"; exit 1; } + echo "$internal" | grep -Eq '^10\.128\.[0-9]+\.[0-9]+/32$' || { echo "internal address ${internal} is not a /32 from 10.128.0.0/16"; exit 1; } + [ "$held" = "${external}/32" ] || { echo "IPAM holds ${held}, claim published ${external}"; exit 1; } + + # The named class must not have leaked into the internal address + # list, and the family entry must not have taken the public class. + class=$(kubectl -n ipam-e2e-alpha get networkinterfaceclaim nic-external -o jsonpath='{.status.addresses[0].class}') + [ -z "$class" ] || { echo "the family address was allocated from class ${class}, want the default class"; exit 1; } diff --git a/test/e2e/networkinterfaceclaim-family-not-carried/chainsaw-test.yaml b/test/e2e/networkinterfaceclaim-family-not-carried/chainsaw-test.yaml new file mode 100644 index 00000000..3524e07a --- /dev/null +++ b/test/e2e/networkinterfaceclaim-family-not-carried/chainsaw-test.yaml @@ -0,0 +1,134 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/kyverno/chainsaw/main/.schemas/json/test-chainsaw-v1alpha1.json +# +# A claim asking for a family its network does not carry fails, naming the +# family, and allocates nothing. +# +# This is the accidental case, not an exotic one: Network.spec.ipFamilies +# defaults to [IPv4] and a claim's defaults to [IPv6], so the most minimal claim +# anyone can write against the most minimal network is exactly this mismatch. +# The check is NSO's own — it compares the two specs before it ever reaches +# IPAM — so a green result here says nothing about IPAM and everything about +# NSO refusing to ask for space it knows is wrong. +# +# A second claim on the SAME network, differing only in the family it asks for, +# allocates for real. It is the control that gives the absence its meaning: an +# IPAM this suite cannot read at all would otherwise look identical to an IPAM +# that was never asked. +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: networkinterfaceclaim-family-not-carried +spec: + cluster: nso-standard + steps: + - name: A minimal claim on a default network is refused + try: + - create: + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: Network + metadata: + name: nic-family-net + namespace: ipam-e2e-alpha + spec: + ipam: + mode: Auto + - assert: + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: Network + metadata: + name: nic-family-net + namespace: ipam-e2e-alpha + spec: + ipFamilies: + - IPv4 + - create: + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: NetworkInterfaceClaim + metadata: + name: nic-family-mismatch + namespace: ipam-e2e-alpha + spec: + network: + name: nic-family-net + - assert: + timeout: 120s + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: NetworkInterfaceClaim + metadata: + name: nic-family-mismatch + namespace: ipam-e2e-alpha + spec: + ipFamilies: + - IPv6 + status: + (conditions[?type == 'Bound'] | [0]): + status: "False" + reason: AddressFamilyNotCarried + (conditions[?type == 'Allocated'] | [0].status): "False" + (conditions[?type == 'Ready'] | [0].status): "False" + (contains(conditions[?type == 'Bound'] | [0].message, 'IPv6')): true + catch: + - script: + content: | + kubectl -n ipam-e2e-alpha get network,networkinterfaceclaim -o yaml + kubectl -n network-services-operator-system logs -l app.kubernetes.io/name=network-services-operator --tail=100 + + - name: The same network satisfies the family it does carry + description: | + The positive control. Identical to the refused claim but for the family + it asks for. + try: + - create: + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: NetworkInterfaceClaim + metadata: + name: nic-family-canary + namespace: ipam-e2e-alpha + spec: + network: + name: nic-family-net + ipFamilies: + - IPv4 + - assert: + timeout: 120s + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: NetworkInterfaceClaim + metadata: + name: nic-family-canary + namespace: ipam-e2e-alpha + status: + (conditions[?type == 'Allocated'] | [0].status): "True" + + - name: The refusal allocated nothing + description: | + The whole IPClaim list is matched against the refused claim's prefix + rather than the one name today's derivation produces, so an allocation + made under a name this suite did not predict is still caught. + try: + - script: + content: | + set -eu + IPAM=$(git rev-parse --show-toplevel)/hack/ipam-tenant-kubectl.sh + ipam() { "$IPAM" project-alpha -n default "$@"; } + + claims=$(ipam get ipclaim -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}') + echo "project-alpha IPClaims: $(echo $claims)" + + echo "$claims" | grep -qx 'nic-family-canary-f-ipv4' \ + || { echo "the canary's IPClaim is not visible in project-alpha — this suite cannot see allocations, so it cannot prove their absence"; exit 1; } + + for found in $(echo "$claims" | grep -E '^nic-family-mismatch-' || true); do + echo "a refused claim still allocated ${found}" + exit 1 + done + + if kubectl -n ipam-e2e-alpha get networkinterface nic-family-mismatch >/dev/null 2>&1; then + echo "a refused claim still created a NetworkInterface" + exit 1 + fi diff --git a/test/e2e/networkinterfaceclaim-interface-release/chainsaw-test.yaml b/test/e2e/networkinterfaceclaim-interface-release/chainsaw-test.yaml new file mode 100644 index 00000000..95bf5475 --- /dev/null +++ b/test/e2e/networkinterfaceclaim-interface-release/chainsaw-test.yaml @@ -0,0 +1,191 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/kyverno/chainsaw/main/.schemas/json/test-chainsaw-v1alpha1.json +# +# Deleting a retained interface hands its addresses back. +# +# A retained interface sits unbound with no claim left to carry a finalizer on +# its behalf, which is exactly when it looks disposable. Before the interface +# grew its own finalizer, deleting one left its IPClaims alive with nothing in +# NSO naming them: capacity gone, nothing reporting it, and the obvious manual +# remedy — deleting those IPClaims by hand — stranded an IPAllocation per claim +# and wedged the name permanently. +# +# So this asserts the release in IPAM, and asserts that the state which made +# the manual remedy tempting no longer exists. +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: networkinterfaceclaim-interface-release +spec: + cluster: nso-standard + steps: + - name: A retained claim binds and its interface carries a finalizer + description: | + The finalizer and the allocation-claim annotation are what make the + release possible at all: the first holds the delete open, the second + records which claim the IPClaims were minted under so they can still be + found once the claim is gone. + try: + - create: + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: Network + metadata: + name: nic-ifrelease-net + namespace: ipam-e2e-alpha + spec: + ipam: + mode: Auto + ipFamilies: + - IPv4 + - IPv6 + - create: + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: NetworkInterfaceClaim + metadata: + name: nic-ifrelease + namespace: ipam-e2e-alpha + spec: + network: + name: nic-ifrelease-net + ipFamilies: + - IPv6 + - IPv4 + reclaimPolicy: Retain + addresses: + - class: datum-public-v4 + - assert: + timeout: 120s + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: NetworkInterfaceClaim + metadata: + name: nic-ifrelease + namespace: ipam-e2e-alpha + status: + (conditions[?type == 'Allocated'] | [0].status): "True" + - assert: + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: NetworkInterface + metadata: + name: nic-ifrelease + namespace: ipam-e2e-alpha + annotations: + networking.datumapis.com/allocation-claim: nic-ifrelease + (contains(finalizers, 'networking.datumapis.com/networkinterface-release')): true + catch: + - script: + content: | + kubectl -n ipam-e2e-alpha get networkinterfaceclaim,networkinterface -o yaml + kubectl -n network-services-operator-system logs -l app.kubernetes.io/name=network-services-operator --tail=100 + cleanup: + - description: > + Safety net. Everything this test creates outlives its claim by + design, so a failure part-way through would otherwise hold three + addresses in project-alpha for every later run. + script: + content: | + set -u + IPAM=$(git rev-parse --show-toplevel)/hack/ipam-tenant-kubectl.sh + ipam() { "$IPAM" project-alpha -n default "$@"; } + + kubectl -n ipam-e2e-alpha delete networkinterfaceclaim nic-ifrelease --ignore-not-found --timeout=60s + kubectl -n ipam-e2e-alpha delete networkinterface nic-ifrelease --ignore-not-found --timeout=60s + leftover=$(ipam get ipclaim -o jsonpath='{range .items[*]}{.metadata.name} {end}' | tr ' ' '\n' | grep -E '^nic-ifrelease-' || true) + [ -z "$leftover" ] || ipam delete ipclaim $leftover --ignore-not-found --timeout=60s + orphans=$(ipam get ipallocation -o name) + [ -z "$orphans" ] || ipam delete $orphans --ignore-not-found + echo "orphaned IPAllocations purged: $(echo "$orphans" | grep -c . || true)" + + - name: The addresses are held in IPAM and survive the claim + try: + - script: + content: | + set -eu + IPAM=$(git rev-parse --show-toplevel)/hack/ipam-tenant-kubectl.sh + ipam() { "$IPAM" project-alpha -n default "$@"; } + held() { ipam get ipclaim -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' | grep -cE '^nic-ifrelease-' || true; } + + [ "$(held)" -eq 3 ] || { echo "expected 3 IPClaims for this interface, found $(held)"; exit 1; } + - delete: + timeout: 120s + ref: + apiVersion: networking.datumapis.com/v1alpha + kind: NetworkInterfaceClaim + name: nic-ifrelease + namespace: ipam-e2e-alpha + - assert: + timeout: 60s + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: NetworkInterface + metadata: + name: nic-ifrelease + namespace: ipam-e2e-alpha + spec: + (claimRef == null): true + status: + phase: Available + - script: + content: | + set -eu + IPAM=$(git rev-parse --show-toplevel)/hack/ipam-tenant-kubectl.sh + ipam() { "$IPAM" project-alpha -n default "$@"; } + count=$(ipam get ipclaim -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' | grep -cE '^nic-ifrelease-' || true) + [ "$count" -eq 3 ] || { echo "a retained interface should still hold its 3 addresses, found ${count}"; exit 1; } + + - name: Deleting the unheld interface releases every address + description: | + The delete blocks on the interface's finalizer until the release has + happened, so an assertion after it returns is asserting a completed + release rather than racing one. + try: + - delete: + timeout: 120s + ref: + apiVersion: networking.datumapis.com/v1alpha + kind: NetworkInterface + name: nic-ifrelease + namespace: ipam-e2e-alpha + - script: + content: | + set -eu + IPAM=$(git rev-parse --show-toplevel)/hack/ipam-tenant-kubectl.sh + ipam() { "$IPAM" project-alpha -n default "$@"; } + + ipam get ipclass datum-public-v4 >/dev/null \ + || { echo "project-alpha's fixtures are not readable through this context, so an empty IPClaim list means nothing"; exit 1; } + + for found in $(ipam get ipclaim -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' | grep -E '^nic-ifrelease-' || true); do + echo "IPClaim ${found} survived the interface that held it" + exit 1 + done + + - name: Nothing is left for an operator to clean up by hand + description: | + The composition that made this dangerous: stranded IPClaims invite a + manual delete, and deleting a Retain-allocated IPClaim leaves an + IPAllocation holding the address and blocking the name forever. There + is no longer a stranded IPClaim to invite it. + + The IPAllocations themselves still outlive the release — IPAM freezes + Retain onto the allocation and nothing revisits it — so the cleanup on + the first step purges them and prints the count. That residue is + IPAM-side and is not something NSO can assert away today. + try: + - error: + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: NetworkInterface + metadata: + name: nic-ifrelease + namespace: ipam-e2e-alpha + - script: + content: | + set -eu + IPAM=$(git rev-parse --show-toplevel)/hack/ipam-tenant-kubectl.sh + ipam() { "$IPAM" project-alpha -n default "$@"; } + stranded=$(ipam get ipclaim -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' | grep -cE '^nic-ifrelease-' || true) + [ "$stranded" -eq 0 ] || { echo "${stranded} IPClaims are stranded with nothing in NSO naming them"; exit 1; } + echo "IPAllocations still holding addresses after the release: $(ipam get ipallocation -o name | grep -c . || true)" diff --git a/test/e2e/networkinterfaceclaim-interface-then-claim-delete/chainsaw-test.yaml b/test/e2e/networkinterfaceclaim-interface-then-claim-delete/chainsaw-test.yaml new file mode 100644 index 00000000..f9846ea2 --- /dev/null +++ b/test/e2e/networkinterfaceclaim-interface-then-claim-delete/chainsaw-test.yaml @@ -0,0 +1,132 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/kyverno/chainsaw/main/.schemas/json/test-chainsaw-v1alpha1.json +# +# Delete the interface, then the claim. Nothing is left holding an address. +# +# This order used to leak everything: release() read the addresses off the +# interface, so a claim whose interface was already gone found nothing to +# release, dropped its finalizer and left every IPClaim alive with no NSO object +# naming it. A claim now releases what it ASKED for — its ipFamilies and its +# addresses[].class — which is what the names were minted from, so the interface +# no longer has to survive for the addresses to come back. +# +# The interface delete does not wait: whether the claim controller rebuilds the +# interface before the claim delete lands is a race, and the end state has to be +# empty either way. +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: networkinterfaceclaim-interface-then-claim-delete +spec: + cluster: nso-standard + steps: + - name: A claim holds a family address and a named class + try: + - create: + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: Network + metadata: + name: nic-reverse-net + namespace: ipam-e2e-alpha + spec: + ipam: + mode: Auto + ipFamilies: + - IPv4 + - IPv6 + - create: + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: NetworkInterfaceClaim + metadata: + name: nic-reverse + namespace: ipam-e2e-alpha + spec: + network: + name: nic-reverse-net + ipFamilies: + - IPv6 + - IPv4 + addresses: + - class: datum-public-v4 + - assert: + timeout: 120s + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: NetworkInterfaceClaim + metadata: + name: nic-reverse + namespace: ipam-e2e-alpha + status: + (conditions[?type == 'Allocated'] | [0].status): "True" + - script: + content: | + set -eu + IPAM=$(git rev-parse --show-toplevel)/hack/ipam-tenant-kubectl.sh + ipam() { "$IPAM" project-alpha -n default "$@"; } + count=$(ipam get ipclaim -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' | grep -cE '^nic-reverse-' || true) + [ "$count" -eq 3 ] || { echo "expected 3 IPClaims before the deletes, found ${count}"; exit 1; } + catch: + - script: + content: | + kubectl -n ipam-e2e-alpha get networkinterfaceclaim,networkinterface -o yaml + kubectl -n network-services-operator-system logs -l app.kubernetes.io/name=network-services-operator --tail=100 + cleanup: + - description: > + Safety net. If the release regresses, the addresses would be held in + project-alpha for every later run; the count printed here is zero + whenever the test itself passed. + script: + content: | + set -u + IPAM=$(git rev-parse --show-toplevel)/hack/ipam-tenant-kubectl.sh + ipam() { "$IPAM" project-alpha -n default "$@"; } + + kubectl -n ipam-e2e-alpha delete networkinterfaceclaim nic-reverse --ignore-not-found --timeout=60s + kubectl -n ipam-e2e-alpha delete networkinterface nic-reverse --ignore-not-found --timeout=60s + stranded=$(ipam get ipclaim -o jsonpath='{range .items[*]}{.metadata.name} {end}' | tr ' ' '\n' | grep -E '^nic-reverse-' || true) + echo "IPClaims left holding addresses: $(echo "$stranded" | grep -c . || true)" + [ -z "$stranded" ] || ipam delete ipclaim $stranded --ignore-not-found --timeout=60s + orphans=$(ipam get ipallocation -o name) + [ -z "$orphans" ] || ipam delete $orphans --ignore-not-found + + - name: Deleting the interface and then the claim releases everything + try: + - script: + timeout: 120s + content: | + set -eu + # --wait=false on purpose: the claim controller rebuilds the + # interface within seconds, so a waiting delete can watch the + # replacement appear under the same name and never return. + kubectl -n ipam-e2e-alpha delete networkinterface nic-reverse --wait=false + - delete: + timeout: 120s + ref: + apiVersion: networking.datumapis.com/v1alpha + kind: NetworkInterfaceClaim + name: nic-reverse + namespace: ipam-e2e-alpha + - script: + content: | + set -eu + IPAM=$(git rev-parse --show-toplevel)/hack/ipam-tenant-kubectl.sh + ipam() { "$IPAM" project-alpha -n default "$@"; } + + ipam get ipclass datum-public-v4 >/dev/null \ + || { echo "project-alpha's fixtures are not readable through this context, so an empty IPClaim list means nothing"; exit 1; } + + for found in $(ipam get ipclaim -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' | grep -E '^nic-reverse-' || true); do + echo "IPClaim ${found} outlived both the interface and the claim" + exit 1 + done + + orphans=$(ipam get ipallocation -o name | grep -c . || true) + [ "$orphans" -eq 0 ] || { echo "${orphans} IPAllocations are still holding addresses"; exit 1; } + - error: + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: NetworkInterface + metadata: + name: nic-reverse + namespace: ipam-e2e-alpha diff --git a/test/e2e/networkinterfaceclaim-missing-allocation/chainsaw-test.yaml b/test/e2e/networkinterfaceclaim-missing-allocation/chainsaw-test.yaml new file mode 100644 index 00000000..588810a5 --- /dev/null +++ b/test/e2e/networkinterfaceclaim-missing-allocation/chainsaw-test.yaml @@ -0,0 +1,276 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/kyverno/chainsaw/main/.schemas/json/test-chainsaw-v1alpha1.json +# +# An interface advertising an address IPAM no longer holds is reported loudly +# and gates nothing. +# +# Both halves matter. IPAM considers the address free and will hand it to the +# next claimant — measured: a fresh datum-endpoint-v4 claim in project-alpha +# came back with the very address the interface was still advertising — so an +# operator has to find out. But re-allocating would renumber a workload that is +# running on that address right now, which is worse than the duplicate, so the +# claim must stay Bound and keep advertising what it is actually using. +# +# The state is built by deleting the IPClaim out from under a bound interface. +# That is not a state NSO can reach on its own; it stands in for an address lost +# to an IPAM-side repair, restore, or operator error. +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: networkinterfaceclaim-missing-allocation +spec: + cluster: nso-standard + steps: + - name: A claim binds and holds one address + try: + - create: + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: Network + metadata: + name: nic-missingalloc-net + namespace: ipam-e2e-alpha + spec: + ipam: + mode: Auto + ipFamilies: + - IPv4 + - create: + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: NetworkInterfaceClaim + metadata: + name: nic-missingalloc + namespace: ipam-e2e-alpha + spec: + network: + name: nic-missingalloc-net + ipFamilies: + - IPv4 + - assert: + timeout: 120s + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: NetworkInterfaceClaim + metadata: + name: nic-missingalloc + namespace: ipam-e2e-alpha + status: + (conditions[?type == 'Bound'] | [0].status): "True" + (conditions[?type == 'Allocated'] | [0].status): "True" + catch: + - script: + content: | + kubectl -n ipam-e2e-alpha get networkinterfaceclaim,networkinterface -o yaml + kubectl -n ipam-e2e-alpha get events --field-selector involvedObject.name=nic-missingalloc + kubectl -n network-services-operator-system logs -l app.kubernetes.io/name=network-services-operator --tail=100 + cleanup: + - script: + content: | + set -u + IPAM=$(git rev-parse --show-toplevel)/hack/ipam-tenant-kubectl.sh + ipam() { "$IPAM" project-alpha -n default "$@"; } + + kubectl -n ipam-e2e-alpha delete networkinterfaceclaim nic-missingalloc --ignore-not-found --timeout=60s + kubectl -n ipam-e2e-alpha delete networkinterface nic-missingalloc --ignore-not-found --timeout=60s + leftover=$(ipam get ipclaim -o jsonpath='{range .items[*]}{.metadata.name} {end}' | tr ' ' '\n' | grep -E '^nic-missingalloc-' || true) + [ -z "$leftover" ] || ipam delete ipclaim $leftover --ignore-not-found --timeout=60s + orphans=$(ipam get ipallocation -o name) + [ -z "$orphans" ] || ipam delete $orphans --ignore-not-found + rm -f "${TMPDIR:-/tmp}/nic-missingalloc-address" + + - name: The allocation disappears from IPAM + try: + - script: + content: | + set -eu + IPAM=$(git rev-parse --show-toplevel)/hack/ipam-tenant-kubectl.sh + ipam() { "$IPAM" project-alpha -n default "$@"; } + + address=$(kubectl -n ipam-e2e-alpha get networkinterfaceclaim nic-missingalloc -o jsonpath='{.status.addresses[0].address}') + [ -n "$address" ] || { echo "the claim never published an address"; exit 1; } + echo "$address" > "${TMPDIR:-/tmp}/nic-missingalloc-address" + + held=$(ipam get ipclaim nic-missingalloc-f-ipv4 -o jsonpath='{.status.allocatedCIDR}') + [ "$held" = "$address" ] || { echo "IPAM holds ${held}, the claim publishes ${address}"; exit 1; } + + ipam delete ipclaim nic-missingalloc-f-ipv4 --timeout=60s + ! ipam get ipclaim nic-missingalloc-f-ipv4 >/dev/null 2>&1 || { echo "the IPClaim is still there"; exit 1; } + - description: > + Every event for this claim is cleared first. The claim name is fixed + and events outlive a run by about an hour, so an event from an + earlier run would otherwise satisfy the next one — the assertion + would stop testing anything the day the controller went quiet. + + Then the claim is nudged, because nothing watches IPAM: this forces + the reconcile an operator's next edit would have caused anyway. + script: + content: | + set -eu + kubectl -n ipam-e2e-alpha delete events \ + --field-selector involvedObject.name=nic-missingalloc --ignore-not-found + kubectl -n ipam-e2e-alpha annotate networkinterfaceclaim nic-missingalloc \ + e2e.datum.net/nudge="$(date +%s)" --overwrite + + - name: The loss is reported as a Warning on the claim + description: | + The event is re-emitted every reconcile so the signal stays alive rather + than ageing out, so this asserts at least one rather than exactly one. + try: + - script: + timeout: 120s + content: | + set -eu + address=$(cat "${TMPDIR:-/tmp}/nic-missingalloc-address") + + for _ in $(seq 1 30); do + count=$(kubectl -n ipam-e2e-alpha get events \ + --field-selector involvedObject.name=nic-missingalloc,reason=AddressAllocationMissing \ + -o jsonpath='{range .items[*]}{.type}{"\n"}{end}' | grep -c '^Warning$' || true) + [ "$count" -ge 1 ] && break + sleep 3 + done + + [ "$count" -ge 1 ] || { echo "no Warning/AddressAllocationMissing event was recorded on the claim"; exit 1; } + + message=$(kubectl -n ipam-e2e-alpha get events \ + --field-selector involvedObject.name=nic-missingalloc,reason=AddressAllocationMissing \ + -o jsonpath='{.items[0].message}') + echo "event: ${message}" + case "$message" in + *"$address"*) ;; + *) echo "the event does not name the address ${address}"; exit 1 ;; + esac + case "$message" in + *nic-missingalloc-f-ipv4*) ;; + *) echo "the event does not name the missing IPClaim"; exit 1 ;; + esac + case "$message" in + *project-alpha*) ;; + *) echo "the event does not name the project"; exit 1 ;; + esac + + - name: Allocated goes False and the address is kept anyway + description: | + Allocated says every requested address is held. It is not, so it reads + False — reporting True here would be the status contradicting a check the + controller had just failed. Ready follows it down. + + Bound stays True and the address does not move: a workload is running on + it, and re-allocating to make the status tidy would renumber a live + interface to report a problem the operator has to resolve in IPAM either + way. Loud, and still not destructive. + try: + - assert: + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: NetworkInterfaceClaim + metadata: + name: nic-missingalloc + namespace: ipam-e2e-alpha + status: + (conditions[?type == 'Bound'] | [0].status): "True" + (conditions[?type == 'Allocated'] | [0]): + status: "False" + reason: AddressAllocationMissing + (conditions[?type == 'Ready'] | [0]): + status: "False" + reason: NotAllocated + - script: + content: | + set -eu + address=$(cat "${TMPDIR:-/tmp}/nic-missingalloc-address") + published=$(kubectl -n ipam-e2e-alpha get networkinterfaceclaim nic-missingalloc -o jsonpath='{.status.addresses[0].address}') + advertised=$(kubectl -n ipam-e2e-alpha get networkinterface nic-missingalloc -o jsonpath='{.spec.addresses[0].address}') + [ "$published" = "$address" ] || { echo "the claim renumbered to ${published}"; exit 1; } + [ "$advertised" = "$address" ] || { echo "the interface renumbered to ${advertised}"; exit 1; } + + message=$(kubectl -n ipam-e2e-alpha get networkinterfaceclaim nic-missingalloc \ + -o "jsonpath={.status.conditions[?(@.type=='Allocated')].message}") + echo "Allocated: ${message}" + case "$message" in + *"$address"*) ;; + *) echo "the condition does not name the address ${address}"; exit 1 ;; + esac + + - name: An allocation that moved to another address is caught too + description: | + The harder half of the same check: the IPClaim exists, under the right + name, and holds a DIFFERENT address. Only comparing what IPAM holds + against what the interface publishes catches it — a check that stopped + at "the IPClaim is there" would call this healthy. + + The blocker claim takes the freed address first, so the recreated one is + guaranteed to land somewhere else; without it IPAM hands back the very + address that was just released and there is nothing to detect. + try: + - script: + content: | + set -eu + IPAM=$(git rev-parse --show-toplevel)/hack/ipam-tenant-kubectl.sh + ipam() { "$IPAM" project-alpha -n default "$@"; } + address=$(cat "${TMPDIR:-/tmp}/nic-missingalloc-address") + + for name in nic-missingalloc-blocker nic-missingalloc-f-ipv4; do + ipam apply -f - >/dev/null </dev/null 2>&1; } + + visible project-alpha nic-isolation-alpha-f-ipv4 || { echo "project-alpha cannot see its own IPClaim"; exit 1; } + visible project-beta nic-isolation-beta-f-ipv4 || { echo "project-beta cannot see its own IPClaim"; exit 1; } + visible project-alpha nic-isolation-legacy-f-ipv4 || { echo "the legacy-labelled namespace did not allocate in project-alpha"; exit 1; } + + noproject get --raw /apis/ipam.miloapis.com/v1alpha1 >/dev/null \ + || { echo "the project-less request cannot reach IPAM at all, so it proves nothing about scoping"; exit 1; } + classes=$(ipam project-alpha get ipclass -o name | wc -l | tr -d ' ') + [ "$classes" -gt 0 ] || { echo "project-alpha sees no IPClass, so the fixtures are not readable as that project"; exit 1; } + platform_classes=$(noproject get ipclass -o name | wc -l | tr -d ' ') + [ "$platform_classes" -eq 0 ] || { echo "the project-less request reads ${platform_classes} IPClass objects; it is meant to hold no project"; exit 1; } + denial=$(noproject -n default get ipclaim nic-isolation-alpha-f-ipv4 2>&1 || true) + echo "$denial" | grep -Eq 'NotFound|Forbidden|forbidden|not found' \ + || { echo "the project-less refusal did not come from the server: ${denial}"; exit 1; } + + ! visible project-beta nic-isolation-alpha-f-ipv4 || { echo "project-beta can read project-alpha's IPClaim"; exit 1; } + ! visible project-alpha nic-isolation-beta-f-ipv4 || { echo "project-alpha can read project-beta's IPClaim"; exit 1; } + ! noproject -n default get ipclaim nic-isolation-alpha-f-ipv4 >/dev/null 2>&1 || { echo "the project-less request can read project-alpha's IPClaim"; exit 1; } + ! noproject -n default get ipclaim nic-isolation-beta-f-ipv4 >/dev/null 2>&1 || { echo "the project-less request can read project-beta's IPClaim"; exit 1; } + + - name: The address IPAM holds is the address the claim published + try: + - script: + content: | + set -eu + IPAM=$(git rev-parse --show-toplevel)/hack/ipam-tenant-kubectl.sh + allocated() { "$IPAM" "$1" -n default get ipclaim "$2" -o jsonpath='{.status.allocatedCIDR}'; } + + check() { + published=$(kubectl -n "$1" get networkinterfaceclaim "$2" -o jsonpath='{.status.addresses[0].address}') + held=$(allocated "$3" "$2-f-ipv4") + [ "$published" = "$held" ] || { echo "$2 published ${published}, IPAM holds ${held}"; exit 1; } + } + + check ipam-e2e-alpha nic-isolation-alpha project-alpha + check ipam-e2e-beta nic-isolation-beta project-beta + check ipam-e2e-legacy nic-isolation-legacy project-alpha diff --git a/test/e2e/networkinterfaceclaim-reclaim-delete/chainsaw-test.yaml b/test/e2e/networkinterfaceclaim-reclaim-delete/chainsaw-test.yaml new file mode 100644 index 00000000..0116f739 --- /dev/null +++ b/test/e2e/networkinterfaceclaim-reclaim-delete/chainsaw-test.yaml @@ -0,0 +1,104 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/kyverno/chainsaw/main/.schemas/json/test-chainsaw-v1alpha1.json +# +# reclaimPolicy: Delete returns every address to IPAM when the claim goes away. +# +# The interface disappearing proves nothing on its own — an address that stays +# claimed in IPAM after its consumer is gone is leaked space that no one can +# name. The release is therefore asserted in IPAM, over all three claim kinds +# the interface holds: both family entries and the named class. +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: networkinterfaceclaim-reclaim-delete +spec: + cluster: nso-standard + steps: + - name: Claim two families and a named class with reclaimPolicy Delete + try: + - create: + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: Network + metadata: + name: nic-delete-net + namespace: ipam-e2e-alpha + spec: + ipam: + mode: Auto + ipFamilies: + - IPv4 + - IPv6 + - create: + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: NetworkInterfaceClaim + metadata: + name: nic-delete + namespace: ipam-e2e-alpha + spec: + network: + name: nic-delete-net + ipFamilies: + - IPv6 + - IPv4 + reclaimPolicy: Delete + addresses: + - class: datum-public-v4 + - assert: + timeout: 120s + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: NetworkInterfaceClaim + metadata: + name: nic-delete + namespace: ipam-e2e-alpha + status: + (conditions[?type == 'Allocated'] | [0].status): "True" + catch: + - script: + content: | + kubectl -n ipam-e2e-alpha get networkinterfaceclaim,networkinterface -o yaml + kubectl -n network-services-operator-system logs -l app.kubernetes.io/name=network-services-operator --tail=100 + + - name: All three addresses are held in IPAM + try: + - script: + content: | + set -eu + IPAM=$(git rev-parse --show-toplevel)/hack/ipam-tenant-kubectl.sh + for name in nic-delete-f-ipv6 nic-delete-f-ipv4 nic-delete-c-datum-public-v4; do + phase=$("$IPAM" project-alpha -n default \ + get ipclaim "$name" -o jsonpath='{.status.phase}' 2>/dev/null || true) + echo "${name}: ${phase:-missing}" + [ "$phase" = "Bound" ] || { echo "IPClaim ${name} is ${phase:-missing}, want Bound"; exit 1; } + done + + - name: Deleting the claim releases every address + try: + - delete: + timeout: 120s + ref: + apiVersion: networking.datumapis.com/v1alpha + kind: NetworkInterfaceClaim + name: nic-delete + namespace: ipam-e2e-alpha + - script: + content: | + set -eu + IPAM=$(git rev-parse --show-toplevel)/hack/ipam-tenant-kubectl.sh + ipam() { "$IPAM" project-alpha -n default "$@"; } + + ipam get ipclass datum-public-v4 >/dev/null \ + || { echo "project-alpha's fixtures are not readable through this context, so an empty IPClaim list means nothing"; exit 1; } + + for found in $(ipam get ipclaim -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' | grep -E '^nic-delete-' || true); do + echo "IPClaim ${found} survived the claim it was allocated for" + exit 1 + done + - error: + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: NetworkInterface + metadata: + name: nic-delete + namespace: ipam-e2e-alpha diff --git a/test/e2e/networkinterfaceclaim-reclaim-retain/chainsaw-test.yaml b/test/e2e/networkinterfaceclaim-reclaim-retain/chainsaw-test.yaml new file mode 100644 index 00000000..98a99fe9 --- /dev/null +++ b/test/e2e/networkinterfaceclaim-reclaim-retain/chainsaw-test.yaml @@ -0,0 +1,289 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/kyverno/chainsaw/main/.schemas/json/test-chainsaw-v1alpha1.json +# +# reclaimPolicy: Retain is a promise about specific addresses: a claim of the +# same name that comes back gets the SAME interface holding the SAME addresses, +# because a workload that is rescheduled keeps the address its peers, its DNS, +# and its firewall rules already know. +# +# The addresses are captured before the claim is deleted and compared verbatim +# after it is recreated. Asserting that the new claim merely has "an" address +# would pass against a controller that reallocated, which is the exact failure +# this policy exists to prevent. +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: networkinterfaceclaim-reclaim-retain +spec: + cluster: nso-standard + steps: + - name: A retained claim binds and its addresses are recorded + try: + - create: + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: Network + metadata: + name: nic-retain-net + namespace: ipam-e2e-alpha + spec: + ipam: + mode: Auto + ipFamilies: + - IPv4 + - IPv6 + - create: + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: NetworkInterfaceClaim + metadata: + name: nic-retain + namespace: ipam-e2e-alpha + spec: + network: + name: nic-retain-net + ipFamilies: + - IPv6 + - IPv4 + reclaimPolicy: Retain + addresses: + - class: datum-public-v4 + - assert: + timeout: 120s + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: NetworkInterfaceClaim + metadata: + name: nic-retain + namespace: ipam-e2e-alpha + status: + (conditions[?type == 'Allocated'] | [0].status): "True" + - script: + content: | + set -eu + record="${TMPDIR:-/tmp}/nic-retain-addresses" + kubectl -n ipam-e2e-alpha get networkinterfaceclaim nic-retain -o jsonpath='{.status.addresses[?(@.family=="IPv6")].address} {.status.addresses[?(@.family=="IPv4")].address} {.status.externalAddresses[0].address} {.status.networkInterfaceRef.name}' > "$record" + cat "$record" + grep -Eq '^2001:db8:a[0-9a-f]{3}:[^ ]*/96 10\.128\.[0-9]+\.[0-9]+/32 198\.51\.100\.[0-9]+ nic-retain$' "$record" \ + || { echo "recorded an unexpected address set"; exit 1; } + catch: + - script: + content: | + kubectl -n ipam-e2e-alpha get networkinterfaceclaim,networkinterface -o yaml + kubectl -n network-services-operator-system logs -l app.kubernetes.io/name=network-services-operator --tail=100 + cleanup: + - description: > + Safety net, and not optional. A retained interface and its IPClaims + outlive the claim by design, and deleting a retained IPClaim leaves + its IPAllocation behind in IPAM — which keeps holding the address + AND blocks the name, so the next run of this test is refused with + RetainedAddressConflict. Every layer therefore has to be cleared + here: claim, interface, IPClaims, then the allocations that held the + recorded addresses. + script: + content: | + set -u + IPAM=$(git rev-parse --show-toplevel)/hack/ipam-tenant-kubectl.sh + record="${TMPDIR:-/tmp}/nic-retain-addresses" + ipam() { "$IPAM" project-alpha -n default "$@"; } + + kubectl -n ipam-e2e-alpha delete networkinterfaceclaim nic-retain --ignore-not-found --timeout=60s + kubectl -n ipam-e2e-alpha delete networkinterface nic-retain --ignore-not-found --timeout=60s + ipam delete ipclaim nic-retain-f-ipv6 nic-retain-f-ipv4 nic-retain-c-datum-public-v4 \ + --ignore-not-found --timeout=60s + + if [ -f "$record" ]; then + read -r want6 want4 wantext _ < "$record" || true + purged=0 + for cidr in "$want6" "$want4" "${wantext}/32"; do + leftover=$(ipam get ipallocation -o "jsonpath={range .items[?(@.status.allocatedCIDR=='${cidr}')]}{.metadata.name} {end}") + if [ -n "$leftover" ]; then + echo "orphaned IPAllocation holding ${cidr}: ${leftover}" + ipam delete ipallocation $leftover --ignore-not-found + purged=$((purged + 1)) + fi + done + echo "orphaned IPAllocations purged: ${purged}" + fi + rm -f "$record" + + - name: Deleting the claim leaves the interface and its addresses in place + try: + - delete: + timeout: 120s + ref: + apiVersion: networking.datumapis.com/v1alpha + kind: NetworkInterfaceClaim + name: nic-retain + namespace: ipam-e2e-alpha + - assert: + timeout: 60s + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: NetworkInterface + metadata: + name: nic-retain + namespace: ipam-e2e-alpha + spec: + (claimRef == null): true + status: + phase: Available + - script: + content: | + set -eu + IPAM=$(git rev-parse --show-toplevel)/hack/ipam-tenant-kubectl.sh + read -r want6 want4 wantext _ < "${TMPDIR:-/tmp}/nic-retain-addresses" || true + + held() { "$IPAM" project-alpha -n default get ipclaim "$1" -o jsonpath='{.status.allocatedCIDR}'; } + + [ "$(held nic-retain-f-ipv6)" = "$want6" ] || { echo "IPv6 address was not retained in IPAM"; exit 1; } + [ "$(held nic-retain-f-ipv4)" = "$want4" ] || { echo "IPv4 address was not retained in IPAM"; exit 1; } + [ "$(held nic-retain-c-datum-public-v4)" = "${wantext}/32" ] || { echo "the external address was not retained in IPAM"; exit 1; } + + - name: A claim asking for a different reclaim policy cannot adopt it + description: | + The delete-and-recreate route to the wedge the CEL immutability closes + in place. These addresses were allocated under Retain and IPAM freezes + that onto the allocation, so a Delete claim adopting them would strand + what it was told to release. It has to be refused at bind. + try: + - create: + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: NetworkInterfaceClaim + metadata: + name: nic-retain + namespace: ipam-e2e-alpha + spec: + network: + name: nic-retain-net + ipFamilies: + - IPv6 + - IPv4 + reclaimPolicy: Delete + addresses: + - class: datum-public-v4 + - assert: + timeout: 120s + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: NetworkInterfaceClaim + metadata: + name: nic-retain + namespace: ipam-e2e-alpha + status: + (conditions[?type == 'Bound'] | [0]): + status: "False" + reason: ReclaimPolicyMismatch + (conditions[?type == 'Allocated'] | [0].status): "False" + (contains(conditions[?type == 'Bound'] | [0].message, 'reclaimPolicy')): true + - assert: + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: NetworkInterface + metadata: + name: nic-retain + namespace: ipam-e2e-alpha + spec: + reclaimPolicy: Retain + (claimRef == null): true + - delete: + timeout: 120s + ref: + apiVersion: networking.datumapis.com/v1alpha + kind: NetworkInterfaceClaim + name: nic-retain + namespace: ipam-e2e-alpha + + - name: A claim of the same name rebinds the same interface with the same addresses + try: + - create: + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: NetworkInterfaceClaim + metadata: + name: nic-retain + namespace: ipam-e2e-alpha + spec: + network: + name: nic-retain-net + ipFamilies: + - IPv6 + - IPv4 + reclaimPolicy: Retain + addresses: + - class: datum-public-v4 + - assert: + timeout: 120s + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: NetworkInterfaceClaim + metadata: + name: nic-retain + namespace: ipam-e2e-alpha + status: + networkInterfaceRef: + name: nic-retain + (conditions[?type == 'Bound'] | [0].status): "True" + (conditions[?type == 'Allocated'] | [0].status): "True" + - script: + content: | + set -eu + read -r want6 want4 wantext wantiface < "${TMPDIR:-/tmp}/nic-retain-addresses" || true + + got6=$(kubectl -n ipam-e2e-alpha get networkinterfaceclaim nic-retain -o 'jsonpath={.status.addresses[?(@.family=="IPv6")].address}') + got4=$(kubectl -n ipam-e2e-alpha get networkinterfaceclaim nic-retain -o 'jsonpath={.status.addresses[?(@.family=="IPv4")].address}') + gotext=$(kubectl -n ipam-e2e-alpha get networkinterfaceclaim nic-retain -o jsonpath='{.status.externalAddresses[0].address}') + gotiface=$(kubectl -n ipam-e2e-alpha get networkinterfaceclaim nic-retain -o jsonpath='{.status.networkInterfaceRef.name}') + echo "before: ${want6} ${want4} ${wantext} ${wantiface}" + echo "after: ${got6} ${got4} ${gotext} ${gotiface}" + + [ "$got6" = "$want6" ] || { echo "IPv6 address changed across the rebind"; exit 1; } + [ "$got4" = "$want4" ] || { echo "IPv4 address changed across the rebind"; exit 1; } + [ "$gotext" = "$wantext" ] || { echo "the external address changed across the rebind"; exit 1; } + [ "$gotiface" = "$wantiface" ] || { echo "the claim bound a different interface"; exit 1; } + - assert: + timeout: 60s + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: NetworkInterface + metadata: + name: nic-retain + namespace: ipam-e2e-alpha + annotations: + networking.datumapis.com/allocation-claim: nic-retain + spec: + claimRef: + name: nic-retain + status: + phase: Bound + + - name: The reclaim policy cannot be edited after creation + description: | + A retained interface is given up by deleting the interface, not by + relaxing the claim that holds it. Editing reclaimPolicy in place is + refused at admission: NSO freezes the policy into IPAM when the address + is first allocated and cannot revise it there, so a claim that changed + its mind would delete an IPClaim that IPAM still believes is retained + and strand the allocation. + try: + - patch: + expect: + - check: + ($error != null): true + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: NetworkInterfaceClaim + metadata: + name: nic-retain + namespace: ipam-e2e-alpha + spec: + reclaimPolicy: Delete + - assert: + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: NetworkInterfaceClaim + metadata: + name: nic-retain + namespace: ipam-e2e-alpha + spec: + reclaimPolicy: Retain diff --git a/test/e2e/networkinterfaceclaim-unlabeled-namespace/chainsaw-test.yaml b/test/e2e/networkinterfaceclaim-unlabeled-namespace/chainsaw-test.yaml new file mode 100644 index 00000000..5e31562f --- /dev/null +++ b/test/e2e/networkinterfaceclaim-unlabeled-namespace/chainsaw-test.yaml @@ -0,0 +1,137 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/kyverno/chainsaw/main/.schemas/json/test-chainsaw-v1alpha1.json +# +# A namespace that names no project allocates nothing. ipam-e2e-unlabeled +# carries no meta.datumapis.com/upstream-cluster-name label, and the only safe +# outcome is a refusal naming the missing label — a fallback to some default +# project would hand one tenant another tenant's address space. +# +# The absence of an allocation is only evidence if this suite can see an +# allocation when there is one, so a canary claim in ipam-e2e-alpha allocates +# for real and is read back through the same context, in the same step, before +# any absence is asserted. Without it a dead IPAM, a stale kubeconfig or a +# typo'd context all read as a pass. +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: networkinterfaceclaim-unlabeled-namespace +spec: + cluster: nso-standard + steps: + - name: A claim in an unlabelled namespace is refused + try: + - create: + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: Network + metadata: + name: nic-unlabeled-net + namespace: ipam-e2e-unlabeled + spec: + ipam: + mode: Auto + ipFamilies: + - IPv4 + - create: + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: NetworkInterfaceClaim + metadata: + name: nic-unlabeled + namespace: ipam-e2e-unlabeled + spec: + network: + name: nic-unlabeled-net + ipFamilies: + - IPv4 + - assert: + timeout: 120s + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: NetworkInterfaceClaim + metadata: + name: nic-unlabeled + namespace: ipam-e2e-unlabeled + status: + (conditions[?type == 'Bound'] | [0]): + status: "False" + reason: ProjectUnresolved + (conditions[?type == 'Allocated'] | [0].status): "False" + (conditions[?type == 'Ready'] | [0].status): "False" + (contains(conditions[?type == 'Bound'] | [0].message, 'meta.datumapis.com/upstream-cluster-name')): true + catch: + - script: + content: | + kubectl -n ipam-e2e-unlabeled get networkinterfaceclaim -o yaml + kubectl -n network-services-operator-system logs -l app.kubernetes.io/name=network-services-operator --tail=100 + + - name: A labelled namespace allocates, so an empty result means empty + description: | + The positive control. This claim differs from the refused one only in + the namespace it lives in. + try: + - create: + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: Network + metadata: + name: nic-unlabeled-canary-net + namespace: ipam-e2e-alpha + spec: + ipam: + mode: Auto + ipFamilies: + - IPv4 + - create: + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: NetworkInterfaceClaim + metadata: + name: nic-unlabeled-canary + namespace: ipam-e2e-alpha + spec: + network: + name: nic-unlabeled-canary-net + ipFamilies: + - IPv4 + - assert: + timeout: 120s + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: NetworkInterfaceClaim + metadata: + name: nic-unlabeled-canary + namespace: ipam-e2e-alpha + status: + (conditions[?type == 'Allocated'] | [0].status): "True" + + - name: Nothing was bound and nothing was allocated + description: | + Every IPClaim in both project namespaces is listed and matched against + the refused claim's name prefix, rather than probing the two names the + naming scheme happens to derive today — a leak under an unexpected name + is exactly what a derivation bug produces. + try: + - script: + content: | + set -eu + IPAM=$(git rev-parse --show-toplevel)/hack/ipam-tenant-kubectl.sh + ipam() { proj=$1; shift; "$IPAM" "$proj" -n default "$@"; } + names() { ipam "$1" get ipclaim -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}'; } + + alpha=$(names project-alpha) + beta=$(names project-beta) + echo "project-alpha IPClaims: $(echo $alpha)" + echo "project-beta IPClaims: $(echo $beta)" + + echo "$alpha" | grep -qx 'nic-unlabeled-canary-f-ipv4' \ + || { echo "the canary's IPClaim is not visible in project-alpha — this suite cannot see allocations, so it cannot prove their absence"; exit 1; } + + for found in $(printf '%s\n%s\n' "$alpha" "$beta" | grep -E '^nic-unlabeled-(f|c)-' || true); do + echo "an unresolvable claim allocated ${found}" + exit 1 + done + + if kubectl -n ipam-e2e-unlabeled get networkinterface nic-unlabeled >/dev/null 2>&1; then + echo "an unresolvable claim still created a NetworkInterface" + exit 1 + fi