diff --git a/.github/workflows/test-e2e.yml b/.github/workflows/test-e2e.yml index 9bede775..d3a520b1 100644 --- a/.github/workflows/test-e2e.yml +++ b/.github/workflows/test-e2e.yml @@ -1,13 +1,51 @@ name: E2E Tests +# In-cluster federation e2e (issue #149). +# +# Stands up the full topology on the shared test-infra federation foundation — +# three Kind clusters (one management/control-plane hosting Karmada, two POP +# cells) with the real production kustomize overlays and hub RBAC — then runs the +# Chainsaw suites against it. This exercises the operators as deployed pods +# authenticating to Karmada as a non-admin identity, not as an in-process test +# binary. + on: push: + branches: [main] pull_request: +# The e2e Taskfile pulls the test-infra foundation in as a remote Taskfile +# include; opt into Task's remote-taskfile support for every step that runs a +# task (paired with `task --yes` for the non-interactive trust prompt). +env: + TASK_X_REMOTE_TASKFILES: "1" + +# Cancel a superseded run on the same ref. The e2e job is expensive (three Kind +# clusters + a Karmada control plane), so we don't want stale pushes burning a +# runner. Unlike the cheaper test/lint workflows this only triggers on PRs and +# main pushes rather than every branch push, for the same cost reason. +concurrency: + group: e2e-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + jobs: test-e2e: name: Run on Ubuntu + # ubuntu-latest is 4 vCPU / 16 GB. The harness is deliberately engineered for + # a constrained/busy host — single-replica Karmada, leader election disabled, + # generous 10m component waits, join retries — so this fits without a larger + # (paid) runner. If real runs show OOM or repeated timeouts, switch to a + # larger hosted runner label (e.g. ubuntu-latest-8-cores); that carries a + # billing implication, hence starting on the free tier. runs-on: ubuntu-latest + # Env standup can spend up to ~20m if both Karmada waits approach their 10m + # ceilings on a slow runner; deploy ~5m; the Chainsaw suites ~15m (the + # referenced-data GC-sweep suite alone floors around 6m). 50m leaves headroom + # over the ~35m expected without letting a genuinely hung run idle too long. + timeout-minutes: 50 steps: - name: Clone the code uses: actions/checkout@v4 @@ -17,19 +55,93 @@ jobs: with: go-version: '~1.25.0' - - name: Install the latest version of kind - run: | - curl -Lo ./kind https://kind.sigs.k8s.io/dl/latest/kind-linux-amd64 - chmod +x ./kind - sudo mv ./kind /usr/local/bin/kind + # go-task drives the entire harness (task e2e:env:up / e2e:deploy / + # e2e:test). The action publishes major refs as branches (v1/v2/v3); the + # version input pins the go-task binary itself. repo-token avoids GitHub + # API rate limiting when the action resolves the release. + - name: Install go-task + uses: arduino/setup-task@v3 + with: + version: 3.52.0 + repo-token: ${{ secrets.GITHUB_TOKEN }} + + # kind, kubectl, helm, and karmadactl are installed by the test-infra + # federation foundation's own ensure-tools; chainsaw is fetched into ./bin + # by the compute task tooling. No harness-side python/PyYAML is used any + # more (the foundation rewrites kubeconfigs with kubectl; the remaining + # python3 calls parse `go list` JSON with the stdlib only). - - name: Verify kind installation - run: kind version + # Split into env / deploy / test so a failure lands on the phase that broke + # rather than a single opaque "task e2e:up" step. e2e:env:up == e2e:up minus + # the deploy; the two together are exactly what e2e:up runs. `--yes` accepts + # Task's one-time trust prompt for the remote test-infra include. + - name: Provision Kind + Karmada environment + run: task --yes e2e:env:up - - name: Create kind cluster - run: kind create cluster + - name: Build image and deploy operators + run: task --yes e2e:deploy - - name: Running Test e2e + - name: Run Chainsaw e2e suites + run: task --yes e2e:test + + # Always capture what the clusters looked like when a step failed. The + # runner is ephemeral so teardown is unnecessary; diagnostics are the only + # thing worth keeping. + - name: Collect diagnostics + if: failure() run: | - go mod tidy - make test-e2e + set +e + DIAG=tmp/e2e/diagnostics + KDIR=.test-infra/kubeconfigs/federation + mkdir -p "$DIAG" + + # Host / kind level: container state plus a full per-cluster export + # (kubelet, containerd, and every pod log). + kind get clusters > "$DIAG/kind-clusters.txt" 2>&1 + docker ps -a > "$DIAG/docker-ps.txt" 2>&1 + for c in compute-control-plane compute-pop-dfw compute-pop-ord; do + kind export logs "$DIAG/kind-$c" --name "$c" 2>&1 | tail -n 2 + done + + # Per-cluster Kubernetes state + the compute-manager operator logs + # (current and previous, all containers) from every plane. + for kc in compute-control-plane compute-pop-dfw compute-pop-ord karmada; do + cfg="$KDIR/$kc.yaml" + [ -f "$cfg" ] || continue + out="$DIAG/$kc"; mkdir -p "$out" + kubectl --kubeconfig="$cfg" get pods -A -o wide > "$out/pods.txt" 2>&1 + kubectl --kubeconfig="$cfg" get events -A --sort-by=.lastTimestamp > "$out/events.txt" 2>&1 + kubectl --kubeconfig="$cfg" -n compute-system describe deploy compute-manager \ + > "$out/compute-manager-describe.txt" 2>&1 + kubectl --kubeconfig="$cfg" -n compute-system logs deploy/compute-manager \ + --all-containers --tail=-1 > "$out/compute-manager.log" 2>&1 + kubectl --kubeconfig="$cfg" -n compute-system logs deploy/compute-manager \ + --all-containers --previous --tail=-1 > "$out/compute-manager-previous.log" 2>&1 + done + + # Karmada control-plane pods + component logs (they live in the + # management cluster) and the federation view from the Karmada API. + if [ -f "$KDIR/compute-control-plane.yaml" ]; then + kubectl --kubeconfig="$KDIR/compute-control-plane.yaml" -n karmada-system get pods -o wide \ + > "$DIAG/karmada-pods.txt" 2>&1 + for d in karmada-apiserver karmada-controller-manager karmada-scheduler; do + kubectl --kubeconfig="$KDIR/compute-control-plane.yaml" -n karmada-system logs deploy/$d \ + --tail=-1 > "$DIAG/karmada-$d.log" 2>&1 + done + fi + if [ -f "$KDIR/karmada.yaml" ]; then + kubectl --kubeconfig="$KDIR/karmada.yaml" get clusters -o wide \ + > "$DIAG/karmada-clusters.txt" 2>&1 + kubectl --kubeconfig="$KDIR/karmada.yaml" get workloaddeployments -A -o wide \ + > "$DIAG/karmada-workloaddeployments.txt" 2>&1 + fi + echo "Diagnostics collected under $DIAG" + + - name: Upload diagnostics + if: failure() + uses: actions/upload-artifact@v4 + with: + name: e2e-diagnostics + path: tmp/e2e/diagnostics + retention-days: 7 + if-no-files-found: warn diff --git a/.gitignore b/.gitignore index d5cc564d..592fed60 100644 --- a/.gitignore +++ b/.gitignore @@ -28,3 +28,9 @@ bin/ # Local e2e environment artefacts (Kind kubeconfigs, etc.) tmp/ + +# test-infra federation foundation: its self-clone and the kubeconfigs it writes +.test-infra/ + +# Task's remote-taskfile cache (populated with TASK_X_REMOTE_TASKFILES=1) +.task/ diff --git a/Taskfile.yaml b/Taskfile.yaml new file mode 100644 index 00000000..1a19308f --- /dev/null +++ b/Taskfile.yaml @@ -0,0 +1,492 @@ +version: '3' + +# ─── Variables ────────────────────────────────────────────────────────────── + +vars: + # Pin for the shared test-infra federation foundation. The remote Taskfile + # include below (and the foundation's own self-clone) both resolve against + # this ref. Must be moved to a release tag before this branch merges — see the + # re-pin chain in the PR description. + TEST_INFRA_REF: feat/federation-topology + + # Chainsaw version for e2e testing (kyverno/chainsaw) + CHAINSAW_VERSION: v0.2.15 + + # Local directory for e2e tooling. Deliberately NOT named LOCALBIN: the + # test-infra foundation defines its own LOCALBIN, and a remote include's root + # var clobbers a same-named var in this Taskfile — compute's value would be + # overwritten by the foundation's REPO_DIR-based path, which is undefined in + # this scope and renders as a nil path. + E2E_BIN: '{{.ROOT_DIR}}/bin' + CHAINSAW: '{{.ROOT_DIR}}/bin/chainsaw' + + # Kind cluster names. These are also the Karmada member/hub cluster names the + # federation foundation registers, and they name the kubeconfig files the + # foundation writes into KUBECONFIG_DIR (.yaml). + KIND_CONTROL_PLANE: compute-control-plane + KIND_POP_DFW: compute-pop-dfw + KIND_POP_ORD: compute-pop-ord + + # Dev image tag built locally and side-loaded into every Kind cluster. This + # replaces the ghcr.io/datum-cloud/compute:latest reference baked into + # config/base/manager so e2e never pulls from a registry. + IMAGE: compute:e2e-dev + + # Working directory for compute-minted e2e artefacts (gitignored): the + # federation kubeconfigs the operators mount and per-run diagnostics. The + # cluster/Karmada kubeconfigs themselves are owned by the foundation and land + # in KUBECONFIG_DIR below. + E2E_DIR: '{{.ROOT_DIR}}/tmp/e2e' + + # Federation kubeconfig directory written by the test-infra foundation. The + # remote include self-clones test-infra into .test-infra/ and drops the hub, + # member, and Karmada kubeconfigs here; compute reads them from the same path. + KUBECONFIG_DIR: '{{.ROOT_DIR}}/.test-infra/kubeconfigs/federation' + + # Fixed NodePort for the Karmada API server, passed through to the foundation. + # The hub Kind cluster is created with an extraPortMapping for this port so it + # is reachable at https://localhost: from the developer's machine. + # Override (e.g. KARMADA_API_NODEPORT=32643) to dodge a host-port collision + # with another local cluster; the value propagates into the include. + KARMADA_API_NODEPORT: "32443" + + # In-cluster Service address of the Karmada API server. The management + # compute-manager runs in the SAME Kind cluster as Karmada, so it reaches the + # hub over cluster DNS rather than the host-exposed NodePort. Matches the + # audience the production management-plane overlay projects its token for. + KARMADA_INCLUSTER_SERVER: "https://karmada-apiserver.karmada-system.svc.cluster.local:5443" + + # Namespace the operators deploy into (matches config/overlays/* namespace:). + COMPUTE_NAMESPACE: compute-system + +# ─── Includes ─────────────────────────────────────────────────────────────── + +includes: + # Shared test-infra federation foundation. Standing up the Kind clusters, + # installing + tuning Karmada, and joining the cells all live here now, so the + # same path runs on a laptop and in CI. Requires TASK_X_REMOTE_TASKFILES=1 in + # the environment and `--yes` on first/non-interactive runs (Task's + # remote-taskfile trust prompt). + infra: + taskfile: https://raw.githubusercontent.com/datum-cloud/test-infra/{{.TEST_INFRA_REF}}/Taskfile.yml + vars: + # The foundation's ensure-repo self-clone must match the include ref. + REPO_REF: '{{.TEST_INFRA_REF}}' + # Compute's topology: a management/control-plane hub and two POP cells + # labelled with their city code. + FEDERATION_HUB_CLUSTER: '{{.KIND_CONTROL_PLANE}}' + FEDERATION_MEMBERS: '{{.KIND_POP_DFW}}=dfw {{.KIND_POP_ORD}}=ord' + KARMADA_API_NODEPORT: '{{.KARMADA_API_NODEPORT}}' + +# ─── Tasks ────────────────────────────────────────────────────────────────── + +tasks: + + default: + cmds: + - task --list + silent: true + + # ════════════════════════════════════════════════════════════════════════ + # e2e environment lifecycle + # ════════════════════════════════════════════════════════════════════════ + + e2e:up: + desc: "Create the Kind+Karmada environment AND deploy the compute operators (idempotent)" + cmds: + - task: e2e:env:up + - task: e2e:deploy + - cmd: | + echo "" + echo "╔══════════════════════════════════════════════════════════╗" + echo "║ e2e environment ready — operators deployed in-cluster ║" + echo "╠══════════════════════════════════════════════════════════╣" + echo "║ Control plane: {{.KUBECONFIG_DIR}}/{{.KIND_CONTROL_PLANE}}.yaml" + echo "║ Karmada API: {{.KUBECONFIG_DIR}}/karmada.yaml" + echo "║ POP DFW: {{.KUBECONFIG_DIR}}/{{.KIND_POP_DFW}}.yaml" + echo "║ POP ORD: {{.KUBECONFIG_DIR}}/{{.KIND_POP_ORD}}.yaml" + echo "╠══════════════════════════════════════════════════════════╣" + echo "║ Inspect operators: ║" + echo "║ kubectl --kubeconfig {{.KUBECONFIG_DIR}}/{{.KIND_CONTROL_PLANE}}.yaml -n {{.COMPUTE_NAMESPACE}} get deploy" + echo "╚══════════════════════════════════════════════════════════╝" + silent: false + + e2e:env:up: + desc: "Stand up the federation foundation, configure Karmada, install CRDs (no operators)" + cmds: + - task: e2e:tools + # The shared foundation creates the Kind clusters, installs+tunes Karmada, + # and joins the POP cells with their city-code labels — everything the + # compute harness used to build by hand. + - task: infra:federation-up + - task: e2e:karmada:configure + - task: e2e:crds:install + + e2e:down: + desc: "Tear down the local e2e environment" + cmds: + - task: infra:federation-down + - rm -rf {{.E2E_DIR}} + - cmd: echo "✓ e2e environment torn down" + silent: false + + e2e:test: + desc: "Run Chainsaw e2e tests against the local Kind+Karmada environment" + deps: [e2e:tools:chainsaw] + cmds: + # --parallel 1 (sequential) on purpose: the federation controllers are a + # single manager, and running all suites concurrently floods it — many + # WorkloadDeployments federate at once and individual suites' downstream + # assertions time out ("resource not found") even though federation works. + # Serial keeps the federation path uncontended on a constrained runner. + # Override for a faster local run with `task e2e:test -- --parallel N`. + - | + KUBECONFIG={{.KUBECONFIG_DIR}}/{{.KIND_CONTROL_PLANE}}.yaml \ + {{.CHAINSAW}} test \ + --config test/e2e/chainsaw-config.yaml \ + --parallel 1 \ + test/e2e/ \ + {{.CLI_ARGS}} + + e2e:test:filter: + desc: "Run a subset of e2e tests by name regex (e.g. task e2e:test:filter -- --include-test-regex federation)" + deps: [e2e:tools:chainsaw] + cmds: + - | + KUBECONFIG={{.KUBECONFIG_DIR}}/{{.KIND_CONTROL_PLANE}}.yaml \ + {{.CHAINSAW}} test \ + --config test/e2e/chainsaw-config.yaml \ + {{.CLI_ARGS}} \ + test/e2e/ + + # ════════════════════════════════════════════════════════════════════════ + # Tool installation + # ════════════════════════════════════════════════════════════════════════ + + e2e:tools: + desc: "Install e2e-specific tooling (chainsaw)" + cmds: + # kind/kubectl/helm/karmadactl are installed by the federation foundation's + # own ensure-tools; chainsaw is the only compute-specific e2e tool left. + - task: e2e:tools:chainsaw + + e2e:tools:chainsaw: + desc: "Download chainsaw {{.CHAINSAW_VERSION}}" + cmds: + - mkdir -p {{.E2E_BIN}} + - | + if [ ! -f "{{.CHAINSAW}}" ]; then + OS=$(uname -s | tr '[:upper:]' '[:lower:]') + ARCH=$(uname -m | sed 's/x86_64/amd64/;s/aarch64/arm64/') + URL="https://github.com/kyverno/chainsaw/releases/download/{{.CHAINSAW_VERSION}}/chainsaw_${OS}_${ARCH}.tar.gz" + echo "Downloading chainsaw {{.CHAINSAW_VERSION}} (${OS}/${ARCH}) from ${URL}..." + curl -sSfL "${URL}" | tar -xz -C {{.E2E_BIN}} chainsaw + chmod +x {{.CHAINSAW}} + echo "chainsaw installed → {{.CHAINSAW}}" + else + echo "chainsaw already present at {{.CHAINSAW}}" + fi + status: + - test -f {{.CHAINSAW}} + + # ════════════════════════════════════════════════════════════════════════ + # Karmada federation configuration (compute-specific) + # ════════════════════════════════════════════════════════════════════════ + + e2e:karmada:configure: + desc: "Apply federation component config to the Karmada API server (idempotent)" + cmds: + # Retry the apply: this runs right after the Karmada install, and the + # apiserver reached over the NodePort can briefly drop the connection + # ("EOF" / API discovery failure) while it settles, which otherwise fails + # provisioning on an otherwise-healthy control plane. + - | + echo "Applying federation component to Karmada..." + for attempt in 1 2 3 4 5 6; do + if kubectl --kubeconfig={{.KUBECONFIG_DIR}}/karmada.yaml apply \ + -k config/components/federation/ --server-side --validate=false; then + echo "Federation component applied (attempt ${attempt})"; exit 0 + fi + echo "configure attempt ${attempt} failed (Karmada apiserver may still be settling); retrying in 10s..." + sleep 10 + done + echo "ERROR: failed to apply federation component after 6 attempts"; exit 1 + + # ════════════════════════════════════════════════════════════════════════ + # CRD installation + # ════════════════════════════════════════════════════════════════════════ + + e2e:crds:install: + desc: "Install compute + NSO + quota CRDs to all clusters" + cmds: + - task: _e2e:crds:compute + - task: _e2e:crds:nso + - task: _e2e:crds:quota + + _e2e:crds:compute: + internal: true + desc: "Apply compute CRDs to all clusters and the Karmada API server" + cmds: + # All three Kind clusters + the Karmada API server get the compute CRDs. + # The Karmada API server needs them so it can store and propagate + # WorkloadDeployment objects. + - | + for KC in \ + {{.KUBECONFIG_DIR}}/{{.KIND_CONTROL_PLANE}}.yaml \ + {{.KUBECONFIG_DIR}}/karmada.yaml \ + {{.KUBECONFIG_DIR}}/{{.KIND_POP_DFW}}.yaml \ + {{.KUBECONFIG_DIR}}/{{.KIND_POP_ORD}}.yaml; do + echo "Installing compute CRDs → $(basename $KC .yaml)..." + kubectl --kubeconfig="$KC" apply -k config/base/crd --server-side --validate=false + done + + _e2e:crds:nso: + internal: true + desc: "Apply NSO CRDs to control-plane and POP cell clusters" + cmds: + # NSO CRDs (NetworkBinding, SubnetClaim, etc.) are installed on the + # control-plane as well as POP cells. The control-plane operator needs them + # so that Subnet/SubnetClaim informer watches can start without cache errors, + # even though NSO controllers themselves only run on POP cells. + - | + go mod download go.datum.net/network-services-operator + NSO_VERSION=$(go list -m -json go.datum.net/network-services-operator \ + | python3 -c "import sys, json; print(json.load(sys.stdin)['Version'])") + NSO_CRD_PATH="$(go env GOMODCACHE)/go.datum.net/network-services-operator@${NSO_VERSION}/config/crd" + echo "NSO CRDs from: ${NSO_CRD_PATH}" + for KC in \ + {{.KUBECONFIG_DIR}}/{{.KIND_CONTROL_PLANE}}.yaml \ + {{.KUBECONFIG_DIR}}/{{.KIND_POP_DFW}}.yaml \ + {{.KUBECONFIG_DIR}}/{{.KIND_POP_ORD}}.yaml; do + echo "Installing NSO CRDs → $(basename $KC .yaml)..." + kubectl --kubeconfig="$KC" apply -k "${NSO_CRD_PATH}" --server-side --validate=false + done + + _e2e:crds:quota: + internal: true + desc: "Apply Milo quota CRDs to all clusters and the Karmada API server" + cmds: + # Quota CRDs (ResourceClaim, ResourceGrant, etc.) are required on all + # clusters so the InstanceReconciler can create and watch ResourceClaims + # against project Milo control planes without cache startup errors. + - | + go mod download go.miloapis.com/milo + MILO_VERSION=$(go list -m -json go.miloapis.com/milo \ + | python3 -c "import sys, json; print(json.load(sys.stdin)['Version'])") + QUOTA_CRD_PATH="$(go env GOMODCACHE)/go.miloapis.com/milo@${MILO_VERSION}/config/crd/bases/quota" + echo "Milo quota CRDs from: ${QUOTA_CRD_PATH}" + for KC in \ + {{.KUBECONFIG_DIR}}/{{.KIND_CONTROL_PLANE}}.yaml \ + {{.KUBECONFIG_DIR}}/karmada.yaml \ + {{.KUBECONFIG_DIR}}/{{.KIND_POP_DFW}}.yaml \ + {{.KUBECONFIG_DIR}}/{{.KIND_POP_ORD}}.yaml; do + echo "Installing Milo quota CRDs → $(basename $KC .yaml)..." + kubectl --kubeconfig="$KC" apply -k "${QUOTA_CRD_PATH}" --server-side --validate=false + done + + # ════════════════════════════════════════════════════════════════════════ + # Operator image build + side-load + # ════════════════════════════════════════════════════════════════════════ + + e2e:image:build: + desc: "Build the compute-manager image with the local dev tag ({{.IMAGE}})" + cmds: + - | + echo "Building {{.IMAGE}} from {{.ROOT_DIR}}/Dockerfile..." + docker build -t {{.IMAGE}} {{.ROOT_DIR}} + + e2e:image:load: + desc: "Side-load {{.IMAGE}} into every Kind cluster (no registry pull)" + cmds: + # The foundation owns the cluster inventory, so it loads the image into the + # hub + every member rather than compute re-deriving the cluster list. + - task: infra:federation-load-image + vars: + IMAGES: '{{.IMAGE}}' + + # ════════════════════════════════════════════════════════════════════════ + # Federation access for the management operator + # ════════════════════════════════════════════════════════════════════════ + + e2e:federation:setup: + desc: "Bind hub RBAC and mint the management operator's Karmada kubeconfig" + cmds: + # ── Hub-side RBAC (the real production manifest) ──────────────────── + # config/base/downstream-rbac grants the compute-manager ClusterRole on + # the Karmada hub and binds it to the user + # system:serviceaccount:compute-system:compute-manager. Applying the real + # manifest is the whole point of #149 — the management operator authenticates + # to Karmada as a non-admin identity and every missing grant surfaces as a + # forbidden error rather than being masked by cluster-admin. + - | + echo "Applying hub RBAC (config/base/downstream-rbac) to Karmada..." + for attempt in 1 2 3 4 5; do + if kubectl --kubeconfig={{.KUBECONFIG_DIR}}/karmada.yaml apply -k config/base/downstream-rbac --server-side --validate=false; then + break + fi + echo "hub RBAC apply attempt ${attempt} failed (Karmada apiserver over NodePort may be settling); retrying in 8s..." + sleep 8 + if [ "${attempt}" = "5" ]; then echo "ERROR: hub RBAC apply failed after 5 attempts"; exit 1; fi + done + # ── Karmada-native identity for the management operator ───────────── + # Production federates the management cluster's projected ServiceAccount + # token into Karmada (Karmada trusts the host cluster's token issuer). We + # do not configure cross-cluster token trust in the Kind environment, so + # instead we create a Karmada-native ServiceAccount whose authenticated + # username is identical — system:serviceaccount:compute-system:compute-manager + # — and therefore matches the very same ClusterRoleBinding subject. Same + # RBAC surface, no issuer federation required. + - | + kubectl --kubeconfig={{.KUBECONFIG_DIR}}/karmada.yaml create namespace {{.COMPUTE_NAMESPACE}} \ + --dry-run=client -o yaml | kubectl --kubeconfig={{.KUBECONFIG_DIR}}/karmada.yaml apply -f - + kubectl --kubeconfig={{.KUBECONFIG_DIR}}/karmada.yaml -n {{.COMPUTE_NAMESPACE}} \ + create serviceaccount compute-manager \ + --dry-run=client -o yaml | kubectl --kubeconfig={{.KUBECONFIG_DIR}}/karmada.yaml apply -f - + # ── Mint the federation kubeconfig ────────────────────────────────── + # A bound token from the Karmada SA, embedded in a kubeconfig that targets + # the Karmada in-cluster Service (the management pod runs alongside Karmada). + # We re-mint on every run so a re-deploy always ships a fresh, unexpired + # token. insecure-skip-tls-verify mirrors the karmada.yaml the foundation + # builds — the served cert does not cover the Service DNS name. + - | + mkdir -p {{.E2E_DIR}} + echo "Minting Karmada token for compute-manager..." + TOKEN=$(kubectl --kubeconfig={{.KUBECONFIG_DIR}}/karmada.yaml -n {{.COMPUTE_NAMESPACE}} \ + create token compute-manager --duration=720h) + cat > {{.E2E_DIR}}/downstream-kubeconfig.yaml < {{.E2E_DIR}}/cell-federation-kubeconfig.yaml </dev/null + kubectl --kubeconfig="$KC" -n {{.COMPUTE_NAMESPACE}} \ + create configmap compute-cell-federation-kubeconfig \ + --from-file=kubeconfig={{.E2E_DIR}}/cell-federation-kubeconfig.yaml \ + --dry-run=client -o yaml | kubectl --kubeconfig="$KC" apply -f - + done + echo "Published compute-cell-federation-kubeconfig on both POP cells" + + # ════════════════════════════════════════════════════════════════════════ + # Operator deployment (real kustomize overlays + e2e patches) + # ════════════════════════════════════════════════════════════════════════ + + e2e:deploy: + desc: "Build+load the image, wire federation, and deploy the operators to all clusters" + cmds: + - task: e2e:image:build + - task: e2e:image:load + - task: e2e:federation:setup + - task: e2e:deploy:management + - task: e2e:deploy:cells + + e2e:deploy:management: + desc: "Deploy the management-plane overlay to the control-plane cluster" + cmds: + - | + echo "Deploying management-plane operator → {{.KIND_CONTROL_PLANE}}..." + kubectl --kubeconfig={{.KUBECONFIG_DIR}}/{{.KIND_CONTROL_PLANE}}.yaml apply -k test/e2e/deploy/management --server-side --validate=false + - task: _e2e:deploy:wait + vars: + KUBECONFIG_FILE: "{{.KUBECONFIG_DIR}}/{{.KIND_CONTROL_PLANE}}.yaml" + LABEL: control-plane + + e2e:deploy:cells: + desc: "Deploy the cell overlay to both POP cell clusters" + cmds: + - task: _e2e:deploy:cell + vars: + KUBECONFIG_FILE: "{{.KUBECONFIG_DIR}}/{{.KIND_POP_DFW}}.yaml" + CLUSTER_NAME: "{{.KIND_POP_DFW}}" + - task: _e2e:deploy:cell + vars: + KUBECONFIG_FILE: "{{.KUBECONFIG_DIR}}/{{.KIND_POP_ORD}}.yaml" + CLUSTER_NAME: "{{.KIND_POP_ORD}}" + + _e2e:deploy:cell: + internal: true + cmds: + - | + echo "Deploying cell operator → {{.CLUSTER_NAME}}..." + kubectl --kubeconfig={{.KUBECONFIG_FILE}} apply -k test/e2e/deploy/cell --server-side --validate=false + - task: _e2e:deploy:wait + vars: + KUBECONFIG_FILE: "{{.KUBECONFIG_FILE}}" + LABEL: "{{.CLUSTER_NAME}}" + + _e2e:deploy:wait: + internal: true + cmds: + - | + echo "Waiting for compute-manager rollout on {{.LABEL}}..." + kubectl --kubeconfig={{.KUBECONFIG_FILE}} -n {{.COMPUTE_NAMESPACE}} \ + rollout status deployment/compute-manager --timeout=180s diff --git a/config/components/controller_rbac/role.yaml b/config/components/controller_rbac/role.yaml index 425eecf6..81147741 100644 --- a/config/components/controller_rbac/role.yaml +++ b/config/components/controller_rbac/role.yaml @@ -31,6 +31,7 @@ rules: verbs: - get - list + - watch - apiGroups: - compute.datumapis.com resources: @@ -68,6 +69,7 @@ rules: resources: - locations - networkcontexts + - networks - subnets verbs: - get diff --git a/test/e2e/chainsaw-config.yaml b/test/e2e/chainsaw-config.yaml new file mode 100644 index 00000000..d7140cd1 --- /dev/null +++ b/test/e2e/chainsaw-config.yaml @@ -0,0 +1,52 @@ +# Chainsaw global configuration for the compute federation e2e test suite. +# +# Prerequisites +# ───────────── +# Run `task e2e:up` to create the Kind clusters and populate kubeconfigs under +# .test-infra/kubeconfigs/federation/ before running Chainsaw. +# +# Running +# ─────── +# From the repository root via Taskfile (recommended): +# +# task e2e:test +# +# Or directly: +# +# KUBECONFIG=.test-infra/kubeconfigs/federation/compute-control-plane.yaml \ +# chainsaw test --config test/e2e/chainsaw-config.yaml test/e2e/ +# +# The KUBECONFIG env var sets the "default" cluster (control-plane cell). +# Additional clusters (downstream, pop-dfw, pop-ord) are declared below and +# referenced by name in individual test steps via `cluster: downstream` etc. +# +# Kubeconfig paths below are relative to the working directory where Chainsaw is +# invoked (the project root), NOT relative to this config file's location. +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Configuration +metadata: + name: chainsaw +spec: + # Timeouts are widened from chainsaw's defaults for the constrained local/CI + # node that co-hosts the Karmada control plane: Karmada propagation of a + # WorkloadDeployment to a cell routinely needs more than the default 60s under + # load, and finalizer-driven cascade deletes (the federator releasing hub + # companions) exceed the default 30s. + timeouts: + apply: 60s + assert: 120s + cleanup: 120s + delete: 120s + error: 60s + exec: 30s + clusters: + # Downstream control plane (the Karmada hub API server). WorkloadDeployments, + # PropagationPolicies, and Instance write-backs live here. + downstream: + kubeconfig: .test-infra/kubeconfigs/federation/karmada.yaml + # POP DFW cell — downstream member cluster labelled topology.datum.net/city-code=dfw. + pop-dfw: + kubeconfig: .test-infra/kubeconfigs/federation/compute-pop-dfw.yaml + # POP ORD cell — downstream member cluster labelled topology.datum.net/city-code=ord. + pop-ord: + kubeconfig: .test-infra/kubeconfigs/federation/compute-pop-ord.yaml diff --git a/test/e2e/deletion-cascade/chainsaw-test.yaml b/test/e2e/deletion-cascade/chainsaw-test.yaml new file mode 100644 index 00000000..e7026842 --- /dev/null +++ b/test/e2e/deletion-cascade/chainsaw-test.yaml @@ -0,0 +1,79 @@ +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: deletion-cascade +spec: + description: | + Verifies that deleting a WorkloadDeployment from the project namespace causes + the federator to remove the corresponding WorkloadDeployment from Karmada. + + The WorkloadDeploymentFederator adds a finalizer + (compute.datumapis.com/federator) to every project WD it manages. When the + project WD is deleted: + 1. The finalizer's Finalize method runs (blocking deletion until complete). + 2. It deletes the Karmada-side WorkloadDeployment. + 3. It removes the PropagationPolicy if no other WDs for the city remain. + 4. It removes the finalizer, allowing the project WD to be garbage-collected. + + This test validates: project WD deletion → Karmada WD deletion. + + template: true + + steps: + - name: create-wd + description: Create a WorkloadDeployment on the control-plane cluster. + try: + - apply: + file: workload-deployment.yaml + + - name: wait-for-federation + description: Wait for the WorkloadDeployment to appear in Karmada. + cluster: downstream + try: + - script: + content: | + kubectl --kubeconfig=../../../.test-infra/kubeconfigs/federation/compute-control-plane.yaml \ + get namespace "$NAMESPACE" \ + -o template='{{printf "ns-%s" .metadata.uid}}' + outputs: + - name: downstreamNS + value: ($stdout) + - assert: + timeout: 30s + resource: + apiVersion: compute.datumapis.com/v1alpha + kind: WorkloadDeployment + metadata: + namespace: ($downstreamNS) + name: test-cascade-wd + + - name: delete-wd + description: Delete the WorkloadDeployment from the control-plane cluster. + try: + - delete: + ref: + apiVersion: compute.datumapis.com/v1alpha + kind: WorkloadDeployment + namespace: ($namespace) + name: test-cascade-wd + + - name: assert-downstream-wd-deleted + description: Confirm the Karmada copy is removed by the finalizer. + cluster: downstream + try: + - script: + content: | + kubectl --kubeconfig=../../../.test-infra/kubeconfigs/federation/compute-control-plane.yaml \ + get namespace "$NAMESPACE" \ + -o template='{{printf "ns-%s" .metadata.uid}}' + outputs: + - name: downstreamNS + value: ($stdout) + - wait: + apiVersion: compute.datumapis.com/v1alpha + kind: WorkloadDeployment + namespace: ($downstreamNS) + name: test-cascade-wd + timeout: 30s + for: + deletion: {} diff --git a/test/e2e/deletion-cascade/workload-deployment.yaml b/test/e2e/deletion-cascade/workload-deployment.yaml new file mode 100644 index 00000000..39d68a1d --- /dev/null +++ b/test/e2e/deletion-cascade/workload-deployment.yaml @@ -0,0 +1,21 @@ +apiVersion: compute.datumapis.com/v1alpha +kind: WorkloadDeployment +metadata: + name: test-cascade-wd +spec: + cityCode: dfw + placementName: default + workloadRef: + name: test-workload + uid: "00000000-0000-0000-0000-000000000001" + template: + spec: + runtime: + resources: + instanceType: datumcloud/d1-standard-2 + networkInterfaces: + - network: + name: test-network + + scaleSettings: + minReplicas: 1 diff --git a/test/e2e/deploy/cell/config_patch.yaml b/test/e2e/deploy/cell/config_patch.yaml new file mode 100644 index 00000000..7e94f3a8 --- /dev/null +++ b/test/e2e/deploy/cell/config_patch.yaml @@ -0,0 +1,36 @@ +# DEVIATION (required for Kind): overrides the operator config the production +# cell overlay ships (config/overlays/cell/disable_webhook_patch.yaml) to drop +# discovery.quotaKubeconfigPath. +# +# The production cell config points quotaKubeconfigPath at +# /etc/quota-credentials/kubeconfig, delivered by config/components/quota-credentials +# from the compute-edge-milo-client-cert Secret + compute-quota-kubeconfig +# ConfigMap. Those are cluster-environment secrets that do not exist in Kind, and +# the projected volume sources are optional — so the file is simply absent. The +# operator treats a configured-but-missing quota kubeconfig as fatal (os.Exit), +# which would crash-loop the cell pod. Dropping the path takes the documented +# opt-out branch instead: quota enforcement is disabled and the operator boots. +# +# Impact: the cell InstanceReconciler skips ResourceClaim creation/quota checks. +# The federation delivery path under test (WorkloadDeployment → Instance) is +# unaffected; edge quota enforcement is covered separately. +# +# featureFlags.enableReferencedDataGate is turned on because the cell +# WorkloadDeploymentReconciler is the sole consumer of this flag: it stamps the +# "ReferencedData" scheduling gate onto Instances whose template references a +# ConfigMap or Secret, and the cell InstanceReconciler clears the gate once the +# companions land on the cell. The referenced-data-mounts and +# referenced-data-delete-cascade suites assert that stamp-then-clear behaviour, +# so without the flag those suites would observe an Instance that is never gated. +apiVersion: v1 +kind: ConfigMap +metadata: + name: compute-config +data: + config.yaml: | + apiVersion: apiserver.config.datumapis.com/v1alpha1 + kind: WorkloadOperator + metricsServer: + bindAddress: "0" + featureFlags: + enableReferencedDataGate: true diff --git a/test/e2e/deploy/cell/deployment_patch.yaml b/test/e2e/deploy/cell/deployment_patch.yaml new file mode 100644 index 00000000..9fd08dd5 --- /dev/null +++ b/test/e2e/deploy/cell/deployment_patch.yaml @@ -0,0 +1,13 @@ +# DEVIATION (required for Kind): imagePullPolicy: IfNotPresent — the dev image is +# side-loaded into the Kind node (task e2e:image:load); never attempt a registry +# pull. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: compute-manager +spec: + template: + spec: + containers: + - name: manager + imagePullPolicy: IfNotPresent diff --git a/test/e2e/deploy/cell/federation_patch.yaml b/test/e2e/deploy/cell/federation_patch.yaml new file mode 100644 index 00000000..c3512185 --- /dev/null +++ b/test/e2e/deploy/cell/federation_patch.yaml @@ -0,0 +1,30 @@ +# Give the cell operator a Karmada hub credential, mirroring how real cell +# deployments are wired: infra's apps/compute-system/edge/manager-patch.yaml sets +# FEDERATION_KUBECONFIG and mounts a client-cert kubeconfig pointed at the hub. +# With it, the cell InstanceReconciler writes each Instance back to the Karmada +# hub, where the management InstanceProjector consumes it — the sole path by +# which cell-created Instances become visible upstream (Karmada never propagates +# Instances, only WorkloadDeployments/ConfigMaps/Secrets, so status aggregation +# cannot surface them). The mount path matches infra's; the kubeconfig content +# (a Karmada-native SA token reaching the kind hub over the control-plane node's +# docker-bridge IP + NodePort) is minted by task e2e:federation:setup. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: compute-manager +spec: + template: + spec: + containers: + - name: manager + env: + - name: FEDERATION_KUBECONFIG + value: /etc/kubernetes/upstream/auth/kubeconfig + volumeMounts: + - name: upstream-kubeconfig + mountPath: /etc/kubernetes/upstream/auth + readOnly: true + volumes: + - name: upstream-kubeconfig + configMap: + name: compute-cell-federation-kubeconfig diff --git a/test/e2e/deploy/cell/kustomization.yaml b/test/e2e/deploy/cell/kustomization.yaml new file mode 100644 index 00000000..b63ce808 --- /dev/null +++ b/test/e2e/deploy/cell/kustomization.yaml @@ -0,0 +1,41 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +# ───────────────────────────────────────────────────────────────────────────── +# e2e cell deploy layer. +# +# References the REAL production cell overlay verbatim and applies only the +# deviations required to run inside a local Kind environment (image source, +# quota-credential neutralisation) plus the cell hub credential that infra +# supplies in production. +# +# The production cell overlay ships FEDERATION_KUBECONFIG empty on purpose — it +# is the *infra* layer that patches it in per-cell (apps/compute-system/edge/ +# manager-patch.yaml mounts a hub client-cert kubeconfig), NOT the overlay. So +# leaving it empty is NOT production-exact: it disables the cell's Instance +# write-back to the Karmada hub, and cell-created Instances then never reach the +# hub or the management InstanceProjector. federation_patch.yaml restores that +# credential the way infra does, so the full cell→hub→projector path is exercised. +# ───────────────────────────────────────────────────────────────────────────── + +resources: + - ../../../../config/overlays/cell + +# Use the locally built image side-loaded into the Kind node (task e2e:image:load) +# instead of pulling ghcr.io/datum-cloud/compute:latest from a registry. +images: + - name: ghcr.io/datum-cloud/compute + newName: compute + newTag: e2e-dev + +patches: + # DEVIATION 1 (required): drop discovery.quotaKubeconfigPath. + # See config_patch.yaml for the full rationale. + - path: config_patch.yaml + + # DEVIATION 2 (required): force IfNotPresent pulls of the side-loaded image. + - path: deployment_patch.yaml + + # Cell hub credential, mirroring infra's per-cell federation patch so the + # Instance write-back path is exercised. See federation_patch.yaml. + - path: federation_patch.yaml diff --git a/test/e2e/deploy/management/config_patch.yaml b/test/e2e/deploy/management/config_patch.yaml new file mode 100644 index 00000000..d3a0abf7 --- /dev/null +++ b/test/e2e/deploy/management/config_patch.yaml @@ -0,0 +1,37 @@ +# DEVIATION (required for Kind): overrides the operator config the production +# management-plane overlay ships (config/overlays/management-plane/discovery_mode_patch.yaml). +# +# - discovery.mode: milo → single. Milo discovery enumerates Projects from a Milo +# control plane to build per-project clients; the Kind environment has no Milo. +# Single-cluster discovery runs the management controllers against the local +# control-plane cluster while still federating WorkloadDeployments to Karmada +# through FEDERATION_KUBECONFIG (unchanged from the overlay). This matches how +# the retired host-run harness drove the management operator. +# +# - webhookServer is intentionally omitted, which disables the admission webhook +# server. The production overlay serves it with a cert delivered by the +# cert-manager CSI driver, which is not installed in Kind and whose issuer is +# supplied by infra rather than the overlay. The e2e suites never create +# Workload objects, so the Workload webhook is never exercised. +# +# - featureFlags.enableReferencedDataGate mirrors the flag set on the cell so the +# two operators share one feature-flag surface, matching the retired host-run +# harness which passed the same --server-config flag to both. Its only consumer +# is the cell WorkloadDeploymentReconciler, which this management overlay does +# not run (it enables management controllers only), so the flag is inert here; +# the cell config_patch carries the copy that actually gates Instances. Kept +# explicit so the flag does not silently diverge between the two operators. +apiVersion: v1 +kind: ConfigMap +metadata: + name: compute-config +data: + config.yaml: | + apiVersion: apiserver.config.datumapis.com/v1alpha1 + kind: WorkloadOperator + metricsServer: + bindAddress: "0" + discovery: + mode: single + featureFlags: + enableReferencedDataGate: true diff --git a/test/e2e/deploy/management/deployment_patch.yaml b/test/e2e/deploy/management/deployment_patch.yaml new file mode 100644 index 00000000..d92134c4 --- /dev/null +++ b/test/e2e/deploy/management/deployment_patch.yaml @@ -0,0 +1,27 @@ +# DEVIATION (required for Kind): +# 1. imagePullPolicy: IfNotPresent — the dev image is side-loaded into the Kind +# node (task e2e:image:load); never attempt a registry pull. +# 2. Back the webhook-server-tls volume with an emptyDir instead of the +# cert-manager CSI driver (csi.cert-manager.io) injected by +# config/components/csi-webhook-cert. That driver is not installed in Kind, +# so the CSI volume would fail to mount and the pod would never start. The +# webhook server itself is disabled in config_patch.yaml, so the serving +# cert is never read — an empty directory at the mount path is enough to let +# the pod boot. We override only the volume source (csi -> emptyDir) and +# leave the container's volumeMounts untouched: a strategic-merge +# "$patch: delete" on a single volumeMount drops the whole list, which would +# also unmount the federation kubeconfig and config volumes. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: compute-manager +spec: + template: + spec: + containers: + - name: manager + imagePullPolicy: IfNotPresent + volumes: + - name: webhook-server-tls + csi: null + emptyDir: {} diff --git a/test/e2e/deploy/management/kustomization.yaml b/test/e2e/deploy/management/kustomization.yaml new file mode 100644 index 00000000..f93b5158 --- /dev/null +++ b/test/e2e/deploy/management/kustomization.yaml @@ -0,0 +1,73 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +# ───────────────────────────────────────────────────────────────────────────── +# e2e management-plane deploy layer. +# +# References the REAL production management-plane overlay verbatim and applies +# only the deviations required to run inside a local Kind environment (no Milo +# control plane, no cert-manager, no image registry). Every deviation is +# annotated below with the reason it exists so drift from production stays +# visible. Exercising the real overlay + real hub RBAC is the point of #149. +# ───────────────────────────────────────────────────────────────────────────── + +resources: + - ../../../../config/overlays/management-plane + +# Use the locally built image side-loaded into the Kind node (task e2e:image:load) +# instead of pulling ghcr.io/datum-cloud/compute:latest from a registry. +images: + - name: ghcr.io/datum-cloud/compute + newName: compute + newTag: e2e-dev + +patches: + # DEVIATION 1 (required): discovery.mode milo → single, webhook server off. + # See config_patch.yaml for the full rationale. + - path: config_patch.yaml + + # DEVIATION 2 (required): drop the cert-manager CSI serving-cert volume and + # force IfNotPresent pulls. See deployment_patch.yaml. + - path: deployment_patch.yaml + + # DEVIATION 3 (required): remove the admission webhook configurations. + # The webhook server is disabled (DEVIATION 1) because no serving cert is + # available in Kind, so these failurePolicy=Fail configurations would reject + # every Workload write. The e2e suites create WorkloadDeployments, never + # Workloads, so nothing here is exercised; removing the configs keeps the + # cluster clean rather than leaving them dangling against a dead endpoint. + - patch: | + $patch: delete + apiVersion: admissionregistration.k8s.io/v1 + kind: ValidatingWebhookConfiguration + metadata: + name: compute-validating + target: + kind: ValidatingWebhookConfiguration + name: compute-validating + - patch: | + $patch: delete + apiVersion: admissionregistration.k8s.io/v1 + kind: MutatingWebhookConfiguration + metadata: + name: compute-mutating + target: + kind: MutatingWebhookConfiguration + name: compute-mutating + + # DEVIATION 4 (required): remove the ResourceMetricsPolicy. + # config/components/resource-metrics ships a ResourceMetricsPolicy + # (resourcemetrics.miloapis.com), whose CRD is owned by a separate + # resource-metrics operator that is not installed in Kind (it is not in + # compute's module deps). Applying it fails with "no matches for kind + # ResourceMetricsPolicy". It is orthogonal to the federation path under test, + # so drop it from the deploy set. + - patch: | + $patch: delete + apiVersion: resourcemetrics.miloapis.com/v1alpha1 + kind: ResourceMetricsPolicy + metadata: + name: compute-metrics + target: + kind: ResourceMetricsPolicy + name: compute-metrics diff --git a/test/e2e/full-federation-ord/chainsaw-test.yaml b/test/e2e/full-federation-ord/chainsaw-test.yaml new file mode 100644 index 00000000..388bb0bb --- /dev/null +++ b/test/e2e/full-federation-ord/chainsaw-test.yaml @@ -0,0 +1,192 @@ +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: full-federation-ord +spec: + description: | + Second-cell federation chain test (city code ord → pop-ord). + + The dfw path is covered by full-federation. This suite proves the same chain + routes independently to the OTHER cell: an ord-placed WorkloadDeployment must + produce PropagationPolicy city-ord (not city-dfw), propagate to pop-ord (not + pop-dfw), and create its Instance on pop-ord. Both cells run the same operator + image, so this is the coverage that catches city-code routing regressions and + a mis-registered second cell. + + 1. Create WorkloadDeployment (cityCode: ord) on control-plane. + 2. WorkloadDeploymentFederator replicates it to Karmada (ns- namespace) + and lazily creates PropagationPolicy city-ord routing to city-code=ord cells. + 3. Karmada propagates the WD to pop-ord. + 4. WorkloadDeploymentReconciler on pop-ord creates Instance test-fullfed-ord-wd-0. + 5. InstanceReconciler on pop-ord writes the Instance back to Karmada with + label meta.datumapis.com/upstream-cluster-name: cluster-single. + 6. InstanceProjector on control-plane projects the Instance into the project + namespace. + + Cluster-name label "cluster-single" is the management operator's federating + cluster name, not the cell's — see full-federation for the full rationale. + + Prerequisites: the management + both cell operators are deployed in-cluster + (task e2e:up). + + template: true + + steps: + - name: create-workload-deployment + description: Create the ord-placed WorkloadDeployment on the control-plane cluster. + try: + - apply: + file: workload-deployment.yaml + + - name: assert-wd-and-policy-in-downstream + description: Assert the WD federated to Karmada and PropagationPolicy city-ord was created for ord. + cluster: downstream + try: + - script: + content: | + kubectl --kubeconfig=../../../.test-infra/kubeconfigs/federation/compute-control-plane.yaml \ + get namespace "$NAMESPACE" \ + -o template='{{printf "ns-%s" .metadata.uid}}' + outputs: + - name: downstreamNS + value: ($stdout) + - assert: + timeout: 120s + resource: + apiVersion: compute.datumapis.com/v1alpha + kind: WorkloadDeployment + metadata: + namespace: ($downstreamNS) + name: test-fullfed-ord-wd + labels: + topology.datum.net/city-code: ord + - assert: + # The federator names the policy city- and routes it to cells + # carrying the same city-code label, so ord must land on pop-ord alone. + timeout: 120s + resource: + apiVersion: policy.karmada.io/v1alpha1 + kind: PropagationPolicy + metadata: + namespace: ($downstreamNS) + name: city-ord + spec: + # Three selectors: the WorkloadDeployment plus the always-on ConfigMap + # and Secret referenced-data selectors. Chainsaw matches list length + # exactly, so all three must be present. + resourceSelectors: + - apiVersion: compute.datumapis.com/v1alpha + kind: WorkloadDeployment + labelSelector: + matchLabels: + topology.datum.net/city-code: ord + - apiVersion: v1 + kind: ConfigMap + labelSelector: + matchLabels: + compute.datumapis.com/referenced-data: "true" + - apiVersion: v1 + kind: Secret + labelSelector: + matchLabels: + compute.datumapis.com/referenced-data: "true" + placement: + clusterAffinity: + labelSelector: + matchLabels: + topology.datum.net/city-code: ord + + - name: assert-wd-on-pop-ord + description: Assert Karmada propagated the WD to pop-ord and the cell reconciler set status. + cluster: pop-ord + try: + - script: + content: | + kubectl --kubeconfig=../../../.test-infra/kubeconfigs/federation/compute-control-plane.yaml \ + get namespace "$NAMESPACE" \ + -o template='{{printf "ns-%s" .metadata.uid}}' + outputs: + - name: downstreamNS + value: ($stdout) + - assert: + # Karmada propagation can take longer than a local apply. + timeout: 60s + resource: + apiVersion: compute.datumapis.com/v1alpha + kind: WorkloadDeployment + metadata: + namespace: ($downstreamNS) + name: test-fullfed-ord-wd + status: + replicas: 1 + desiredReplicas: 1 + + - name: assert-instance-on-pop-ord + description: Assert WorkloadDeploymentReconciler created an Instance on pop-ord. + cluster: pop-ord + try: + - script: + content: | + kubectl --kubeconfig=../../../.test-infra/kubeconfigs/federation/compute-control-plane.yaml \ + get namespace "$NAMESPACE" \ + -o template='{{printf "ns-%s" .metadata.uid}}' + outputs: + - name: downstreamNS + value: ($stdout) + - assert: + timeout: 30s + resource: + apiVersion: compute.datumapis.com/v1alpha + kind: Instance + metadata: + namespace: ($downstreamNS) + name: test-fullfed-ord-wd-0 + # The Instance is blocked before scheduling completes (a scheduling + # gate, or no matching Location in the e2e environment), so Ready is + # definitively False — not indeterminate. The exact reason is + # environment-dependent, so only the status is asserted. + (status.conditions[?type == 'Ready'] | [0]): + status: "False" + + - name: assert-instance-writeback-in-downstream + description: Assert the pop-ord InstanceReconciler wrote the Instance back to Karmada. + cluster: downstream + try: + - script: + content: | + kubectl --kubeconfig=../../../.test-infra/kubeconfigs/federation/compute-control-plane.yaml \ + get namespace "$NAMESPACE" \ + -o template='{{printf "ns-%s" .metadata.uid}}' + outputs: + - name: downstreamNS + value: ($stdout) + - assert: + timeout: 30s + resource: + apiVersion: compute.datumapis.com/v1alpha + kind: Instance + metadata: + namespace: ($downstreamNS) + name: test-fullfed-ord-wd-0 + labels: + meta.datumapis.com/upstream-cluster-name: cluster-single + + - name: assert-instance-projected-to-control-plane + description: Assert InstanceProjector created a projection on the control-plane. + try: + - assert: + timeout: 30s + resource: + apiVersion: compute.datumapis.com/v1alpha + kind: Instance + metadata: + namespace: ($namespace) + name: test-fullfed-ord-wd-0 + labels: + meta.datumapis.com/upstream-cluster-name: cluster-single + # The Instance is blocked before scheduling completes (a scheduling + # gate, or no matching Location in the e2e environment), so Ready is + # definitively False — not indeterminate. The exact reason is + # environment-dependent, so only the status is asserted. + (status.conditions[?type == 'Ready'] | [0]): + status: "False" diff --git a/test/e2e/full-federation-ord/workload-deployment.yaml b/test/e2e/full-federation-ord/workload-deployment.yaml new file mode 100644 index 00000000..74663331 --- /dev/null +++ b/test/e2e/full-federation-ord/workload-deployment.yaml @@ -0,0 +1,21 @@ +apiVersion: compute.datumapis.com/v1alpha +kind: WorkloadDeployment +metadata: + name: test-fullfed-ord-wd + # namespace is injected by Chainsaw from ($namespace) +spec: + cityCode: ord + placementName: default + workloadRef: + name: test-workload + uid: "00000000-0000-0000-0000-000000000002" + template: + spec: + runtime: + resources: + instanceType: datumcloud/d1-standard-2 + networkInterfaces: + - network: + name: test-network + scaleSettings: + minReplicas: 1 diff --git a/test/e2e/full-federation/chainsaw-test.yaml b/test/e2e/full-federation/chainsaw-test.yaml new file mode 100644 index 00000000..ec8a57b1 --- /dev/null +++ b/test/e2e/full-federation/chainsaw-test.yaml @@ -0,0 +1,173 @@ +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: full-federation +spec: + description: | + End-to-end federation chain test. + + Exercises the complete path from WorkloadDeployment creation through to + Instance projection on the control-plane cluster: + + 1. Create WorkloadDeployment on control-plane. + 2. WorkloadDeploymentFederator replicates it to Karmada (ns- namespace). + 3. Karmada PropagationPolicy routes the WD to pop-dfw. + 4. WorkloadDeploymentReconciler on pop-dfw creates Instance test-full-fed-wd-0. + 5. InstanceReconciler on pop-dfw writes Instance back to Karmada with + label meta.datumapis.com/upstream-cluster-name: cluster-single. + 6. InstanceProjector on control-plane creates a projection of the Instance + in the project namespace. + + Cluster-name label: "cluster-single". + The value is NOT the cell's cluster name. The management operator federates + the WorkloadDeployment from its local control-plane cluster, which it + registers with multicluster-runtime under the name "single" (both operators + run discovery.mode=single in this environment, defaulting the cluster name to + "single" per cmd/main.go singleClusterName). The federator stamps + EncodeClusterName("single") = "cluster-single" onto the hub ns- + namespace; the cell write-back and the projection both copy that namespace + label verbatim, so the assertion holds regardless of which cell reconciles. + + Prerequisites: the management + both cell operators are deployed in-cluster + (task e2e:up). + + template: true + + steps: + - name: create-workload-deployment + description: Create the WorkloadDeployment on the control-plane cluster. + try: + - apply: + file: workload-deployment.yaml + + - name: assert-wd-in-downstream + description: Assert WorkloadDeploymentFederator replicated the WD to Karmada and status is synced back. + cluster: downstream + try: + - script: + content: | + kubectl --kubeconfig=../../../.test-infra/kubeconfigs/federation/compute-control-plane.yaml \ + get namespace "$NAMESPACE" \ + -o template='{{printf "ns-%s" .metadata.uid}}' + outputs: + - name: downstreamNS + value: ($stdout) + - assert: + timeout: 30s + resource: + apiVersion: compute.datumapis.com/v1alpha + kind: WorkloadDeployment + metadata: + namespace: ($downstreamNS) + name: test-full-fed-wd + - assert: + # Wait for the cell operator to write status back to the Karmada WD. + timeout: 60s + resource: + apiVersion: compute.datumapis.com/v1alpha + kind: WorkloadDeployment + metadata: + namespace: ($downstreamNS) + name: test-full-fed-wd + status: + replicas: 1 + desiredReplicas: 1 + + - name: assert-wd-on-pop-dfw + description: Assert Karmada propagated the WD to pop-dfw and the cell reconciler set status. + cluster: pop-dfw + try: + - script: + content: | + kubectl --kubeconfig=../../../.test-infra/kubeconfigs/federation/compute-control-plane.yaml \ + get namespace "$NAMESPACE" \ + -o template='{{printf "ns-%s" .metadata.uid}}' + outputs: + - name: downstreamNS + value: ($stdout) + - assert: + # Karmada propagation can take longer than a local apply. + timeout: 60s + resource: + apiVersion: compute.datumapis.com/v1alpha + kind: WorkloadDeployment + metadata: + namespace: ($downstreamNS) + name: test-full-fed-wd + status: + replicas: 1 + desiredReplicas: 1 + + - name: assert-instance-on-pop-dfw + description: Assert WorkloadDeploymentReconciler created an Instance on pop-dfw with a Ready condition. + cluster: pop-dfw + try: + - script: + content: | + kubectl --kubeconfig=../../../.test-infra/kubeconfigs/federation/compute-control-plane.yaml \ + get namespace "$NAMESPACE" \ + -o template='{{printf "ns-%s" .metadata.uid}}' + outputs: + - name: downstreamNS + value: ($stdout) + - assert: + # This is the tail of the full chain (project WD → federate → Karmada + # propagate to the cell → cell reconcile → Instance). On a constrained CI + # runner that whole sequence can exceed 30s; 120s gives it room without + # masking a real stall (the smoke path confirms the Instance does appear). + timeout: 120s + resource: + apiVersion: compute.datumapis.com/v1alpha + kind: Instance + metadata: + namespace: ($downstreamNS) + name: test-full-fed-wd-0 + # The Instance is blocked before scheduling completes (a scheduling + # gate, or no matching Location in the e2e environment), so Ready is + # definitively False — not indeterminate. The exact reason is + # environment-dependent, so only the status is asserted. + (status.conditions[?type == 'Ready'] | [0]): + status: "False" + + - name: assert-instance-writeback-in-downstream + description: Assert InstanceReconciler wrote the Instance back to Karmada. + cluster: downstream + try: + - script: + content: | + kubectl --kubeconfig=../../../.test-infra/kubeconfigs/federation/compute-control-plane.yaml \ + get namespace "$NAMESPACE" \ + -o template='{{printf "ns-%s" .metadata.uid}}' + outputs: + - name: downstreamNS + value: ($stdout) + - assert: + timeout: 30s + resource: + apiVersion: compute.datumapis.com/v1alpha + kind: Instance + metadata: + namespace: ($downstreamNS) + name: test-full-fed-wd-0 + labels: + meta.datumapis.com/upstream-cluster-name: cluster-single + + - name: assert-instance-projected-to-control-plane + description: Assert InstanceProjector created a projection with status on the control-plane. + try: + - assert: + timeout: 30s + resource: + apiVersion: compute.datumapis.com/v1alpha + kind: Instance + metadata: + namespace: ($namespace) + name: test-full-fed-wd-0 + labels: + meta.datumapis.com/upstream-cluster-name: cluster-single + # The Instance is blocked before scheduling completes (a scheduling + # gate, or no matching Location in the e2e environment), so Ready is + # definitively False — not indeterminate. The exact reason is + # environment-dependent, so only the status is asserted. + (status.conditions[?type == 'Ready'] | [0]): + status: "False" diff --git a/test/e2e/full-federation/workload-deployment.yaml b/test/e2e/full-federation/workload-deployment.yaml new file mode 100644 index 00000000..70b4cb94 --- /dev/null +++ b/test/e2e/full-federation/workload-deployment.yaml @@ -0,0 +1,21 @@ +apiVersion: compute.datumapis.com/v1alpha +kind: WorkloadDeployment +metadata: + name: test-full-fed-wd + # namespace is injected by Chainsaw from ($namespace) +spec: + cityCode: dfw + placementName: default + workloadRef: + name: test-workload + uid: "00000000-0000-0000-0000-000000000001" + template: + spec: + runtime: + resources: + instanceType: datumcloud/d1-standard-2 + networkInterfaces: + - network: + name: test-network + scaleSettings: + minReplicas: 1 diff --git a/test/e2e/instance-projection/chainsaw-test.yaml b/test/e2e/instance-projection/chainsaw-test.yaml new file mode 100644 index 00000000..89c57a49 --- /dev/null +++ b/test/e2e/instance-projection/chainsaw-test.yaml @@ -0,0 +1,146 @@ +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: instance-projection +spec: + description: | + Verifies that the InstanceProjector watches Instances written back to the + Karmada API server and creates corresponding read-only projections in the + project namespace on the control-plane cluster. + + Flow: + 1. Create a WorkloadDeployment → triggers federator → Karmada namespace created. + 2. Write an Instance to Karmada (simulating a POP-cell InstanceReconciler write-back). + 3. InstanceProjector detects the Karmada Instance and creates a projection in the + project namespace (the Chainsaw test namespace on the control-plane cluster). + 4. Assert the projection exists with the upstream tracking label and an owner + reference to the WorkloadDeployment (for cascading deletion). + + Cluster name label: "cluster-single". + The management operator runs discovery.mode=single in this environment, which + registers the control-plane cluster with the multicluster-runtime manager + under the fixed name "single" (cmd/main.go singleClusterName, wired via + mcsingle.New). The InstanceProjector decodes the upstream-cluster-name label + ("cluster-single" → "single") to pick the project cluster to project into, so + the label this test writes onto the hub Instance must encode "single". + + template: true + + steps: + - name: create-wd + description: Create the WorkloadDeployment to trigger federation and namespace creation. + try: + - apply: + file: workload-deployment.yaml + + - name: wait-for-downstream-namespace + description: Wait for the federated WorkloadDeployment to appear in Karmada. + cluster: downstream + try: + - script: + content: | + kubectl --kubeconfig=../../../.test-infra/kubeconfigs/federation/compute-control-plane.yaml \ + get namespace "$NAMESPACE" \ + -o template='{{printf "ns-%s" .metadata.uid}}' + outputs: + - name: downstreamNS + value: ($stdout) + - assert: + # Federation of a freshly-created namespace depends on the management + # operator's namespace cache being current; with the namespaces-watch RBAC + # fix it stays live, but 30s is still tight on a loaded CI runner, so allow 120s. + timeout: 120s + resource: + apiVersion: compute.datumapis.com/v1alpha + kind: WorkloadDeployment + metadata: + namespace: ($downstreamNS) + name: test-projector-wd + + - name: write-instance-to-downstream + description: | + Write an Instance to Karmada simulating InstanceReconciler write-back. + Uses explicit control-plane kubeconfig to derive downstreamNS and WD UID. + cluster: downstream + try: + - script: + content: | + kubectl --kubeconfig=../../../.test-infra/kubeconfigs/federation/compute-control-plane.yaml \ + get namespace "$NAMESPACE" \ + -o template='{{printf "ns-%s" .metadata.uid}}' + outputs: + - name: downstreamNS + value: ($stdout) + - script: + content: | + kubectl --kubeconfig=../../../.test-infra/kubeconfigs/federation/compute-control-plane.yaml \ + get workloaddeployment test-projector-wd \ + --namespace "$NAMESPACE" \ + -o jsonpath='{.metadata.uid}' + outputs: + - name: wdUID + value: ($stdout) + - script: + env: + - name: KARMADA_NS + value: ($downstreamNS) + - name: PROJECT_NS + value: ($namespace) + - name: WD_UID + value: ($wdUID) + content: | + # These labels mirror what the real cell InstanceReconciler stamps on a + # write-back copy. The projector resolves the project cluster from + # upstream-cluster-name, the project NAMESPACE from upstream-namespace + # (the project namespace where the WD lives — NOT the hub ns-), and + # looks up the owning WorkloadDeployment by workload-deployment-name to + # build the projection's owner reference. + kubectl apply -f - < namespace, which the management + federator stamped: meta.datumapis.com/upstream-cluster-name carries + EncodeClusterName of the federating cluster ("single" here → "cluster-single") + and meta.datumapis.com/upstream-namespace records the originating namespace. + + Note: this test requires the cell InstanceReconciler to be running in the DFW + POP cell cluster with federation configured. + + Runtime prerequisite (see verification phase): the write-back path errors + unless the hub ns- namespace carries the two upstream-* labels above and + the cell Instance carries the full set of linking labels the stateful control + strategy stamps at creation. This suite hand-crafts a bare namespace and a + bare Instance, so the write-back may not fire until those labels are seeded + (or the Instance is created via a real WorkloadDeployment, as full-federation + does). + + template: true + + steps: + - name: setup-namespaces + description: Create the Instance namespace in the DFW POP cell and Karmada. + try: + - script: + content: | + kubectl get namespace "$NAMESPACE" \ + -o template='{{printf "ns-%s" .metadata.uid}}' + outputs: + - name: instanceNS + value: ($stdout) + - script: + env: + - name: INSTANCE_NS + value: ($instanceNS) + content: | + kubectl --kubeconfig=../../../.test-infra/kubeconfigs/federation/compute-pop-dfw.yaml \ + create namespace "$INSTANCE_NS" \ + --dry-run=client -o yaml | \ + kubectl --kubeconfig=../../../.test-infra/kubeconfigs/federation/compute-pop-dfw.yaml apply -f - + - script: + env: + - name: INSTANCE_NS + value: ($instanceNS) + content: | + # The cell write-back reads its identity (upstream cluster + project + # namespace) from THIS hub namespace's labels, exactly as it would from + # the ns- namespace the federator stamps in the real path. Seed them + # so writeBackToUpstream can resolve identity instead of erroring. + kubectl --kubeconfig=../../../.test-infra/kubeconfigs/federation/karmada.yaml apply -f - < convention so the InstanceProjector can resolve it later. +# +# The linking labels below mirror what the stateful control strategy stamps on a +# real cell Instance at creation. The write-back path requires all of them to be +# present and non-empty before it will copy the Instance upstream; a real Instance +# always carries them, so a hand-crafted one must too. +apiVersion: compute.datumapis.com/v1alpha +kind: Instance +metadata: + name: test-writeback-instance + namespace: ($instanceNS) + labels: + compute.datumapis.com/workload-uid: "00000000-0000-0000-0000-000000000001" + compute.datumapis.com/workload-deployment-uid: "00000000-0000-0000-0000-000000000002" + compute.datumapis.com/instance-index: "0" + compute.datumapis.com/workload-deployment-name: test-writeback-wd + compute.datumapis.com/city-code: dfw + compute.datumapis.com/workload-name: test-workload + compute.datumapis.com/placement-name: default +spec: + runtime: + resources: + instanceType: datumcloud/d1-standard-2 + networkInterfaces: + - network: + name: test-network diff --git a/test/e2e/propagation-policy-lifecycle/chainsaw-test.yaml b/test/e2e/propagation-policy-lifecycle/chainsaw-test.yaml new file mode 100644 index 00000000..d19a8e97 --- /dev/null +++ b/test/e2e/propagation-policy-lifecycle/chainsaw-test.yaml @@ -0,0 +1,133 @@ +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: propagation-policy-lifecycle +spec: + description: | + Verifies the PropagationPolicy lifecycle managed by the WorkloadDeploymentFederator: + + - A PropagationPolicy (city-dfw) is lazily created when the first WorkloadDeployment + for city code "dfw" is federated to Karmada. + - The PropagationPolicy is RETAINED while at least one WorkloadDeployment for + that city code remains in the Karmada namespace. + - The PropagationPolicy is DELETED when the last deployment for the city is removed. + + The test creates two WDs (wd-alpha, wd-beta) both targeting cityCode=dfw, verifies + the PP appears, deletes wd-alpha and asserts the PP is still present, then deletes + wd-beta and waits for the PP to disappear. + + template: true + + steps: + - name: create-deployments + description: Create two WorkloadDeployments targeting dfw on the control-plane. + try: + - apply: + file: workload-deployment-alpha.yaml + - apply: + file: workload-deployment-beta.yaml + + - name: assert-policy-created + description: | + Assert both WDs are federated to Karmada and the PropagationPolicy exists. + Both WDs must be present in Karmada before proceeding to the deletion steps; + otherwise wd-alpha's finalizer could see an empty Karmada list and prematurely + delete the PP before wd-beta has been federated. + cluster: downstream + try: + - script: + content: | + kubectl --kubeconfig=../../../.test-infra/kubeconfigs/federation/compute-control-plane.yaml \ + get namespace "$NAMESPACE" \ + -o template='{{printf "ns-%s" .metadata.uid}}' + outputs: + - name: downstreamNS + value: ($stdout) + - assert: + timeout: 30s + resource: + apiVersion: compute.datumapis.com/v1alpha + kind: WorkloadDeployment + metadata: + namespace: ($downstreamNS) + name: wd-alpha + - assert: + timeout: 30s + resource: + apiVersion: compute.datumapis.com/v1alpha + kind: WorkloadDeployment + metadata: + namespace: ($downstreamNS) + name: wd-beta + - assert: + timeout: 30s + resource: + apiVersion: policy.karmada.io/v1alpha1 + kind: PropagationPolicy + metadata: + namespace: ($downstreamNS) + name: city-dfw + + - name: delete-alpha + description: Delete wd-alpha; wd-beta still targets dfw so the PP must be retained. + try: + - delete: + ref: + apiVersion: compute.datumapis.com/v1alpha + kind: WorkloadDeployment + namespace: ($namespace) + name: wd-alpha + + - name: assert-policy-retained + description: Assert the PropagationPolicy is still present after wd-alpha is deleted. + cluster: downstream + try: + - script: + content: | + kubectl --kubeconfig=../../../.test-infra/kubeconfigs/federation/compute-control-plane.yaml \ + get namespace "$NAMESPACE" \ + -o template='{{printf "ns-%s" .metadata.uid}}' + outputs: + - name: downstreamNS + value: ($stdout) + - sleep: + duration: 8s + - assert: + timeout: 5s + resource: + apiVersion: policy.karmada.io/v1alpha1 + kind: PropagationPolicy + metadata: + namespace: ($downstreamNS) + name: city-dfw + + - name: delete-beta + description: Delete wd-beta (the last WD for city dfw). + try: + - delete: + ref: + apiVersion: compute.datumapis.com/v1alpha + kind: WorkloadDeployment + namespace: ($namespace) + name: wd-beta + + - name: assert-policy-deleted + description: Wait for the PropagationPolicy to be removed once no WDs remain. + cluster: downstream + try: + - script: + content: | + kubectl --kubeconfig=../../../.test-infra/kubeconfigs/federation/compute-control-plane.yaml \ + get namespace "$NAMESPACE" \ + -o template='{{printf "ns-%s" .metadata.uid}}' + outputs: + - name: downstreamNS + value: ($stdout) + - wait: + apiVersion: policy.karmada.io/v1alpha1 + kind: PropagationPolicy + namespace: ($downstreamNS) + name: city-dfw + timeout: 30s + for: + deletion: {} diff --git a/test/e2e/propagation-policy-lifecycle/workload-deployment-alpha.yaml b/test/e2e/propagation-policy-lifecycle/workload-deployment-alpha.yaml new file mode 100644 index 00000000..f9eb27fd --- /dev/null +++ b/test/e2e/propagation-policy-lifecycle/workload-deployment-alpha.yaml @@ -0,0 +1,21 @@ +apiVersion: compute.datumapis.com/v1alpha +kind: WorkloadDeployment +metadata: + name: wd-alpha +spec: + cityCode: dfw + placementName: default + workloadRef: + name: test-workload + uid: "00000000-0000-0000-0000-000000000001" + template: + spec: + runtime: + resources: + instanceType: datumcloud/d1-standard-2 + networkInterfaces: + - network: + name: test-network + + scaleSettings: + minReplicas: 1 diff --git a/test/e2e/propagation-policy-lifecycle/workload-deployment-beta.yaml b/test/e2e/propagation-policy-lifecycle/workload-deployment-beta.yaml new file mode 100644 index 00000000..fd1d65c1 --- /dev/null +++ b/test/e2e/propagation-policy-lifecycle/workload-deployment-beta.yaml @@ -0,0 +1,21 @@ +apiVersion: compute.datumapis.com/v1alpha +kind: WorkloadDeployment +metadata: + name: wd-beta +spec: + cityCode: dfw + placementName: default + workloadRef: + name: test-workload + uid: "00000000-0000-0000-0000-000000000001" + template: + spec: + runtime: + resources: + instanceType: datumcloud/d1-standard-2 + networkInterfaces: + - network: + name: test-network + + scaleSettings: + minReplicas: 1 diff --git a/test/e2e/referenced-data-delete-cascade/chainsaw-test.yaml b/test/e2e/referenced-data-delete-cascade/chainsaw-test.yaml new file mode 100644 index 00000000..e0ac9b34 --- /dev/null +++ b/test/e2e/referenced-data-delete-cascade/chainsaw-test.yaml @@ -0,0 +1,287 @@ +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: referenced-data-delete-cascade +spec: + description: | + Validates the happy-path delete-cascade for referenced-data companions. + + Create a WorkloadDeployment referencing a ConfigMap + Secret. Assert companions + materialize and propagate to the member cluster. Delete the WD (last referrer). + Assert the hub companion, its Karmada ResourceBinding, and the member-cluster + copy are all deleted and stay deleted (no re-create loop). + + A second scenario — the stranded-companion backstop, where a level-triggered + companion GC reclaims a companion whose referenced-by annotation points at a + non-existent WorkloadDeployment — is intentionally omitted: that hub-side GC + ships in PR #144 (refdata-hub-gc), which was still unmerged when this branch + was cut. Restore the scenario once #144 lands. + + Prerequisites: + - task e2e:up completed (deploys the management + both cell operators in-cluster) + - the cell operator runs with featureFlags.enableReferencedDataGate: true, set + by the e2e cell deploy layer (test/e2e/deploy/cell/config_patch.yaml) + - .test-infra/kubeconfigs/federation/karmada.yaml exists + + template: true + + steps: + + # ═══════════════════════════════════════════════════════════════════════════ + # SCENARIO 1: HAPPY-PATH CASCADE + # ═══════════════════════════════════════════════════════════════════════════ + + - name: s1-create-source-data + description: Create the source ConfigMap and Secret in the project namespace. + try: + - apply: + file: source-configmap.yaml + - apply: + file: source-secret.yaml + + - name: s1-create-workload-deployment + description: Create the WorkloadDeployment referencing both sources. + try: + - apply: + file: workload-deployment.yaml + + - name: s1-assert-companions-on-hub + description: | + Assert companions materialize in ns-{project-uid} on the Karmada hub + with the referenced-data label. + cluster: downstream + try: + - script: + content: | + kubectl --kubeconfig=../../../.test-infra/kubeconfigs/federation/compute-control-plane.yaml \ + get namespace "$NAMESPACE" \ + -o template='{{printf "ns-%s" .metadata.uid}}' + outputs: + - name: companionNS + value: ($stdout) + - assert: + timeout: 120s + resource: + apiVersion: v1 + kind: ConfigMap + metadata: + namespace: ($companionNS) + name: gc-test-config + labels: + compute.datumapis.com/referenced-data: "true" + - assert: + timeout: 120s + resource: + apiVersion: v1 + kind: Secret + metadata: + namespace: ($companionNS) + name: gc-test-secret + labels: + compute.datumapis.com/referenced-data: "true" + + - name: s1-assert-companion-rbs-on-hub + description: | + Assert ResourceBindings for both companions exist on the hub. + These RBs will be deleted by the ReferencedDataController's explicit + teardown (Component 3) when the WD is deleted. + cluster: downstream + try: + - script: + content: | + kubectl --kubeconfig=../../../.test-infra/kubeconfigs/federation/compute-control-plane.yaml \ + get namespace "$NAMESPACE" \ + -o template='{{printf "ns-%s" .metadata.uid}}' + outputs: + - name: companionNS + value: ($stdout) + - assert: + timeout: 30s + resource: + apiVersion: work.karmada.io/v1alpha2 + kind: ResourceBinding + metadata: + namespace: ($companionNS) + name: gc-test-config-configmap + - assert: + timeout: 30s + resource: + apiVersion: work.karmada.io/v1alpha2 + kind: ResourceBinding + metadata: + namespace: ($companionNS) + name: gc-test-secret-secret + + - name: s1-assert-companions-on-cell + description: Assert Karmada propagated companions to the pop-dfw cell. + cluster: pop-dfw + try: + - script: + content: | + kubectl --kubeconfig=../../../.test-infra/kubeconfigs/federation/compute-control-plane.yaml \ + get namespace "$NAMESPACE" \ + -o template='{{printf "ns-%s" .metadata.uid}}' + outputs: + - name: cellNS + value: ($stdout) + - assert: + timeout: 60s + resource: + apiVersion: v1 + kind: ConfigMap + metadata: + namespace: ($cellNS) + name: gc-test-config + - assert: + timeout: 60s + resource: + apiVersion: v1 + kind: Secret + metadata: + namespace: ($cellNS) + name: gc-test-secret + + - name: s1-delete-workload-deployment + description: Delete the WorkloadDeployment (the sole referrer). + try: + - delete: + ref: + apiVersion: compute.datumapis.com/v1alpha + kind: WorkloadDeployment + namespace: ($namespace) + name: gc-test-wd + + - name: s1-assert-hub-companion-cm-deleted + description: | + Assert the hub companion ConfigMap is deleted by the ReferencedDataController + finalizer after the WD is gone. + cluster: downstream + try: + - script: + content: | + kubectl --kubeconfig=../../../.test-infra/kubeconfigs/federation/compute-control-plane.yaml \ + get namespace "$NAMESPACE" \ + -o template='{{printf "ns-%s" .metadata.uid}}' + outputs: + - name: companionNS + value: ($stdout) + - wait: + apiVersion: v1 + kind: ConfigMap + namespace: ($companionNS) + name: gc-test-config + timeout: 60s + for: + deletion: {} + + - name: s1-assert-hub-companion-secret-deleted + description: Assert the hub companion Secret is also deleted. + cluster: downstream + try: + - script: + content: | + kubectl --kubeconfig=../../../.test-infra/kubeconfigs/federation/compute-control-plane.yaml \ + get namespace "$NAMESPACE" \ + -o template='{{printf "ns-%s" .metadata.uid}}' + outputs: + - name: companionNS + value: ($stdout) + - wait: + apiVersion: v1 + kind: Secret + namespace: ($companionNS) + name: gc-test-secret + timeout: 60s + for: + deletion: {} + + - name: s1-assert-rbs-deleted + description: | + Assert the ResourceBindings are deleted (Component 3 explicit teardown). + Deletion of the RB drives Karmada to remove the Work and cell copies. + cluster: downstream + try: + - script: + content: | + kubectl --kubeconfig=../../../.test-infra/kubeconfigs/federation/compute-control-plane.yaml \ + get namespace "$NAMESPACE" \ + -o template='{{printf "ns-%s" .metadata.uid}}' + outputs: + - name: companionNS + value: ($stdout) + - wait: + apiVersion: work.karmada.io/v1alpha2 + kind: ResourceBinding + namespace: ($companionNS) + name: gc-test-config-configmap + timeout: 60s + for: + deletion: {} + - wait: + apiVersion: work.karmada.io/v1alpha2 + kind: ResourceBinding + namespace: ($companionNS) + name: gc-test-secret-secret + timeout: 60s + for: + deletion: {} + + - name: s1-assert-cell-copies-deleted-and-stay-deleted + description: | + Assert the cell copies are gone and STAY gone for 30 seconds. + If a Work were still present, Karmada would re-create the cell copy + within ~5 seconds — the poll window catches any recreate loop. + cluster: pop-dfw + try: + - script: + content: | + kubectl --kubeconfig=../../../.test-infra/kubeconfigs/federation/compute-control-plane.yaml \ + get namespace "$NAMESPACE" \ + -o template='{{printf "ns-%s" .metadata.uid}}' + outputs: + - name: cellNS + value: ($stdout) + - wait: + apiVersion: v1 + kind: ConfigMap + namespace: ($cellNS) + name: gc-test-config + timeout: 90s + for: + deletion: {} + - wait: + apiVersion: v1 + kind: Secret + namespace: ($cellNS) + name: gc-test-secret + timeout: 90s + for: + deletion: {} + - script: + # Poll for 30 seconds to confirm no recreate loop. If Karmada re-creates + # the cell copy from a still-live Work, this script catches it. + timeout: 40s + content: | + CELL_NS=$(kubectl --kubeconfig=../../../.test-infra/kubeconfigs/federation/compute-control-plane.yaml \ + get namespace "$NAMESPACE" \ + -o template='{{printf "ns-%s" .metadata.uid}}') + for i in $(seq 1 6); do + sleep 5 + CM=$(kubectl --kubeconfig=../../../.test-infra/kubeconfigs/federation/compute-pop-dfw.yaml \ + get configmap gc-test-config \ + --namespace "$CELL_NS" \ + --ignore-not-found 2>/dev/null) + if [ -n "$CM" ]; then + echo "ERROR: gc-test-config ConfigMap was re-created on cell (Karmada recreate loop!)" + exit 1 + fi + SEC=$(kubectl --kubeconfig=../../../.test-infra/kubeconfigs/federation/compute-pop-dfw.yaml \ + get secret gc-test-secret \ + --namespace "$CELL_NS" \ + --ignore-not-found 2>/dev/null) + if [ -n "$SEC" ]; then + echo "ERROR: gc-test-secret Secret was re-created on cell (Karmada recreate loop!)" + exit 1 + fi + done + echo "OK: cell copies absent for 30+ seconds — no recreate loop" diff --git a/test/e2e/referenced-data-delete-cascade/source-configmap.yaml b/test/e2e/referenced-data-delete-cascade/source-configmap.yaml new file mode 100644 index 00000000..e9da4de8 --- /dev/null +++ b/test/e2e/referenced-data-delete-cascade/source-configmap.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: gc-test-config + # namespace injected by Chainsaw from ($namespace) +data: + config.yaml: "env=test" diff --git a/test/e2e/referenced-data-delete-cascade/source-secret.yaml b/test/e2e/referenced-data-delete-cascade/source-secret.yaml new file mode 100644 index 00000000..e5eb7500 --- /dev/null +++ b/test/e2e/referenced-data-delete-cascade/source-secret.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: Secret +metadata: + name: gc-test-secret + # namespace injected by Chainsaw from ($namespace) +stringData: + token: "test-token-value" diff --git a/test/e2e/referenced-data-delete-cascade/workload-deployment.yaml b/test/e2e/referenced-data-delete-cascade/workload-deployment.yaml new file mode 100644 index 00000000..99534bae --- /dev/null +++ b/test/e2e/referenced-data-delete-cascade/workload-deployment.yaml @@ -0,0 +1,38 @@ +apiVersion: compute.datumapis.com/v1alpha +kind: WorkloadDeployment +metadata: + name: gc-test-wd + # namespace injected by Chainsaw from ($namespace) +spec: + cityCode: dfw + placementName: default + workloadRef: + name: gc-test-workload + uid: "00000000-0000-0000-0000-000000000099" + template: + spec: + runtime: + resources: + instanceType: datumcloud/d1-standard-2 + sandbox: + containers: + - name: app + image: docker.io/library/busybox:stable + env: + - name: TOKEN + valueFrom: + secretKeyRef: + name: gc-test-secret + key: token + volumeAttachments: + - name: cfg-vol + mountPath: /etc/config + volumes: + - name: cfg-vol + configMap: + name: gc-test-config + networkInterfaces: + - network: + name: test-network + scaleSettings: + minReplicas: 1 diff --git a/test/e2e/referenced-data-mounts/README.md b/test/e2e/referenced-data-mounts/README.md new file mode 100644 index 00000000..e5468208 --- /dev/null +++ b/test/e2e/referenced-data-mounts/README.md @@ -0,0 +1,89 @@ +# referenced-data-mounts — Federated Delivery E2E Test + +This Chainsaw scenario validates the **cross-plane delivery path** for referenced +ConfigMap and Secret data, exercised end-to-end across the Kind+Karmada topology. + +## What this test validates + +The test covers Hops 1–5 of the federated delivery chain: + +| Hop | Cluster | What is asserted | +|-----|---------|-----------------| +| 1 | control-plane | Source ConfigMap + Secret created in the project namespace | +| 2 | control-plane | Companion `app-config` + `app-secret` appear in `ns-{project-uid}` with `compute.datumapis.com/referenced-data: "true"`; WD carries `expected-referenced-data` annotation; WD condition `ReferencedDataReady=True` | +| 3 | downstream (Karmada hub) | Companion ConfigMap + Secret present in `ns-{project-uid}` on the hub; WD carries the annotation; `PropagationPolicy city-dfw` has ConfigMap and Secret resource selectors | +| 4 | pop-dfw (cell) | WD + companions propagated to the cell in `ns-{project-uid}` | +| 5 | pop-dfw (cell) | Instance `test-refdata-wd-0` exists; `ReferencedData` scheduling gate cleared; `ReferencedDataReady=True` condition set | + +## What this test does NOT validate + +Actual env-var injection and file mounting inside a running Instance is the +**provider + kubelet layer**, not the delivery layer. That path requires the +unikraft-provider to be running with `SameCluster=true` or `SameCluster=false` +against a downstream cluster. See `docs/compute/development/plans/configmap-secret-mounts-e2e.md` +(same-cluster provider path) and `configmap-secret-mounts-e2e-multicluster.md` +(cross-cluster provider path, 4-cluster topology) for the full mount-validation scope. + +## Prerequisites + +**`task e2e:up` has completed successfully.** In the in-cluster harness this one +target brings up the Kind clusters + Karmada AND deploys the operators from the +real production overlays (plus the local e2e deviations), so there is no separate +operator-start step. The shared test-infra federation foundation writes these +kubeconfigs under `.test-infra/kubeconfigs/federation/`: + +- `compute-control-plane.yaml` — management cluster (also hosts the Karmada hub) +- `karmada.yaml` — the Karmada hub API server; `chainsaw-config.yaml` maps the + `cluster: downstream` steps directly at this file +- `compute-pop-dfw.yaml`, `compute-pop-ord.yaml` — the POP cell clusters + +**The cell operator runs with `enableReferencedDataGate: true`.** The e2e cell +deploy layer sets it in `test/e2e/deploy/cell/config_patch.yaml`. The flag's sole +consumer is the cell `WorkloadDeploymentReconciler` (it stamps the `ReferencedData` +scheduling gate); without it the Instance is never gated and Hop 5 cannot pass. +The management deploy layer sets the same flag for parity, though it is inert +there because that overlay enables management controllers only. + +## Running just this scenario + +```sh +KUBECONFIG=.test-infra/kubeconfigs/federation/compute-control-plane.yaml \ +bin/chainsaw test \ + --config test/e2e/chainsaw-config.yaml \ + --include-test-regex "referenced-data-mounts" \ + test/e2e/ +``` + +Or via the Taskfile filter target: + +```sh +task e2e:test:filter -- --include-test-regex referenced-data-mounts +``` + +## Harness notes + +This test targets the in-cluster harness, where `task e2e:up` builds the operator +image, side-loads it into every Kind node, and deploys the management + cell +operators from the real overlays. Points worth knowing: + +1. The federation foundation writes the hub kubeconfig as `karmada.yaml`, and + `chainsaw-config.yaml` points the `cluster: downstream` steps straight at it, + so those steps work out of the box. +2. The `enableReferencedDataGate` feature flag is delivered through the deploy + layer (`test/e2e/deploy/{cell,management}/config_patch.yaml`), not a host-side + `--server-config`. There is no separate operator-start task to run. +3. `e2e:crds:install` installs Milo quota CRDs to all clusters so the + InstanceReconciler's ResourceClaim watches start cleanly. + +## Companion naming convention + +The `ReferencedDataController` derives companion names deterministically. When +the source name is already a valid DNS subdomain within the length budget, the +companion keeps that name unchanged (kind is not prefixed): + +| Source | Companion name | +|--------|---------------| +| `ConfigMap/app-config` | `app-config` | +| `Secret/app-secret` | `app-secret` | + +These names are asserted directly in the test steps. diff --git a/test/e2e/referenced-data-mounts/chainsaw-test.yaml b/test/e2e/referenced-data-mounts/chainsaw-test.yaml new file mode 100644 index 00000000..72b12b24 --- /dev/null +++ b/test/e2e/referenced-data-mounts/chainsaw-test.yaml @@ -0,0 +1,352 @@ +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: referenced-data-mounts +spec: + description: | + Validates the FEDERATED DELIVERY path for referenced ConfigMap and Secret data. + + This test exercises the cross-plane chain introduced by the referenced-data + feature (feat/configmap-secret-mounts-federated): + + [Hop 1] Source ConfigMap and Secret are created in the project namespace + (control-plane, Chainsaw default cluster). + + [Hop 2] WorkloadDeployment is created; ReferencedDataController materialises + companion objects in ns-{project-uid} on the Karmada hub (downstream) + and stamps the expected-referenced-data annotation on the WD. + Companions are NOT written to the control-plane cluster. + + [Hop 3] Karmada hub holds the companion ConfigMap + Secret in ns-{project-uid}, + propagated by the always-on label selector in the city-dfw + PropagationPolicy (which includes ConfigMap and Secret selectors). + + [Hop 4] Karmada propagates the WD + companions to pop-dfw. + Companions appear in ns-{project-uid} on the cell alongside the WD. + + [Hop 5] WorkloadDeploymentReconciler on pop-dfw creates Instance test-refdata-wd-0 + with the ReferencedData scheduling gate (feature flag on). + InstanceReconciler clears the gate once companions are present and sets + ReferencedDataReady=True on the Instance. + + Scope: cross-plane DELIVERY up to the Instance gate-cleared state. + Actual env-var and file mounting is the provider+kubelet layer and is NOT + asserted here. See README.md for full scope and prerequisites. + + Prerequisites: + - task e2e:up completed: brings up control-plane + Karmada + pop-dfw + pop-ord + and deploys the management and both cell operators in-cluster. + - the cell operator runs with featureFlags.enableReferencedDataGate: true; the + e2e cell deploy layer sets it (test/e2e/deploy/cell/config_patch.yaml), so + no separate operator-start step is needed. + - .test-infra/kubeconfigs/federation/karmada.yaml exists (the Taskfile writes it as a copy + of the Karmada hub kubeconfig). + + template: true + + steps: + + # ─── Hop 1: create source data ───────────────────────────────────────────── + + - name: create-source-data + description: Create the source ConfigMap and Secret in the project namespace. + try: + - apply: + file: source-configmap.yaml + - apply: + file: source-secret.yaml + + # ─── Hop 2: create WD; assert companion materialisation on Karmada hub ───── + + - name: create-workload-deployment + description: Create the WorkloadDeployment referencing the source ConfigMap and Secret. + try: + - apply: + file: workload-deployment.yaml + + - name: assert-companion-configmap-on-hub + description: | + Assert the ReferencedDataController materialised companion app-config + in ns-{project-uid} on the Karmada hub (downstream cluster). + Companions are written to the hub namespace, NOT the control-plane namespace. + cluster: downstream + try: + - script: + content: | + kubectl --kubeconfig=../../../.test-infra/kubeconfigs/federation/compute-control-plane.yaml \ + get namespace "$NAMESPACE" \ + -o template='{{printf "ns-%s" .metadata.uid}}' + outputs: + - name: companionNS + value: ($stdout) + - assert: + timeout: 60s + resource: + apiVersion: v1 + kind: ConfigMap + metadata: + namespace: ($companionNS) + name: app-config + labels: + compute.datumapis.com/referenced-data: "true" + + - name: assert-companion-secret-on-hub + description: | + Assert the ReferencedDataController materialised companion app-secret + in ns-{project-uid} on the Karmada hub (downstream cluster). + Companions are written to the hub namespace, NOT the control-plane namespace. + cluster: downstream + try: + - script: + content: | + kubectl --kubeconfig=../../../.test-infra/kubeconfigs/federation/compute-control-plane.yaml \ + get namespace "$NAMESPACE" \ + -o template='{{printf "ns-%s" .metadata.uid}}' + outputs: + - name: companionNS + value: ($stdout) + - assert: + timeout: 60s + resource: + apiVersion: v1 + kind: Secret + metadata: + namespace: ($companionNS) + name: app-secret + labels: + compute.datumapis.com/referenced-data: "true" + + - name: assert-wd-annotation-and-condition-on-control-plane + description: | + Assert the WD carries the expected-referenced-data annotation and has + ReferencedDataReady=True condition set by the ReferencedDataController. + try: + - script: + # Verify the annotation is present and non-empty. + content: | + ANNO=$(kubectl --kubeconfig=../../../.test-infra/kubeconfigs/federation/compute-control-plane.yaml \ + get workloaddeployment test-refdata-wd \ + --namespace "$NAMESPACE" \ + -o jsonpath='{.metadata.annotations.compute\.datumapis\.com/expected-referenced-data}') + if [ -z "$ANNO" ] || [ "$ANNO" = "[]" ]; then + echo "ERROR: expected-referenced-data annotation is missing or empty: '$ANNO'" + exit 1 + fi + echo "annotation: $ANNO" + timeout: 60s + - assert: + timeout: 60s + resource: + apiVersion: compute.datumapis.com/v1alpha + kind: WorkloadDeployment + metadata: + namespace: ($namespace) + name: test-refdata-wd + (status.conditions[?type == 'ReferencedDataReady'] | [0]): + status: "True" + reason: "Ready" + + # ─── Hop 3: assert companions on the Karmada hub (downstream) ────────────── + + - name: assert-wd-and-companions-on-hub + description: | + Assert the Karmada hub (downstream cluster) holds the WD with the + expected-referenced-data annotation and the companion ConfigMap + Secret + in ns-{project-uid}. + cluster: downstream + try: + - script: + content: | + kubectl --kubeconfig=../../../.test-infra/kubeconfigs/federation/compute-control-plane.yaml \ + get namespace "$NAMESPACE" \ + -o template='{{printf "ns-%s" .metadata.uid}}' + outputs: + - name: downstreamNS + value: ($stdout) + - assert: + # WD federated to Karmada with the expected-referenced-data annotation forwarded + timeout: 30s + resource: + apiVersion: compute.datumapis.com/v1alpha + kind: WorkloadDeployment + metadata: + namespace: ($downstreamNS) + name: test-refdata-wd + - assert: + # Companion ConfigMap on Karmada hub + timeout: 30s + resource: + apiVersion: v1 + kind: ConfigMap + metadata: + namespace: ($downstreamNS) + name: app-config + labels: + compute.datumapis.com/referenced-data: "true" + - assert: + # Companion Secret on Karmada hub + timeout: 30s + resource: + apiVersion: v1 + kind: Secret + metadata: + namespace: ($downstreamNS) + name: app-secret + labels: + compute.datumapis.com/referenced-data: "true" + + - name: assert-propagation-policy-has-companion-selectors + description: | + Assert the PropagationPolicy city-dfw on the hub includes ConfigMap and + Secret resource selectors so companions co-propagate with the WD. + cluster: downstream + try: + - script: + content: | + kubectl --kubeconfig=../../../.test-infra/kubeconfigs/federation/compute-control-plane.yaml \ + get namespace "$NAMESPACE" \ + -o template='{{printf "ns-%s" .metadata.uid}}' + outputs: + - name: downstreamNS + value: ($stdout) + - assert: + timeout: 30s + resource: + apiVersion: policy.karmada.io/v1alpha1 + kind: PropagationPolicy + metadata: + namespace: ($downstreamNS) + name: city-dfw + spec: + resourceSelectors: + - apiVersion: compute.datumapis.com/v1alpha + kind: WorkloadDeployment + namespace: ($downstreamNS) + labelSelector: + matchLabels: + topology.datum.net/city-code: dfw + - apiVersion: v1 + kind: ConfigMap + namespace: ($downstreamNS) + labelSelector: + matchLabels: + compute.datumapis.com/referenced-data: "true" + - apiVersion: v1 + kind: Secret + namespace: ($downstreamNS) + labelSelector: + matchLabels: + compute.datumapis.com/referenced-data: "true" + + # ─── Hop 4: assert companions propagated to pop-dfw cell ─────────────────── + + - name: assert-wd-and-companions-on-cell + description: | + Assert Karmada propagated the WD and companion ConfigMap + Secret to + pop-dfw. All three objects must appear in ns-{project-uid} on the cell. + cluster: pop-dfw + try: + - script: + content: | + kubectl --kubeconfig=../../../.test-infra/kubeconfigs/federation/compute-control-plane.yaml \ + get namespace "$NAMESPACE" \ + -o template='{{printf "ns-%s" .metadata.uid}}' + outputs: + - name: downstreamNS + value: ($stdout) + - assert: + timeout: 60s + resource: + apiVersion: compute.datumapis.com/v1alpha + kind: WorkloadDeployment + metadata: + namespace: ($downstreamNS) + name: test-refdata-wd + - assert: + timeout: 60s + resource: + apiVersion: v1 + kind: ConfigMap + metadata: + namespace: ($downstreamNS) + name: app-config + labels: + compute.datumapis.com/referenced-data: "true" + - assert: + timeout: 60s + resource: + apiVersion: v1 + kind: Secret + metadata: + namespace: ($downstreamNS) + name: app-secret + labels: + compute.datumapis.com/referenced-data: "true" + + # ─── Hop 5: assert Instance gate cleared on pop-dfw ──────────────────────── + + - name: assert-instance-exists-on-cell + description: Assert the WorkloadDeploymentReconciler created Instance test-refdata-wd-0. + cluster: pop-dfw + try: + - script: + content: | + kubectl --kubeconfig=../../../.test-infra/kubeconfigs/federation/compute-control-plane.yaml \ + get namespace "$NAMESPACE" \ + -o template='{{printf "ns-%s" .metadata.uid}}' + outputs: + - name: downstreamNS + value: ($stdout) + - assert: + timeout: 60s + resource: + apiVersion: compute.datumapis.com/v1alpha + kind: Instance + metadata: + namespace: ($downstreamNS) + name: test-refdata-wd-0 + + - name: assert-referenced-data-gate-cleared + description: | + Assert the ReferencedData scheduling gate is removed from the Instance and + the ReferencedDataReady=True condition is set by the InstanceReconciler. + The gate is stamped when the Instance is created (feature flag on) and + cleared once the cell InstanceReconciler confirms all expected companions + are present in ns-{project-uid}. + cluster: pop-dfw + try: + - script: + content: | + kubectl --kubeconfig=../../../.test-infra/kubeconfigs/federation/compute-control-plane.yaml \ + get namespace "$NAMESPACE" \ + -o template='{{printf "ns-%s" .metadata.uid}}' + outputs: + - name: downstreamNS + value: ($stdout) + - assert: + timeout: 60s + resource: + apiVersion: compute.datumapis.com/v1alpha + kind: Instance + metadata: + namespace: ($downstreamNS) + name: test-refdata-wd-0 + (status.conditions[?type == 'ReferencedDataReady'] | [0]): + status: "True" + reason: "Ready" + - script: + # Verify the ReferencedData gate is absent from the Instance spec. + timeout: 60s + content: | + DOWNSTREAMS_NS=$(kubectl --kubeconfig=../../../.test-infra/kubeconfigs/federation/compute-control-plane.yaml \ + get namespace "$NAMESPACE" \ + -o template='{{printf "ns-%s" .metadata.uid}}') + GATES=$(kubectl --kubeconfig=../../../.test-infra/kubeconfigs/federation/compute-pop-dfw.yaml \ + get instance test-refdata-wd-0 \ + --namespace "$DOWNSTREAMS_NS" \ + -o jsonpath='{.spec.schedulingGates[*].name}') + if echo "$GATES" | grep -qw "ReferencedData"; then + echo "ERROR: ReferencedData gate still present in schedulingGates: '$GATES'" + exit 1 + fi + echo "ReferencedData gate cleared. Remaining gates: '$GATES'" diff --git a/test/e2e/referenced-data-mounts/source-configmap.yaml b/test/e2e/referenced-data-mounts/source-configmap.yaml new file mode 100644 index 00000000..3664e9ae --- /dev/null +++ b/test/e2e/referenced-data-mounts/source-configmap.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: app-config + # namespace injected by Chainsaw from ($namespace) +data: + app.properties: "environment=test" + log.level: "info" diff --git a/test/e2e/referenced-data-mounts/source-secret.yaml b/test/e2e/referenced-data-mounts/source-secret.yaml new file mode 100644 index 00000000..afde8426 --- /dev/null +++ b/test/e2e/referenced-data-mounts/source-secret.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: Secret +metadata: + name: app-secret + # namespace injected by Chainsaw from ($namespace) +type: Opaque +stringData: + db.password: "test-db-password" diff --git a/test/e2e/referenced-data-mounts/workload-deployment.yaml b/test/e2e/referenced-data-mounts/workload-deployment.yaml new file mode 100644 index 00000000..d849be0b --- /dev/null +++ b/test/e2e/referenced-data-mounts/workload-deployment.yaml @@ -0,0 +1,38 @@ +apiVersion: compute.datumapis.com/v1alpha +kind: WorkloadDeployment +metadata: + name: test-refdata-wd + # namespace injected by Chainsaw from ($namespace) +spec: + cityCode: dfw + placementName: default + workloadRef: + name: test-workload + uid: "00000000-0000-0000-0000-000000000002" + template: + spec: + runtime: + resources: + instanceType: datumcloud/d1-standard-2 + sandbox: + containers: + - name: app + image: docker.io/library/busybox:stable + env: + - name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: app-secret + key: db.password + volumeAttachments: + - name: config-vol + mountPath: /etc/config + volumes: + - name: config-vol + configMap: + name: app-config + networkInterfaces: + - network: + name: test-network + scaleSettings: + minReplicas: 1 diff --git a/test/e2e/workload-deployment-federation/chainsaw-test.yaml b/test/e2e/workload-deployment-federation/chainsaw-test.yaml new file mode 100644 index 00000000..f759072b --- /dev/null +++ b/test/e2e/workload-deployment-federation/chainsaw-test.yaml @@ -0,0 +1,98 @@ +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: workload-deployment-federation +spec: + description: | + Verifies that the WorkloadDeploymentFederator replicates a WorkloadDeployment + from the project namespace (control-plane cluster) to the Karmada API server + with the correct city-code label and PropagationPolicy. + + The federator follows the ns- convention for Karmada namespaces, + matching the MappedNamespaceResourceStrategy used by NSO. The test derives + the expected Karmada namespace dynamically from the Chainsaw test namespace UID. + + Verified: + - WorkloadDeployment exists in Karmada at ns- + - Karmada copy carries label topology.datum.net/city-code: dfw + - PropagationPolicy city-dfw exists in the Karmada namespace, + selecting WDs by city-code and routing them to matching POP-cell clusters. + + template: true + + steps: + - name: derive-ns-and-create-wd + description: Derive Karmada namespace and create the WorkloadDeployment. + try: + - apply: + file: workload-deployment.yaml + + - name: assert-wd-in-downstream + description: Assert WorkloadDeployment federated to Karmada with city-code label. + cluster: downstream + try: + - script: + content: | + kubectl --kubeconfig=../../../.test-infra/kubeconfigs/federation/compute-control-plane.yaml \ + get namespace "$NAMESPACE" \ + -o template='{{printf "ns-%s" .metadata.uid}}' + outputs: + - name: downstreamNS + value: ($stdout) + - assert: + timeout: 30s + resource: + apiVersion: compute.datumapis.com/v1alpha + kind: WorkloadDeployment + metadata: + namespace: ($downstreamNS) + name: test-federation-wd + labels: + topology.datum.net/city-code: dfw + + - name: assert-propagation-policy-in-downstream + description: Assert PropagationPolicy created for city-dfw. + cluster: downstream + try: + - script: + content: | + kubectl --kubeconfig=../../../.test-infra/kubeconfigs/federation/compute-control-plane.yaml \ + get namespace "$NAMESPACE" \ + -o template='{{printf "ns-%s" .metadata.uid}}' + outputs: + - name: downstreamNS + value: ($stdout) + - assert: + timeout: 30s + resource: + apiVersion: policy.karmada.io/v1alpha1 + kind: PropagationPolicy + metadata: + namespace: ($downstreamNS) + name: city-dfw + spec: + # The federator emits three resource selectors, not one: the + # WorkloadDeployment plus the always-on ConfigMap and Secret + # referenced-data selectors that co-propagate companions to the cell. + # Chainsaw matches list length exactly, so all three must be asserted. + resourceSelectors: + - apiVersion: compute.datumapis.com/v1alpha + kind: WorkloadDeployment + labelSelector: + matchLabels: + topology.datum.net/city-code: dfw + - apiVersion: v1 + kind: ConfigMap + labelSelector: + matchLabels: + compute.datumapis.com/referenced-data: "true" + - apiVersion: v1 + kind: Secret + labelSelector: + matchLabels: + compute.datumapis.com/referenced-data: "true" + placement: + clusterAffinity: + labelSelector: + matchLabels: + topology.datum.net/city-code: dfw diff --git a/test/e2e/workload-deployment-federation/workload-deployment.yaml b/test/e2e/workload-deployment-federation/workload-deployment.yaml new file mode 100644 index 00000000..0cd2347a --- /dev/null +++ b/test/e2e/workload-deployment-federation/workload-deployment.yaml @@ -0,0 +1,22 @@ +apiVersion: compute.datumapis.com/v1alpha +kind: WorkloadDeployment +metadata: + name: test-federation-wd + # namespace is injected by Chainsaw from ($namespace) +spec: + cityCode: dfw + placementName: default + workloadRef: + name: test-workload + uid: "00000000-0000-0000-0000-000000000001" + template: + spec: + runtime: + resources: + instanceType: datumcloud/d1-standard-2 + networkInterfaces: + - network: + name: test-network + + scaleSettings: + minReplicas: 1