From fe2906629e835cedc9e5f2301fd9b046e1dbff7c Mon Sep 17 00:00:00 2001 From: Alex Savanovich <40720931+savme@users.noreply.github.com> Date: Tue, 4 Aug 2026 14:56:36 +0200 Subject: [PATCH] test: add WorkloadDeployment HPA lifecycle chainsaw tests --- .github/workflows/test-e2e.yml | 13 +- .gitignore | 2 + Makefile | 22 +-- Taskfile.yaml | 132 ++++++++++++++++++ test/e2e/.chainsaw.yaml | 14 ++ test/e2e/e2e_test.go | 6 - .../hpa-lifecycle/chainsaw-test.yaml | 21 +++ .../workloaddeployment/hpa-lifecycle/hpa.yaml | 25 ++++ .../hpa-lifecycle/workloaddeployment.yaml | 32 +++++ .../hpa-stray/chainsaw-test.yaml | 24 ++++ .../hpa-stray/stray-hpa.yaml | 21 +++ .../hpa-stray/workloaddeployment.yaml | 32 +++++ .../hpa-update/chainsaw-test.yaml | 20 +++ .../hpa-update/hpa-initial.yaml | 15 ++ .../hpa-update/hpa-updated.yaml | 15 ++ .../hpa-update/workloaddeployment.yaml | 32 +++++ 16 files changed, 399 insertions(+), 27 deletions(-) create mode 100644 Taskfile.yaml create mode 100644 test/e2e/.chainsaw.yaml delete mode 100644 test/e2e/e2e_test.go create mode 100644 test/e2e/workloaddeployment/hpa-lifecycle/chainsaw-test.yaml create mode 100644 test/e2e/workloaddeployment/hpa-lifecycle/hpa.yaml create mode 100644 test/e2e/workloaddeployment/hpa-lifecycle/workloaddeployment.yaml create mode 100644 test/e2e/workloaddeployment/hpa-stray/chainsaw-test.yaml create mode 100644 test/e2e/workloaddeployment/hpa-stray/stray-hpa.yaml create mode 100644 test/e2e/workloaddeployment/hpa-stray/workloaddeployment.yaml create mode 100644 test/e2e/workloaddeployment/hpa-update/chainsaw-test.yaml create mode 100644 test/e2e/workloaddeployment/hpa-update/hpa-initial.yaml create mode 100644 test/e2e/workloaddeployment/hpa-update/hpa-updated.yaml create mode 100644 test/e2e/workloaddeployment/hpa-update/workloaddeployment.yaml diff --git a/.github/workflows/test-e2e.yml b/.github/workflows/test-e2e.yml index b2eda8c3..837e97fc 100644 --- a/.github/workflows/test-e2e.yml +++ b/.github/workflows/test-e2e.yml @@ -1,5 +1,8 @@ name: E2E Tests +env: + TASK_X_REMOTE_TASKFILES: 1 + on: push: pull_request: @@ -17,6 +20,9 @@ jobs: with: go-version-file: go.mod + - name: Setup Task + uses: arduino/setup-task@v2 + - name: Install the latest version of kind run: | curl -Lo ./kind https://kind.sigs.k8s.io/dl/latest/kind-linux-amd64 @@ -26,10 +32,5 @@ jobs: - name: Verify kind installation run: kind version - - name: Create kind cluster - run: kind create cluster - - name: Running Test e2e - run: | - go mod tidy - make test-e2e + run: make test-e2e diff --git a/.gitignore b/.gitignore index d5cc564d..3092d0a0 100644 --- a/.gitignore +++ b/.gitignore @@ -28,3 +28,5 @@ bin/ # Local e2e environment artefacts (Kind kubeconfigs, etc.) tmp/ +.test-infra/ +.task/ diff --git a/Makefile b/Makefile index 3d6a3e2e..6a5a68b5 100644 --- a/Makefile +++ b/Makefile @@ -64,22 +64,14 @@ vet: ## Run go vet against code. test: manifests generate fmt vet envtest ## Run tests. KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test $$(go list ./... | grep -v /e2e) -coverprofile cover.out -# TODO(user): To use a different vendor for e2e tests, modify the setup under 'tests/e2e'. -# The default setup assumes Kind is pre-installed and builds/loads the Manager Docker image locally. -# Prometheus and CertManager are installed by default; skip with: -# - PROMETHEUS_INSTALL_SKIP=true -# - CERT_MANAGER_INSTALL_SKIP=true .PHONY: test-e2e -test-e2e: manifests generate fmt vet ## Run the e2e tests. Expected an isolated environment using Kind. - # @command -v kind >/dev/null 2>&1 || { \ - # echo "Kind is not installed. Please install Kind manually."; \ - # exit 1; \ - # } - # @kind get clusters | grep -q 'kind' || { \ - # echo "No Kind cluster is running. Please start a Kind cluster before running the e2e tests."; \ - # exit 1; \ - # } - # go test ./test/e2e/ -v -ginkgo.v +test-e2e: manifests generate fmt vet ## Run Chainsaw e2e tests in an isolated test-infra Kind cluster. + @set -e; \ + export TASK_X_REMOTE_TASKFILES=1; \ + cleanup() { task e2e:down || true; }; \ + trap cleanup EXIT; \ + task e2e:up; \ + task e2e .PHONY: lint lint: golangci-lint ## Run golangci-lint linter diff --git a/Taskfile.yaml b/Taskfile.yaml new file mode 100644 index 00000000..b53f9004 --- /dev/null +++ b/Taskfile.yaml @@ -0,0 +1,132 @@ +version: '3' + +dotenv: ['.env'] + +includes: + # Remote Taskfile from datum-cloud/test-infra. Requires + # TASK_X_REMOTE_TASKFILES=1. See: https://taskfile.dev/experiments/remote-taskfiles + test-infra: + taskfile: https://raw.githubusercontent.com/datum-cloud/test-infra/{{.TEST_INFRA_REPO_REF}}/Taskfile.yml + checksum: 1d057f69ef11b9fb7a94bbe7723f50f9508098cb2481f375f38ca5d55533d93e + vars: + CLUSTER_NAME: '{{.E2E_CLUSTER}}' + REPO_REF: '{{.TEST_INFRA_REPO_REF}}' + WAIT_TIMEOUT: '{{.WAIT_TIMEOUT}}' + +vars: + WAIT_TIMEOUT: '300s' + TOOL_DIR: '{{.USER_WORKING_DIR}}/bin' + IMG_NAME: 'compute' + IMG_TAG: 'e2e' + IMG: '{{.IMG_NAME}}:{{.IMG_TAG}}' + E2E_CLUSTER: 'test-infra' + E2E_KUBECONFIG: '.test-infra/kubeconfig' + TEST_INFRA_REPO_REF: 'v0.7.1' + CHAINSAW_VERSION: 'v0.2.15' + +tasks: + default: + cmds: + - task --list + silent: true + + dev:build: + desc: Build the compute manager container image + silent: true + cmds: + - docker build -t {{.IMG}} . + + e2e:up: + desc: Bring up a test-infra Kind cluster with the compute cell controller + silent: true + env: + KUBECONFIG: '{{.E2E_KUBECONFIG}}' + cmds: + - task: test-infra:cluster-up + vars: + CLUSTER_NAME: '{{.E2E_CLUSTER}}' + - task: dev:build + - task: test-infra:kind-load-image + vars: + CLUSTER_NAME: '{{.E2E_CLUSTER}}' + IMAGES: '{{.IMG}}' + - task test-infra:kubectl -- apply -k config/base/crd + - task test-infra:kubectl -- create namespace compute-system --dry-run=client -o yaml | task test-infra:kubectl -- apply -f - + - task test-infra:kubectl -- apply -k config/overlays/cell + - | + task test-infra:kubectl -- -n compute-system apply -f - <<'EOF' + apiVersion: v1 + kind: ConfigMap + metadata: + name: compute-config + data: + config.yaml: | + apiVersion: apiserver.config.datumapis.com/v1alpha1 + kind: WorkloadOperator + metricsServer: + bindAddress: "0" + discovery: {} + EOF + - task test-infra:kubectl -- -n compute-system set image deployment/compute-manager manager={{.IMG}} + - task test-infra:kubectl -- -n compute-system patch deployment compute-manager --type=json -p='[{"op":"replace","path":"/spec/template/spec/containers/0/imagePullPolicy","value":"IfNotPresent"}]' + - task test-infra:kubectl -- -n compute-system rollout status deployment/compute-manager --timeout=180s + + e2e: + desc: Run Chainsaw e2e tests against the isolated Kind cluster + silent: true + deps: + - e2e:preflight + - install:chainsaw + env: + KUBECONFIG: '{{.E2E_KUBECONFIG}}' + cmds: + - '{{.TOOL_DIR}}/chainsaw test test/e2e --config test/e2e/.chainsaw.yaml' + + install:chainsaw: + desc: Install Chainsaw locally + silent: true + cmds: + - | + mkdir -p {{.TOOL_DIR}} + if [ ! -f "{{.TOOL_DIR}}/chainsaw-{{.CHAINSAW_VERSION}}" ]; then + GOBIN={{.TOOL_DIR}} go install github.com/kyverno/chainsaw@{{.CHAINSAW_VERSION}} + mv {{.TOOL_DIR}}/chainsaw {{.TOOL_DIR}}/chainsaw-{{.CHAINSAW_VERSION}} + fi + ln -sf {{.TOOL_DIR}}/chainsaw-{{.CHAINSAW_VERSION}} {{.TOOL_DIR}}/chainsaw + status: + - test -f {{.TOOL_DIR}}/chainsaw-{{.CHAINSAW_VERSION}} + + e2e:preflight: + internal: true + silent: true + env: + KUBECONFIG: '{{.E2E_KUBECONFIG}}' + cmds: + - | + if [ ! -f "$KUBECONFIG" ]; then + echo "missing e2e kubeconfig at $KUBECONFIG" + echo "run: TASK_X_REMOTE_TASKFILES=1 task e2e:up" + exit 1 + fi + - | + context="$(kubectl config current-context 2>/dev/null || true)" + if [ "$context" != "kind-{{.E2E_CLUSTER}}" ]; then + echo "e2e kubeconfig points at $context, expected kind-{{.E2E_CLUSTER}}" + echo "run: TASK_X_REMOTE_TASKFILES=1 task e2e:up" + exit 1 + fi + - | + if ! kubectl get crd workloaddeployments.compute.datumapis.com >/dev/null 2>&1; then + echo "Compute CRDs are not installed in the e2e cluster" + echo "run: TASK_X_REMOTE_TASKFILES=1 task e2e:up" + exit 1 + fi + - kubectl -n compute-system rollout status deployment/compute-manager --timeout=30s + + e2e:down: + desc: Tear down the e2e test-infra Kind cluster + silent: true + cmds: + - task: test-infra:cluster-down + vars: + CLUSTER_NAME: '{{.E2E_CLUSTER}}' diff --git a/test/e2e/.chainsaw.yaml b/test/e2e/.chainsaw.yaml new file mode 100644 index 00000000..bd19c7a6 --- /dev/null +++ b/test/e2e/.chainsaw.yaml @@ -0,0 +1,14 @@ +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Configuration +metadata: + name: compute-e2e +spec: + timeouts: + apply: 60s + assert: 120s + cleanup: 120s + delete: 60s + error: 60s + exec: 60s + parallel: 1 + skipDelete: false diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go deleted file mode 100644 index 86de3c02..00000000 --- a/test/e2e/e2e_test.go +++ /dev/null @@ -1,6 +0,0 @@ -package e2e - -// This file exists due to the inability to bypass e2e generation when using -// kubebuilder to generate resources. -// -// See: https://github.com/kubernetes-sigs/kubebuilder/issues/4509 diff --git a/test/e2e/workloaddeployment/hpa-lifecycle/chainsaw-test.yaml b/test/e2e/workloaddeployment/hpa-lifecycle/chainsaw-test.yaml new file mode 100644 index 00000000..b247c4ee --- /dev/null +++ b/test/e2e/workloaddeployment/hpa-lifecycle/chainsaw-test.yaml @@ -0,0 +1,21 @@ +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: workloaddeployment-hpa-lifecycle +spec: + steps: + - name: create autoscaled WorkloadDeployment + try: + - apply: + file: workloaddeployment.yaml + - assert: + file: hpa.yaml + - name: disable autoscaling + try: + - script: + content: | + kubectl -n "$NAMESPACE" patch workloaddeployment.compute.datumapis.com chainsaw-hpa --type=json \ + -p='[{"op":"remove","path":"/spec/scaleSettings/maxReplicas"},{"op":"remove","path":"/spec/scaleSettings/metrics"}]' + - script: + content: | + kubectl -n "$NAMESPACE" wait --for=delete horizontalpodautoscaler.autoscaling/chainsaw-hpa --timeout=60s diff --git a/test/e2e/workloaddeployment/hpa-lifecycle/hpa.yaml b/test/e2e/workloaddeployment/hpa-lifecycle/hpa.yaml new file mode 100644 index 00000000..91abd3eb --- /dev/null +++ b/test/e2e/workloaddeployment/hpa-lifecycle/hpa.yaml @@ -0,0 +1,25 @@ +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: chainsaw-hpa + namespace: ($namespace) + labels: + services.miloapis.com/service-name: compute.datumapis.com + compute.datumapis.com/workload-deployment-name: chainsaw-hpa + compute.datumapis.com/workload-name: chainsaw-workload + compute.datumapis.com/placement-name: chainsaw-placement + compute.datumapis.com/city-code: DFW +spec: + scaleTargetRef: + apiVersion: compute.datumapis.com/v1alpha + kind: WorkloadDeployment + name: chainsaw-hpa + minReplicas: 2 + maxReplicas: 10 + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 75 diff --git a/test/e2e/workloaddeployment/hpa-lifecycle/workloaddeployment.yaml b/test/e2e/workloaddeployment/hpa-lifecycle/workloaddeployment.yaml new file mode 100644 index 00000000..3813b88d --- /dev/null +++ b/test/e2e/workloaddeployment/hpa-lifecycle/workloaddeployment.yaml @@ -0,0 +1,32 @@ +apiVersion: compute.datumapis.com/v1alpha +kind: WorkloadDeployment +metadata: + name: chainsaw-hpa + namespace: ($namespace) +spec: + workloadRef: + name: chainsaw-workload + uid: chainsaw-workload-uid + placementName: chainsaw-placement + cityCode: DFW + template: + spec: + runtime: + resources: + instanceType: datumcloud/d1-standard-2 + sandbox: + containers: + - name: app + image: registry.k8s.io/pause:3.10 + networkInterfaces: + - network: + name: default + scaleSettings: + minReplicas: 2 + maxReplicas: 10 + metrics: + - resource: + name: cpu + target: + averageUtilization: 75 + instanceManagementPolicy: OrderedReady diff --git a/test/e2e/workloaddeployment/hpa-stray/chainsaw-test.yaml b/test/e2e/workloaddeployment/hpa-stray/chainsaw-test.yaml new file mode 100644 index 00000000..d9fd6f6d --- /dev/null +++ b/test/e2e/workloaddeployment/hpa-stray/chainsaw-test.yaml @@ -0,0 +1,24 @@ +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: workloaddeployment-hpa-stray +spec: + steps: + - name: create stray HPA + try: + - apply: + file: stray-hpa.yaml + - name: create matching autoscaled WorkloadDeployment + try: + - apply: + file: workloaddeployment.yaml + - assert: + file: stray-hpa.yaml + - name: disable autoscaling + try: + - script: + content: | + kubectl -n "$NAMESPACE" patch workloaddeployment.compute.datumapis.com chainsaw-stray --type=json \ + -p='[{"op":"remove","path":"/spec/scaleSettings/maxReplicas"},{"op":"remove","path":"/spec/scaleSettings/metrics"}]' + - assert: + file: stray-hpa.yaml diff --git a/test/e2e/workloaddeployment/hpa-stray/stray-hpa.yaml b/test/e2e/workloaddeployment/hpa-stray/stray-hpa.yaml new file mode 100644 index 00000000..ecb69f49 --- /dev/null +++ b/test/e2e/workloaddeployment/hpa-stray/stray-hpa.yaml @@ -0,0 +1,21 @@ +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: chainsaw-stray + namespace: ($namespace) + labels: + e2e.compute.datumapis.com/stray: "true" +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: not-owned-by-compute + minReplicas: 1 + maxReplicas: 4 + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 50 diff --git a/test/e2e/workloaddeployment/hpa-stray/workloaddeployment.yaml b/test/e2e/workloaddeployment/hpa-stray/workloaddeployment.yaml new file mode 100644 index 00000000..3a855fa1 --- /dev/null +++ b/test/e2e/workloaddeployment/hpa-stray/workloaddeployment.yaml @@ -0,0 +1,32 @@ +apiVersion: compute.datumapis.com/v1alpha +kind: WorkloadDeployment +metadata: + name: chainsaw-stray + namespace: ($namespace) +spec: + workloadRef: + name: chainsaw-workload + uid: chainsaw-workload-uid + placementName: chainsaw-placement + cityCode: DFW + template: + spec: + runtime: + resources: + instanceType: datumcloud/d1-standard-2 + sandbox: + containers: + - name: app + image: registry.k8s.io/pause:3.10 + networkInterfaces: + - network: + name: default + scaleSettings: + minReplicas: 2 + maxReplicas: 10 + metrics: + - resource: + name: cpu + target: + averageUtilization: 75 + instanceManagementPolicy: OrderedReady diff --git a/test/e2e/workloaddeployment/hpa-update/chainsaw-test.yaml b/test/e2e/workloaddeployment/hpa-update/chainsaw-test.yaml new file mode 100644 index 00000000..9f73939c --- /dev/null +++ b/test/e2e/workloaddeployment/hpa-update/chainsaw-test.yaml @@ -0,0 +1,20 @@ +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: workloaddeployment-hpa-update +spec: + steps: + - name: create autoscaled WorkloadDeployment + try: + - apply: + file: workloaddeployment.yaml + - assert: + file: hpa-initial.yaml + - name: update autoscaling settings + try: + - script: + content: | + kubectl -n "$NAMESPACE" patch workloaddeployment.compute.datumapis.com chainsaw-hpa-update --type=merge \ + -p='{"spec":{"scaleSettings":{"minReplicas":3,"maxReplicas":12,"metrics":[{"resource":{"name":"memory","target":{"averageValue":"512Mi"}}}]}}}' + - assert: + file: hpa-updated.yaml diff --git a/test/e2e/workloaddeployment/hpa-update/hpa-initial.yaml b/test/e2e/workloaddeployment/hpa-update/hpa-initial.yaml new file mode 100644 index 00000000..8c07db42 --- /dev/null +++ b/test/e2e/workloaddeployment/hpa-update/hpa-initial.yaml @@ -0,0 +1,15 @@ +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: chainsaw-hpa-update + namespace: ($namespace) +spec: + minReplicas: 2 + maxReplicas: 10 + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 75 diff --git a/test/e2e/workloaddeployment/hpa-update/hpa-updated.yaml b/test/e2e/workloaddeployment/hpa-update/hpa-updated.yaml new file mode 100644 index 00000000..592f246f --- /dev/null +++ b/test/e2e/workloaddeployment/hpa-update/hpa-updated.yaml @@ -0,0 +1,15 @@ +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: chainsaw-hpa-update + namespace: ($namespace) +spec: + minReplicas: 3 + maxReplicas: 12 + metrics: + - type: Resource + resource: + name: memory + target: + type: AverageValue + averageValue: 512Mi diff --git a/test/e2e/workloaddeployment/hpa-update/workloaddeployment.yaml b/test/e2e/workloaddeployment/hpa-update/workloaddeployment.yaml new file mode 100644 index 00000000..eea78f22 --- /dev/null +++ b/test/e2e/workloaddeployment/hpa-update/workloaddeployment.yaml @@ -0,0 +1,32 @@ +apiVersion: compute.datumapis.com/v1alpha +kind: WorkloadDeployment +metadata: + name: chainsaw-hpa-update + namespace: ($namespace) +spec: + workloadRef: + name: chainsaw-workload + uid: chainsaw-workload-uid + placementName: chainsaw-placement + cityCode: DFW + template: + spec: + runtime: + resources: + instanceType: datumcloud/d1-standard-2 + sandbox: + containers: + - name: app + image: registry.k8s.io/pause:3.10 + networkInterfaces: + - network: + name: default + scaleSettings: + minReplicas: 2 + maxReplicas: 10 + metrics: + - resource: + name: cpu + target: + averageUtilization: 75 + instanceManagementPolicy: OrderedReady