test: add WorkloadDeployment HPA lifecycle chainsaw tests

.github/workflows/test-e2e.yml

@@ -1,5 +1,8 @@
 name: E2E Tests
 
+env:
+  TASK_X_REMOTE_TASKFILES: 1
+
 on:
   push:
   pull_request:
@@ -17,6 +20,9 @@ jobs:
         with:
           go-version-file: go.mod
 
+      - name: Setup Task
+        uses: arduino/setup-task@v2
+
       - name: Install the latest version of kind
         run: |
           curl -Lo ./kind https://kind.sigs.k8s.io/dl/latest/kind-linux-amd64
@@ -26,10 +32,5 @@ jobs:
       - name: Verify kind installation
         run: kind version
 
-      - name: Create kind cluster
-        run: kind create cluster
-
       - name: Running Test e2e
-        run: |
-          go mod tidy
-          make test-e2e
+        run: make test-e2e

.gitignore

@@ -28,3 +28,5 @@ bin/
 
 # Local e2e environment artefacts (Kind kubeconfigs, etc.)
 tmp/
+.test-infra/
+.task/

Makefile

@@ -64,22 +64,14 @@ vet: ## Run go vet against code.
 test: manifests generate fmt vet envtest ## Run tests.
 	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test $$(go list ./... | grep -v /e2e) -coverprofile cover.out
 
-# TODO(user): To use a different vendor for e2e tests, modify the setup under 'tests/e2e'.
-# The default setup assumes Kind is pre-installed and builds/loads the Manager Docker image locally.
-# Prometheus and CertManager are installed by default; skip with:
-# - PROMETHEUS_INSTALL_SKIP=true
-# - CERT_MANAGER_INSTALL_SKIP=true
 .PHONY: test-e2e
-test-e2e: manifests generate fmt vet ## Run the e2e tests. Expected an isolated environment using Kind.
-	# @command -v kind >/dev/null 2>&1 || { \
-	# 	echo "Kind is not installed. Please install Kind manually."; \
-	# 	exit 1; \
-	# }
-	# @kind get clusters | grep -q 'kind' || { \
-	# 	echo "No Kind cluster is running. Please start a Kind cluster before running the e2e tests."; \
-	# 	exit 1; \
-	# }
-	# go test ./test/e2e/ -v -ginkgo.v
+test-e2e: manifests generate fmt vet ## Run Chainsaw e2e tests in an isolated test-infra Kind cluster.
+	@set -e; \
+	export TASK_X_REMOTE_TASKFILES=1; \
+	cleanup() { task e2e:down || true; }; \
+	trap cleanup EXIT; \
+	task e2e:up; \
+	task e2e
 
 .PHONY: lint
 lint: golangci-lint ## Run golangci-lint linter

Taskfile.yaml

@@ -0,0 +1,132 @@
+version: '3'
+
+dotenv: ['.env']
+
+includes:
+  # Remote Taskfile from datum-cloud/test-infra. Requires
+  # TASK_X_REMOTE_TASKFILES=1. See: https://taskfile.dev/experiments/remote-taskfiles
+  test-infra:
+    taskfile: https://raw.githubusercontent.com/datum-cloud/test-infra/{{.TEST_INFRA_REPO_REF}}/Taskfile.yml
+    checksum: 1d057f69ef11b9fb7a94bbe7723f50f9508098cb2481f375f38ca5d55533d93e
+    vars:
+      CLUSTER_NAME: '{{.E2E_CLUSTER}}'
+      REPO_REF: '{{.TEST_INFRA_REPO_REF}}'
+      WAIT_TIMEOUT: '{{.WAIT_TIMEOUT}}'
+
+vars:
+  WAIT_TIMEOUT: '300s'
+  TOOL_DIR: '{{.USER_WORKING_DIR}}/bin'
+  IMG_NAME: 'compute'
+  IMG_TAG: 'e2e'
+  IMG: '{{.IMG_NAME}}:{{.IMG_TAG}}'
+  E2E_CLUSTER: 'test-infra'
+  E2E_KUBECONFIG: '.test-infra/kubeconfig'
+  TEST_INFRA_REPO_REF: 'v0.7.1'
+  CHAINSAW_VERSION: 'v0.2.15'
+
+tasks:
+  default:
+    cmds:
+      - task --list
+    silent: true
+
+  dev:build:
+    desc: Build the compute manager container image
+    silent: true
+    cmds:
+      - docker build -t {{.IMG}} .
+
+  e2e:up:
+    desc: Bring up a test-infra Kind cluster with the compute cell controller
+    silent: true
+    env:
+      KUBECONFIG: '{{.E2E_KUBECONFIG}}'
+    cmds:
+      - task: test-infra:cluster-up
+        vars:
+          CLUSTER_NAME: '{{.E2E_CLUSTER}}'
+      - task: dev:build
+      - task: test-infra:kind-load-image
+        vars:
+          CLUSTER_NAME: '{{.E2E_CLUSTER}}'
+          IMAGES: '{{.IMG}}'
+      - task test-infra:kubectl -- apply -k config/base/crd
+      - task test-infra:kubectl -- create namespace compute-system --dry-run=client -o yaml | task test-infra:kubectl -- apply -f -
+      - task test-infra:kubectl -- apply -k config/overlays/cell
+      - |
+        task test-infra:kubectl -- -n compute-system apply -f - <<'EOF'
+        apiVersion: v1
+        kind: ConfigMap
+        metadata:
+          name: compute-config
+        data:
+          config.yaml: |
+            apiVersion: apiserver.config.datumapis.com/v1alpha1
+            kind: WorkloadOperator
+            metricsServer:
+              bindAddress: "0"
+            discovery: {}
+        EOF
+      - task test-infra:kubectl -- -n compute-system set image deployment/compute-manager manager={{.IMG}}
+      - task test-infra:kubectl -- -n compute-system patch deployment compute-manager --type=json -p='[{"op":"replace","path":"/spec/template/spec/containers/0/imagePullPolicy","value":"IfNotPresent"}]'
+      - task test-infra:kubectl -- -n compute-system rollout status deployment/compute-manager --timeout=180s
+
+  e2e:
+    desc: Run Chainsaw e2e tests against the isolated Kind cluster
+    silent: true
+    deps:
+      - e2e:preflight
+      - install:chainsaw
+    env:
+      KUBECONFIG: '{{.E2E_KUBECONFIG}}'
+    cmds:
+      - '{{.TOOL_DIR}}/chainsaw test test/e2e --config test/e2e/.chainsaw.yaml'
+
+  install:chainsaw:
+    desc: Install Chainsaw locally
+    silent: true
+    cmds:
+      - |
+        mkdir -p {{.TOOL_DIR}}
+        if [ ! -f "{{.TOOL_DIR}}/chainsaw-{{.CHAINSAW_VERSION}}" ]; then
+          GOBIN={{.TOOL_DIR}} go install github.com/kyverno/chainsaw@{{.CHAINSAW_VERSION}}
+          mv {{.TOOL_DIR}}/chainsaw {{.TOOL_DIR}}/chainsaw-{{.CHAINSAW_VERSION}}
+        fi
+        ln -sf {{.TOOL_DIR}}/chainsaw-{{.CHAINSAW_VERSION}} {{.TOOL_DIR}}/chainsaw
+    status:
+      - test -f {{.TOOL_DIR}}/chainsaw-{{.CHAINSAW_VERSION}}
+
+  e2e:preflight:
+    internal: true
+    silent: true
+    env:
+      KUBECONFIG: '{{.E2E_KUBECONFIG}}'
+    cmds:
+      - |
+        if [ ! -f "$KUBECONFIG" ]; then
+          echo "missing e2e kubeconfig at $KUBECONFIG"
+          echo "run: TASK_X_REMOTE_TASKFILES=1 task e2e:up"
+          exit 1
+        fi
+      - |
+        context="$(kubectl config current-context 2>/dev/null || true)"
+        if [ "$context" != "kind-{{.E2E_CLUSTER}}" ]; then
+          echo "e2e kubeconfig points at $context, expected kind-{{.E2E_CLUSTER}}"
+          echo "run: TASK_X_REMOTE_TASKFILES=1 task e2e:up"
+          exit 1
+        fi
+      - |
+        if ! kubectl get crd workloaddeployments.compute.datumapis.com >/dev/null 2>&1; then
+          echo "Compute CRDs are not installed in the e2e cluster"
+          echo "run: TASK_X_REMOTE_TASKFILES=1 task e2e:up"
+          exit 1
+        fi
+      - kubectl -n compute-system rollout status deployment/compute-manager --timeout=30s
+
+  e2e:down:
+    desc: Tear down the e2e test-infra Kind cluster
+    silent: true
+    cmds:
+      - task: test-infra:cluster-down
+        vars:
+          CLUSTER_NAME: '{{.E2E_CLUSTER}}'

test/e2e/.chainsaw.yaml

@@ -0,0 +1,14 @@
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: Configuration
+metadata:
+  name: compute-e2e
+spec:
+  timeouts:
+    apply: 60s
+    assert: 120s
+    cleanup: 120s
+    delete: 60s
+    error: 60s
+    exec: 60s
+  parallel: 1
+  skipDelete: false

test/e2e/workloaddeployment/hpa-lifecycle/chainsaw-test.yaml

@@ -0,0 +1,21 @@
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: Test
+metadata:
+  name: workloaddeployment-hpa-lifecycle
+spec:
+  steps:
+    - name: create autoscaled WorkloadDeployment
+      try:
+        - apply:
+            file: workloaddeployment.yaml
+        - assert:
+            file: hpa.yaml
+    - name: disable autoscaling
+      try:
+        - script:
+            content: |
+              kubectl -n "$NAMESPACE" patch workloaddeployment.compute.datumapis.com chainsaw-hpa --type=json \
+                -p='[{"op":"remove","path":"/spec/scaleSettings/maxReplicas"},{"op":"remove","path":"/spec/scaleSettings/metrics"}]'
+        - script:
+            content: |
+              kubectl -n "$NAMESPACE" wait --for=delete horizontalpodautoscaler.autoscaling/chainsaw-hpa --timeout=60s

test/e2e/workloaddeployment/hpa-lifecycle/hpa.yaml

@@ -0,0 +1,25 @@
+apiVersion: autoscaling/v2
+kind: HorizontalPodAutoscaler
+metadata:
+  name: chainsaw-hpa
+  namespace: ($namespace)
+  labels:
+    services.miloapis.com/service-name: compute.datumapis.com
+    compute.datumapis.com/workload-deployment-name: chainsaw-hpa
+    compute.datumapis.com/workload-name: chainsaw-workload
+    compute.datumapis.com/placement-name: chainsaw-placement
+    compute.datumapis.com/city-code: DFW
+spec:
+  scaleTargetRef:
+    apiVersion: compute.datumapis.com/v1alpha
+    kind: WorkloadDeployment
+    name: chainsaw-hpa
+  minReplicas: 2
+  maxReplicas: 10
+  metrics:
+    - type: Resource
+      resource:
+        name: cpu
+        target:
+          type: Utilization
+          averageUtilization: 75

test/e2e/workloaddeployment/hpa-lifecycle/workloaddeployment.yaml

@@ -0,0 +1,32 @@
+apiVersion: compute.datumapis.com/v1alpha
+kind: WorkloadDeployment
+metadata:
+  name: chainsaw-hpa
+  namespace: ($namespace)
+spec:
+  workloadRef:
+    name: chainsaw-workload
+    uid: chainsaw-workload-uid
+  placementName: chainsaw-placement
+  cityCode: DFW
+  template:
+    spec:
+      runtime:
+        resources:
+          instanceType: datumcloud/d1-standard-2
+        sandbox:
+          containers:
+            - name: app
+              image: registry.k8s.io/pause:3.10
+      networkInterfaces:
+        - network:
+            name: default
+  scaleSettings:
+    minReplicas: 2
+    maxReplicas: 10
+    metrics:
+      - resource:
+          name: cpu
+          target:
+            averageUtilization: 75
+    instanceManagementPolicy: OrderedReady

test/e2e/workloaddeployment/hpa-stray/chainsaw-test.yaml

@@ -0,0 +1,24 @@
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: Test
+metadata:
+  name: workloaddeployment-hpa-stray
+spec:
+  steps:
+    - name: create stray HPA
+      try:
+        - apply:
+            file: stray-hpa.yaml
+    - name: create matching autoscaled WorkloadDeployment
+      try:
+        - apply:
+            file: workloaddeployment.yaml
+        - assert:
+            file: stray-hpa.yaml
+    - name: disable autoscaling
+      try:
+        - script:
+            content: |
+              kubectl -n "$NAMESPACE" patch workloaddeployment.compute.datumapis.com chainsaw-stray --type=json \
+                -p='[{"op":"remove","path":"/spec/scaleSettings/maxReplicas"},{"op":"remove","path":"/spec/scaleSettings/metrics"}]'
+        - assert:
+            file: stray-hpa.yaml

test/e2e/workloaddeployment/hpa-stray/stray-hpa.yaml

@@ -0,0 +1,21 @@
+apiVersion: autoscaling/v2
+kind: HorizontalPodAutoscaler
+metadata:
+  name: chainsaw-stray
+  namespace: ($namespace)
+  labels:
+    e2e.compute.datumapis.com/stray: "true"
+spec:
+  scaleTargetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: not-owned-by-compute
+  minReplicas: 1
+  maxReplicas: 4
+  metrics:
+    - type: Resource
+      resource:
+        name: cpu
+        target:
+          type: Utilization
+          averageUtilization: 50

test/e2e/workloaddeployment/hpa-stray/workloaddeployment.yaml

@@ -0,0 +1,32 @@
+apiVersion: compute.datumapis.com/v1alpha
+kind: WorkloadDeployment
+metadata:
+  name: chainsaw-stray
+  namespace: ($namespace)
+spec:
+  workloadRef:
+    name: chainsaw-workload
+    uid: chainsaw-workload-uid
+  placementName: chainsaw-placement
+  cityCode: DFW
+  template:
+    spec:
+      runtime:
+        resources:
+          instanceType: datumcloud/d1-standard-2
+        sandbox:
+          containers:
+            - name: app
+              image: registry.k8s.io/pause:3.10
+      networkInterfaces:
+        - network:
+            name: default
+  scaleSettings:
+    minReplicas: 2
+    maxReplicas: 10
+    metrics:
+      - resource:
+          name: cpu
+          target:
+            averageUtilization: 75
+    instanceManagementPolicy: OrderedReady