diff --git a/.github/workflows/test-e2e-cluster.yml b/.github/workflows/test-e2e-cluster.yml new file mode 100644 index 000000000..60fe00fe5 --- /dev/null +++ b/.github/workflows/test-e2e-cluster.yml @@ -0,0 +1,32 @@ +# SPDX-FileCopyrightText: 2026 SAP SE or an SAP affiliate company and IronCore contributors +# SPDX-License-Identifier: Apache-2.0 + +name: Test E2E (Cluster) + +on: + pull_request: + branches: + - main + paths-ignore: + - 'docs/**' + - '**/*.md' + +jobs: + test-e2e-cluster: + name: Run E2E Tests (Cluster) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + - uses: actions/setup-go@v6 + with: + go-version-file: 'go.mod' + - name: Create kind cluster + uses: helm/kind-action@v1 + with: + version: v0.31.0 + kubectl_version: v1.35.0 + cluster_name: network + - name: Running E2E Cluster Tests + run: | + go mod download + make test-e2e-cluster GINKGO_PROCS=8 diff --git a/.github/workflows/test-e2e-envtest.yml b/.github/workflows/test-e2e-envtest.yml new file mode 100644 index 000000000..550de5f88 --- /dev/null +++ b/.github/workflows/test-e2e-envtest.yml @@ -0,0 +1,26 @@ +# SPDX-FileCopyrightText: 2026 SAP SE or an SAP affiliate company and IronCore contributors +# SPDX-License-Identifier: Apache-2.0 + +name: Test E2E (Envtest) + +on: + pull_request: + branches: + - main + paths-ignore: + - 'docs/**' + - '**/*.md' + +jobs: + test-e2e-envtest: + name: Run E2E Tests (Envtest) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + - uses: actions/setup-go@v6 + with: + go-version-file: 'go.mod' + - name: Running E2E Envtest Tests + run: | + go mod download + make test-e2e-envtest diff --git a/.license-scan-overrides.jsonl b/.license-scan-overrides.jsonl index faa62bcb5..f6e9877b5 100644 --- a/.license-scan-overrides.jsonl +++ b/.license-scan-overrides.jsonl @@ -1,6 +1,7 @@ {"name": "github.com/chzyer/logex", "licenceType": "MIT"} {"name": "github.com/grpc-ecosystem/go-grpc-middleware/v2", "licenceType": "Apache-2.0"} {"name": "github.com/hashicorp/vault/api/auth/approle", "licenceType": "MPL-2.0"} +{"name": "github.com/ironcore-dev/gnmi-test-server", "licenceType": "Apache-2.0"} {"name": "github.com/jpillora/longestcommon", "licenceType": "MIT"} {"name": "github.com/logrusorgru/aurora", "licenceType": "Unlicense"} {"name": "github.com/mattn/go-localereader", "licenceType": "MIT"} diff --git a/Dockerfile b/Dockerfile index 8619d4333..ab5dad76b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,6 +16,7 @@ WORKDIR /workspace RUN --mount=type=cache,target=/go/pkg/mod \ --mount=type=bind,source=go.mod,target=go.mod \ --mount=type=bind,source=go.sum,target=go.sum \ + --mount=type=bind,source=test/gnmi,target=test/gnmi \ go mod download -x RUN --mount=type=bind,target=. \ diff --git a/Makefile b/Makefile index 1c13ac2b4..6a9221c8f 100644 --- a/Makefile +++ b/Makefile @@ -50,6 +50,9 @@ $(LOCALBIN): install-gofumpt: FORCE @if ! hash gofumpt 2>/dev/null; then printf "\e[1;36m>> Installing gofumpt...\e[0m\n"; go install mvdan.cc/gofumpt@latest; fi +install-ginkgo: FORCE + @if ! hash ginkgo 2>/dev/null; then printf "\e[1;36m>> Installing ginkgo...\e[0m\n"; go install github.com/onsi/ginkgo/v2/ginkgo@latest; fi + install-kubebuilder: FORCE @set -eou pipefail; if ! hash kubebuilder 2>/dev/null; then printf "\e[1;36m>> Installing kubebuilder...\e[0m\n"; if command -v curl >/dev/null 2>&1; then GET="curl -sLo"; elif command -v wget >/dev/null 2>&1; then GET="wget -O"; else echo "Didn't find curl or wget to download kubebuilder"; exit 2; fi; BIN=$$(go env GOBIN); if [[ -z $$BIN ]]; then BIN=$$(go env GOPATH)/bin; fi; $$GET "$$BIN/kubebuilder" "https://go.kubebuilder.io/dl/latest/$$(go env GOOS)/$$(go env GOARCH)"; chmod +x "$$BIN/kubebuilder"; fi @@ -72,7 +75,11 @@ fmt: FORCE install-gofumpt @printf "\e[1;36m>> gofumpt -l -w .\e[0m\n" @gofumpt -l -w $(shell git ls-files '*.go' | grep -v '^internal/provider/openconfig') -# Run the e2e tests against a k8s cluster. +# PROVIDER defines which provider to test (cisco-nxos-gnmi, cisco-iosxr-gnmi, openconfig). +# Used by test-e2e-cluster and test-e2e-envtest to filter tests. +PROVIDER ?= cisco-nxos-gnmi + +# Run the scaffolded e2e tests (unchanged from Kubebuilder). test-e2e: FORCE @command -v kind >/dev/null 2>&1 || { \ echo "Kind is not installed. Please install Kind manually."; \ @@ -85,6 +92,26 @@ test-e2e: FORCE @printf "\e[1;36m>> go test ./test/e2e/ -v -ginkgo.v\e[0m\n" @KIND_CLUSTER=$(KIND_CLUSTER) go test ./test/e2e/ -v -ginkgo.v +# Run gNMI controller tests in cluster mode (requires Kind cluster). +# Uses ginkgo for parallel execution. +GINKGO_PROCS ?= 4 +test-e2e-cluster: FORCE install-ginkgo + @command -v kind >/dev/null 2>&1 || { \ + echo "Kind is not installed. Please install Kind manually."; \ + exit 1; \ + } + @kind get clusters | grep -q $(KIND_CLUSTER) || { \ + echo "No Kind cluster is running. Please start a Kind cluster before running the e2e tests."; \ + exit 1; \ + } + @printf "\e[1;36m>> ginkgo -procs=$(GINKGO_PROCS) -tags=cluster -timeout=20m -v ./test/e2e/ (PROVIDER=$(PROVIDER))\e[0m\n" + @KIND_CLUSTER=$(KIND_CLUSTER) E2E_PROVIDER=$(PROVIDER) ginkgo -procs=$(GINKGO_PROCS) -tags=cluster -timeout=20m -v ./test/e2e/ + +# Run gNMI controller tests in envtest mode (no cluster required). +test-e2e-envtest: FORCE install-setup-envtest + @printf "\e[1;36m>> go test ./test/e2e/ -tags=envtest -v -ginkgo.v (PROVIDER=$(PROVIDER))\e[0m\n" + @KUBEBUILDER_ASSETS=$$(setup-envtest use 1.32 -p path) E2E_PROVIDER=$(PROVIDER) go test ./test/e2e/ -tags=envtest -v -ginkgo.v + docker-build: FORCE @printf "\e[1;36m>> $(CONTAINER_TOOL) build --tag=$(IMG) .\e[0m\n" @$(CONTAINER_TOOL) build --build-arg=BININFO_BUILD_DATE=$(BININFO_BUILD_DATE) --build-arg=BININFO_COMMIT_HASH=$(BININFO_COMMIT_HASH) --build-arg=BININFO_VERSION=$(BININFO_VERSION) --tag=$(IMG) . @@ -98,15 +125,16 @@ build-installer: FORCE generate install-kustomize @printf "\e[1;36m>> kustomize build config/default > dist/install.yaml\e[0m\n" @mkdir -p dist; kustomize build config/default > dist/install.yaml -# Deploy controller to the k8s cluster +# Deploy controller to the k8s cluster. +# Use PROVIDER to set the provider (default: cisco-nxos-gnmi). deploy: FORCE generate install-kustomize - @printf "\e[1;36m>> kustomize build config/default | kubectl apply -f -\e[0m\n" - @kustomize build config/default | kubectl apply -f - + @printf "\e[1;36m>> deploying controller-manager (PROVIDER=$(PROVIDER))\e[0m\n" + @kustomize build config/develop | sed 's/--provider=openconfig/--provider=$(PROVIDER)/' | kubectl apply -f - # Undeploy controller from the k8s cluster undeploy: FORCE install-kustomize - @printf "\e[1;36m>> kustomize build config/default | kubectl delete -f -\e[0m\n" - @kustomize build config/default | kubectl delete --ignore-not-found=true -f - + @printf "\e[1;36m>> undeploying controller-manager\e[0m\n" + @kustomize build config/develop | kubectl delete --ignore-not-found=true -f - # Install CRDs into the k8s cluster deploy-crds: FORCE generate install-kustomize diff --git a/Makefile.maker.yaml b/Makefile.maker.yaml index 53143db03..3ecaefc5b 100644 --- a/Makefile.maker.yaml +++ b/Makefile.maker.yaml @@ -92,6 +92,9 @@ verbatim: | install-gofumpt: FORCE @if ! hash gofumpt 2>/dev/null; then printf "\e[1;36m>> Installing gofumpt...\e[0m\n"; go install mvdan.cc/gofumpt@latest; fi + install-ginkgo: FORCE + @if ! hash ginkgo 2>/dev/null; then printf "\e[1;36m>> Installing ginkgo...\e[0m\n"; go install github.com/onsi/ginkgo/v2/ginkgo@latest; fi + install-kubebuilder: FORCE @set -eou pipefail; if ! hash kubebuilder 2>/dev/null; then printf "\e[1;36m>> Installing kubebuilder...\e[0m\n"; if command -v curl >/dev/null 2>&1; then GET="curl -sLo"; elif command -v wget >/dev/null 2>&1; then GET="wget -O"; else echo "Didn't find curl or wget to download kubebuilder"; exit 2; fi; BIN=$$(go env GOBIN); if [[ -z $$BIN ]]; then BIN=$$(go env GOPATH)/bin; fi; $$GET "$$BIN/kubebuilder" "https://go.kubebuilder.io/dl/latest/$$(go env GOOS)/$$(go env GOARCH)"; chmod +x "$$BIN/kubebuilder"; fi @@ -114,7 +117,11 @@ verbatim: | @printf "\e[1;36m>> gofumpt -l -w .\e[0m\n" @gofumpt -l -w $(shell git ls-files '*.go' | grep -v '^internal/provider/openconfig') - # Run the e2e tests against a k8s cluster. + # PROVIDER defines which provider to test (cisco-nxos-gnmi, cisco-iosxr-gnmi, openconfig). + # Used by test-e2e-cluster and test-e2e-envtest to filter tests. + PROVIDER ?= cisco-nxos-gnmi + + # Run the scaffolded e2e tests (unchanged from Kubebuilder). test-e2e: FORCE @command -v kind >/dev/null 2>&1 || { \ echo "Kind is not installed. Please install Kind manually."; \ @@ -127,6 +134,26 @@ verbatim: | @printf "\e[1;36m>> go test ./test/e2e/ -v -ginkgo.v\e[0m\n" @KIND_CLUSTER=$(KIND_CLUSTER) go test ./test/e2e/ -v -ginkgo.v + # Run gNMI controller tests in cluster mode (requires Kind cluster). + # Uses ginkgo for parallel execution. + GINKGO_PROCS ?= 4 + test-e2e-cluster: FORCE install-ginkgo + @command -v kind >/dev/null 2>&1 || { \ + echo "Kind is not installed. Please install Kind manually."; \ + exit 1; \ + } + @kind get clusters | grep -q $(KIND_CLUSTER) || { \ + echo "No Kind cluster is running. Please start a Kind cluster before running the e2e tests."; \ + exit 1; \ + } + @printf "\e[1;36m>> ginkgo -procs=$(GINKGO_PROCS) -tags=cluster -timeout=20m -v ./test/e2e/ (PROVIDER=$(PROVIDER))\e[0m\n" + @KIND_CLUSTER=$(KIND_CLUSTER) E2E_PROVIDER=$(PROVIDER) ginkgo -procs=$(GINKGO_PROCS) -tags=cluster -timeout=20m -v ./test/e2e/ + + # Run gNMI controller tests in envtest mode (no cluster required). + test-e2e-envtest: FORCE install-setup-envtest + @printf "\e[1;36m>> go test ./test/e2e/ -tags=envtest -v -ginkgo.v (PROVIDER=$(PROVIDER))\e[0m\n" + @KUBEBUILDER_ASSETS=$$(setup-envtest use 1.32 -p path) E2E_PROVIDER=$(PROVIDER) go test ./test/e2e/ -tags=envtest -v -ginkgo.v + docker-build: FORCE @printf "\e[1;36m>> $(CONTAINER_TOOL) build --tag=$(IMG) .\e[0m\n" @$(CONTAINER_TOOL) build --build-arg=BININFO_BUILD_DATE=$(BININFO_BUILD_DATE) --build-arg=BININFO_COMMIT_HASH=$(BININFO_COMMIT_HASH) --build-arg=BININFO_VERSION=$(BININFO_VERSION) --tag=$(IMG) . @@ -140,15 +167,16 @@ verbatim: | @printf "\e[1;36m>> kustomize build config/default > dist/install.yaml\e[0m\n" @mkdir -p dist; kustomize build config/default > dist/install.yaml - # Deploy controller to the k8s cluster + # Deploy controller to the k8s cluster. + # Use PROVIDER to set the provider (default: cisco-nxos-gnmi). deploy: FORCE generate install-kustomize - @printf "\e[1;36m>> kustomize build config/default | kubectl apply -f -\e[0m\n" - @kustomize build config/default | kubectl apply -f - + @printf "\e[1;36m>> deploying controller-manager (PROVIDER=$(PROVIDER))\e[0m\n" + @kustomize build config/develop | sed 's/--provider=openconfig/--provider=$(PROVIDER)/' | kubectl apply -f - # Undeploy controller from the k8s cluster undeploy: FORCE install-kustomize - @printf "\e[1;36m>> kustomize build config/default | kubectl delete -f -\e[0m\n" - @kustomize build config/default | kubectl delete --ignore-not-found=true -f - + @printf "\e[1;36m>> undeploying controller-manager\e[0m\n" + @kustomize build config/develop | kubectl delete --ignore-not-found=true -f - # Install CRDs into the k8s cluster deploy-crds: FORCE generate install-kustomize diff --git a/config/develop/gnmi-test-server.yaml b/config/develop/gnmi-test-server.yaml index 789b76681..9f2d4d1c8 100644 --- a/config/develop/gnmi-test-server.yaml +++ b/config/develop/gnmi-test-server.yaml @@ -33,7 +33,7 @@ spec: containers: - name: gnmi-test-server image: ghcr.io/ironcore-dev/gnmi-test-server:latest - imagePullPolicy: IfNotPresent + imagePullPolicy: Never ports: - containerPort: 9339 name: grpc diff --git a/config/develop/manager_patch.yaml b/config/develop/manager_patch.yaml index e07e73a48..162d5e9de 100644 --- a/config/develop/manager_patch.yaml +++ b/config/develop/manager_patch.yaml @@ -1,6 +1,7 @@ - op: replace path: /spec/template/spec/containers/0/args value: + - --metrics-bind-address=:8443 - --leader-elect=false - --health-probe-bind-address=:8081 - --provider=openconfig diff --git a/go.mod b/go.mod index 31f543d34..22aa5b0a9 100644 --- a/go.mod +++ b/go.mod @@ -72,6 +72,7 @@ require ( github.com/google/uuid v1.6.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.7 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/ironcore-dev/gnmi-test-server v0.0.0-00010101000000-000000000000 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/kylelemons/godebug v1.1.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect @@ -90,6 +91,7 @@ require ( github.com/stretchr/objx v0.5.2 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.1 // indirect + github.com/tidwall/sjson v1.2.5 // indirect github.com/x448/float16 v0.8.4 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.65.0 // indirect @@ -129,3 +131,5 @@ require ( sigs.k8s.io/randfill v1.0.0 // indirect sigs.k8s.io/structured-merge-diff/v6 v6.3.2 // indirect ) + +replace github.com/ironcore-dev/gnmi-test-server => ./test/gnmi diff --git a/go.sum b/go.sum index 4ae727d67..cf17074ff 100644 --- a/go.sum +++ b/go.sum @@ -191,10 +191,12 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/gjson v1.19.0 h1:xwxm7n691Uf3u5OFjzngavjGTh55KX5q/9w9xHW88JU= github.com/tidwall/gjson v1.19.0/go.mod h1:V37/opeE/JbLUOfH0QTXiNez2l0RUjYUhpT4szFQAfc= github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= diff --git a/test/e2e/AGENTS.md b/test/e2e/AGENTS.md new file mode 100644 index 000000000..86c917bf4 --- /dev/null +++ b/test/e2e/AGENTS.md @@ -0,0 +1,146 @@ +# E2E Tests + +Integration tests that validate the full reconciliation pipeline from Kubernetes CRD to gNMI JSON output. + +## Rationale + +Unit tests verify individual functions but miss the gaps between layers: CRD validation, controller logic, provider field mapping, and gNMI path construction. For example, a controller passing the wrong value to a provider. + +Two test modes via Go build tags (`//go:build envtest` vs `//go:build !envtest`): + +- **Envtest mode** — runs tests sequantially without infrastructure using envtest + in-process gnmi-test-server. Tests complete in seconds. +- **Cluster mode** — runs tests in parallel against Kind with full operator installation. Validates RBAC, webhooks, metrics. +Each reconciliation test is isolated: unique namespace + dedicated gnmi-test-server pod. + +## Commands + +```bash +make test-e2e-envtest PROVIDER=cisco-nxos-gnmi # no cluster required +make test-e2e-kind PROVIDER=cisco-nxos-gnmi # requires Kind cluster "network" +``` + +**Provider options:** `openconfig`, `cisco-nxos-gnmi` + +## Mode Comparison + +| Aspect | Envtest | Cluster | +|--------|---------|---------| +| **K8s API** | In-process (envtest) | Kind cluster | +| **gNMI server** | In-process (gnmi-test-server) | Deployed pod per test | +| **Controllers** | In-process per test | Deployed operator pod | +| **Speed** | ~10s | ~3-4min (parallel) | +| **Dependencies** | None | Docker + Kind | + +| Coverage | Envtest | Cluster | +|----------|:-------:|:-------:| +| Controller reconciliation | ✅ | ✅ | +| Status conditions | ✅ | ✅ | +| gNMI payload generation | ✅ | ✅ | +| RBAC / ServiceAccount | ❌ | ✅ | +| Webhook TLS + cert-manager | ❌ | ✅ | +| Container image build | ❌ | ✅ | +| Metrics endpoint | ❌ | ✅ | + +## Environment Variables + +| Variable | Purpose | +|----------|---------| +| `PROVIDER` / `E2E_PROVIDER` | Filter tests to specific provider | +| `PROMETHEUS_INSTALL_SKIP=true` | Skip Prometheus installation (cluster mode) | +| `CERT_MANAGER_INSTALL_SKIP=true` | Skip CertManager installation (cluster mode) | + +## Architecture + +Both modes implement `TestEnvironment` interface via `testutil/`: + +| Method | Purpose | +|--------|---------| +| `Setup()` / `Teardown()` | Initialize/cleanup K8s + gNMI server | +| `Client()` / `RESTConfig()` | K8s client for resources and managers | +| `GNMIAddress()` | Endpoint for Device CRDs | +| `GetGNMIState()` / `ClearGNMIState()` | Verify and reset gNMI state | +| `PreloadGNMIState()` | Set initial state before reconciliation | +| `IsEnvtest()` | Detect mode for conditional logic | + +### GNMI test server + +The server accumulates gNMI Set operations and exposes state via `GetState()`. + +When configured with `WithNXOSBehavior()`: +- Strips fields with value `"DME_UNSET_PROPERTY_MARKER"` when storing (the marker means "unset this field", not "store this literal string") +- Returns empty TypedValue for non-existent paths (instead of NOT_FOUND error), matching real NX-OS behavior + + +## Testdata Format + +Location: `test/e2e/testdata//.txt` — auto-discovered, txtar format. + +Each test file contains K8s YAML resources and expected gNMI JSON state: + +``` +-- state/preload -- # OPTIONAL: initial gNMI state +{"System": {"procsys-items": {"bootTime": "1700000000"}}} + +-- / -- # Resource to create +apiVersion: networking.metal.ironcore.dev/v1alpha1 +kind: Interface +spec: + deviceRef: + name: device # substituted at runtime + ... + +-- / -- # Multiple resources supported +apiVersion: networking.metal.ironcore.dev/v1alpha1 +kind: BGPPeer +spec: + deviceRef: + name: device + ... + +-- state/expect -- # Expected gNMI JSON (array order ignored) +{"System": {"intf-items": ...}} +``` + +### Test Flow + +1. Parse txtar file → extract resources + expected state +2. Create test namespace +3. Deploy/connect to gnmi-test-server +4. Preload gNMI state if `state/preload` section exists +5. Create Device pointing to gnmi-test-server address +6. Apply resources **sequentially**, waiting for `Configured` condition before next +7. Compare final gNMI state vs `state/expect` using semantic JSON comparison + +### Important Notes + +> **gNMI State Differences:** The JSON stored in gnmi-test-server may differ from actual provider output. For example, NXOS `DME_UNSET_PROPERTY_MARKER` values are filtered out by the test server. + +> **Resource Dependencies:** Resources are created in file order. Use ordering to handle dependencies (e.g., BGPPeer after BGP, RoutingPolicy after PrefixSet). + +## Existing Test Files + +``` +test/e2e/testdata/cisco-nxos-gnmi/ +├── acl.txt +├── banner.txt +├── bgp_bgppeer.txt +├── dhcprelay.txt +├── dns.txt +├── evpninstance.txt +├── interfaceconfig.txt +├── interfaces.txt +├── isis.txt +├── lldp.txt +├── managementaccess.txt +├── ntp.txt +├── nve.txt +├── ospf.txt +├── pim.txt +├── routedvlan.txt +├── routingpolicy_prefixset.txt +├── snmp.txt +├── subinterface.txt +├── syslog.txt +├── vpcdomain.txt +└── vrf.txt +``` diff --git a/test/e2e/cluster_suite_test.go b/test/e2e/cluster_suite_test.go new file mode 100644 index 000000000..1782e905e --- /dev/null +++ b/test/e2e/cluster_suite_test.go @@ -0,0 +1,99 @@ +// SPDX-FileCopyrightText: 2025 SAP SE or an SAP affiliate company and IronCore contributors +// SPDX-License-Identifier: Apache-2.0 + +//go:build cluster + +package e2e + +import ( + "fmt" + "testing" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + + "github.com/ironcore-dev/network-operator/test/e2e/testutil" +) + +// TestCluster runs the e2e test suite in cluster mode. +// Named differently from TestE2E to avoid conflict with scaffolded e2e_suite_test.go. +func TestCluster(t *testing.T) { + RegisterFailHandler(Fail) + _, _ = fmt.Fprintf(GinkgoWriter, "Starting network-operator tests in CLUSTER mode\n") + RunSpecs(t, "e2e suite (cluster)") +} + +// SynchronizedBeforeSuite enables parallel test execution: +// - Process 1: Builds images, installs Prometheus/CertManager, deploys manager (runs first, alone) +// - All processes: Create ClusterEnvironment connection (runs after process 1 completes) +var _ = SynchronizedBeforeSuite( + // First function: runs ONLY on process 1, before other processes start + func(ctx SpecContext) []byte { + logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) + // Note: Timeout is set in the second function for all processes + + By("Ensure that Prometheus is enabled") + cwd, err := testutil.GetProjectDir() + Expect(err).NotTo(HaveOccurred(), "Failed to get project directory") + + err = testutil.UncommentCode(cwd+"/config/default/kustomization.yaml", "#- ../prometheus", "#") + Expect(err).NotTo(HaveOccurred(), "Failed to enable Prometheus") + + // Build and load images to Kind (only process 1) + buildAndLoadImages(ctx) + + // Setup Prometheus and CertManager (only process 1) + setupClusterDependencies(ctx) + + // Deploy controller-manager (includes CRDs via make deploy) + By("deploying controller-manager") + tmpEnv := testutil.NewClusterEnvironment() + Expect(tmpEnv.Setup(ctx)).To(Succeed()) + Expect(tmpEnv.DeployManager(ctx)).To(Succeed()) + + return nil // No data to pass to other processes + }, + // Second function: runs on ALL processes after the first function completes + func(ctx SpecContext, _ []byte) { + SetDefaultEventuallyTimeout(testutil.DefaultTimeout) + SetDefaultEventuallyPollingInterval(time.Second) + + // All processes create their own ClusterEnvironment connection + By("initializing cluster environment") + testEnv = testutil.NewClusterEnvironment() + Expect(testEnv.Setup(ctx)).To(Succeed()) + }, +) + +// SynchronizedAfterSuite enables parallel test cleanup: +// - All processes: Local cleanup (runs on all processes) +// - Process 1: Uninstall shared dependencies (runs last, alone) +var _ = SynchronizedAfterSuite( + // First function: runs on ALL processes + func(ctx SpecContext) { + // Perform local cleanup (will run only once even if called from signal handler) + performCleanup() + + // Wait for all test namespaces to be fully deleted before returning. + // This ensures DeferCleanup hooks have finished deleting resources and their + // finalizers have been processed by the controller. Without this, the second + // function (UndeployManager) may delete the CRDs while resources still exist, + // causing finalizers to be stuck forever. + if testEnv != nil { + _ = testEnv.WaitForTestNamespacesGone(ctx) //nolint:errcheck // best-effort cleanup + } + }, + // Second function: runs ONLY on process 1, after all other processes complete + func(ctx SpecContext) { + // Undeploy the controller-manager + tmpEnv := testutil.NewClusterEnvironment() + _ = tmpEnv.Setup(ctx) //nolint:errcheck // best-effort cleanup + _ = tmpEnv.UndeployManager(ctx) //nolint:errcheck // best-effort cleanup + + // Uninstall Prometheus and CertManager + cleanupClusterDependencies(ctx) + }, +) diff --git a/test/e2e/cluster_test.go b/test/e2e/cluster_test.go new file mode 100644 index 000000000..4483974c9 --- /dev/null +++ b/test/e2e/cluster_test.go @@ -0,0 +1,577 @@ +// SPDX-FileCopyrightText: 2025 SAP SE or an SAP affiliate company and IronCore contributors +// SPDX-License-Identifier: Apache-2.0 + +//go:build cluster + +package e2e + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "golang.org/x/tools/txtar" + + "github.com/ironcore-dev/network-operator/test/e2e/testutil" +) + +// namespace where the project is deployed in +// tests create resources in separate namespaces +const namespace = "network-operator-system" + +// serviceAccountName created for the project +const serviceAccountName = "network-operator-controller-manager" + +// metricsServiceName is the name of the metrics service of the project +const metricsServiceName = "network-operator-controller-manager-metrics-service" + +// metricsRoleBindingName is the name of the RBAC that will be created to allow get the metrics data +const metricsRoleBindingName = "network-operator-metrics-binding" + +// image is the name of the image which will be build and loaded +// with the code source changes to be tested. +const image = "ghcr.io/ironcore-dev/network-operator:latest" + +// serverImage is the name of the image which will be built and loaded +// with the gNMI test server. +const serverImage = "ghcr.io/ironcore-dev/gnmi-test-server:latest" + +var ( + // Optional Environment Variables: + // - PROMETHEUS_INSTALL_SKIP=true: Skips Prometheus Operator installation during test setup. + // - CERT_MANAGER_INSTALL_SKIP=true: Skips CertManager installation during test setup. + // These variables are useful if Prometheus or CertManager is already installed, avoiding re-installation and conflicts. + skipPrometheusInstall = os.Getenv("PROMETHEUS_INSTALL_SKIP") == "true" + skipCertManagerInstall = os.Getenv("CERT_MANAGER_INSTALL_SKIP") == "true" + // isPrometheusOperatorAlreadyInstalled will be set true when prometheus CRDs be found on the cluster + isPrometheusOperatorAlreadyInstalled = false + // isCertManagerAlreadyInstalled will be set true when CertManager CRDs be found on the cluster + isCertManagerAlreadyInstalled = false +) + +var ( + cleanupOnce sync.Once + // testEnv is the cluster test environment. + testEnv *testutil.ClusterEnvironment +) + +func init() { + _, _ = fmt.Fprintf(GinkgoWriter, "Starting network-operator tests in CLUSTER mode\n") +} + +// Manager Setup tests run serially on a single Ginkgo process. +// These tests deploy and verify the controller-manager before reconciliation tests run in parallel. +var _ = Describe("Manager Setup", Serial, Ordered, func() { + var controllerPodName string + + // Before running the tests, set up the environment by creating the namespace, + // enforce the restricted security policy to the namespace, installing CRDs, + // and deploying the controller. + BeforeAll(func(ctx SpecContext) { + By("creating manager namespace") + cmd := exec.CommandContext(ctx, "kubectl", "create", "ns", namespace, "--dry-run=client", "-o", "yaml") + nsYaml, err := testutil.Run(cmd, GinkgoWriter) + Expect(err).NotTo(HaveOccurred(), "Failed to generate namespace YAML") + cmd = exec.CommandContext(ctx, "kubectl", "apply", "-f", "-") + cmd.Stdin = bytes.NewBufferString(nsYaml) + _, err = testutil.Run(cmd, GinkgoWriter) + Expect(err).NotTo(HaveOccurred(), "Failed to create namespace") + + By("labeling the namespace to enforce the restricted security policy") + cmd = exec.CommandContext(ctx, "kubectl", "label", "--overwrite", "ns", namespace, "pod-security.kubernetes.io/enforce=restricted") + _, err = testutil.Run(cmd, GinkgoWriter) + Expect(err).NotTo(HaveOccurred(), "Failed to label namespace with restricted policy") + + By("installing CRDs") + cmd = exec.CommandContext(ctx, "make", "deploy-crds") + _, err = testutil.Run(cmd, GinkgoWriter) + Expect(err).NotTo(HaveOccurred(), "Failed to install CRDs") + + By("deploying the controller-manager") + cmd = exec.CommandContext(ctx, "make", "deploy") + _, err = testutil.Run(cmd, GinkgoWriter) + Expect(err).NotTo(HaveOccurred(), "Failed to deploy the controller-manager") + }) + + // After all setup tests complete, clean up the manager. + // Note: CRDs are left installed for the parallel reconciliation tests. + AfterAll(func(ctx SpecContext) { + By("cleaning up the ClusterRoleBinding of the service account to allow access to metrics") + cmd := exec.CommandContext(ctx, "kubectl", "delete", "clusterrolebinding", metricsRoleBindingName, "--ignore-not-found") + _, err := testutil.Run(cmd, GinkgoWriter) + Expect(err).NotTo(HaveOccurred(), "Failed to delete ClusterRoleBinding") + + By("cleaning up the curl pod for metrics") + cmd = exec.CommandContext(ctx, "kubectl", "delete", "pod", "curl-metrics", "-n", namespace, "--ignore-not-found") + _, err = testutil.Run(cmd, GinkgoWriter) + Expect(err).NotTo(HaveOccurred(), "Failed to delete curl-metrics pod") + }) + + // After each test, check for failures and collect logs, events, + // and pod descriptions for debugging. + AfterEach(func(ctx SpecContext) { + if specReport := CurrentSpecReport(); specReport.Failed() { + By("Fetching controller manager pod logs") + cmd := exec.CommandContext(ctx, "kubectl", "logs", controllerPodName, "-n", namespace) + controllerLogs, err := testutil.Run(cmd, GinkgoWriter) + if err == nil { + _, _ = fmt.Fprintf(GinkgoWriter, "Controller logs:\n %s", controllerLogs) + } else { + _, _ = fmt.Fprintf(GinkgoWriter, "Failed to get Controller logs: %s", err) + } + + By("Fetching Kubernetes events") + cmd = exec.CommandContext(ctx, "kubectl", "get", "events", "-n", namespace, "--sort-by=.lastTimestamp") + eventsOutput, err := testutil.Run(cmd, GinkgoWriter) + if err == nil { + _, _ = fmt.Fprintf(GinkgoWriter, "Kubernetes events:\n%s", eventsOutput) + } else { + _, _ = fmt.Fprintf(GinkgoWriter, "Failed to get Kubernetes events: %s", err) + } + + By("Fetching curl-metrics logs") + cmd = exec.CommandContext(ctx, "kubectl", "logs", "curl-metrics", "-n", namespace) + metricsOutput, err := testutil.Run(cmd, GinkgoWriter) + if err == nil { + _, _ = fmt.Fprintf(GinkgoWriter, "Metrics logs:\n %s", metricsOutput) + } else { + _, _ = fmt.Fprintf(GinkgoWriter, "Failed to get curl-metrics logs: %s", err) + } + + By("Fetching controller manager pod description") + cmd = exec.CommandContext(ctx, "kubectl", "describe", "pod", controllerPodName, "-n", namespace) + podDescription, err := testutil.Run(cmd, GinkgoWriter) + if err == nil { + fmt.Println("Pod description:\n", podDescription) + } else { + fmt.Println("Failed to describe controller pod") + } + } + }) + + BeforeEach(func() { + SetDefaultEventuallyTimeout(testutil.LongTimeout) + SetDefaultEventuallyPollingInterval(time.Second) + }) + + It("should run successfully", func(ctx SpecContext) { + By("validating that the controller-manager pod is running as expected") + verifyControllerUp := func(g Gomega) { + // Get the name of the controller-manager pod + cmd := exec.CommandContext( + ctx, "kubectl", "get", + "pods", "-l", "control-plane=controller-manager", + "-o", "go-template={{ range .items }}"+ + "{{ if not .metadata.deletionTimestamp }}"+ + "{{ .metadata.name }}"+ + "{{ \"\\n\" }}{{ end }}{{ end }}", + "-n", namespace, + ) + + podOutput, err := testutil.Run(cmd, GinkgoWriter) + g.Expect(err).NotTo(HaveOccurred(), "Failed to retrieve controller-manager pod information") + podNames := testutil.GetNonEmptyLines(podOutput) + g.Expect(podNames).To(HaveLen(1), "expected 1 controller pod running") + controllerPodName = podNames[0] + g.Expect(controllerPodName).To(ContainSubstring("controller-manager")) + + // Validate the pod's status + cmd = exec.CommandContext(ctx, "kubectl", "get", "pods", controllerPodName, "-o", "jsonpath={.status.phase}", "-n", namespace) + output, err := testutil.Run(cmd, GinkgoWriter) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(output).To(Equal("Running"), "Incorrect controller-manager pod status") + } + Eventually(verifyControllerUp).Should(Succeed()) + }) + + It("should ensure the metrics endpoint is serving metrics", func(ctx SpecContext) { + By("creating a ClusterRoleBinding for the service account to allow access to metrics") + // #nosec G204 + cmd := exec.CommandContext(ctx, "kubectl", "create", "clusterrolebinding", metricsRoleBindingName, "--clusterrole=network-operator-metrics-reader", fmt.Sprintf("--serviceaccount=%s:%s", namespace, serviceAccountName)) + _, err := testutil.Run(cmd, GinkgoWriter) + Expect(err).NotTo(HaveOccurred(), "Failed to create ClusterRoleBinding") + + By("validating that the metrics service is available") + cmd = exec.CommandContext(ctx, "kubectl", "get", "service", metricsServiceName, "-n", namespace) + _, err = testutil.Run(cmd, GinkgoWriter) + Expect(err).NotTo(HaveOccurred(), "Metrics service should exist") + + By("validating that the ServiceMonitor for Prometheus is applied in the namespace") + cmd = exec.CommandContext(ctx, "kubectl", "get", "ServiceMonitor", "-n", namespace) + _, err = testutil.Run(cmd, GinkgoWriter) + Expect(err).NotTo(HaveOccurred(), "ServiceMonitor should exist") + + By("getting the service account token") + token, err := serviceAccountToken(ctx) + Expect(err).NotTo(HaveOccurred()) + Expect(token).NotTo(BeEmpty()) + + By("waiting for the metrics endpoint to be ready") + verifyMetricsEndpointReady := func(g Gomega) { + kcmd := exec.CommandContext(ctx, "kubectl", "get", "endpoints", metricsServiceName, "-n", namespace) + output, kErr := testutil.Run(kcmd, GinkgoWriter) + g.Expect(kErr).NotTo(HaveOccurred()) + g.Expect(output).To(ContainSubstring("8443"), "Metrics endpoint is not ready") + } + Eventually(verifyMetricsEndpointReady).Should(Succeed()) + + By("verifying that the controller manager has started") + verifyManagerStarted := func(g Gomega) { + kcmd := exec.CommandContext(ctx, "kubectl", "logs", controllerPodName, "-n", namespace) + output, kErr := testutil.Run(kcmd, GinkgoWriter) + g.Expect(kErr).NotTo(HaveOccurred()) + g.Expect(output).To(ContainSubstring("starting manager"), "Manager not yet started") + } + Eventually(verifyManagerStarted).Should(Succeed()) + + By("creating the curl-metrics pod to access the metrics endpoint") + // #nosec G204 + cmd = exec.CommandContext(ctx, "kubectl", "run", "curl-metrics", "--restart=Never", + "--namespace", namespace, + "--image=curlimages/curl:latest", + "--overrides", + fmt.Sprintf(`{ + "spec": { + "containers": [{ + "name": "curl", + "image": "curlimages/curl:latest", + "command": ["/bin/sh", "-c"], + "args": ["curl -v -k -H 'Authorization: Bearer %s' https://%s.%s.svc.cluster.local:8443/metrics"], + "securityContext": { + "allowPrivilegeEscalation": false, + "capabilities": { + "drop": ["ALL"] + }, + "runAsNonRoot": true, + "runAsUser": 1000, + "seccompProfile": { + "type": "RuntimeDefault" + } + } + }], + "serviceAccount": "%s" + } + }`, token, metricsServiceName, namespace, serviceAccountName)) + _, err = testutil.Run(cmd, GinkgoWriter) + Expect(err).NotTo(HaveOccurred(), "Failed to create curl-metrics pod") + + By("waiting for the curl-metrics pod to complete.") + verifyCurlUp := func(g Gomega) { + cmd := exec.CommandContext(ctx, "kubectl", "get", "pods", "curl-metrics", "-o", "jsonpath={.status.phase}", "-n", namespace) + output, err := testutil.Run(cmd, GinkgoWriter) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(output).To(Equal("Succeeded"), "curl pod in wrong status") + } + Eventually(verifyCurlUp, testutil.VeryLongTimeout).Should(Succeed()) + + By("getting the metrics by checking curl-metrics logs") + metricsOutput := getMetricsOutput(ctx) + Expect(metricsOutput).To(ContainSubstring("controller_runtime_webhook_panics_total")) + }) + + It("should provisioned cert-manager", func(ctx SpecContext) { + By("validating that cert-manager has the certificate Secret") + verifyCertManager := func(g Gomega) { + cmd := exec.CommandContext(ctx, "kubectl", "get", "secrets", "webhook-server-cert", "-n", namespace) + _, err := testutil.Run(cmd, GinkgoWriter) + g.Expect(err).NotTo(HaveOccurred()) + } + Eventually(verifyCertManager).Should(Succeed()) + }) + + It("should have CA injection for validating webhooks", func(ctx SpecContext) { + By("checking CA injection for validating webhooks") + verifyCAInjection := func(g Gomega) { + cmd := exec.CommandContext(ctx, "kubectl", "get", + "validatingwebhookconfigurations.admissionregistration.k8s.io", + "network-operator-validating-webhook-configuration", + "-o", "go-template={{ range .webhooks }}{{ .clientConfig.caBundle }}{{ end }}") + vwhOutput, err := testutil.Run(cmd, GinkgoWriter) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(len(vwhOutput)).To(BeNumerically(">", 10)) + } + Eventually(verifyCAInjection).Should(Succeed()) + }) + + // +kubebuilder:scaffold:e2e-webhooks-checks +}) + +// Reconciliation tests run in parallel across multiple Ginkgo processes. +// Each test creates its own namespace and gnmi-test-server instance for isolation. +var _ = Describe("Reconciliation", func() { + projectDir, err := testutil.GetProjectDir() + if err != nil { + Fail(fmt.Sprintf("Failed to get project directory: %v", err)) + } + + // Get provider filter from environment (set by Makefile) + providerFilter := os.Getenv("E2E_PROVIDER") + + testdataRoot := filepath.Join(projectDir, "test", "e2e", "testdata") + providerDirs, err := os.ReadDir(testdataRoot) + if err != nil { + Fail(fmt.Sprintf("Failed to read testdata directory: %v", err)) + } + + var testFiles []string + var providerName string + for _, providerDir := range providerDirs { + if !providerDir.IsDir() { + continue + } + providerName = providerDir.Name() + + if providerFilter != "" && providerName != providerFilter { + continue + } + + providerTestdataDir := filepath.Join(testdataRoot, providerName) + + testFiles, err = filepath.Glob(filepath.Join(providerTestdataDir, "*.txt")) + if err != nil { + Fail(fmt.Sprintf("Failed to glob testdata: %v", err)) + } + break + } + + for _, testFile := range testFiles { + testName := filepath.Base(testFile) + testName = testName[:len(testName)-4] // remove .txt + + It(fmt.Sprintf("should reconcile %s/%s", providerName, testName), func(ctx SpecContext) { + By("parsing testdata file") + a, err := txtar.ParseFile(testFile) + Expect(err).NotTo(HaveOccurred(), "Failed to parse test file: %s", testFile) + + var state, preload []byte + var resources []txtar.File + for _, f := range a.Files { + switch f.Name { + case "state/expect": + state = f.Data + case "state/preload": + preload = f.Data + default: + resources = append(resources, f) + } + } + Expect(state).NotTo(BeEmpty(), "Expected '-- state/expect --' section in testdata") + Expect(resources).NotTo(BeEmpty(), "Expected at least one resource in testdata") + + By("creating test namespace") + testNamespace := fmt.Sprintf("test-%s-%s-%s", providerName, strings.ReplaceAll(testName, "_", "-"), time.Now().Format("20060102150405")) + // Truncate to 63 chars max (K8s namespace limit) + if len(testNamespace) > 63 { + testNamespace = testNamespace[:63] + } + Expect(testEnv.CreateNamespace(ctx, testNamespace)).NotTo(HaveOccurred(), "Failed to create test namespace") + + DeferCleanup(func(_ SpecContext) { + // Use a fresh context with generous timeout for cleanup + // The SpecContext may be nearly exhausted after test timeout + cleanupCtx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + + // Clean up test resources before deleting the gnmi-test-server pod to avoid issues with finalizers that require API access. + By("deleting test resources") + _ = testEnv.DeleteCustomResources(cleanupCtx, testNamespace) //nolint:errcheck // best-effort cleanup + By("deleting test namespace") + _ = testEnv.DeleteNamespace(cleanupCtx, testNamespace) //nolint:errcheck // best-effort cleanup + }) + + deviceName := fmt.Sprintf("test-device-%d", time.Now().UnixNano()) + + By("deploying a gnmi-test-server instance for this test") + gnmiAddr, err := testEnv.DeployGNMIServer(ctx, testNamespace) + Expect(err).NotTo(HaveOccurred(), "Failed to deploy gnmi-test-server") + Expect(gnmiAddr).ToNot(BeNil()) + + By("preloading gNMI state if specified") + if len(preload) > 0 { + err = testEnv.PreloadGNMIState(ctx, testNamespace, preload) + Expect(err).NotTo(HaveOccurred(), "Failed to preload gNMI state") + } + + By("creating a test device") + device := fmt.Sprintf(` +apiVersion: networking.metal.ironcore.dev/v1alpha1 +kind: Device +metadata: + name: %s + namespace: %s + labels: + %s: "" +spec: + endpoint: + address: "%s"`, deviceName, testNamespace, testutil.E2ETestLabel, gnmiAddr.String()) + err = testutil.Apply(ctx, device, GinkgoWriter) + Expect(err).NotTo(HaveOccurred(), "Failed to apply Device") + + By("applying resources from testdata") + _, _ = fmt.Fprintf(GinkgoWriter, "DEBUG: Found %d resources to apply\n", len(resources)) + for _, res := range resources { + _, _ = fmt.Fprintf(GinkgoWriter, "DEBUG: Applying resource: %s\n", res.Name) + err = testutil.ApplyWithPatch(ctx, string(res.Data), testNamespace, deviceName, GinkgoWriter) + Expect(err).NotTo(HaveOccurred(), "Failed to apply resource: %s", res.Name) + } + + By("waiting for resources to be configured") + for _, res := range resources { + // Extract actual kind/name from YAML since section name may differ from metadata.name + resourceID, err := testutil.ExtractResourceIdentifier(string(res.Data)) + Expect(err).NotTo(HaveOccurred(), "Failed to extract resource identifier from: %s", res.Name) + err = testutil.WaitForCondition(ctx, resourceID, testNamespace, GinkgoWriter) + Expect(err).NotTo(HaveOccurred(), "Resource not configured: %s", resourceID) + } + + By("verifying gNMI state matches expected JSON") + gnmiState, err := testEnv.GetGNMIState(ctx, testNamespace) + Expect(err).NotTo(HaveOccurred(), "Failed to get gNMI state") + + err = testutil.CompareJSON(string(gnmiState), string(state)) + Expect(err).NotTo(HaveOccurred(), "gNMI state does not match expected JSON") + }) + } +}) + +// serviceAccountToken returns a token for the specified service account in the given namespace. +// It uses the Kubernetes TokenRequest API to generate a token by directly sending a request +// and parsing the resulting token from the API response. +func serviceAccountToken(ctx context.Context) (string, error) { + // #nosec G101 + const tokenRequestRawString = `{ + "apiVersion": "authentication.k8s.io/v1", + "kind": "TokenRequest" + }` + + // Temporary file to store the token request + secretName := serviceAccountName + "-token-request" + tokenRequestFile := filepath.Join(os.TempDir(), secretName) + if err := os.WriteFile(tokenRequestFile, []byte(tokenRequestRawString), os.FileMode(0o644)); err != nil { + return "", err + } + + var out string + verifyTokenCreation := func(g Gomega) { + // Execute kubectl command to create the token + // #nosec G204 + cmd := exec.CommandContext(ctx, "kubectl", "create", "--raw", fmt.Sprintf("/api/v1/namespaces/%s/serviceaccounts/%s/token", namespace, serviceAccountName), "-f", tokenRequestFile) + output, err := cmd.CombinedOutput() + g.Expect(err).NotTo(HaveOccurred()) + + // Parse the JSON output to extract the token + var token tokenRequest + err = json.Unmarshal(output, &token) + g.Expect(err).NotTo(HaveOccurred()) + + out = token.Status.Token + } + Eventually(verifyTokenCreation).Should(Succeed()) + + return out, nil +} + +// getMetricsOutput retrieves and returns the logs from the curl pod used to access the metrics endpoint. +func getMetricsOutput(ctx context.Context) string { + By("getting the curl-metrics logs") + cmd := exec.CommandContext(ctx, "kubectl", "logs", "curl-metrics", "-n", namespace) + metricsOutput, err := testutil.Run(cmd, GinkgoWriter) + Expect(err).NotTo(HaveOccurred(), "Failed to retrieve logs from curl pod") + Expect(metricsOutput).To(ContainSubstring("< HTTP/1.1 200 OK")) + return metricsOutput +} + +// tokenRequest is a simplified representation of the Kubernetes TokenRequest API response, +// containing only the token field that we need to extract. +type tokenRequest struct { + Status struct { + Token string `json:"token"` + } `json:"status"` +} + +// performCleanup ensures testEnv.Teardown is called exactly once +func performCleanup() { + cleanupOnce.Do(func() { + if testEnv != nil { + fmt.Fprintf(os.Stderr, "Tearing down test environment...\n") + ctx, cancel := context.WithTimeout(context.Background(), testutil.DefaultTimeout) + defer cancel() + if err := testEnv.Teardown(ctx); err != nil { + fmt.Fprintf(os.Stderr, "Warning: failed to teardown test environment: %v\n", err) + } + } + }) +} + +// setupClusterDependencies installs Prometheus and CertManager if needed. +// Called by cluster_suite_test.go. +func setupClusterDependencies(ctx SpecContext) { + if !skipPrometheusInstall { + By("checking if prometheus is installed already") + isPrometheusOperatorAlreadyInstalled = testutil.IsPrometheusCRDsInstalled(ctx, GinkgoWriter) + if !isPrometheusOperatorAlreadyInstalled { + _, _ = fmt.Fprintf(GinkgoWriter, "Installing Prometheus Operator...\n") + Expect(testutil.InstallPrometheusOperator(ctx, GinkgoWriter)).To(Succeed(), "Failed to install Prometheus Operator") + } else { + _, _ = fmt.Fprintf(GinkgoWriter, "WARNING: Prometheus Operator is already installed. Skipping installation...\n") + } + } + if !skipCertManagerInstall { + By("checking if cert manager is installed already") + isCertManagerAlreadyInstalled = testutil.IsCertManagerCRDsInstalled(ctx, GinkgoWriter) + if !isCertManagerAlreadyInstalled { + _, _ = fmt.Fprintf(GinkgoWriter, "Installing CertManager...\n") + Expect(testutil.InstallCertManager(ctx, GinkgoWriter)).To(Succeed(), "Failed to install CertManager") + // Fresh install - need to wait for webhook to be ready (can take up to 90s) + By("waiting for cert-manager webhook to be ready (fresh install)") + Expect(testutil.WaitForCertManagerWebhook(ctx, GinkgoWriter)).To(Succeed(), "Cert-manager webhook not ready") + } else { + _, _ = fmt.Fprintf(GinkgoWriter, "WARNING: CertManager is already installed. Skipping installation...\n") + // Already installed - webhook should be ready, but verify quickly + } + } +} + +// cleanupClusterDependencies uninstalls Prometheus and CertManager if we installed them. +// Called by cluster_suite_test.go. +func cleanupClusterDependencies(ctx SpecContext) { + if !skipPrometheusInstall && !isPrometheusOperatorAlreadyInstalled { + _, _ = fmt.Fprintf(GinkgoWriter, "Uninstalling Prometheus Operator...\n") + testutil.UninstallPrometheusOperator(ctx, GinkgoWriter) + } + if !skipCertManagerInstall && !isCertManagerAlreadyInstalled { + _, _ = fmt.Fprintf(GinkgoWriter, "Uninstalling CertManager...\n") + testutil.UninstallCertManager(ctx, GinkgoWriter) + } +} + +// buildAndLoadImages builds and loads Docker images to Kind. +// Called by cluster_suite_test.go. +func buildAndLoadImages(ctx SpecContext) { + By("building the manager(Operator) image") + cmd := exec.CommandContext(ctx, "make", "docker-build", "IMG="+image) + _, err := testutil.Run(cmd, GinkgoWriter) + ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to build the manager(Operator) image") + + By("loading the manager(Operator) image on Kind") + err = testutil.LoadImageToKindClusterWithName(ctx, image, GinkgoWriter) + ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to load the manager(Operator) image into Kind") + + By("building the gnmi-test-server image") + cmd = exec.CommandContext(ctx, "make", "docker-build-test-gnmi-server", "TEST_SERVER_IMG="+serverImage) + _, err = testutil.Run(cmd, GinkgoWriter) + ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to build the gnmi-test-server image") + + By("loading the gnmi-test-server image on Kind") + err = testutil.LoadImageToKindClusterWithName(ctx, serverImage, GinkgoWriter) + ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to load the gnmi-test-server image into Kind") +} diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 3702bb802..f59c18c69 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -1,3 +1,5 @@ +//go:build !cluster && !envtest + // SPDX-FileCopyrightText: 2025 SAP SE or an SAP affiliate company and IronCore contributors // SPDX-License-Identifier: Apache-2.0 diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index e583ec9cb..2dabadeff 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -1,3 +1,5 @@ +//go:build !cluster && !envtest + // SPDX-FileCopyrightText: 2025 SAP SE or an SAP affiliate company and IronCore contributors // SPDX-License-Identifier: Apache-2.0 diff --git a/test/e2e/envtest_suite_test.go b/test/e2e/envtest_suite_test.go new file mode 100644 index 000000000..c8cfa522c --- /dev/null +++ b/test/e2e/envtest_suite_test.go @@ -0,0 +1,42 @@ +// SPDX-FileCopyrightText: 2025 SAP SE or an SAP affiliate company and IronCore contributors +// SPDX-License-Identifier: Apache-2.0 + +//go:build envtest + +package e2e + +import ( + "fmt" + "testing" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + + "github.com/ironcore-dev/network-operator/test/e2e/testutil" +) + +// TestE2E runs the e2e test suite in envtest mode. +func TestEnvtest(t *testing.T) { + RegisterFailHandler(Fail) + _, _ = fmt.Fprintf(GinkgoWriter, "Starting network-operator tests in ENVTEST mode\n") + RunSpecs(t, "e2e suite (envtest)") +} + +// BeforeSuite initializes the envtest environment. +// Envtest runs in-process, so no special parallel handling is needed. +var _ = BeforeSuite(func(ctx SpecContext) { + logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) + SetDefaultEventuallyTimeout(testutil.DefaultTimeout) + SetDefaultEventuallyPollingInterval(time.Second) + + initTestEnv(ctx) +}) + +// AfterSuite cleans up the envtest environment. +var _ = AfterSuite(func(ctx SpecContext) { + fmt.Fprintf(GinkgoWriter, "Tearing down test environment...\n") + cleanupTestEnv(ctx) +}) diff --git a/test/e2e/envtest_test.go b/test/e2e/envtest_test.go new file mode 100644 index 000000000..852089b6e --- /dev/null +++ b/test/e2e/envtest_test.go @@ -0,0 +1,634 @@ +// SPDX-FileCopyrightText: 2025 SAP SE or an SAP affiliate company and IronCore contributors +// SPDX-License-Identifier: Apache-2.0 + +//go:build envtest + +package e2e + +import ( + "context" + "fmt" + "os" + "path/filepath" + "slices" + "strings" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "golang.org/x/tools/txtar" + + corev1 "k8s.io/api/core/v1" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/events" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/cache" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/yaml" + + "github.com/ironcore-dev/network-operator/api/core/v1alpha1" + nx "github.com/ironcore-dev/network-operator/internal/controller/cisco/nx" + "github.com/ironcore-dev/network-operator/internal/controller/core" + "github.com/ironcore-dev/network-operator/internal/resourcelock" + "github.com/ironcore-dev/network-operator/test/e2e/testutil" +) + +// reconcileTestNamespacePrefix is used with GenerateName to create unique test namespaces. +// This isolates tests from other resources in the cluster (e.g., from the deployed operator). +const reconcileTestNamespacePrefix = "reconcile-test-" + +// testEnv is the envtest test environment. +var testEnv *testutil.EnvtestEnvironment + +func init() { + _, _ = fmt.Fprintf(GinkgoWriter, "Starting network-operator tests in ENVTEST mode\n") +} + +// initTestEnv initializes the envtest environment. +func initTestEnv(ctx SpecContext) { + By("initializing envtest environment") + testEnv = testutil.NewEnvtestEnvironment() + Expect(testEnv.Setup(ctx)).To(Succeed()) +} + +// cleanupTestEnv performs envtest-specific cleanup (none needed). +func cleanupTestEnv(_ SpecContext) { + // No additional cleanup needed for envtest +} + +// ============================================================================ +// Provider test helpers +// ============================================================================ + +// ProviderTestContext holds the context for a provider-specific test run. +type ProviderTestContext struct { + Provider testutil.ProviderType + Manager ctrl.Manager + Locker *resourcelock.ResourceLocker + Namespace string + CancelFunc context.CancelFunc +} + +// SetupProviderTest creates a new manager with controllers for the given provider. +// The manager only watches the specified namespace to avoid conflicts with other controllers. +// Call the returned cleanup function in AfterEach/AfterAll. +func SetupProviderTest(providerCfg testutil.ProviderConfig, k8sClient client.Client, restCfg *rest.Config, namespace string) *ProviderTestContext { + GinkgoHelper() + + providerCtx, providerCancel := context.WithCancel(context.Background()) //nolint:gosec // cancel stored in ProviderTestContext + + mgr, err := ctrl.NewManager(restCfg, ctrl.Options{ + Scheme: k8sClient.Scheme(), + Logger: GinkgoLogr, + Cache: cache.Options{ + DefaultNamespaces: map[string]cache.Config{ + namespace: {}, + }, + }, + }) + Expect(err).ToNot(HaveOccurred()) + + // Ignore events during tests + recorder := events.NewFakeRecorder(0) + go func() { + for range recorder.Events { //nolint:revive // intentionally drain events + } + }() + + locker, err := resourcelock.NewResourceLocker(mgr.GetClient(), namespace, 15*time.Second, 10*time.Second) + Expect(err).NotTo(HaveOccurred()) + + err = mgr.Add(locker) + Expect(err).NotTo(HaveOccurred()) + + providerFunc := providerCfg.NewProvider + + // Register all controllers + registerControllers(providerCtx, mgr, recorder, providerFunc, locker) + + go func() { + defer GinkgoRecover() + err = mgr.Start(providerCtx) + if providerCtx.Err() == nil { + Expect(err).ToNot(HaveOccurred(), "failed to run manager") + } + }() + + return &ProviderTestContext{ + Provider: providerCfg.Name, + Manager: mgr, + Locker: locker, + Namespace: namespace, + CancelFunc: providerCancel, + } +} + +// TeardownProviderTest stops the manager for a provider test. +func TeardownProviderTest(ptc *ProviderTestContext) { + if ptc != nil && ptc.CancelFunc != nil { + ptc.CancelFunc() + } +} + +// registerControllers registers all controllers with the manager. +func registerControllers(ctx context.Context, mgr ctrl.Manager, recorder *events.FakeRecorder, providerFunc testutil.ProviderFactory, locker *resourcelock.ResourceLocker) { + var err error + + err = (&core.PrefixSetReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Recorder: recorder, + Provider: providerFunc, + Locker: locker, + }).SetupWithManager(ctx, mgr) + Expect(err).NotTo(HaveOccurred()) + + err = (&core.RoutingPolicyReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Recorder: recorder, + Provider: providerFunc, + Locker: locker, + }).SetupWithManager(ctx, mgr) + Expect(err).NotTo(HaveOccurred()) + + err = (&core.InterfaceReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Recorder: recorder, + Provider: providerFunc, + Locker: locker, + RequeueInterval: time.Minute, + }).SetupWithManager(ctx, mgr) + Expect(err).NotTo(HaveOccurred()) + + err = (&core.VLANReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Recorder: recorder, + Provider: providerFunc, + Locker: locker, + RequeueInterval: time.Minute, + }).SetupWithManager(ctx, mgr) + Expect(err).NotTo(HaveOccurred()) + + err = (&core.VRFReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Recorder: recorder, + Provider: providerFunc, + Locker: locker, + }).SetupWithManager(ctx, mgr) + Expect(err).NotTo(HaveOccurred()) + + err = (&core.NTPReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Recorder: recorder, + Provider: providerFunc, + Locker: locker, + }).SetupWithManager(ctx, mgr) + Expect(err).NotTo(HaveOccurred()) + + err = (&core.DNSReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Recorder: recorder, + Provider: providerFunc, + Locker: locker, + }).SetupWithManager(ctx, mgr) + Expect(err).NotTo(HaveOccurred()) + + err = (&core.LLDPReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Recorder: recorder, + Provider: providerFunc, + Locker: locker, + RequeueInterval: time.Minute, + }).SetupWithManager(ctx, mgr) + Expect(err).NotTo(HaveOccurred()) + + err = (&core.BannerReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Recorder: recorder, + Provider: providerFunc, + Locker: locker, + }).SetupWithManager(ctx, mgr) + Expect(err).NotTo(HaveOccurred()) + + err = (&core.OSPFReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Recorder: recorder, + Provider: providerFunc, + Locker: locker, + RequeueInterval: time.Minute, + }).SetupWithManager(ctx, mgr) + Expect(err).NotTo(HaveOccurred()) + + err = (&core.PIMReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Recorder: recorder, + Provider: providerFunc, + Locker: locker, + }).SetupWithManager(ctx, mgr) + Expect(err).NotTo(HaveOccurred()) + + err = (&core.NetworkVirtualizationEdgeReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Recorder: recorder, + Provider: providerFunc, + Locker: locker, + RequeueInterval: time.Minute, + }).SetupWithManager(ctx, mgr) + Expect(err).NotTo(HaveOccurred()) + + err = (&core.EVPNInstanceReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Recorder: recorder, + Provider: providerFunc, + Locker: locker, + }).SetupWithManager(ctx, mgr) + Expect(err).NotTo(HaveOccurred()) + + // NX-OS specific controllers + err = (&nx.VPCDomainReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Recorder: recorder, + Provider: providerFunc, + Locker: locker, + RequeueInterval: time.Minute, + }).SetupWithManager(ctx, mgr) + Expect(err).NotTo(HaveOccurred()) + + err = (&core.BGPReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Recorder: recorder, + Provider: providerFunc, + Locker: locker, + RequeueInterval: time.Minute, + }).SetupWithManager(ctx, mgr) + Expect(err).NotTo(HaveOccurred()) + + err = (&core.BGPPeerReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Recorder: recorder, + Provider: providerFunc, + Locker: locker, + RequeueInterval: time.Minute, + }).SetupWithManager(ctx, mgr) + Expect(err).NotTo(HaveOccurred()) + + err = (&core.SyslogReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Recorder: recorder, + Provider: providerFunc, + Locker: locker, + }).SetupWithManager(ctx, mgr) + Expect(err).NotTo(HaveOccurred()) + + err = (&core.SNMPReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Recorder: recorder, + Provider: providerFunc, + Locker: locker, + }).SetupWithManager(ctx, mgr) + Expect(err).NotTo(HaveOccurred()) + + err = (&core.ManagementAccessReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Recorder: recorder, + Provider: providerFunc, + Locker: locker, + }).SetupWithManager(ctx, mgr) + Expect(err).NotTo(HaveOccurred()) + + err = (&core.AccessControlListReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Recorder: recorder, + Provider: providerFunc, + Locker: locker, + }).SetupWithManager(ctx, mgr) + Expect(err).NotTo(HaveOccurred()) + + err = (&core.DHCPRelayReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Recorder: recorder, + Provider: providerFunc, + Locker: locker, + RequeueInterval: time.Minute, + }).SetupWithManager(ctx, mgr) + Expect(err).NotTo(HaveOccurred()) + + err = (&core.ISISReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Recorder: recorder, + Provider: providerFunc, + Locker: locker, + }).SetupWithManager(ctx, mgr) + Expect(err).NotTo(HaveOccurred()) +} + +// CreateTestDevice creates a Device pointing to the gNMI server with a generated name. +func CreateTestDevice(ctx context.Context, c client.Client, gnmiAddr, namespace string) (*v1alpha1.Device, error) { + device := &v1alpha1.Device{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "test-device-", + Namespace: namespace, + }, + Spec: v1alpha1.DeviceSpec{ + Endpoint: v1alpha1.Endpoint{ + Address: gnmiAddr, + }, + }, + } + if err := c.Create(ctx, device); err != nil { + return nil, err + } + + // Set the device status to Running so that dependent resources can reconcile + device.Status.Phase = v1alpha1.DevicePhaseRunning + if err := c.Status().Update(ctx, device); err != nil { + return nil, err + } + + return device, nil +} + +// ============================================================================ +// Reconciliation tests +// ============================================================================ + +var _ = Describe("gNMI requests tests", func() { + // Resolve provider during tree construction so we can generate individual It nodes + projectDir, err := testutil.GetProjectDir() + if err != nil { + Fail(fmt.Sprintf("Failed to get project directory: %v", err)) + } + + provider := os.Getenv("E2E_PROVIDER") + providerNames := make([]string, len(testutil.SupportedProviders)) + for i, cfg := range testutil.SupportedProviders { + providerNames[i] = string(cfg.Name) + } + + // If provider is invalid, create a failing test with clear message + if provider == "" { + It("requires E2E_PROVIDER to be set", func() { + Fail(fmt.Sprintf("E2E_PROVIDER not set. Please set E2E_PROVIDER to one of: %s", strings.Join(providerNames, ", "))) + }) + return + } + + providerIdx := slices.IndexFunc(testutil.SupportedProviders, func(cfg testutil.ProviderConfig) bool { + return string(cfg.Name) == provider + }) + if providerIdx < 0 { + It("requires valid E2E_PROVIDER", func() { + Fail(fmt.Sprintf("E2E_PROVIDER=%q is not a supported provider. Valid values: %s", provider, strings.Join(providerNames, ", "))) + }) + return + } + + providerCfg := testutil.SupportedProviders[providerIdx] + testdataDir := filepath.Join(projectDir, "test", "e2e", "testdata", string(providerCfg.Name)) + + if _, err := os.Stat(testdataDir); os.IsNotExist(err) { + It("requires testdata directory", func() { + Fail(fmt.Sprintf("Testdata directory does not exist for provider %q: %s", provider, testdataDir)) + }) + return + } + + // Discover test files during tree construction + testFiles, err := filepath.Glob(filepath.Join(testdataDir, "*.txt")) + if err != nil || len(testFiles) == 0 { + It("requires test files", func() { + if err != nil { + Fail(fmt.Sprintf("Failed to glob testdata: %v", err)) + } + Fail(fmt.Sprintf("No test files found in %s", testdataDir)) + }) + return + } + + Describe(fmt.Sprintf("Provider: %s", providerCfg.Name), Ordered, func() { + var ptc *ProviderTestContext + var device *v1alpha1.Device + var testNamespace string + + BeforeAll(func(ctx SpecContext) { + By("creating dedicated test namespace") + ns := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: reconcileTestNamespacePrefix, + }, + } + Expect(testEnv.Client().Create(ctx, ns)).To(Succeed()) + testNamespace = ns.Name + + By(fmt.Sprintf("setting up %s provider", providerCfg.Name)) + ptc = SetupProviderTest(providerCfg, testEnv.Client(), testEnv.RESTConfig(), testNamespace) + }) + + AfterAll(func(ctx SpecContext) { + By(fmt.Sprintf("tearing down %s provider manager", providerCfg.Name)) + TeardownProviderTest(ptc) + + By("deleting test namespace") + ns := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: testNamespace, + }, + } + _ = testEnv.Client().Delete(ctx, ns) + }) + + AfterEach(func(ctx SpecContext) { + By("cleaning up resources") + cleanupAllResources(testEnv.Client(), testNamespace) + + if device == nil { + return + } + + By("deleting test device") + Expect(client.IgnoreNotFound(testEnv.Client().Delete(ctx, device))).To(Succeed()) + Eventually(func(g Gomega) { + err := testEnv.Client().Get(ctx, client.ObjectKeyFromObject(device), &v1alpha1.Device{}) + g.Expect(client.IgnoreNotFound(err)).To(Succeed()) + g.Expect(err).To(HaveOccurred(), "Device should be deleted") + }).Should(Succeed()) + device = nil + + By("clearing gNMI state for next test") + Expect(testEnv.ClearGNMIState(ctx)).To(Succeed()) + }) + + // Generate individual It nodes for each test file + for _, testFile := range testFiles { + testFile := testFile // capture for closure + testName := filepath.Base(testFile) + testName = testName[:len(testName)-4] // remove .txt + + It("should reconcile "+testName, func(ctx SpecContext) { + By("parsing testdata file") + a, err := txtar.ParseFile(testFile) + Expect(err).NotTo(HaveOccurred(), "Failed to parse test file: %s", testFile) + Expect(len(a.Files)).To(BeNumerically(">=", 2), "Expected at least 2 files (resource(s) and state)") + + var state, preload []byte + var resources []txtar.File + for _, f := range a.Files { + switch f.Name { + case "state/expect": + state = f.Data + case "state/preload": + preload = f.Data + default: + resources = append(resources, f) + } + } + Expect(state).NotTo(BeEmpty(), "Expected '-- state/expect --' section in testdata") + Expect(resources).NotTo(BeEmpty(), "Expected at least one resource in testdata") + + // Preload gNMI state BEFORE creating Device (e.g., bootTime for Device controller) + if len(preload) > 0 { + By("preloading gNMI state") + Expect(testEnv.PreloadGNMIState(ctx, preload)).To(Succeed(), "Failed to preload gNMI state") + } + + By("creating test device") + device, err = testutil.CreateTestDevice(ctx, testEnv.Client(), testEnv.GNMIAddress(), testNamespace) + Expect(err).NotTo(HaveOccurred()) + + By(fmt.Sprintf("creating %d resource(s) from testdata", len(resources))) + for _, res := range resources { + obj := createResourceFromTxtar(ctx, testEnv.Client(), res, device.Name, testNamespace) + waitForResource(ctx, testEnv.Client(), obj) + } + + By("verifying gNMI state matches expected JSON") + gnmiState, err := testEnv.GetGNMIState(ctx) + Expect(err).NotTo(HaveOccurred()) + + err = testutil.CompareJSON(string(gnmiState), string(state)) + Expect(err).NotTo(HaveOccurred(), "gNMI state does not match expected JSON") + }) + } + }) +}) + +// createResourceFromTxtar creates a K8s resource from txtar file data. +// The file name format is "kind/name" (e.g., "prefixset/my-prefixset"). +// It substitutes "device" in deviceRef.name with the actual device name. +func createResourceFromTxtar(ctx SpecContext, c client.Client, res txtar.File, deviceName, namespace string) client.Object { + obj := &unstructured.Unstructured{} + Expect(yaml.Unmarshal(res.Data, obj)).To(Succeed(), "Failed to unmarshal %s", res.Name) + + // Set the namespace + obj.SetNamespace(namespace) + + // Update deviceRef.name to use the actual device name + if spec, ok := obj.Object["spec"].(map[string]any); ok { + if deviceRef, ok := spec["deviceRef"].(map[string]any); ok { + deviceRef["name"] = deviceName + } + } + // Also update the device label + labels := obj.GetLabels() + if labels != nil { + if _, ok := labels[v1alpha1.DeviceLabel]; ok { + labels[v1alpha1.DeviceLabel] = deviceName + obj.SetLabels(labels) + } + } + + Expect(c.Create(ctx, obj)).To(Succeed(), "Failed to create %s", res.Name) + return obj +} + +// waitForResource waits for a resource to be in Ready=True or Configured=True. +// If Configured condition exists, it checks it, otherwise it falls back to Ready condition. +// Skips config-only --controller-less-- resources that don't have status conditions (e.g., InterfaceConfig). +func waitForResource(ctx SpecContext, c client.Client, obj client.Object) { + key := client.ObjectKeyFromObject(obj) + gvk := obj.GetObjectKind().GroupVersionKind() + + // Add as needed. + switch gvk.Kind { + case "InterfaceConfig", "LLDPConfig", "BGPConfig", "NVEConfig", "ManagementAccessConfig": + return + } + + Eventually(func(g Gomega) { + r := &unstructured.Unstructured{} + r.SetGroupVersionKind(gvk) + g.Expect(c.Get(ctx, key, r)).To(Succeed()) + + conditions, err := testutil.ExtractConditions(r) + g.Expect(err).NotTo(HaveOccurred()) + + conditionToCheck := string(v1alpha1.ReadyCondition) + if apimeta.FindStatusCondition(conditions, string(v1alpha1.ConfiguredCondition)) != nil { + conditionToCheck = string(v1alpha1.ConfiguredCondition) + } + + g.Expect(apimeta.IsStatusConditionTrue(conditions, conditionToCheck)).To(BeTrue()) + }).Should(Succeed()) +} + +// cleanupAllResources deletes all test resources and lets the controller handle finalizer cleanup. +// Uses a background context with timeout to ensure cleanup completes even on interrupt. +// Deletion order: CoreResources first (have finalizers), then ConfigResources, then Device. +func cleanupAllResources(c client.Client, namespace string) { + // Use background context with timeout - cleanup must complete even on Ctrl+C + cleanupCtx, cancel := context.WithTimeout(context.Background(), testutil.LongTimeout) + defer cancel() + + deleteResources := func(gvks []schema.GroupVersionKind) { + for _, gvk := range gvks { + list := &unstructured.UnstructuredList{} + list.SetGroupVersionKind(schema.GroupVersionKind{ + Group: gvk.Group, + Version: gvk.Version, + Kind: gvk.Kind + "List", + }) + + if err := c.List(cleanupCtx, list, client.InNamespace(namespace)); err != nil { + if apimeta.IsNoMatchError(err) { + continue // CRD not installed, skip + } + Expect(err).NotTo(HaveOccurred(), "Failed to list %s", gvk.Kind) + } + + // Delete all resources - controller will handle finalizer removal + for _, item := range list.Items { + Expect(client.IgnoreNotFound(c.Delete(cleanupCtx, &item))).To(Succeed()) + } + } + } + + // Delete core resources first (have finalizers that need Device + configs) + deleteResources(testutil.CoreResources) + // Then delete config resources + deleteResources(testutil.ConfigResources) +} diff --git a/test/e2e/testdata/cisco-nxos-gnmi/acl.txt b/test/e2e/testdata/cisco-nxos-gnmi/acl.txt new file mode 100644 index 000000000..95e5a8841 --- /dev/null +++ b/test/e2e/testdata/cisco-nxos-gnmi/acl.txt @@ -0,0 +1,79 @@ +# NX-OS Access Control List Integration Test +# +# Tests ACL configuration with permit/deny entries. + + + +-- accesscontrollist/test-acl -- +apiVersion: networking.metal.ironcore.dev/v1alpha1 +kind: AccessControlList +metadata: + name: test-acl + namespace: default + labels: + networking.metal.ironcore.dev/device: device +spec: + deviceRef: + name: device + name: BLOCK-EXTERNAL + entries: + - sequence: 10 + action: Permit + protocol: IP + sourceAddress: 10.0.0.0/8 + destinationAddress: 0.0.0.0/0 + description: "Allow internal" + - sequence: 20 + action: Deny + protocol: IP + sourceAddress: 0.0.0.0/0 + destinationAddress: 0.0.0.0/0 + description: "Deny all" + +-- state/preload -- +{ + "System": { + "procsys-items": { + "bootTime": "1700000000" + } + } +} + +-- state/expect -- +{ + "System": { + "acl-items": { + "ipv4-items": { + "name-items": { + "ACL-list": [ + { + "name": "BLOCK-EXTERNAL", + "seq-items": { + "ACE-list": [ + { + "action": "deny", + "dstPrefix": "0.0.0.0", + "protocol": 0, + "seqNum": 20, + "srcPrefix": "0.0.0.0" + }, + { + "action": "permit", + "dstPrefix": "0.0.0.0", + "protocol": 0, + "seqNum": 10, + "srcPrefix": "10.0.0.0", + "srcPrefixLength": 8 + } + ] + } + } + ] + } + } + }, + "procsys-items": { + "bootTime": "1700000000" + } + } +} diff --git a/test/e2e/testdata/cisco-nxos-gnmi/banner.txt b/test/e2e/testdata/cisco-nxos-gnmi/banner.txt new file mode 100644 index 000000000..239703a56 --- /dev/null +++ b/test/e2e/testdata/cisco-nxos-gnmi/banner.txt @@ -0,0 +1,41 @@ +# NX-OS Banner Integration Test + + +-- state/preload -- +{ + "System": { + "procsys-items": { + "bootTime": "1700000000" + } + } +} + +-- banner/prelogin -- +apiVersion: networking.metal.ironcore.dev/v1alpha1 +kind: Banner +metadata: + name: prelogin + namespace: default + labels: + networking.metal.ironcore.dev/device: device +spec: + deviceRef: + name: device + type: PreLogin + message: + inline: "Authorized users only. All activity is monitored." + +-- state/expect -- +{ + "System": { + "userext-items": { + "preloginbanner-items": { + "delimiter": "^", + "message": "Authorized users only. All activity is monitored." + } + }, + "procsys-items": { + "bootTime": "1700000000" + } + } +} diff --git a/test/e2e/testdata/cisco-nxos-gnmi/bgp_bgppeer.txt b/test/e2e/testdata/cisco-nxos-gnmi/bgp_bgppeer.txt new file mode 100644 index 000000000..990761e8f --- /dev/null +++ b/test/e2e/testdata/cisco-nxos-gnmi/bgp_bgppeer.txt @@ -0,0 +1,226 @@ +# NX-OS BGP and BGPPeer Integration Test +# +# Resource dependency chain: +# loopback-vtep -> Loopback interface (spec.name: lo0) for BGP router-id/update-source +# evpn-settings -> BGPConfig (NX-OS provider config) for advertise-pip +# evpn-bgp -> BGP instance with L2VPN EVPN, references BGPConfig +# spine1 -> BGPPeer referencing BGP instance and loopback for localAddress + +-- state/preload -- +{ + "System": { + "procsys-items": { + "bootTime": "1700000000" + } + } +} + +-- interface/loopback-vtep -- +apiVersion: networking.metal.ironcore.dev/v1alpha1 +kind: Interface +metadata: + name: loopback-vtep + namespace: default + labels: + networking.metal.ironcore.dev/device: device +spec: + deviceRef: + name: device + name: lo0 + adminState: Up + type: Loopback + ipv4: + addresses: + - 10.0.0.1/32 + +-- bgpconfig/evpn-settings -- +apiVersion: nx.cisco.networking.metal.ironcore.dev/v1alpha1 +kind: BGPConfig +metadata: + name: evpn-settings + namespace: default +spec: + addressFamilies: + l2vpnEvpn: + advertisePIP: true + +-- bgp/evpn-bgp -- +apiVersion: networking.metal.ironcore.dev/v1alpha1 +kind: BGP +metadata: + name: evpn-bgp + namespace: default + labels: + networking.metal.ironcore.dev/device: device +spec: + deviceRef: + name: device + asNumber: 65000 + routerId: "10.0.0.1" + providerConfigRef: + apiVersion: nx.cisco.networking.metal.ironcore.dev/v1alpha1 + kind: BGPConfig + name: evpn-settings + addressFamilies: + l2vpnEvpn: + enabled: true + routeTargetPolicy: + retainAll: true + +-- bgppeer/spine1 -- +apiVersion: networking.metal.ironcore.dev/v1alpha1 +kind: BGPPeer +metadata: + name: spine1 + namespace: default + labels: + networking.metal.ironcore.dev/device: device +spec: + deviceRef: + name: device + bgpRef: + name: evpn-bgp + address: "10.0.0.2" + asNumber: 65000 + description: "EVPN peering with spine1" + localAddress: + interfaceRef: + name: loopback-vtep + addressFamilies: + l2vpnEvpn: + enabled: true + sendCommunity: Both + routeReflectorClient: true + +-- state/expect -- +{ + "System": { + "bgp-items": { + "inst-items": { + "adminSt": "enabled", + "asn": "65000", + "dom-items": { + "Dom-list": [ + { + "name": "default", + "rtrId": "10.0.0.1", + "rtrIdAuto": "disabled", + "af-items": { + "DomAf-list": [ + { + "advPip": "enabled", + "exportGwIp": "disabled", + "maxExtEcmp": 1, + "maxExtIntEcmp": 1, + "retainRttAll": "enabled", + "type": "l2vpn-evpn" + } + ] + }, + "peer-items": { + "Peer-list": [ + { + "addr": "10.0.0.2", + "adminSt": "enabled", + "asn": "65000", + "asnType": "none", + "name": "EVPN peering with spine1", + "srcIf": "lo0", + "af-items": { + "PeerAf-list": [ + { + "ctrl": "rr-client", + "sendComExt": "enabled", + "sendComStd": "enabled", + "type": "l2vpn-evpn" + } + ] + } + } + ] + }, + "peercont-items": { + "PeerCont-list": [ + { + "name": "__operator-managed--default__" + } + ] + } + } + ] + } + } + }, + "fm-items": { + "bgp-items": { + "adminSt": "enabled" + }, + "evpn-items": { + "adminSt": "enabled" + } + }, + "icmpv4-items": { + "inst-items": { + "dom-items": { + "Dom-list": [ + { + "name": "default", + "if-items": { + "If-list": [ + { + "ctrl": "port-unreachable,redirect", + "id": "lo0" + } + ] + } + } + ] + } + } + }, + "intf-items": { + "lb-items": { + "LbRtdIf-list": [ + { + "adminSt": "up", + "id": "lo0", + "rtvrfMbr-items": { + "tDn": "/System/inst-items/Inst-list[name='default']" + } + } + ] + } + }, + "ipv4-items": { + "inst-items": { + "dom-items": { + "Dom-list": [ + { + "name": "default", + "if-items": { + "If-list": [ + { + "addr-items": { + "Addr-list": [ + { + "addr": "10.0.0.1/32", + "pref": 0, + "tag": 0, + "type": "primary" + } + ] + }, + "id": "lo0" + } + ] + } + } + ] + } + } + }, + "procsys-items": { + "bootTime": "1700000000" + } + } +} diff --git a/test/e2e/testdata/cisco-nxos-gnmi/dhcprelay.txt b/test/e2e/testdata/cisco-nxos-gnmi/dhcprelay.txt new file mode 100644 index 000000000..232f45866 --- /dev/null +++ b/test/e2e/testdata/cisco-nxos-gnmi/dhcprelay.txt @@ -0,0 +1,178 @@ +# NX-OS DHCP Relay Integration Test +# +# Resource dependency chain: +# vlan100 -> VLAN (id: 100, name: DHCP-VLAN) +# svi100 -> RoutedVLAN Interface referencing vlan100 (spec.name: Vlan100) +# relay-config -> DHCPRelay with server and interface reference + + +-- state/preload -- +{ + "System": { + "procsys-items": { + "bootTime": "1700000000" + } + } +} + +-- vlan/vlan100 -- +apiVersion: networking.metal.ironcore.dev/v1alpha1 +kind: VLAN +metadata: + name: vlan100 + namespace: default + labels: + networking.metal.ironcore.dev/device: device +spec: + deviceRef: + name: device + id: 100 + name: DHCP-VLAN + +-- interface/svi100 -- +apiVersion: networking.metal.ironcore.dev/v1alpha1 +kind: Interface +metadata: + name: svi100 + namespace: default + labels: + networking.metal.ironcore.dev/device: device +spec: + deviceRef: + name: device + name: Vlan100 + adminState: Up + type: RoutedVLAN + vlanRef: + name: vlan100 + ipv4: + addresses: + - 192.168.100.1/24 + +-- dhcprelay/relay-config -- +apiVersion: networking.metal.ironcore.dev/v1alpha1 +kind: DHCPRelay +metadata: + name: relay-config + namespace: default + labels: + networking.metal.ironcore.dev/device: device +spec: + deviceRef: + name: device + servers: + - "10.0.0.10" + interfaceRefs: + - name: svi100 + +-- state/expect -- +{ + "System": { + "bd-items": { + "bd-items": { + "BD-list": [ + { + "BdState": "active", + "adminSt": "active", + "fabEncap": "vlan-100", + "name": "DHCP-VLAN" + } + ] + } + }, + "dhcp-items": { + "inst-items": { + "relayif-items": { + "RelayIf-list": [ + { + "addr-items": { + "RelayAddr-list": [ + { + "address": "10.0.0.10", + "vrf": "!unspecified" + } + ] + }, + "id": "vlan100" + } + ] + } + } + }, + "fm-items": { + "dhcp-items": { + "adminSt": "enabled" + }, + "ifvlan-items": { + "adminSt": "enabled" + } + }, + "icmpv4-items": { + "inst-items": { + "dom-items": { + "Dom-list": [ + { + "if-items": { + "If-list": [ + { + "ctrl": "port-unreachable", + "id": "vlan100" + } + ] + }, + "name": "default" + } + ] + } + } + }, + "intf-items": { + "svi-items": { + "If-list": [ + { + "adminSt": "up", + "descr": "", + "id": "vlan100", + "medium": "bcast", + "mtu": 1500, + "rtvrfMbr-items": { + "tDn": "/System/inst-items/Inst-list[name='default']" + }, + "vlanId": 100 + } + ] + } + }, + "ipv4-items": { + "inst-items": { + "dom-items": { + "Dom-list": [ + { + "if-items": { + "If-list": [ + { + "addr-items": { + "Addr-list": [ + { + "addr": "192.168.100.1/24", + "pref": 0, + "tag": 0, + "type": "primary" + } + ] + }, + "id": "vlan100" + } + ] + }, + "name": "default" + } + ] + } + } + }, + "procsys-items": { + "bootTime": "1700000000" + } + } +} diff --git a/test/e2e/testdata/cisco-nxos-gnmi/dns.txt b/test/e2e/testdata/cisco-nxos-gnmi/dns.txt new file mode 100644 index 000000000..68c640c3c --- /dev/null +++ b/test/e2e/testdata/cisco-nxos-gnmi/dns.txt @@ -0,0 +1,63 @@ +# NX-OS DNS Integration Test + +-- state/preload -- +{ + "System": { + "procsys-items": { + "bootTime": "1700000000" + } + } +} + +-- dns/config -- +apiVersion: networking.metal.ironcore.dev/v1alpha1 +kind: DNS +metadata: + name: config + namespace: default + labels: + networking.metal.ironcore.dev/device: device +spec: + deviceRef: + name: device + adminState: Up + domain: example.com + servers: + - address: 10.10.10.10 + vrfName: management + +-- state/expect -- +{ + "System": { + "dns-items": { + "adminSt": "enabled", + "prof-items": { + "Prof-list": [ + { + "name": "default", + "dom-items": { + "name": "example.com" + }, + "vrf-items": { + "Vrf-list": [ + { + "name": "management", + "prov-items": { + "Provider-list": [ + { + "addr": "10.10.10.10" + } + ] + } + } + ] + } + } + ] + } + }, + "procsys-items": { + "bootTime": "1700000000" + } + } +} diff --git a/test/e2e/testdata/cisco-nxos-gnmi/evpninstance.txt b/test/e2e/testdata/cisco-nxos-gnmi/evpninstance.txt new file mode 100644 index 000000000..c54ba30f8 --- /dev/null +++ b/test/e2e/testdata/cisco-nxos-gnmi/evpninstance.txt @@ -0,0 +1,64 @@ +# NX-OS EVPNInstance Integration Test +# +# Tests EVPN Instance (L3VNI) for IP-VRF in VXLAN fabric. +# Using Routed type which doesn't require a VLAN dependency. + +-- state/preload -- +{ + "System": { + "procsys-items": { + "bootTime": "1700000000" + } + } +} + +-- evpninstance/l3vni -- +apiVersion: networking.metal.ironcore.dev/v1alpha1 +kind: EVPNInstance +metadata: + name: l3vni + namespace: default + labels: + networking.metal.ironcore.dev/device: device +spec: + deviceRef: + name: device + vni: 50000 + type: Routed + routeDistinguisher: "auto" + +-- state/expect -- +{ + "System": { + "eps-items": { + "epId-items": { + "Ep-list": [ + { + "epId": "1", + "nws-items": { + "vni-items": { + "Nw-list": [ + { + "vni": 50000, + "associateVrfFlag": true + } + ] + } + } + } + ] + } + }, + "fm-items": { + "nvo-items": { + "adminSt": "enabled" + }, + "vnsegment-items": { + "adminSt": "enabled" + } + }, + "procsys-items": { + "bootTime": "1700000000" + } + } +} diff --git a/test/e2e/testdata/cisco-nxos-gnmi/interfaces.txt b/test/e2e/testdata/cisco-nxos-gnmi/interfaces.txt new file mode 100644 index 000000000..d2c520759 --- /dev/null +++ b/test/e2e/testdata/cisco-nxos-gnmi/interfaces.txt @@ -0,0 +1,358 @@ +# Integration test for Interface resources +# +# Tests interface types and NX-OS specific InterfaceConfig: +# loopback-vtep -> Loopback with IPv4 address +# uplink-spine1 -> Physical L3 with unnumbered IPv4 + BFD +# edge-port -> Physical L2 with InterfaceConfig (STP edge, BufferBoost disabled) +# host-pc -> Aggregate L2 with vPC, LACP, InterfaceConfig (STP network, LACP options) + +-- state/preload -- +{ + "System": { + "procsys-items": { + "bootTime": "1700000000" + } + } +} + +-- interfaceconfig/edge-port-config -- +apiVersion: nx.cisco.networking.metal.ironcore.dev/v1alpha1 +kind: InterfaceConfig +metadata: + name: interface-nxconfig-edge + namespace: default +spec: + bufferBoost: + enabled: false + spanningTree: + portType: Edge + bpduGuard: true + +-- interfaceconfig/host-pc-config -- +apiVersion: nx.cisco.networking.metal.ironcore.dev/v1alpha1 +kind: InterfaceConfig +metadata: + name: interface-nxconfig-po + namespace: default +spec: + lacp: + vpcConvergence: true + suspendIndividual: false + spanningTree: + portType: Network + bpduFilter: true + +-- interface/loopback-vtep -- +apiVersion: networking.metal.ironcore.dev/v1alpha1 +kind: Interface +metadata: + labels: + networking.metal.ironcore.dev/device-name: device + name: loopback-vtep + namespace: default +spec: + deviceRef: + name: device + name: lo0 + description: NVE/VTEP Leaf1 + adminState: Up + type: Loopback + ipv4: + addresses: + - 10.0.0.10/32 + +-- interface/uplink-spine1 -- +apiVersion: networking.metal.ironcore.dev/v1alpha1 +kind: Interface +metadata: + labels: + networking.metal.ironcore.dev/device-name: device + name: uplink-spine1 + namespace: default +spec: + deviceRef: + name: device + name: eth1/1 + description: Leaf1 to Spine1 + adminState: Up + type: Physical + mtu: 9216 + ipv4: + unnumbered: + interfaceRef: + name: loopback-vtep + bfd: + enabled: true + desiredMinimumTxInterval: 300ms + requiredMinimumReceive: 300ms + detectionMultiplier: 3 + +-- interface/edge-port -- +apiVersion: networking.metal.ironcore.dev/v1alpha1 +kind: Interface +metadata: + name: edge-port + namespace: default + labels: + networking.metal.ironcore.dev/device: device +spec: + deviceRef: + name: device + name: eth1/2 + type: Physical + adminState: Up + mtu: 1500 + description: "Edge port with STP and BufferBoost config" + providerConfigRef: + apiVersion: nx.cisco.networking.metal.ironcore.dev/v1alpha1 + kind: InterfaceConfig + name: interface-nxconfig-edge + switchport: + mode: Trunk + nativeVlan: 1 + allowedVlans: + - 10 + +-- interface/host-pc -- +apiVersion: networking.metal.ironcore.dev/v1alpha1 +kind: Interface +metadata: + name: host-pc + namespace: default + labels: + networking.metal.ironcore.dev/device: device +spec: + deviceRef: + name: device + name: po10 + type: Aggregate + adminState: Up + mtu: 1500 + description: "vPC to Host with STP and LACP config" + providerConfigRef: + apiVersion: nx.cisco.networking.metal.ironcore.dev/v1alpha1 + kind: InterfaceConfig + name: interface-nxconfig-po + switchport: + mode: Trunk + nativeVlan: 1 + allowedVlans: + - 10 + aggregation: + controlProtocol: + mode: Active + memberInterfaceRefs: + - name: edge-port + multichassis: + enabled: true + id: 2 + +-- state/expect -- +{ + "System": { + "bfd-items": { + "inst-items": { + "if-items": { + "If-list": [ + { + "adminSt": "enabled", + "id": "eth1/1", + "ifka-items": { + "detectMult": 3, + "minRxIntvl": 300, + "minTxIntvl": 300 + } + } + ] + } + } + }, + "fm-items": { + "bfd-items": { + "adminSt": "enabled" + }, + "lacp-items": { + "adminSt": "enabled" + } + }, + "icmpv4-items": { + "inst-items": { + "dom-items": { + "Dom-list": [ + { + "name": "default", + "if-items": { + "If-list": [ + { + "ctrl": "port-unreachable,redirect", + "id": "lo0" + }, + { + "ctrl": "port-unreachable", + "id": "eth1/1" + } + ] + } + } + ] + } + } + }, + "intf-items": { + "aggr-items": { + "AggrIf-list": [ + { + "accessVlan": "vlan-1", + "adminSt": "up", + "aggrExtd-items": { + "bufferBoost": "enable" + }, + "descr": "vPC to Host with STP and LACP config", + "id": "po10", + "lacpVpcConvergence": "enable", + "layer": "Layer2", + "medium": "broadcast", + "mode": "trunk", + "mtu": 1500, + "nativeVlan": "vlan-1", + "pcMode": "active", + "rsmbrIfs-items": { + "RsMbrIfs-list": [ + { + "tDn": "/System/intf-items/phys-items/PhysIf-list[id='eth1/2']" + } + ] + }, + "suspIndividual": "disable", + "trunkVlans": "10", + "userCfgdFlags": "admin_layer,admin_mtu,admin_state" + } + ] + }, + "lb-items": { + "LbRtdIf-list": [ + { + "adminSt": "up", + "descr": "NVE/VTEP Leaf1", + "id": "lo0", + "rtvrfMbr-items": { + "tDn": "/System/inst-items/Inst-list[name='default']" + } + } + ] + }, + "phys-items": { + "PhysIf-list": [ + { + "FECMode": "auto", + "accessVlan": "unknown", + "adminSt": "up", + "descr": "Leaf1 to Spine1", + "id": "eth1/1", + "layer": "Layer3", + "medium": "p2p", + "mode": "access", + "mtu": 9216, + "nativeVlan": "unknown", + "physExtd-items": { + "bufferBoost": "enable" + }, + "rtvrfMbr-items": { + "tDn": "/System/inst-items/Inst-list[name='default']" + }, + "trunkVlans": "1-4094", + "userCfgdFlags": "admin_layer,admin_mtu,admin_state" + }, + { + "accessVlan": "vlan-1", + "adminSt": "up", + "descr": "Edge port with STP and BufferBoost config", + "FECMode": "auto", + "id": "eth1/2", + "layer": "Layer2", + "medium": "broadcast", + "mode": "trunk", + "mtu": 1500, + "nativeVlan": "vlan-1", + "trunkVlans": "10", + "userCfgdFlags": "admin_layer,admin_mtu,admin_state", + "physExtd-items": { + "bufferBoost": "disable" + } + } + ] + } + }, + "ipv4-items": { + "inst-items": { + "dom-items": { + "Dom-list": [ + { + "name": "default", + "if-items": { + "If-list": [ + { + "addr-items": { + "Addr-list": [ + { + "addr": "10.0.0.10/32", + "pref": 0, + "tag": 0, + "type": "primary" + } + ] + }, + "id": "lo0" + }, + { + "id": "eth1/1", + "unnumbered": "lo0" + } + ] + } + } + ] + } + } + }, + "stp-items": { + "inst-items": { + "if-items": { + "If-list": [ + { + "id": "eth1/2", + "mode": "edge", + "bpdufilter": "default", + "bpduguard": "enable" + }, + { + "id": "po10", + "mode": "network", + "bpdufilter": "enable", + "bpduguard": "default" + } + ] + } + } + }, + "vpc-items": { + "inst-items": { + "dom-items": { + "if-items": { + "If-list": [ + { + "id": 2, + "rsvpcConf-items": { + "tDn": "/System/intf-items/aggr-items/AggrIf-list[id='po10']" + } + } + ] + } + } + } + }, + "procsys-items": { + "bootTime": "1700000000" + } + } +} diff --git a/test/e2e/testdata/cisco-nxos-gnmi/isis.txt b/test/e2e/testdata/cisco-nxos-gnmi/isis.txt new file mode 100644 index 000000000..36a8361d8 --- /dev/null +++ b/test/e2e/testdata/cisco-nxos-gnmi/isis.txt @@ -0,0 +1,158 @@ +# NX-OS ISIS Integration Test +# +# Resource dependency chain: +# ethernet1 -> Physical Interface (spec.name: Ethernet1/1 -> eth1/1) +# fabric-isis -> ISIS instance referencing ethernet1 + +-- state/preload -- +{ + "System": { + "procsys-items": { + "bootTime": "1700000000" + } + } +} + +-- interface/ethernet1 -- +apiVersion: networking.metal.ironcore.dev/v1alpha1 +kind: Interface +metadata: + name: ethernet1 + namespace: default + labels: + networking.metal.ironcore.dev/device: device +spec: + deviceRef: + name: device + name: Ethernet1/1 + adminState: Up + type: Physical + ipv4: + addresses: + - 10.0.1.1/30 + +-- isis/fabric-isis -- +apiVersion: networking.metal.ironcore.dev/v1alpha1 +kind: ISIS +metadata: + name: fabric-isis + namespace: default + labels: + networking.metal.ironcore.dev/device: device +spec: + deviceRef: + name: device + adminState: Up + instance: FABRIC + networkEntityTitle: "49.0001.0000.0000.0001.00" + type: Level2 + addressFamilies: + - IPv4Unicast + interfaceRefs: + - name: ethernet1 + +-- state/expect -- +{ + "System": { + "fm-items": { + "isis-items": { + "adminSt": "enabled" + } + }, + "intf-items": { + "phys-items": { + "PhysIf-list": [ + { + "FECMode": "auto", + "accessVlan": "unknown", + "adminSt": "up", + "id": "eth1/1", + "layer": "Layer3", + "medium": "broadcast", + "mode": "access", + "mtu": 1500, + "nativeVlan": "unknown", + "physExtd-items": { + "bufferBoost": "enable" + }, + "rtvrfMbr-items": { + "tDn": "/System/inst-items/Inst-list[name='default']" + }, + "trunkVlans": "1-4094", + "userCfgdFlags": "admin_layer,admin_state" + } + ] + } + }, + "ipv4-items": { + "inst-items": { + "dom-items": { + "Dom-list": [ + { + "if-items": { + "If-list": [ + { + "addr-items": { + "Addr-list": [ + { + "addr": "10.0.1.1/30", + "pref": 0, + "tag": 0, + "type": "primary" + } + ] + }, + "id": "eth1/1" + } + ] + }, + "name": "default" + } + ] + } + } + }, + "isis-items": { + "inst-items": { + "Inst-list": [ + { + "adminSt": "enabled", + "dom-items": { + "Dom-list": [ + { + "af-items": { + "DomAf-list": [ + { + "type": "v4" + } + ] + }, + "if-items": { + "If-list": [ + { + "id": "eth1/1", + "networkTypeP2P": "on", + "v4Bfd": "inheritVrf", + "v4enable": true, + "v6Bfd": "", + "v6enable": false + } + ] + }, + "isType": "l2", + "name": "default", + "net": "49.0001.0000.0000.0001.00", + "passiveDflt": "l2" + } + ] + }, + "name": "FABRIC" + } + ] + } + }, + "procsys-items": { + "bootTime": "1700000000" + } + } +} diff --git a/test/e2e/testdata/cisco-nxos-gnmi/lldp.txt b/test/e2e/testdata/cisco-nxos-gnmi/lldp.txt new file mode 100644 index 000000000..f3527bad2 --- /dev/null +++ b/test/e2e/testdata/cisco-nxos-gnmi/lldp.txt @@ -0,0 +1,58 @@ +# NX-OS LLDP Integration Test + + +-- state/preload -- +{ + "System": { + "procsys-items": { + "bootTime": "1700000000" + } + } +} + +-- lldpconfig/lldp-settings -- +apiVersion: nx.cisco.networking.metal.ironcore.dev/v1alpha1 +kind: LLDPConfig +metadata: + name: lldp-settings + namespace: default +spec: + holdTime: 180 + initDelay: 5 + +-- lldp/config -- +apiVersion: networking.metal.ironcore.dev/v1alpha1 +kind: LLDP +metadata: + name: config + namespace: default + labels: + networking.metal.ironcore.dev/device: device +spec: + deviceRef: + name: device + adminState: Up + providerConfigRef: + apiVersion: nx.cisco.networking.metal.ironcore.dev/v1alpha1 + kind: LLDPConfig + name: lldp-settings + +-- state/expect -- +{ + "System": { + "fm-items": { + "lldp-items": { + "adminSt": "enabled" + } + }, + "lldp-items": { + "inst-items": { + "holdTime": 180, + "initDelayTime": 5 + } + }, + "procsys-items": { + "bootTime": "1700000000" + } + } +} diff --git a/test/e2e/testdata/cisco-nxos-gnmi/managementaccess.txt b/test/e2e/testdata/cisco-nxos-gnmi/managementaccess.txt new file mode 100644 index 000000000..f4932744c --- /dev/null +++ b/test/e2e/testdata/cisco-nxos-gnmi/managementaccess.txt @@ -0,0 +1,75 @@ +# NX-OS ManagementAccess Integration Test + +-- state/preload -- +{ + "System": { + "procsys-items": { + "bootTime": "1700000000" + } + } +} + +-- managementaccess/config -- +apiVersion: networking.metal.ironcore.dev/v1alpha1 +kind: ManagementAccess +metadata: + name: config + namespace: default + labels: + networking.metal.ironcore.dev/device: device +spec: + deviceRef: + name: device + grpc: + enabled: true + port: 9339 + vrfName: default + gnmi: + maxConcurrentCall: 8 + keepAliveTimeout: 600s + ssh: + enabled: true + timeout: 10m + sessionLimit: 32 + +-- state/expect -- +{ + "System": { + "fm-items": { + "grpc-items": { + "adminSt": "enabled" + }, + "ssh-items": { + "adminSt": "enabled" + } + }, + "grpc-items": { + "gnmi-items": { + "keepAliveTimeout": 600, + "maxCalls": 8 + }, + "port": 9339, + "useVrf": "default" + }, + "terml-items": { + "ln-items": { + "cons-items": { + "execTmeout-items": { + "timeout": 0 + } + }, + "vty-items": { + "execTmeout-items": { + "timeout": 10 + }, + "ssLmt-items": { + "sesLmt": 32 + } + } + } + }, + "procsys-items": { + "bootTime": "1700000000" + } + } +} diff --git a/test/e2e/testdata/cisco-nxos-gnmi/ntp.txt b/test/e2e/testdata/cisco-nxos-gnmi/ntp.txt new file mode 100644 index 000000000..f0cbbbcc2 --- /dev/null +++ b/test/e2e/testdata/cisco-nxos-gnmi/ntp.txt @@ -0,0 +1,62 @@ +# NX-OS NTP Integration Test + +-- state/preload -- +{ + "System": { + "procsys-items": { + "bootTime": "1700000000" + } + } +} + +-- ntp/config -- +apiVersion: networking.metal.ironcore.dev/v1alpha1 +kind: NTP +metadata: + name: config + namespace: default + labels: + networking.metal.ironcore.dev/device: device +spec: + deviceRef: + name: device + adminState: Up + sourceInterfaceName: mgmt0 + servers: + - address: ntp.example.com + prefer: true + vrfName: management + +-- state/expect -- +{ + "System": { + "fm-items": { + "ntpd-items": { + "adminSt": "enabled" + } + }, + "time-items": { + "adminSt": "enabled", + "logging": "disabled", + "prov-items": { + "NtpProvider-list": [ + { + "keyId": 0, + "maxPoll": 6, + "minPoll": 4, + "name": "ntp.example.com", + "preferred": true, + "provT": "server", + "vrf": "management" + } + ] + }, + "srcIf-items": { + "srcIf": "mgmt0" + } + }, + "procsys-items": { + "bootTime": "1700000000" + } + } +} diff --git a/test/e2e/testdata/cisco-nxos-gnmi/nve.txt b/test/e2e/testdata/cisco-nxos-gnmi/nve.txt new file mode 100644 index 000000000..1eb311208 --- /dev/null +++ b/test/e2e/testdata/cisco-nxos-gnmi/nve.txt @@ -0,0 +1,144 @@ +# NX-OS NVE (Network Virtualization Edge) Integration Test +# +# NVE requires a loopback interface as source. + +-- state/preload -- +{ + "System": { + "procsys-items": { + "bootTime": "1700000000" + } + } +} + +-- interface/lo-nve -- +apiVersion: networking.metal.ironcore.dev/v1alpha1 +kind: Interface +metadata: + name: lo-nve + namespace: default + labels: + networking.metal.ironcore.dev/device: device +spec: + deviceRef: + name: device + name: lo1 + type: Loopback + adminState: Up + ipv4: + addresses: + - 10.0.0.1/32 + +-- nve/nve1 -- +apiVersion: networking.metal.ironcore.dev/v1alpha1 +kind: NetworkVirtualizationEdge +metadata: + name: nve1 + namespace: default + labels: + networking.metal.ironcore.dev/device: device +spec: + deviceRef: + name: device + adminState: Up + sourceInterfaceRef: + name: lo-nve + hostReachability: BGP + +-- state/expect -- +{ + "System": { + "eps-items": { + "epId-items": { + "Ep-list": [ + { + "epId": 1, + "adminSt": "enabled", + "advertiseVmac": false, + "holdDownTime": 180, + "hostReach": "bgp", + "sourceInterface": "lo1", + "suppressARP": false + } + ] + } + }, + "fm-items": { + "evpn-items": { + "adminSt": "enabled" + }, + "nvo-items": { + "adminSt": "enabled" + } + }, + "hmm-items": { + "fwdinst-items": { + "adminSt": "disabled", + "amac": "" + } + }, + "icmpv4-items": { + "inst-items": { + "dom-items": { + "Dom-list": [ + { + "name": "default", + "if-items": { + "If-list": [ + { + "id": "lo1", + "ctrl": "port-unreachable,redirect" + } + ] + } + } + ] + } + } + }, + "intf-items": { + "lb-items": { + "LbRtdIf-list": [ + { + "adminSt": "up", + "id": "lo1", + "rtvrfMbr-items": { + "tDn": "/System/inst-items/Inst-list[name='default']" + } + } + ] + } + }, + "ipv4-items": { + "inst-items": { + "dom-items": { + "Dom-list": [ + { + "name": "default", + "if-items": { + "If-list": [ + { + "id": "lo1", + "addr-items": { + "Addr-list": [ + { + "addr": "10.0.0.1/32", + "pref": 0, + "tag": 0, + "type": "primary" + } + ] + } + } + ] + } + } + ] + } + } + }, + "procsys-items": { + "bootTime": "1700000000" + } + } +} diff --git a/test/e2e/testdata/cisco-nxos-gnmi/ospf.txt b/test/e2e/testdata/cisco-nxos-gnmi/ospf.txt new file mode 100644 index 000000000..305bad3f9 --- /dev/null +++ b/test/e2e/testdata/cisco-nxos-gnmi/ospf.txt @@ -0,0 +1,159 @@ +# NX-OS OSPF Integration Test +# +# OSPF requires at least one interface. + +-- state/preload -- +{ + "System": { + "procsys-items": { + "bootTime": "1700000000" + } + } +} + +-- interface/lo-ospf -- +apiVersion: networking.metal.ironcore.dev/v1alpha1 +kind: Interface +metadata: + name: lo-ospf + namespace: default + labels: + networking.metal.ironcore.dev/device: device +spec: + deviceRef: + name: device + name: lo10 + type: Loopback + adminState: Up + ipv4: + addresses: + - 10.255.255.10/32 + +-- ospf/underlay -- +apiVersion: networking.metal.ironcore.dev/v1alpha1 +kind: OSPF +metadata: + name: underlay + namespace: default + labels: + networking.metal.ironcore.dev/device: device +spec: + deviceRef: + name: device + adminState: Up + instance: UNDERLAY + routerId: 10.255.255.10 + interfaceRefs: + - name: lo-ospf + area: "0.0.0.0" + +-- state/expect -- +{ + "System": { + "fm-items": { + "ospf-items": { + "adminSt": "enabled" + } + }, + "icmpv4-items": { + "inst-items": { + "dom-items": { + "Dom-list": [ + { + "name": "default", + "if-items": { + "If-list": [ + { + "id": "lo10", + "ctrl": "port-unreachable,redirect" + } + ] + } + } + ] + } + } + }, + "intf-items": { + "lb-items": { + "LbRtdIf-list": [ + { + "adminSt": "up", + "id": "lo10", + "rtvrfMbr-items": { + "tDn": "/System/inst-items/Inst-list[name='default']" + } + } + ] + } + }, + "ipv4-items": { + "inst-items": { + "dom-items": { + "Dom-list": [ + { + "name": "default", + "if-items": { + "If-list": [ + { + "id": "lo10", + "addr-items": { + "Addr-list": [ + { + "addr": "10.255.255.10/32", + "pref": 0, + "tag": 0, + "type": "primary" + } + ] + } + } + ] + } + } + ] + } + } + }, + "ospf-items": { + "inst-items": { + "Inst-list": [ + { + "name": "UNDERLAY", + "adminSt": "enabled", + "dom-items": { + "Dom-list": [ + { + "name": "default", + "adminSt": "enabled", + "adjChangeLogLevel": "none", + "bwRef": 40000, + "bwRefUnit": "mbps", + "ctrl": "default-passive", + "dist": 110, + "rtrId": "10.255.255.10", + "if-items": { + "If-list": [ + { + "id": "lo10", + "adminSt": "enabled", + "advertiseSecondaries": true, + "area": "0.0.0.0", + "bfdCtrl": "unspecified", + "nwT": "unspecified", + "passiveCtrl": "disabled" + } + ] + } + } + ] + } + } + ] + } + }, + "procsys-items": { + "bootTime": "1700000000" + } + } +} diff --git a/test/e2e/testdata/cisco-nxos-gnmi/pim.txt b/test/e2e/testdata/cisco-nxos-gnmi/pim.txt new file mode 100644 index 000000000..a1e1e39cf --- /dev/null +++ b/test/e2e/testdata/cisco-nxos-gnmi/pim.txt @@ -0,0 +1,142 @@ +# NX-OS PIM Integration Test +# +# PIM requires at least one interface. + +-- state/preload -- +{ + "System": { + "procsys-items": { + "bootTime": "1700000000" + } + } +} + +-- interface/lo-pim -- +apiVersion: networking.metal.ironcore.dev/v1alpha1 +kind: Interface +metadata: + name: lo-pim + namespace: default + labels: + networking.metal.ironcore.dev/device: device +spec: + deviceRef: + name: device + name: lo20 + type: Loopback + adminState: Up + ipv4: + addresses: + - 10.255.255.20/32 + +-- pim/multicast -- +apiVersion: networking.metal.ironcore.dev/v1alpha1 +kind: PIM +metadata: + name: multicast + namespace: default + labels: + networking.metal.ironcore.dev/device: device +spec: + deviceRef: + name: device + adminState: Up + interfaceRefs: + - name: lo-pim + mode: Sparse + +-- state/expect -- +{ + "System": { + "fm-items": { + "pim-items": { + "adminSt": "enabled" + } + }, + "icmpv4-items": { + "inst-items": { + "dom-items": { + "Dom-list": [ + { + "name": "default", + "if-items": { + "If-list": [ + { + "id": "lo20", + "ctrl": "port-unreachable,redirect" + } + ] + } + } + ] + } + } + }, + "intf-items": { + "lb-items": { + "LbRtdIf-list": [ + { + "adminSt": "up", + "id": "lo20", + "rtvrfMbr-items": { + "tDn": "/System/inst-items/Inst-list[name='default']" + } + } + ] + } + }, + "ipv4-items": { + "inst-items": { + "dom-items": { + "Dom-list": [ + { + "name": "default", + "if-items": { + "If-list": [ + { + "id": "lo20", + "addr-items": { + "Addr-list": [ + { + "addr": "10.255.255.20/32", + "pref": 0, + "tag": 0, + "type": "primary" + } + ] + } + } + ] + } + } + ] + } + } + }, + "pim-items": { + "adminSt": "enabled", + "inst-items": { + "adminSt": "enabled", + "dom-items": { + "Dom-list": [ + { + "name": "default", + "adminSt": "enabled", + "if-items": { + "If-list": [ + { + "id": "lo20", + "pimSparseMode": true + } + ] + } + } + ] + } + } + }, + "procsys-items": { + "bootTime": "1700000000" + } + } +} diff --git a/test/e2e/testdata/cisco-nxos-gnmi/routedvlan.txt b/test/e2e/testdata/cisco-nxos-gnmi/routedvlan.txt new file mode 100644 index 000000000..7568c3628 --- /dev/null +++ b/test/e2e/testdata/cisco-nxos-gnmi/routedvlan.txt @@ -0,0 +1,136 @@ +# NX-OS RoutedVLAN (SVI) Integration Test + +-- state/preload -- +{ + "System": { + "procsys-items": { + "bootTime": "1700000000" + } + } +} + +-- vlan/vlan10 -- +apiVersion: networking.metal.ironcore.dev/v1alpha1 +kind: VLAN +metadata: + name: vlan10 + namespace: default + labels: + networking.metal.ironcore.dev/device: device +spec: + deviceRef: + name: device + id: 10 + name: SERVERS + +-- interface/svi-vlan10 -- +apiVersion: networking.metal.ironcore.dev/v1alpha1 +kind: Interface +metadata: + name: svi-vlan10 + namespace: default + labels: + networking.metal.ironcore.dev/device: device +spec: + deviceRef: + name: device + name: vlan10 + type: RoutedVLAN + adminState: Up + mtu: 9000 + description: "SVI for VLAN 10 - Servers" + vlanRef: + name: vlan10 + ipv4: + addresses: + - 10.10.0.1/24 + +-- state/expect -- +{ + "System": { + "bd-items": { + "bd-items": { + "BD-list": [ + { + "BdState": "active", + "adminSt": "active", + "fabEncap": "vlan-10", + "name": "SERVERS" + } + ] + } + }, + "fm-items": { + "ifvlan-items": { + "adminSt": "enabled" + } + }, + "icmpv4-items": { + "inst-items": { + "dom-items": { + "Dom-list": [ + { + "name": "default", + "if-items": { + "If-list": [ + { + "id": "vlan10", + "ctrl": "port-unreachable" + } + ] + } + } + ] + } + } + }, + "intf-items": { + "svi-items": { + "If-list": [ + { + "adminSt": "up", + "descr": "SVI for VLAN 10 - Servers", + "id": "vlan10", + "medium": "bcast", + "mtu": 9000, + "rtvrfMbr-items": { + "tDn": "/System/inst-items/Inst-list[name='default']" + }, + "vlanId": 10 + } + ] + } + }, + "ipv4-items": { + "inst-items": { + "dom-items": { + "Dom-list": [ + { + "name": "default", + "if-items": { + "If-list": [ + { + "id": "vlan10", + "addr-items": { + "Addr-list": [ + { + "addr": "10.10.0.1/24", + "pref": 0, + "tag": 0, + "type": "primary" + } + ] + } + } + ] + } + } + ] + } + } + }, + "procsys-items": { + "bootTime": "1700000000" + } + } +} diff --git a/test/e2e/testdata/cisco-nxos-gnmi/routingpolicy_prefixset.txt b/test/e2e/testdata/cisco-nxos-gnmi/routingpolicy_prefixset.txt new file mode 100644 index 000000000..dc3a070a7 --- /dev/null +++ b/test/e2e/testdata/cisco-nxos-gnmi/routingpolicy_prefixset.txt @@ -0,0 +1,154 @@ +# RoutingPolicy with PrefixSet reference + +-- state/preload -- +{ + "System": { + "procsys-items": { + "bootTime": "1700000000" + } + } +} + +-- prefixset/test-prefixset-v4 -- +apiVersion: networking.metal.ironcore.dev/v1alpha1 +kind: PrefixSet +metadata: + name: test-prefixset-v4 + namespace: default + labels: + networking.metal.ironcore.dev/device-name: device +spec: + deviceRef: + name: device + name: IPV4-NETWORKS + entries: + - sequence: 10 + prefix: 10.0.0.0/8 +-- prefixset/test-prefixset-v6 -- +apiVersion: networking.metal.ironcore.dev/v1alpha1 +kind: PrefixSet +metadata: + name: test-prefixset-v6 + namespace: default + labels: + networking.metal.ironcore.dev/device-name: device +spec: + deviceRef: + name: device + name: IPV6-NETWORKS + entries: + - sequence: 10 + prefix: "2001:db8::/32" +-- routingpolicy/rm-import -- +apiVersion: networking.metal.ironcore.dev/v1alpha1 +kind: RoutingPolicy +metadata: + name: rm-import + namespace: default + labels: + networking.metal.ironcore.dev/device-name: device +spec: + deviceRef: + name: device + name: RM-IMPORT + statements: + - sequence: 10 + conditions: + matchPrefixSet: + prefixSetRef: + name: test-prefixset-v4 + actions: + routeDisposition: AcceptRoute + - sequence: 20 + conditions: + matchPrefixSet: + prefixSetRef: + name: test-prefixset-v6 + actions: + routeDisposition: AcceptRoute + +-- state/expect -- +{ + "System": { + "rpm-items": { + "pfxlistv4-items": { + "RuleV4-list": [ + { + "name": "IPV4-NETWORKS", + "ent-items": { + "Entry-list": [ + { + "action": "permit", + "criteria": "exact", + "fromPfxLen": 0, + "order": 10, + "pfx": "10.0.0.0/8", + "toPfxLen": 0 + } + ] + } + } + ] + }, + "pfxlistv6-items": { + "RuleV6-list": [ + { + "name": "IPV6-NETWORKS", + "ent-items": { + "Entry-list": [ + { + "action": "permit", + "criteria": "exact", + "fromPfxLen": 0, + "order": 10, + "pfx": "2001:db8::/32", + "toPfxLen": 0 + } + ] + } + } + ] + }, + "rtmap-items": { + "Rule-list": [ + { + "name": "RM-IMPORT", + "ent-items": { + "Entry-list": [ + { + "action": "permit", + "order": 10, + "mrtdst-items": { + "rsrtDstAtt-items": { + "RsRtDstAtt-list": [ + { + "tDn": "/System/rpm-items/pfxlistv4-items/RuleV4-list[name='IPV4-NETWORKS']" + } + ] + } + } + }, + { + "action": "permit", + "order": 20, + "mrtdst-items": { + "rsrtDstAtt-items": { + "RsRtDstAtt-list": [ + { + "tDn": "/System/rpm-items/pfxlistv6-items/RuleV6-list[name='IPV6-NETWORKS']" + } + ] + } + } + } + ] + } + } + ] + } + }, + "procsys-items": { + "bootTime": "1700000000" + } + } +} diff --git a/test/e2e/testdata/cisco-nxos-gnmi/snmp.txt b/test/e2e/testdata/cisco-nxos-gnmi/snmp.txt new file mode 100644 index 000000000..8e8515bff --- /dev/null +++ b/test/e2e/testdata/cisco-nxos-gnmi/snmp.txt @@ -0,0 +1,76 @@ +# NX-OS SNMP Integration Test + +-- state/preload -- +{ + "System": { + "procsys-items": { + "bootTime": "1700000000" + } + } +} + +-- snmp/config -- +apiVersion: networking.metal.ironcore.dev/v1alpha1 +kind: SNMP +metadata: + name: config + namespace: default + labels: + networking.metal.ironcore.dev/device: device +spec: + deviceRef: + name: device + contact: "admin@example.com" + location: "DC1-Rack42" + sourceInterfaceName: "mgmt0" + hosts: + - address: "10.0.0.100" + communities: + - name: "public" + group: "network-operator" + +-- state/expect -- +{ + "System": { + "snmp-items": { + "inst-items": { + "community-items": { + "CommSecP-list": [ + { + "commAcess": "unspecified", + "grpName": "network-operator", + "name": "public" + } + ] + }, + "globals-items": { + "srcInterfaceInforms-items": { + "ifname": "mgmt0" + }, + "srcInterfaceTraps-items": { + "ifname": "mgmt0" + } + }, + "host-items": { + "Host-list": [ + { + "hostName": "10.0.0.100", + "notifType": "traps", + "secLevel": "noauth", + "udpPortID": 162, + "version": "v2c" + } + ] + }, + "sysinfo-items": { + "sysContact": "admin@example.com", + "sysLocation": "DC1-Rack42" + }, + "traps-items": {} + } + }, + "procsys-items": { + "bootTime": "1700000000" + } + } +} diff --git a/test/e2e/testdata/cisco-nxos-gnmi/subinterface.txt b/test/e2e/testdata/cisco-nxos-gnmi/subinterface.txt new file mode 100644 index 000000000..5235e7d77 --- /dev/null +++ b/test/e2e/testdata/cisco-nxos-gnmi/subinterface.txt @@ -0,0 +1,142 @@ +# NX-OS Subinterface Integration Test + +-- state/preload -- +{ + "System": { + "procsys-items": { + "bootTime": "1700000000" + } + } +} + +-- interface/parent-eth -- +apiVersion: networking.metal.ironcore.dev/v1alpha1 +kind: Interface +metadata: + name: parent-eth + namespace: default + labels: + networking.metal.ironcore.dev/device: device +spec: + deviceRef: + name: device + name: eth1/3 + type: Physical + adminState: Up + mtu: 9216 + description: "Parent interface for subinterfaces" + +-- interface/subif-vlan100 -- +apiVersion: networking.metal.ironcore.dev/v1alpha1 +kind: Interface +metadata: + name: subif-vlan100 + namespace: default + labels: + networking.metal.ironcore.dev/device: device +spec: + deviceRef: + name: device + name: eth1/3.100 + type: Subinterface + adminState: Up + mtu: 1500 + description: "L3 Subinterface VLAN 100" + parentInterfaceRef: + name: parent-eth + encapsulation: + type: "802.1q" + tag: 100 + ipv4: + addresses: + - 10.100.0.1/24 + +-- state/expect -- +{ + "System": { + "intf-items": { + "phys-items": { + "PhysIf-list": [ + { + "FECMode": "auto", + "accessVlan": "vlan-1", + "adminSt": "up", + "descr": "Parent interface for subinterfaces", + "id": "eth1/3", + "layer": "Layer2", + "medium": "broadcast", + "mode": "access", + "mtu": 9216, + "nativeVlan": "vlan-1", + "physExtd-items": { + "bufferBoost": "enable" + }, + "trunkVlans": "1-4094", + "userCfgdFlags": "admin_layer,admin_mtu,admin_state" + } + ] + }, + "encrtd-items": { + "EncRtdIf-list": [ + { + "adminSt": "up", + "descr": "L3 Subinterface VLAN 100", + "encap": "vlan-100", + "id": "eth1/3.100", + "mediumType": "broadcast", + "mtu": 1500, + "mtuInherit": false, + "rtvrfMbr-items": { + "tDn": "/System/inst-items/Inst-list[name='default']" + } + } + ] + } + }, + "ipv4-items": { + "inst-items": { + "dom-items": { + "Dom-list": [ + { + "name": "default", + "if-items": { + "If-list": [ + { + "id": "eth1/3.100", + "addr-items": { + "Addr-list": [ + { + "addr": "10.100.0.1/24", + "pref": 0, + "tag": 0, + "type": "primary" + } + ] + } + } + ] + } + } + ] + } + } + }, + "stp-items": { + "inst-items": { + "if-items": { + "If-list": [ + { + "id": "eth1/3", + "mode": "default", + "bpdufilter": "default", + "bpduguard": "default" + } + ] + } + } + }, + "procsys-items": { + "bootTime": "1700000000" + } + } +} diff --git a/test/e2e/testdata/cisco-nxos-gnmi/syslog.txt b/test/e2e/testdata/cisco-nxos-gnmi/syslog.txt new file mode 100644 index 000000000..f490780bd --- /dev/null +++ b/test/e2e/testdata/cisco-nxos-gnmi/syslog.txt @@ -0,0 +1,76 @@ +# NX-OS Syslog Integration Test + +-- state/preload -- +{ + "System": { + "procsys-items": { + "bootTime": "1700000000" + } + } +} + +-- syslog/logging -- +apiVersion: networking.metal.ironcore.dev/v1alpha1 +kind: Syslog +metadata: + name: logging + namespace: default + labels: + networking.metal.ironcore.dev/device: device +spec: + deviceRef: + name: device + servers: + - address: "10.0.0.100" + severity: Warning + vrfName: management + facilities: + - name: Local7 + severity: Info + +-- state/expect -- +{ + "System": { + "logging-items": { + "loglevel-items": { + "facility-items": { + "Facility-list": [ + { + "facilityName": "Local7", + "severityLevel": "information" + } + ] + } + } + }, + "syslog-items": { + "logginghistory-items": { + "level": "information", + "size": 500 + }, + "originid-items": { + "idtype": "string", + "idvalue": "logging" + }, + "rdst-items": { + "RemoteDest-list": [ + { + "forwardingFacility": "local7", + "host": "10.0.0.100", + "port": 514, + "severity": "warnings", + "transport": "udp", + "vrfName": "management" + } + ] + }, + "source-items": { + "adminState": "enabled", + "ifName": "mgmt0" + } + }, + "procsys-items": { + "bootTime": "1700000000" + } + } +} diff --git a/test/e2e/testdata/cisco-nxos-gnmi/vpcdomain.txt b/test/e2e/testdata/cisco-nxos-gnmi/vpcdomain.txt new file mode 100644 index 000000000..22738179d --- /dev/null +++ b/test/e2e/testdata/cisco-nxos-gnmi/vpcdomain.txt @@ -0,0 +1,191 @@ +# NX-OS VPCDomain Integration Test +# +# Tests vPC (Virtual Port Channel) domain configuration. +# VPCDomain is NX-OS specific (nx.cisco.networking.metal.ironcore.dev/v1alpha1). +# Requires a port-channel interface for the peer-link with member interfaces. +# +# Using this payload on a clean device will fail as features must be already enabled. + + +-- state/preload -- +{ + "System": { + "procsys-items": { + "bootTime": "1700000000" + } + } +} + +-- interface/eth1 -- +apiVersion: networking.metal.ironcore.dev/v1alpha1 +kind: Interface +metadata: + name: eth1 + namespace: default + labels: + networking.metal.ironcore.dev/device: device +spec: + deviceRef: + name: device + name: eth1/1 + type: Physical + adminState: Up + +-- interface/vpc-peerlink -- +apiVersion: networking.metal.ironcore.dev/v1alpha1 +kind: Interface +metadata: + name: vpc-peerlink + namespace: default + labels: + networking.metal.ironcore.dev/device: device +spec: + deviceRef: + name: device + name: po1 + type: Aggregate + adminState: Up + aggregation: + memberInterfaceRefs: + - name: eth1 + controlProtocol: + mode: Active + +-- vpcdomain/vpc1 -- +apiVersion: nx.cisco.networking.metal.ironcore.dev/v1alpha1 +kind: VPCDomain +metadata: + name: vpc1 + namespace: default + labels: + networking.metal.ironcore.dev/device: device +spec: + deviceRef: + name: device + domainId: 100 + adminState: Up + rolePriority: 32667 + systemPriority: 32667 + peer: + adminState: Up + interfaceRef: + name: vpc-peerlink + keepalive: + destination: 10.0.0.2 + source: 10.0.0.1 + vrfName: management + +-- state/expect -- +{ + "System": { + "fm-items": { + "lacp-items": { + "adminSt": "enabled" + }, + "vpc-items": { + "adminSt": "enabled" + } + }, + "intf-items": { + "aggr-items": { + "AggrIf-list": [ + { + "id": "po1", + "adminSt": "up", + "accessVlan": "vlan-1", + "aggrExtd-items": { + "bufferBoost": "enable" + }, + "lacpVpcConvergence": "disable", + "layer": "Layer2", + "medium": "broadcast", + "mode": "access", + "mtu": 1500, + "nativeVlan": "vlan-1", + "pcMode": "active", + "rsmbrIfs-items": { + "RsMbrIfs-list": [ + { + "tDn": "/System/intf-items/phys-items/PhysIf-list[id='eth1/1']" + } + ] + }, + "suspIndividual": "enable", + "trunkVlans": "1-4094", + "userCfgdFlags": "admin_layer,admin_state" + } + ] + }, + "phys-items": { + "PhysIf-list": [ + { + "id": "eth1/1", + "FECMode": "auto", + "accessVlan": "vlan-1", + "adminSt": "up", + "layer": "Layer2", + "medium": "broadcast", + "mode": "access", + "mtu": 1500, + "nativeVlan": "vlan-1", + "physExtd-items": { + "bufferBoost": "enable" + }, + "trunkVlans": "1-4094", + "userCfgdFlags": "admin_layer,admin_state" + } + ] + } + }, + "stp-items": { + "inst-items": { + "if-items": { + "If-list": [ + { + "id": "eth1/1", + "bpdufilter": "default", + "bpduguard": "default", + "mode": "default" + }, + { + "id": "po1", + "bpdufilter": "default", + "bpduguard": "default", + "mode": "default" + } + ] + } + } + }, + "vpc-items": { + "inst-items": { + "dom-items": { + "id": 100, + "adminSt": "enabled", + "autoRecovery": "disabled", + "autoRecoveryIntvl": 240, + "delayRestoreSVI": 10, + "delayRestoreVPC": 30, + "fastConvergence": "disabled", + "keepalive-items": { + "destIp": "10.0.0.2", + "srcIp": "10.0.0.1", + "vrf": "management", + "peerlink-items": { + "adminSt": "enabled", + "id": "po1" + } + }, + "l3PeerRouter": "disabled", + "peerGw": "disabled", + "peerSwitch": "disabled", + "rolePrio": 32667, + "sysPrio": 32667 + } + } + }, + "procsys-items": { + "bootTime": "1700000000" + } + } +} diff --git a/test/e2e/testdata/cisco-nxos-gnmi/vrf.txt b/test/e2e/testdata/cisco-nxos-gnmi/vrf.txt new file mode 100644 index 000000000..5bec2e965 --- /dev/null +++ b/test/e2e/testdata/cisco-nxos-gnmi/vrf.txt @@ -0,0 +1,129 @@ +# NX-OS VRF Integration Test +# +# Tests VRF creation with L3 interface membership: +# tenant1 -> VRF with description +# vrf-uplink -> Physical L3 interface assigned to the VRF + +-- state/preload -- +{ + "System": { + "procsys-items": { + "bootTime": "1700000000" + } + } +} + +-- vrf/tenant1 -- +apiVersion: networking.metal.ironcore.dev/v1alpha1 +kind: VRF +metadata: + name: k8s-vrf-tenant1 + namespace: default + labels: + networking.metal.ironcore.dev/device: device +spec: + deviceRef: + name: device + name: TENANT-1 + description: "Tenant 1 VRF" + +-- interface/vrf-uplink -- +apiVersion: networking.metal.ironcore.dev/v1alpha1 +kind: Interface +metadata: + name: vrf-uplink + namespace: default + labels: + networking.metal.ironcore.dev/device: device +spec: + deviceRef: + name: device + name: eth1/10 + type: Physical + adminState: Up + mtu: 9000 + description: "Uplink in TENANT-1 VRF" + vrfRef: + name: k8s-vrf-tenant1 + ipv4: + addresses: + - 10.1.1.1/30 + +-- state/expect -- +{ + "System": { + "inst-items": { + "Inst-list": [ + { + "name": "TENANT-1", + "descr": "Tenant 1 VRF", + "l3vni": false, + "dom-items": { + "Dom-list": [ + { + "name": "TENANT-1" + } + ] + } + } + ] + }, + "intf-items": { + "phys-items": { + "PhysIf-list": [ + { + "FECMode": "auto", + "accessVlan": "unknown", + "adminSt": "up", + "descr": "Uplink in TENANT-1 VRF", + "id": "eth1/10", + "layer": "Layer3", + "medium": "broadcast", + "mode": "access", + "mtu": 9000, + "nativeVlan": "unknown", + "physExtd-items": { + "bufferBoost": "enable" + }, + "rtvrfMbr-items": { + "tDn": "/System/inst-items/Inst-list[name='TENANT-1']" + }, + "trunkVlans": "1-4094", + "userCfgdFlags": "admin_layer,admin_mtu,admin_state" + } + ] + } + }, + "ipv4-items": { + "inst-items": { + "dom-items": { + "Dom-list": [ + { + "name": "TENANT-1", + "if-items": { + "If-list": [ + { + "id": "eth1/10", + "addr-items": { + "Addr-list": [ + { + "addr": "10.1.1.1/30", + "pref": 0, + "tag": 0, + "type": "primary" + } + ] + } + } + ] + } + } + ] + } + } + }, + "procsys-items": { + "bootTime": "1700000000" + } + } +} diff --git a/test/e2e/testdata/openconfig/interface.txt b/test/e2e/testdata/openconfig/interface.txt new file mode 100644 index 000000000..f6055ec7e --- /dev/null +++ b/test/e2e/testdata/openconfig/interface.txt @@ -0,0 +1,61 @@ +# Loopback Interface +-- interfaces/lo0 -- +apiVersion: networking.metal.ironcore.dev/v1alpha1 +kind: Interface +metadata: + name: lo0 + namespace: default + labels: + networking.metal.ironcore.dev/device-name: device +spec: + deviceRef: + name: device + name: lo0 + description: Router-ID Leaf1 + adminState: Up + type: Loopback + mtu: 1500 + ipv4: + addresses: + - 10.0.0.10/32 +-- state/expect -- +{ + "interfaces": { + "interface": [ + { + "config": { + "description": "Router-ID Leaf1", + "enabled": true, + "mtu": 1500, + "name": "lo0", + "type": "softwareLoopback" + }, + "name": "lo0", + "subinterfaces": { + "subinterface": [ + { + "config": { + "index": 0 + }, + "index": 0, + "ipv4": { + "addresses": { + "address": [ + { + "config": { + "ip": "10.0.0.10", + "prefix-length": 32, + "type": "PRIMARY" + }, + "ip": "10.0.0.10" + } + ] + } + } + } + ] + } + } + ] + } +} diff --git a/test/e2e/testutil/cluster.go b/test/e2e/testutil/cluster.go new file mode 100644 index 000000000..bd22fad9b --- /dev/null +++ b/test/e2e/testutil/cluster.go @@ -0,0 +1,341 @@ +// SPDX-FileCopyrightText: 2026 SAP SE or an SAP affiliate company and IronCore contributors +// SPDX-License-Identifier: Apache-2.0 + +package testutil + +import ( + "bytes" + "context" + "fmt" + "net/netip" + "os" + "os/exec" + "strconv" + "strings" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + + nxv1alpha1 "github.com/ironcore-dev/network-operator/api/cisco/nx/v1alpha1" + "github.com/ironcore-dev/network-operator/api/core/v1alpha1" +) + +var ( + // gnmiPort is the port on which the gnmi-test-server listens for gNMI requests. + gnmiPort uint16 = 9339 + // serverImage is the container image for the gnmi-test-server. + // This must match the image built by the Makefile. + serverImage = "ghcr.io/ironcore-dev/gnmi-test-server:latest" +) + +// ClusterEnvironment enables end-to-end tests to run against a real Kubernetes cluster (e.g., Kind). +// TODO: use native library instead of kubectl (follow up) +type ClusterEnvironment struct { + restConfig *rest.Config + k8sClient client.Client +} + +// NewClusterEnvironment creates a new cluster-based test environment. +func NewClusterEnvironment() *ClusterEnvironment { + return &ClusterEnvironment{} +} + +// Setup connects to the existing cluster (CRDs should already be installed). +func (c *ClusterEnvironment) Setup(ctx context.Context) error { + // Register schemes + if err := corev1.AddToScheme(scheme.Scheme); err != nil { + return err + } + if err := v1alpha1.AddToScheme(scheme.Scheme); err != nil { + return err + } + if err := nxv1alpha1.AddToScheme(scheme.Scheme); err != nil { + return err + } + + // Get REST config from kubeconfig + var err error + c.restConfig = ctrl.GetConfigOrDie() + + c.k8sClient, err = client.New(c.restConfig, client.Options{Scheme: scheme.Scheme}) + if err != nil { + return err + } + + return nil +} + +// InstallCRDs installs CRDs into the cluster. Should only be called once (from process 1). +// Uses server-side apply to handle existing CRDs gracefully. +func (c *ClusterEnvironment) InstallCRDs(ctx context.Context) error { + if err := c.runKubectl(ctx, "apply", "-k", "config/crd", "--server-side", "--force-conflicts"); err != nil { + return fmt.Errorf("failed to install CRDs: %w", err) + } + return nil +} + +// DeployManager deploys the controller-manager and waits for it to be ready. +// Should only be called once (from process 1). +// Respects E2E_PROVIDER env var for provider selection. +func (c *ClusterEnvironment) DeployManager(ctx context.Context) error { + dir, _ := GetProjectDir() //nolint:errcheck // uses current dir as fallback + env := os.Environ() + if provider := os.Getenv("E2E_PROVIDER"); provider != "" { + env = append(env, "PROVIDER="+provider) + } + + // First deploy CRDs explicitly (make deploy also does this, but let's be sure) + cmd := exec.CommandContext(ctx, "make", "deploy-crds") + cmd.Dir = dir + cmd.Env = env + cmd.Stdin = nil // Prevent stdin inheritance that can cause hangs + if output, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("failed to deploy CRDs: %s: %w", string(output), err) + } + + // Then deploy the manager + cmd = exec.CommandContext(ctx, "make", "deploy") + cmd.Dir = dir + cmd.Env = env + cmd.Stdin = nil // Prevent stdin inheritance that can cause hangs + if output, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("failed to deploy manager: %s: %w", string(output), err) + } + + if err := c.runKubectl(ctx, "wait", "deployment/network-operator-controller-manager", + "-n", "network-operator-system", + "--for", "condition=Available", + "--timeout", "2m"); err != nil { + return fmt.Errorf("manager not ready: %w", err) + } + return nil +} + +// UndeployManager undeploys the controller-manager. +// Uses kubectl delete -k directly instead of make undeploy to avoid shell pipeline stdin issues. +func (c *ClusterEnvironment) UndeployManager(ctx context.Context) error { + if err := c.runKubectl(ctx, "delete", "-k", "config/develop", "--ignore-not-found"); err != nil { + return fmt.Errorf("failed to undeploy manager: %w", err) + } + return nil +} + +// Teardown cleans up CRDs. +func (c *ClusterEnvironment) Teardown(ctx context.Context) error { + _ = c.runKubectl(ctx, "delete", "-k", "config/crd", "--ignore-not-found") //nolint:errcheck // best-effort cleanup + return nil +} + +// Client returns the Kubernetes client. +func (c *ClusterEnvironment) Client() client.Client { + return c.k8sClient +} + +// RESTConfig returns the REST config. +func (c *ClusterEnvironment) RESTConfig() *rest.Config { + return c.restConfig +} + +// DeployGNMIServer deploys the gnmi-test-server pod and returns its gNMI address. +func (c *ClusterEnvironment) DeployGNMIServer(ctx context.Context, namespace string) (netip.AddrPort, error) { + if err := c.runKubectl( + ctx, + "run", "gnmi-test-server", + "--image", serverImage, + "--image-pull-policy", "Never", + "--namespace", namespace, + "--restart", "Never", + "--port", "8000", + "--port", strconv.FormatUint(uint64(gnmiPort), 10), + ); err != nil { + return netip.AddrPort{}, fmt.Errorf("failed to deploy gnmi-test-server: %w", err) + } + + if err := c.runKubectl( + ctx, + "wait", "pods/gnmi-test-server", + "--for", "condition=Ready", + "--namespace", namespace, + "--timeout", "1m", + ); err != nil { + return netip.AddrPort{}, fmt.Errorf("gnmi-test-server pod not ready: %w", err) + } + + out, err := c.runKubectlOutput( + ctx, + "get", "pod", "gnmi-test-server", + "--output", "jsonpath={.status.podIP}", + "--namespace", namespace, + ) + if err != nil { + return netip.AddrPort{}, fmt.Errorf("failed to get gnmi-test-server IP: %w", err) + } + var s netip.Addr + if s, err = netip.ParseAddr(strings.TrimSpace(out)); err != nil { + return netip.AddrPort{}, fmt.Errorf("invalid IP address from gnmi-test-server pod: %w", err) + } + + return netip.AddrPortFrom(s, gnmiPort), nil +} + +// GetGNMIState fetches state via kubectl exec. +func (c *ClusterEnvironment) GetGNMIState(ctx context.Context, namespace string) ([]byte, error) { + out, err := c.runKubectlOutput( + ctx, + "exec", "gnmi-test-server", + "--namespace", namespace, + "--", + "wget", "-qO-", "http://localhost:8000/v1/state", + ) + if err != nil { + return nil, fmt.Errorf("failed to get gNMI state: %w", err) + } + return []byte(out), nil +} + +// ClearGNMIState clears state. +func (c *ClusterEnvironment) ClearGNMIState(ctx context.Context, namespace string) error { + _, err := c.runKubectlOutput( + ctx, + "exec", "gnmi-test-server", + "--namespace", namespace, + "--", + "wget", "-qO-", "--post-data=", "http://localhost:8000/v1/clear", + ) + return err +} + +// PreloadGNMIState preloads nested JSON into the gnmi-test-server state. +// This allows tests to set up paths like System/procsys-items/bootTime +// before the Device controller reconciles. +func (c *ClusterEnvironment) PreloadGNMIState(ctx context.Context, namespace string, jsonData []byte) error { + _, err := c.runKubectlOutput( + ctx, + "exec", "gnmi-test-server", + "--namespace", namespace, + "--", + "wget", "-qO-", "--post-data="+string(jsonData), "http://localhost:8000/v1/state", + ) + return err +} + +// runKubectl runs a kubectl command and returns an error if it fails. +func (c *ClusterEnvironment) runKubectl(ctx context.Context, args ...string) error { + cmd := exec.CommandContext(ctx, "kubectl", args...) + dir, _ := GetProjectDir() //nolint:errcheck // uses current dir as fallback + cmd.Dir = dir + cmd.Stdin = nil // Prevent stdin inheritance that can cause hangs + output, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("%s: %w", string(output), err) + } + return nil +} + +// runKubectlOutput runs a kubectl command and returns its output. +func (c *ClusterEnvironment) runKubectlOutput(ctx context.Context, args ...string) (string, error) { + cmd := exec.CommandContext(ctx, "kubectl", args...) + dir, _ := GetProjectDir() //nolint:errcheck // uses current dir as fallback + cmd.Dir = dir + cmd.Stdin = nil // Prevent stdin inheritance that can cause hangs + var stdout, stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + if err := cmd.Run(); err != nil { + return "", fmt.Errorf("%s: %w", stderr.String(), err) + } + return stdout.String(), nil +} + +// DeleteNamespace deletes the given namespace. +func (c *ClusterEnvironment) DeleteNamespace(ctx context.Context, namespace string) error { + if err := c.runKubectl(ctx, "delete", "namespace", namespace, "--ignore-not-found"); err != nil { + return fmt.Errorf("failed to delete namespace %s: %w", namespace, err) + } + return nil +} + +// CreateNamespace creates a new namespace with the given name and labels it for cleanup tracking. +func (c *ClusterEnvironment) CreateNamespace(ctx context.Context, namespace string) error { + if err := c.runKubectl(ctx, "create", "namespace", namespace); err != nil { + return fmt.Errorf("failed to create namespace %s: %w", namespace, err) + } + // Label the namespace for cleanup tracking across parallel test processes + if err := c.runKubectl(ctx, "label", "namespace", namespace, E2ETestLabel+"="); err != nil { + return fmt.Errorf("failed to label namespace %s: %w", namespace, err) + } + return nil +} + +// WaitForTestNamespacesGone waits for all labeled test namespaces to be fully deleted. +// This ensures DeferCleanup hooks have completed before the manager is undeployed. +// Without this, the manager (and CRDs) may be deleted while resources with finalizers still exist, +// leaving them stuck forever because the controller can no longer process the finalizers. +func (c *ClusterEnvironment) WaitForTestNamespacesGone(ctx context.Context) error { + // Get all namespaces with our e2e test label + out, err := c.runKubectlOutput( + ctx, + "get", "namespaces", + "-l", E2ETestLabel, + "-o", "jsonpath={.items[*].metadata.name}", + ) + if err != nil { + return fmt.Errorf("failed to list test namespaces: %w", err) + } + + namespaces := strings.Fields(out) + if len(namespaces) == 0 { + return nil + } + + // Wait for each test namespace to be deleted (with timeout) + for _, ns := range namespaces { + _ = c.runKubectl(ctx, "wait", "namespace", ns, //nolint:errcheck // best-effort cleanup + "--for=delete", + "--timeout=120s") + } + + return nil +} + +// DeleteCustomResources deletes all custom resources in the given namespace. +// Deletion order: CoreResources first (have finalizers), then ConfigResources, then Device. +// This allows finalizers to complete while their dependencies still exist. +func (c *ClusterEnvironment) DeleteCustomResources(ctx context.Context, namespace string) error { + deleteResources := func(gvks []schema.GroupVersionKind) { + for _, gvk := range gvks { + _ = c.runKubectl(ctx, "delete", //nolint:errcheck // best-effort cleanup + ResourcePluralName(gvk), "--all", + "--namespace", namespace, + "--ignore-not-found") + } + // Wait for resources to be fully gone (finalizers completed) + for _, gvk := range gvks { + _ = c.runKubectl(ctx, "wait", //nolint:errcheck // best-effort cleanup + ResourcePluralName(gvk), + "--for=delete", "--all", + "--namespace", namespace, + "--timeout=60s") + } + } + + // Delete core resources first (have finalizers that need Device + configs) + deleteResources(CoreResources) + // Then delete config resources + deleteResources(ConfigResources) + + // Delete Device LAST - after all other resources and their finalizers are done + _ = c.runKubectl(ctx, "delete", "devices", "--all", //nolint:errcheck // best-effort cleanup + "--namespace", namespace, + "--ignore-not-found") + _ = c.runKubectl(ctx, "wait", "devices", //nolint:errcheck // best-effort cleanup + "--for=delete", "--all", + "--namespace", namespace, + "--timeout=60s") + + return nil +} diff --git a/test/e2e/testutil/doc.go b/test/e2e/testutil/doc.go new file mode 100644 index 000000000..9098b9793 --- /dev/null +++ b/test/e2e/testutil/doc.go @@ -0,0 +1,16 @@ +// SPDX-FileCopyrightText: 2026 SAP SE or an SAP affiliate company and IronCore contributors +// SPDX-License-Identifier: Apache-2.0 + +// Package testutil provides test infrastructure for e2e tests. +// +// It supports two test modes selected by build tags: +// +// - Envtest (build tag: envtest): Uses controller-runtime's envtest.Environment +// with an in-process gNMI test server. Fast (~10s) but doesn't test deployment. +// +// - Cluster (default): Uses a real Kubernetes cluster (typically Kind) with a +// deployed operator and gnmi-test-server pod. Slower (~2-5min) but tests full stack. +// +// The concrete types ClusterEnvironment and EnvtestEnvironment provide the same +// methods, allowing test logic to work with either mode via build tag selection. +package testutil diff --git a/test/e2e/testutil/envtest.go b/test/e2e/testutil/envtest.go new file mode 100644 index 000000000..244840764 --- /dev/null +++ b/test/e2e/testutil/envtest.go @@ -0,0 +1,186 @@ +// SPDX-FileCopyrightText: 2026 SAP SE or an SAP affiliate company and IronCore contributors +// SPDX-License-Identifier: Apache-2.0 + +package testutil + +import ( + "context" + "os" + "path/filepath" + "slices" + + gnmitestserver "github.com/ironcore-dev/gnmi-test-server/testserver" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" + + nxv1alpha1 "github.com/ironcore-dev/network-operator/api/cisco/nx/v1alpha1" + "github.com/ironcore-dev/network-operator/api/core/v1alpha1" +) + +// EnvtestEnvironment implements TestEnvironment using envtest and an in-process gNMI server. +type EnvtestEnvironment struct { + testEnv *envtest.Environment + restConfig *rest.Config + k8sClient client.Client + gnmiServer *gnmitestserver.Server + gnmiAddr string + cancel context.CancelFunc +} + +// NewEnvtestEnvironment creates a new envtest-based test environment. +func NewEnvtestEnvironment() *EnvtestEnvironment { + return &EnvtestEnvironment{} +} + +// Setup initializes envtest and starts the in-process gNMI server. +func (e *EnvtestEnvironment) Setup(ctx context.Context) error { + ctx, e.cancel = context.WithCancel(ctx) + + // Start in-process gNMI test server with NX-OS behavior + var err error + e.gnmiServer, e.gnmiAddr, _, err = gnmitestserver.NewTestServer(ctx, gnmitestserver.WithNXOSBehavior()) + if err != nil { + return err + } + + // Register schemes + if err := corev1.AddToScheme(scheme.Scheme); err != nil { + return err + } + if err := v1alpha1.AddToScheme(scheme.Scheme); err != nil { + return err + } + if err := nxv1alpha1.AddToScheme(scheme.Scheme); err != nil { + return err + } + + // Start envtest + e.testEnv = &envtest.Environment{ + CRDDirectoryPaths: []string{filepath.Join("..", "..", "config", "crd", "bases")}, + ErrorIfCRDPathMissing: true, + } + + // Detect test binary directory for IDEs + if dir := detectTestBinaryDir(); dir != "" { + e.testEnv.BinaryAssetsDirectory = dir + } + + e.restConfig, err = e.testEnv.Start() + if err != nil { + return err + } + + e.k8sClient, err = client.New(e.restConfig, client.Options{Scheme: scheme.Scheme}) + if err != nil { + return err + } + + // Wait for default namespace to be ready + for { + var ns corev1.Namespace + if err := e.k8sClient.Get(ctx, client.ObjectKey{Name: metav1.NamespaceDefault}, &ns); err == nil { + break + } + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + } + + return nil +} + +// Teardown stops envtest and the gNMI server. +func (e *EnvtestEnvironment) Teardown(_ context.Context) error { + if e.cancel != nil { + e.cancel() + } + + if e.gnmiServer != nil { + if err := e.gnmiServer.Close(); err != nil { + return err + } + } + + if e.testEnv != nil { + if err := e.testEnv.Stop(); err != nil { + return err + } + } + + return nil +} + +// Client returns the Kubernetes client. +func (e *EnvtestEnvironment) Client() client.Client { + return e.k8sClient +} + +// RESTConfig returns the REST config. +func (e *EnvtestEnvironment) RESTConfig() *rest.Config { + return e.restConfig +} + +// GNMIAddress returns the in-process gNMI server address. +func (e *EnvtestEnvironment) GNMIAddress() string { + return e.gnmiAddr +} + +// GetGNMIState fetches state directly from the in-process server. +func (e *EnvtestEnvironment) GetGNMIState(_ context.Context) ([]byte, error) { + return e.gnmiServer.GetState() +} + +// ClearGNMIState clears state directly on the in-process server. +func (e *EnvtestEnvironment) ClearGNMIState(_ context.Context) error { + e.gnmiServer.ClearState() + return nil +} + +// PreloadGNMIState replaces the in-process gNMI server state with the given JSON. +// This resets the server to a clean state for test isolation. +func (e *EnvtestEnvironment) PreloadGNMIState(_ context.Context, jsonData []byte) error { + if len(jsonData) == 0 { + return nil + } + // Replace entire state by directly setting Buf (thread-safe with Lock) + e.gnmiServer.State.Lock() + e.gnmiServer.State.Buf = jsonData + e.gnmiServer.State.Unlock() + return nil +} + +// IsEnvtest returns true for envtest mode. +func (e *EnvtestEnvironment) IsEnvtest() bool { + return true +} + +// detectTestBinaryDir locates the envtest binary directory. +// The structure is: bin/k8s/k8s/--/ +// We need to find the version directory that contains etcd/kube-apiserver. +func detectTestBinaryDir() string { + basePath := filepath.Join("..", "..", "bin", "k8s", "k8s") + entries, err := os.ReadDir(basePath) + if err != nil { + return "" + } + // Find a version directory (e.g., "1.35.0-darwin-arm64") + idx := slices.IndexFunc(entries, func(e os.DirEntry) bool { + if !e.IsDir() { + return false + } + // Check if this directory contains etcd + etcdPath := filepath.Join(basePath, e.Name(), "etcd") + _, err := os.Stat(etcdPath) + return err == nil + }) + if idx >= 0 { + return filepath.Join(basePath, entries[idx].Name()) + } + return "" +} diff --git a/test/e2e/testutil/helpers.go b/test/e2e/testutil/helpers.go new file mode 100644 index 000000000..b9888543d --- /dev/null +++ b/test/e2e/testutil/helpers.go @@ -0,0 +1,548 @@ +// SPDX-FileCopyrightText: 2025 SAP SE or an SAP affiliate company and IronCore contributors +// SPDX-License-Identifier: Apache-2.0 + +package testutil + +import ( + "bufio" + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "os/exec" + "reflect" + "sort" + "strings" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "sigs.k8s.io/yaml" +) + +const ( + prometheusURL = "https://github.com/prometheus-operator/prometheus-operator/releases/download/v0.82.2/bundle.yaml" + certmanagerURL = "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml" +) + +// warnError writes a warning to the provided writer. +func warnError(w io.Writer, err error) { + _, _ = fmt.Fprintf(w, "warning: %v\n", err) +} + +// Run executes the provided command within this context. +// It writes the command to the provided writer for logging. +func Run(cmd *exec.Cmd, w io.Writer) (string, error) { + dir, err := GetProjectDir() + if err != nil { + return "", fmt.Errorf("failed to get project directory: %w", err) + } + + cmd.Dir = dir + cmd.Stdin = nil // Prevent stdin inheritance that can cause hangs + if err = os.Chdir(cmd.Dir); err != nil { + _, _ = fmt.Fprintf(w, "chdir dir: %s\n", err) + } + + command := strings.Join(cmd.Args, " ") + // #nosec G705 + _, _ = fmt.Fprintf(w, "running: %s\n", command) + + output, err := cmd.CombinedOutput() + if err != nil { + return string(output), fmt.Errorf("%s failed with error: (%w) %s", command, err, string(output)) + } + + return string(output), nil +} + +// Apply takes a raw YAML resource and applies it to the cluster by +// creating a temporary file and running 'kubectl apply -f'. +func Apply(ctx context.Context, resource string, w io.Writer) error { + file, err := os.CreateTemp("", "resource-*.yaml") + if err != nil { + return fmt.Errorf("failed to create temp file: %w", err) + } + // #nosec G703 + defer func() { _ = os.Remove(file.Name()) }() + if _, err = file.WriteString(resource); err != nil { + return fmt.Errorf("failed to write to temp file: %w", err) + } + if err = file.Close(); err != nil { + return fmt.Errorf("failed to close temp file: %w", err) + } + // #nosec G204 G702 + cmd := exec.CommandContext(ctx, "kubectl", "apply", "-f", file.Name()) + if _, err = Run(cmd, w); err != nil { + return fmt.Errorf("failed to apply resource: %w", err) + } + return nil +} + +// ExtractResourceIdentifier parses YAML and returns "kind/name" for use with kubectl wait. +// The kind is lowercased to match kubectl's resource type format. +func ExtractResourceIdentifier(resourceYAML string) (string, error) { + var obj unstructured.Unstructured + if err := yaml.Unmarshal([]byte(resourceYAML), &obj); err != nil { + return "", fmt.Errorf("failed to unmarshal YAML: %w", err) + } + + kind := strings.ToLower(obj.GetKind()) + name := obj.GetName() + if kind == "" || name == "" { + return "", errors.New("YAML missing kind or metadata.name") + } + + return kind + "/" + name, nil +} + +// CompareJSON compares two JSON strings and returns an error if they are not equal. +// For comparison, it unmarshals both into interface{} and uses reflect.DeepEqual +// after sorting any arrays and removing empty arrays/objects to ignore ordering +// and cleanup artifacts. +func CompareJSON(got, want string) error { + var gotObj, wantObj any + if err := json.Unmarshal([]byte(got), &gotObj); err != nil { + return fmt.Errorf("failed to unmarshal got JSON: %w", err) + } + if err := json.Unmarshal([]byte(want), &wantObj); err != nil { + return fmt.Errorf("failed to unmarshal want JSON: %w", err) + } + + // Normalize both objects (sort arrays, remove empty containers) + gotObj = normalizeJSON(gotObj) + wantObj = normalizeJSON(wantObj) + + if !reflect.DeepEqual(gotObj, wantObj) { + // For error message, show original compacted JSON (not normalized) + // so empty objects show as {} not null + var gotBuf, wantBuf bytes.Buffer + _ = json.Compact(&gotBuf, []byte(got)) //nolint:errcheck // already parsed successfully above + _ = json.Compact(&wantBuf, []byte(want)) //nolint:errcheck // already parsed successfully above + return fmt.Errorf("JSON mismatch:\ngot: %s\nwant: %s", gotBuf.String(), wantBuf.String()) + } + return nil +} + +// normalizeJSON recursively sorts arrays and removes empty arrays/objects +// to make comparison order-independent and ignore cleanup artifacts. +func normalizeJSON(v any) any { + switch val := v.(type) { + case map[string]any: + result := make(map[string]any) + for k, v := range val { + normalized := normalizeJSON(v) + // Skip empty maps and empty arrays + if !isEmpty(normalized) { + result[k] = normalized + } + } + if len(result) == 0 { + return nil + } + return result + case []any: + var result []any + for _, elem := range val { + normalized := normalizeJSON(elem) + if !isEmpty(normalized) { + result = append(result, normalized) + } + } + if len(result) == 0 { + return nil + } + // Sort the array by JSON representation + sort.Slice(result, func(i, j int) bool { + bi, _ := json.Marshal(result[i]) //nolint:errcheck // sorting comparison, errors treated as equal + bj, _ := json.Marshal(result[j]) //nolint:errcheck // sorting comparison, errors treated as equal + return string(bi) < string(bj) + }) + return result + default: + return v + } +} + +// isEmpty checks if a value is an empty map, empty array, or nil. +func isEmpty(v any) bool { + if v == nil { + return true + } + switch val := v.(type) { + case map[string]any: + return len(val) == 0 + case []any: + return len(val) == 0 + } + return false +} + +// InstallPrometheusOperator installs the prometheus Operator to be used to export the enabled metrics. +func InstallPrometheusOperator(ctx context.Context, w io.Writer) error { + cmd := exec.CommandContext(ctx, "kubectl", "create", "-f", prometheusURL) + _, err := Run(cmd, w) + return err +} + +// UninstallPrometheusOperator uninstalls the prometheus +func UninstallPrometheusOperator(ctx context.Context, w io.Writer) { + cmd := exec.CommandContext(ctx, "kubectl", "delete", "-f", prometheusURL) + if _, err := Run(cmd, w); err != nil { + warnError(w, err) + } +} + +// IsPrometheusCRDsInstalled checks if any Prometheus CRDs are installed +// by verifying the existence of key CRDs related to Prometheus. +func IsPrometheusCRDsInstalled(ctx context.Context, w io.Writer) bool { + // List of common Prometheus CRDs + prometheusCRDs := []string{ + "prometheuses.monitoring.coreos.com", + "prometheusrules.monitoring.coreos.com", + "prometheusagents.monitoring.coreos.com", + } + + cmd := exec.CommandContext(ctx, "kubectl", "get", "crds", "-o", "custom-columns=NAME:.metadata.name") + output, err := Run(cmd, w) + if err != nil { + return false + } + crdList := GetNonEmptyLines(output) + for _, crd := range prometheusCRDs { + for _, line := range crdList { + if strings.Contains(line, crd) { + return true + } + } + } + + return false +} + +// InstallCertManager installs the cert manager bundle. +func InstallCertManager(ctx context.Context, w io.Writer) error { + cmd := exec.CommandContext(ctx, "kubectl", "apply", "-f", certmanagerURL) + if _, err := Run(cmd, w); err != nil { + return err + } + // Wait for cert-manager-webhook to be ready, which can take time if cert-manager + // was re-installed after uninstalling on a cluster. + cmd = exec.CommandContext( + ctx, "kubectl", "wait", "deployment.apps/cert-manager-webhook", + "--for", "condition=Available", + "--namespace", "cert-manager", + "--timeout", "5m", + ) + if _, err := Run(cmd, w); err != nil { + return err + } + + // Wait for webhook to be fully operational (TLS cert ready) + // The deployment being Available doesn't mean the webhook TLS is ready + cmd = exec.CommandContext( + ctx, "kubectl", "wait", "certificate/cert-manager-webhook-ca", + "--for", "condition=Ready", + "--namespace", "cert-manager", + "--timeout", "2m", + ) + _, _ = Run(cmd, w) //nolint:errcheck // cert may not exist in older versions + + // Give the webhook a moment to pick up the cert + time.Sleep(5 * time.Second) + return nil +} + +// UninstallCertManager uninstalls the cert manager +func UninstallCertManager(ctx context.Context, w io.Writer) { + cmd := exec.CommandContext(ctx, "kubectl", "delete", "-f", certmanagerURL) + if _, err := Run(cmd, w); err != nil { + warnError(w, err) + } +} + +// WaitForCertManagerWebhook waits for the cert-manager webhook to be fully operational. +// This should be called before deploying resources that use cert-manager certificates. +func WaitForCertManagerWebhook(ctx context.Context, w io.Writer) error { + // Wait for deployment to be available + cmd := exec.CommandContext( + ctx, "kubectl", "wait", "deployment.apps/cert-manager-webhook", + "--for", "condition=Available", + "--namespace", "cert-manager", + "--timeout", "2m", + ) + if _, err := Run(cmd, w); err != nil { + return err + } + + // Wait for the CA injector to inject the CA bundle into the webhook + cmd = exec.CommandContext( + ctx, "kubectl", "wait", "deployment.apps/cert-manager-cainjector", + "--for", "condition=Available", + "--namespace", "cert-manager", + "--timeout", "2m", + ) + if _, err := Run(cmd, w); err != nil { + return err + } + + // Wait for the cainjector to inject the CA bundle into the webhook configuration + // This is what actually makes the webhook work - the API server needs the CA to verify the webhook's TLS cert + cmd = exec.CommandContext( + ctx, "kubectl", "wait", "validatingwebhookconfiguration/cert-manager-webhook", + "--for", "jsonpath={.webhooks[0].clientConfig.caBundle}", + "--timeout", "2m", + ) + if _, err := Run(cmd, w); err != nil { + return fmt.Errorf("cert-manager webhook CA bundle not injected: %w", err) + } + + return nil +} + +// IsCertManagerCRDsInstalled checks if any Cert Manager CRDs are installed +// by verifying the existence of key CRDs related to Cert Manager. +func IsCertManagerCRDsInstalled(ctx context.Context, w io.Writer) bool { + // List of common Cert Manager CRDs + certManagerCRDs := []string{ + "certificates.cert-manager.io", + "issuers.cert-manager.io", + "clusterissuers.cert-manager.io", + "certificaterequests.cert-manager.io", + "orders.acme.cert-manager.io", + "challenges.acme.cert-manager.io", + } + + // Execute the kubectl command to get all CRDs + cmd := exec.CommandContext(ctx, "kubectl", "get", "crds") + output, err := Run(cmd, w) + if err != nil { + return false + } + + // Check if any of the Cert Manager CRDs are present + crdList := GetNonEmptyLines(output) + for _, crd := range certManagerCRDs { + for _, line := range crdList { + if strings.Contains(line, crd) { + return true + } + } + } + + return false +} + +// LoadImageToKindClusterWithName loads a local docker image to the kind cluster +func LoadImageToKindClusterWithName(ctx context.Context, name string, w io.Writer) error { + cluster := "kind" + if v, ok := os.LookupEnv("KIND_CLUSTER"); ok { + cluster = v + } + // See: https://kind.sigs.k8s.io/docs/user/rootless/#creating-a-kind-cluster-with-rootless-nerdctl + prov, ok := os.LookupEnv("KIND_EXPERIMENTAL_PROVIDER") + if ok && prov != "docker" { + // If kind is configured to not use the docker runtime (e.g. when using podman or nerctl), + // we need to create a temp file to store the image archive and load it as a tarball. + // See: https://github.com/kubernetes-sigs/kind/issues/2760 + file, err := os.CreateTemp("", "operator-image-") + if err != nil { + return fmt.Errorf("failed to create temp file: %w", err) + } + _ = file.Close() + // #nosec G703 + defer func() { _ = os.Remove(file.Name()) }() + + // https://github.com/containerd/nerdctl/blob/main/docs/command-reference.md#whale-nerdctl-save + // https://docs.podman.io/en/v5.3.0/markdown/podman-save.1.html + // #nosec G702 + cmd := exec.CommandContext(ctx, prov, "save", name, "--output", file.Name()) + if _, err = Run(cmd, w); err != nil { + return fmt.Errorf("failed to save image: %w", err) + } + + cmd = exec.CommandContext(ctx, "kind", "load", "image-archive", file.Name(), "--name", cluster) //nolint:gosec + _, err = Run(cmd, w) + return err + } + cmd := exec.CommandContext(ctx, "kind", "load", "docker-image", name, "--name", cluster) + _, err := Run(cmd, w) + return err +} + +// GetNonEmptyLines converts given command output string into individual objects +// according to line breakers, and ignores the empty elements in it. +func GetNonEmptyLines(output string) []string { + var res []string + for element := range strings.SplitSeq(output, "\n") { + if element != "" { + res = append(res, element) + } + } + return res +} + +// GetProjectDir will return the directory where the project is +func GetProjectDir() (string, error) { + wd, err := os.Getwd() + if err != nil { + return wd, err + } + wd = strings.ReplaceAll(wd, "/test/e2e", "") + return wd, nil +} + +// PatchResourceYAML takes a raw YAML resource and patches its namespace and deviceRef. +// This allows txtar test files to have placeholder values that get replaced at runtime. +// It returns the patched YAML string ready for kubectl apply. +func PatchResourceYAML(resourceYAML, namespace, deviceName string) (string, error) { + // Parse YAML into unstructured map + var obj map[string]any + if err := yaml.Unmarshal([]byte(resourceYAML), &obj); err != nil { + return "", fmt.Errorf("failed to unmarshal YAML: %w", err) + } + + // Patch metadata.namespace + metadata, ok := obj["metadata"].(map[string]any) + if !ok { + metadata = make(map[string]any) + obj["metadata"] = metadata + } + metadata["namespace"] = namespace + + // Ensure labels map exists and add e2e test label + labels, ok := metadata["labels"].(map[string]any) + if !ok { + labels = make(map[string]any) + metadata["labels"] = labels + } + labels[E2ETestLabel] = "" + + // Patch spec.deviceRef.name if it exists + if spec, ok := obj["spec"].(map[string]any); ok { + if deviceRef, ok := spec["deviceRef"].(map[string]any); ok { + deviceRef["name"] = deviceName + } + } + + // Patch the device label if it exists + if _, hasDeviceLabel := labels["networking.metal.ironcore.dev/device"]; hasDeviceLabel { + labels["networking.metal.ironcore.dev/device"] = deviceName + } + + // Marshal back to YAML + out, err := yaml.Marshal(obj) + if err != nil { + return "", fmt.Errorf("failed to marshal patched YAML: %w", err) + } + return string(out), nil +} + +// ApplyWithPatch applies a YAML resource after patching its namespace and deviceRef. +// This is the cluster-mode equivalent of envtest's createResourceFromTxtar. +func ApplyWithPatch(ctx context.Context, resourceYAML, namespace, deviceName string, w io.Writer) error { + patched, err := PatchResourceYAML(resourceYAML, namespace, deviceName) + if err != nil { + return err + } + return Apply(ctx, patched, w) +} + +// WaitForCondition waits for a resource to have a condition set to True. +// It tries "Configured" first, falls back to "Ready" if Configured doesn't exist. +// Skips config-only resources that don't have status conditions. +func WaitForCondition(ctx context.Context, resourceName, namespace string, w io.Writer) error { + // Config-only resources don't have status conditions - skip them + // resourceName format is "kind/name" e.g. "bgpconfig/evpn-settings" + kind := strings.Split(resourceName, "/")[0] + switch strings.ToLower(kind) { + case "interfaceconfig", "lldpconfig", "bgpconfig", "nveconfig", "managementaccessconfig": + return nil // No conditions to wait for + } + + // Try Configured first using jsonpath (more reliable than --for condition=X with multiple conditions) + cmd := exec.CommandContext( + ctx, "kubectl", "wait", resourceName, + "--for", `jsonpath={.status.conditions[?(@.type=="Configured")].status}=True`, + "--namespace", namespace, + "--timeout", "10s", + ) + if _, err := Run(cmd, w); err == nil { + return nil + } + + // Fallback to Ready using jsonpath (condition=Ready doesn't work reliably with custom resources) + cmd = exec.CommandContext( + ctx, "kubectl", "wait", resourceName, + "--for", `jsonpath={.status.conditions[?(@.type=="Ready")].status}=True`, + "--namespace", namespace, + "--timeout", "2m", + ) + _, err := Run(cmd, w) + return err +} + +// UncommentCode searches for target in the file and remove the comment prefix +// of the target content. The target content may span multiple lines. +func UncommentCode(filename, target, prefix string) error { + content, err := os.ReadFile(filename) + if err != nil { + return err + } + + before, after, ok := bytes.Cut(content, []byte(target)) + if !ok { + if bytes.Contains(content, []byte(target)[len(prefix):]) { + return nil // already uncommented + } + + return fmt.Errorf("unable to find the code %s to be uncomment", target) + } + + out := new(bytes.Buffer) + if _, err = out.Write(before); err != nil { + return err + } + + scanner := bufio.NewScanner(bytes.NewBufferString(target)) + if !scanner.Scan() { + return nil + } + for { + _, err = out.WriteString(strings.TrimPrefix(scanner.Text(), prefix)) + if err != nil { + return err + } + // Avoid writing a newline in case the previous line was the last in target. + if !scanner.Scan() { + break + } + if _, err = out.WriteString("\n"); err != nil { + return err + } + } + + if _, err = out.Write(after); err != nil { + return err + } + + return os.WriteFile(filename, out.Bytes(), 0o644) +} + +// ExtractConditions extracts status conditions from an unstructured object +// into a typed []metav1.Condition slice for use with apimeta helpers. +func ExtractConditions(obj *unstructured.Unstructured) ([]metav1.Condition, error) { + raw, _, err := unstructured.NestedSlice(obj.Object, "status", "conditions") + if err != nil { + return nil, err + } + data, err := json.Marshal(raw) + if err != nil { + return nil, err + } + var conditions []metav1.Condition + return conditions, json.Unmarshal(data, &conditions) +} diff --git a/test/e2e/testutil/provider.go b/test/e2e/testutil/provider.go new file mode 100644 index 000000000..986747139 --- /dev/null +++ b/test/e2e/testutil/provider.go @@ -0,0 +1,156 @@ +// SPDX-FileCopyrightText: 2026 SAP SE or an SAP affiliate company and IronCore contributors +// SPDX-License-Identifier: Apache-2.0 + +package testutil + +import ( + "context" + "time" + + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/client" + + nxv1alpha1 "github.com/ironcore-dev/network-operator/api/cisco/nx/v1alpha1" + "github.com/ironcore-dev/network-operator/api/core/v1alpha1" + "github.com/ironcore-dev/network-operator/internal/provider" + "github.com/ironcore-dev/network-operator/internal/provider/cisco/iosxr" + "github.com/ironcore-dev/network-operator/internal/provider/cisco/nxos" +) + +// E2ETestLabel is the label key applied to resources created by e2e tests for cleanup tracking. +const E2ETestLabel = "networking.metal.ironcore.dev/e2e-test" + +// ProviderType represents the network device provider to test against. +type ProviderType string + +// ProviderFactory creates a new provider instance. +type ProviderFactory = func() provider.Provider + +// Provider names must match the registered provider names in internal/provider/*/provider.go +const ( + ProviderNXOS ProviderType = "cisco-nxos-gnmi" + ProviderIOSXR ProviderType = "cisco-iosxr-gnmi" +) + +// ProviderConfig holds the configuration for a provider test. +type ProviderConfig struct { + Name ProviderType + NewProvider ProviderFactory +} + +// SupportedProviders lists all providers to test. +var SupportedProviders = []ProviderConfig{ + {Name: ProviderNXOS, NewProvider: func() provider.Provider { return nxos.NewProvider() }}, + {Name: ProviderIOSXR, NewProvider: func() provider.Provider { return iosxr.NewProvider() }}, +} + +// CoreResources are the main API resources with finalizers. +// During cleanup, these are deleted FIRST so their finalizers can +// complete while Device and config resources still exist. +var CoreResources = []schema.GroupVersionKind{ + v1alpha1.GroupVersion.WithKind("Interface"), + v1alpha1.GroupVersion.WithKind("VLAN"), + v1alpha1.GroupVersion.WithKind("VRF"), + v1alpha1.GroupVersion.WithKind("NTP"), + v1alpha1.GroupVersion.WithKind("DNS"), + v1alpha1.GroupVersion.WithKind("LLDP"), + v1alpha1.GroupVersion.WithKind("Banner"), + v1alpha1.GroupVersion.WithKind("OSPF"), + v1alpha1.GroupVersion.WithKind("PIM"), + v1alpha1.GroupVersion.WithKind("NetworkVirtualizationEdge"), + v1alpha1.GroupVersion.WithKind("EVPNInstance"), + v1alpha1.GroupVersion.WithKind("RoutingPolicy"), + v1alpha1.GroupVersion.WithKind("PrefixSet"), + v1alpha1.GroupVersion.WithKind("BGP"), + v1alpha1.GroupVersion.WithKind("BGPPeer"), + v1alpha1.GroupVersion.WithKind("Syslog"), + v1alpha1.GroupVersion.WithKind("SNMP"), + v1alpha1.GroupVersion.WithKind("ManagementAccess"), + v1alpha1.GroupVersion.WithKind("AccessControlList"), + v1alpha1.GroupVersion.WithKind("DHCPRelay"), + v1alpha1.GroupVersion.WithKind("ISIS"), +} + +// ConfigResources are provider-specific config resources (e.g., NX-OS configs). +// During cleanup, these are deleted AFTER core resources. +var ConfigResources = []schema.GroupVersionKind{ + nxv1alpha1.GroupVersion.WithKind("InterfaceConfig"), + nxv1alpha1.GroupVersion.WithKind("LLDPConfig"), + nxv1alpha1.GroupVersion.WithKind("BGPConfig"), + nxv1alpha1.GroupVersion.WithKind("VPCDomain"), +} + +// ResourcePluralName returns the plural resource name for a GVK. +// These must match the CRD spec.names.plural values (from `kubectl api-resources`). +// We can't use meta.UnsafeGuessKindToResource because CRDs define their own plurals +// which don't always follow standard Kubernetes pluralization rules. +func ResourcePluralName(gvk schema.GroupVersionKind) string { + plurals := map[string]string{ + "Interface": "interfaces", + "VLAN": "vlans", + "VRF": "vrfs", + "NTP": "ntp", + "DNS": "dns", + "LLDP": "lldps", + "Banner": "banners", + "OSPF": "ospf", + "PIM": "pim", + "NetworkVirtualizationEdge": "networkvirtualizationedges", + "EVPNInstance": "evpninstances", + "InterfaceConfig": "interfaceconfigs", + "LLDPConfig": "lldpconfigs", + "VPCDomain": "vpcdomains", + "BGPConfig": "bgpconfigs", + "RoutingPolicy": "routingpolicies", + "PrefixSet": "prefixsets", + "BGP": "bgp", + "BGPPeer": "bgppeers", + "Syslog": "syslogs", + "SNMP": "snmp", + "ManagementAccess": "managementaccesses", + "AccessControlList": "accesscontrollists", + "DHCPRelay": "dhcprelays", + "ISIS": "isis", + "Device": "devices", + } + if plural, ok := plurals[gvk.Kind]; ok { + return plural + } + // Fallback to standard pluralization + plural, _ := meta.UnsafeGuessKindToResource(gvk) + return plural.Resource +} + +// CreateTestDevice creates a Device pointing to the gNMI server with a generated name. +func CreateTestDevice(ctx context.Context, c client.Client, gnmiAddr, namespace string) (*v1alpha1.Device, error) { + device := &v1alpha1.Device{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "test-device-", + Namespace: namespace, + }, + Spec: v1alpha1.DeviceSpec{ + Endpoint: v1alpha1.Endpoint{ + Address: gnmiAddr, + }, + }, + } + if err := c.Create(ctx, device); err != nil { + return nil, err + } + + // Set the device status to Running so that dependent resources can reconcile + device.Status.Phase = v1alpha1.DevicePhaseRunning + if err := c.Status().Update(ctx, device); err != nil { + return nil, err + } + + return device, nil +} + +// CleanupTimeout is the timeout for cleanup operations. +const CleanupTimeout = 30 * time.Second + +// CleanupInterval is the polling interval for cleanup operations. +const CleanupInterval = 100 * time.Millisecond diff --git a/test/e2e/testutil/timeouts.go b/test/e2e/testutil/timeouts.go new file mode 100644 index 000000000..4c8e8ef31 --- /dev/null +++ b/test/e2e/testutil/timeouts.go @@ -0,0 +1,17 @@ +// SPDX-FileCopyrightText: 2025 SAP SE or an SAP affiliate company and IronCore contributors +// SPDX-License-Identifier: Apache-2.0 + +package testutil + +import "time" + +const ( + // DefaultTimeout is used for standard resource reconciliation + DefaultTimeout = 30 * time.Second + + // LongTimeout is used for operations that may take longer (deployments, pod starts) + LongTimeout = 2 * time.Minute + + // VeryLongTimeout is used for end-to-end scenarios with multiple dependencies + VeryLongTimeout = 5 * time.Minute +) diff --git a/test/e2e/util_test.go b/test/e2e/util_test.go index 1281d18db..1492aa773 100644 --- a/test/e2e/util_test.go +++ b/test/e2e/util_test.go @@ -1,3 +1,5 @@ +//go:build !cluster && !envtest + // SPDX-FileCopyrightText: 2025 SAP SE or an SAP affiliate company and IronCore contributors // SPDX-License-Identifier: Apache-2.0 diff --git a/test/gnmi/Dockerfile b/test/gnmi/Dockerfile index d80e16ac4..bdc864cc6 100644 --- a/test/gnmi/Dockerfile +++ b/test/gnmi/Dockerfile @@ -12,17 +12,14 @@ ARG TARGETARCH WORKDIR /workspace -# Install dependencies -RUN --mount=type=cache,target=/go/pkg/mod \ - --mount=type=bind,source=go.mod,target=go.mod \ - --mount=type=bind,source=go.sum,target=go.sum \ - go mod download -x +# Copy source files +COPY go.mod go.sum ./ +RUN go mod download -x + +COPY . . # Build the application into a static executable while removing the symbol table and debugging information -RUN --mount=type=bind,target=. \ - --mount=type=cache,target=/go/pkg/mod \ - --mount=type=cache,target=/root/.cache/go-build \ - CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -ldflags="-s -w" -o /usr/bin/server ./main.go +RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -ldflags="-s -w" -o /usr/bin/server ./main.go FROM alpine:${ALPINE_VERSION} @@ -39,5 +36,5 @@ USER 65532:65532 # Switch into workspace WORKDIR / -# Start the server application -CMD ["/server", "--port=9339", "--http-port=8000"] +# Start the server application with NX-OS behavior enabled +CMD ["/server", "--port=9339", "--http-port=8000", "--nxos"] diff --git a/test/gnmi/main.go b/test/gnmi/main.go index ef2018cde..eb5408639 100644 --- a/test/gnmi/main.go +++ b/test/gnmi/main.go @@ -4,362 +4,57 @@ package main import ( - "bytes" "context" - "crypto/tls" - "encoding/json" "flag" - "fmt" - "io" "log" - "net" - "net/http" - "strconv" - "strings" - "sync" - "time" + "os" + "os/signal" + "syscall" - gpb "github.com/openconfig/gnmi/proto/gnmi" - "github.com/tidwall/gjson" - "github.com/tidwall/sjson" - "google.golang.org/grpc" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/credentials" - "google.golang.org/grpc/reflection" - "google.golang.org/grpc/status" - - gtls "github.com/openconfig/gnmi/testing/fake/testing/tls" + "github.com/ironcore-dev/gnmi-test-server/testserver" ) -var _ gpb.GNMIServer = (*Server)(nil) - -// Server implements the GNMI gRPC server -type Server struct { - gpb.UnimplementedGNMIServer - - State *State -} - -func (s *Server) Capabilities(_ context.Context, _ *gpb.CapabilityRequest) (*gpb.CapabilityResponse, error) { - return &gpb.CapabilityResponse{SupportedEncodings: []gpb.Encoding{gpb.Encoding_JSON}}, nil -} - -func (s *Server) Get(_ context.Context, req *gpb.GetRequest) (*gpb.GetResponse, error) { - notifications := make([]*gpb.Notification, 0, len(req.GetPath())) - for _, path := range req.GetPath() { - if len(path.GetElem()) == 0 { - return nil, status.Error(codes.InvalidArgument, "root path is not allowed") - } - log.Printf("Getting path: %v", path) - notifications = append(notifications, &gpb.Notification{ - Timestamp: time.Now().UnixNano(), - Update: []*gpb.Update{ - { - Path: path, - Val: &gpb.TypedValue{ - Value: &gpb.TypedValue_JsonVal{ - JsonVal: s.State.Get(path), - }, - }, - }, - }, - }) - } - return &gpb.GetResponse{ - Notification: notifications, - }, nil -} - -func (s *Server) Set(_ context.Context, req *gpb.SetRequest) (*gpb.SetResponse, error) { - log.Printf("Received Set request: %v", req) - res := make([]*gpb.UpdateResult, 0, len(req.GetDelete())+len(req.GetUpdate())) - for _, del := range req.GetDelete() { - log.Printf("Deleting path: %v", del) - res = append(res, &gpb.UpdateResult{ - Timestamp: time.Now().UnixNano(), - Path: del, - Op: gpb.UpdateResult_DELETE, - }) - s.State.Del(del) - } - for _, replace := range req.GetReplace() { - log.Printf("Replacing path: %v with value: %q", replace.GetPath(), replace.GetVal().GetJsonVal()) - res = append(res, &gpb.UpdateResult{ - Timestamp: time.Now().UnixNano(), - Path: replace.Path, - Op: gpb.UpdateResult_REPLACE, - }) - // Delete the existing value at the path and set the new value. - s.State.Del(replace.GetPath()) - s.State.Set(replace.GetPath(), replace.GetVal().GetJsonVal()) - } - for _, update := range req.GetUpdate() { - log.Printf("Updating path: %v with value: %q", update.GetPath(), update.GetVal().GetJsonVal()) - res = append(res, &gpb.UpdateResult{ - Timestamp: time.Now().UnixNano(), - Path: update.Path, - Op: gpb.UpdateResult_UPDATE, - }) - // The value will automatically be merged into the existing state. - s.State.Set(update.GetPath(), update.GetVal().GetJsonVal()) - } - // TODO: Handle UnionReplace - return &gpb.SetResponse{ - Response: res, - Timestamp: time.Now().UnixNano(), - }, nil -} - -func (s *Server) Subscribe(stream grpc.BidiStreamingServer[gpb.SubscribeRequest, gpb.SubscribeResponse]) error { - req, err := stream.Recv() - switch { - case err == io.EOF: - return nil - case err != nil: - return err - case req.GetSubscribe() == nil: - return status.Errorf(codes.InvalidArgument, "the subscribe request must contain a subscription definition") - } - - switch req.GetRequest().(type) { - case *gpb.SubscribeRequest_Poll: - return status.Errorf(codes.InvalidArgument, "invalid request type: %T", req.GetRequest()) - case *gpb.SubscribeRequest_Subscribe: - } - - switch mode := req.GetSubscribe().GetMode(); mode { - case gpb.SubscriptionList_ONCE: - log.Printf("Received Subscribe request with ONCE mode") - - paths := make([]*gpb.Path, 0, len(req.GetSubscribe().GetSubscription())) - for _, r := range req.GetSubscribe().GetSubscription() { - paths = append(paths, r.GetPath()) - } - - res, err := s.Get(stream.Context(), &gpb.GetRequest{ - Prefix: req.GetSubscribe().GetPrefix(), - Path: paths, - Encoding: req.GetSubscribe().GetEncoding(), - UseModels: req.GetSubscribe().GetUseModels(), - Extension: req.GetExtension(), - }) - if err != nil { - return err - } - - for _, notification := range res.GetNotification() { - if err := stream.Send(&gpb.SubscribeResponse{ - Response: &gpb.SubscribeResponse_Update{ - Update: notification, - }, - }); err != nil { - return status.Errorf(codes.Internal, "failed to send response: %v", err) - } - } - - case gpb.SubscriptionList_STREAM: - return status.Errorf(codes.Unimplemented, "subscribe method Stream not implemented") - case gpb.SubscriptionList_POLL: - return status.Errorf(codes.Unimplemented, "subscribe method Poll not implemented") - default: - return status.Errorf(codes.InvalidArgument, "unknown subscribe request mode: %v", mode) - } - - return nil -} - -// handleState handles HTTP requests to the /v1/state endpoint -func (s *Server) handleState(w http.ResponseWriter, r *http.Request) { - switch r.Method { - case http.MethodGet: - s.State.RLock() - defer s.State.RUnlock() - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(http.StatusOK) - if len(s.State.Buf) == 0 { - w.Write([]byte("{}")) - return - } - var buf bytes.Buffer - if err := json.Compact(&buf, s.State.Buf); err != nil { - log.Printf("Failed to compact JSON: %v", err) - w.WriteHeader(http.StatusInternalServerError) - w.Write([]byte("Internal Server Error")) - return - } - w.Write(buf.Bytes()) - case http.MethodDelete: - s.State.Lock() - defer s.State.Unlock() - s.State.Buf = nil - w.WriteHeader(http.StatusNoContent) - default: - w.WriteHeader(http.StatusMethodNotAllowed) - } -} - -// State represents a JSON body that can be manipulated using [sjson] syntax. -type State struct { - sync.RWMutex - - Buf []byte -} - -func (s State) Get(path *gpb.Path) []byte { - s.RLock() - defer s.RUnlock() - var sb strings.Builder - for _, elem := range path.GetElem() { - if elem.GetName() == "" { - continue - } - if sb.Len() > 0 { - sb.WriteByte('|') - } - sb.WriteString(elem.GetName()) - if len(elem.GetKey()) == 0 { - continue - } - for k, v := range elem.GetKey() { - sb.WriteByte('|') - sb.WriteString(`#(`) - sb.WriteString(k) - sb.WriteString(`=="`) - sb.WriteString(v) - sb.WriteString(`")#`) - } - } - res := gjson.GetBytes(s.Buf, sb.String()) - if !res.Exists() || (res.IsArray() && len(res.Array()) == 0) { - return []byte("null") - } - return []byte(res.Raw) -} - -func (s *State) Set(path *gpb.Path, raw []byte) { - s.Lock() - defer s.Unlock() - var sb strings.Builder - for _, elem := range path.GetElem() { - if elem.GetName() == "" { - continue - } - if sb.Len() > 0 { - sb.WriteByte('.') - } - sb.WriteString(elem.GetName()) - if len(elem.GetKey()) == 0 { - continue - } - var idx int - gjson.GetBytes(s.Buf, sb.String()).ForEach(func(_, r gjson.Result) bool { - for k, v := range elem.GetKey() { - if r.Get(k).String() != v { - idx++ - return true - } - } - return false - }) - sb.WriteByte('.') - sb.WriteString(strconv.Itoa(idx)) - } - s.Buf, _ = sjson.SetRawBytes(s.Buf, sb.String(), raw) //nolint:errcheck -} - -func (s *State) Del(path *gpb.Path) { - s.Lock() - defer s.Unlock() - var sb strings.Builder - for _, elem := range path.GetElem() { - if elem.GetName() == "" { - continue - } - if sb.Len() > 0 { - sb.WriteByte('.') - } - sb.WriteString(elem.GetName()) - if len(elem.GetKey()) == 0 { - continue - } - var ( - idx int - found bool - ) - gjson.GetBytes(s.Buf, sb.String()).ForEach(func(_, r gjson.Result) bool { - for k, v := range elem.GetKey() { - if r.Get(k).String() != v { - idx++ - return true - } - } - found = true - return false - }) - if !found { - return - } - sb.WriteByte('.') - sb.WriteString(strconv.Itoa(idx)) - } - - s.Buf, _ = sjson.DeleteBytes(s.Buf, sb.String()) //nolint:errcheck -} - func main() { // Parse command line flags port := flag.Int("port", 9339, "The gRPC server port") httpPort := flag.Int("http-port", 8000, "The HTTP server port") + nxos := flag.Bool("nxos", false, "Enable NX-OS behavior (strip DME markers)") flag.Parse() - // Create a listener on the specified port - lis, err := net.Listen("tcp", fmt.Sprintf(":%d", *port)) - if err != nil { - log.Fatalf("Failed to listen on port %d: %v", *port, err) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + // Build server options + opts := []testserver.ServerOption{ + testserver.WithGRPCPort(*port), + testserver.WithHTTPPort(*httpPort), + testserver.WithBindAddress("0.0.0.0"), + } + if *nxos { + opts = append(opts, testserver.WithNXOSBehavior()) } - // Create a TLS certificate for gRPC server - // This is a self-signed certificate for testing purposes. - cert, err := gtls.NewCert() + // Start the server using the reusable NewTestServer function + // Bind to 0.0.0.0 to accept connections from other pods in the cluster + server, grpcAddr, httpAddr, err := testserver.NewTestServer(ctx, opts...) if err != nil { - log.Fatalf("Failed to create TLS certificate: %v", err) + log.Fatalf("Failed to start server: %v", err) } - // Create a new gRPC server with TLS - grpcServer := grpc.NewServer(grpc.Creds(credentials.NewTLS(&tls.Config{ - Certificates: []tls.Certificate{cert}, - }))) - - // Create our server implementation - server := &Server{State: &State{}} - - // Register the GNMIService with our server implementation - gpb.RegisterGNMIServer(grpcServer, server) - - // Enable reflection for easier testing with tools like grpcurl - reflection.Register(grpcServer) - - // Setup HTTP server - http.HandleFunc("/v1/state", server.handleState) - httpServer := &http.Server{Addr: fmt.Sprintf(":%d", *httpPort)} - - // Start HTTP server in a goroutine - go func() { - log.Printf("Starting HTTP server on port %d", *httpPort) - log.Printf("HTTP endpoint available at: /v1/state") - if err := httpServer.ListenAndServe(); err != nil && err != http.ErrServerClosed { - log.Fatalf("Failed to serve HTTP server: %v", err) - } - }() - - log.Printf("Starting gRPC server on port %d", *port) - log.Printf("Server is ready to accept connections...") + log.Printf("gRPC server listening on %s", grpcAddr) + log.Printf("HTTP server listening on %s", httpAddr) + log.Printf("HTTP endpoint available at: /v1/state") log.Printf("Use --port flag to specify a different gRPC port (default: 9339)") log.Printf("Use --http-port flag to specify a different HTTP port (default: 8000)") log.Printf("Available services: GNMI") - // Start serving - if err := grpcServer.Serve(lis); err != nil { - log.Fatalf("Failed to serve gRPC server: %v", err) + // Wait for interrupt signal + sigCh := make(chan os.Signal, 1) + signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM) + <-sigCh + + log.Println("Shutting down...") + if err := server.Close(); err != nil { + log.Printf("Error during shutdown: %v", err) } } diff --git a/test/gnmi/testserver/server.go b/test/gnmi/testserver/server.go new file mode 100644 index 000000000..c8a7aa734 --- /dev/null +++ b/test/gnmi/testserver/server.go @@ -0,0 +1,636 @@ +// SPDX-FileCopyrightText: 2026 SAP SE or an SAP affiliate company and IronCore contributors +// SPDX-License-Identifier: Apache-2.0 + +package testserver + +import ( + "bytes" + "context" + "crypto/tls" + "encoding/json" + "fmt" + "io" + "log" + "net" + "net/http" + "strconv" + "strings" + "sync" + "time" + + gpb "github.com/openconfig/gnmi/proto/gnmi" + "github.com/tidwall/gjson" + "github.com/tidwall/sjson" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/reflection" + "google.golang.org/grpc/status" + + gtls "github.com/openconfig/gnmi/testing/fake/testing/tls" +) + +var _ gpb.GNMIServer = (*Server)(nil) + +// Server implements the GNMI gRPC server +type Server struct { + gpb.UnimplementedGNMIServer + + State *State + + grpcServer *grpc.Server + httpServer *http.Server + grpcAddr string + httpAddr string +} + +// ServerOption configures the test server +type ServerOption func(*serverConfig) + +type serverConfig struct { + grpcPort int + httpPort int + bindAddress string + stripDMEMarkers bool + dmeMarkerValue string +} + +// WithGRPCPort sets a specific gRPC port (default: 0 for random) +func WithGRPCPort(port int) ServerOption { + return func(c *serverConfig) { + c.grpcPort = port + } +} + +// WithHTTPPort sets a specific HTTP port (default: 0 for random) +func WithHTTPPort(port int) ServerOption { + return func(c *serverConfig) { + c.httpPort = port + } +} + +// WithBindAddress sets the address to bind to (default: 127.0.0.1). +// Use "0.0.0.0" to listen on all interfaces (required for container/pod deployments). +func WithBindAddress(addr string) ServerOption { + return func(c *serverConfig) { + c.bindAddress = addr + } +} + +// WithNXOSBehavior configures the server to emulate NX-OS device behavior: +// - Strips fields with DME_UNSET_PROPERTY_MARKER value when storing (the marker +// means "unset this field", not "store this literal string") +// - Returns empty TypedValue for non-existent paths (instead of NOT_FOUND error) +func WithNXOSBehavior() ServerOption { + return func(c *serverConfig) { + c.stripDMEMarkers = true + c.dmeMarkerValue = "DME_UNSET_PROPERTY_MARKER" + } +} + +// NewTestServer starts an in-process gNMI + HTTP server. +// By default, it uses random available ports. Use WithGRPCPort/WithHTTPPort to specify ports. +// Returns the server, gRPC address, HTTP address, and any error. +func NewTestServer(ctx context.Context, opts ...ServerOption) (*Server, string, string, error) { + cfg := &serverConfig{ + grpcPort: 0, // Random port by default + httpPort: 0, // Random port by default + bindAddress: "127.0.0.1", // Localhost by default (safe for in-process tests) + } + for _, opt := range opts { + opt(cfg) + } + + // Create a listener on the specified port + grpcLis, err := net.Listen("tcp", fmt.Sprintf("%s:%d", cfg.bindAddress, cfg.grpcPort)) + if err != nil { + return nil, "", "", fmt.Errorf("failed to listen for gRPC: %w", err) + } + + httpLis, err := net.Listen("tcp", fmt.Sprintf("%s:%d", cfg.bindAddress, cfg.httpPort)) + if err != nil { + grpcLis.Close() + return nil, "", "", fmt.Errorf("failed to listen for HTTP: %w", err) + } + + // Create a TLS certificate for gRPC server + cert, err := gtls.NewCert() + if err != nil { + grpcLis.Close() + httpLis.Close() + return nil, "", "", fmt.Errorf("failed to create TLS certificate: %w", err) + } + + // Create a new gRPC server with TLS + grpcServer := grpc.NewServer(grpc.Creds(credentials.NewTLS(&tls.Config{ + Certificates: []tls.Certificate{cert}, + }))) + + // Create our server implementation + server := &Server{ + State: &State{ + stripDMEMarkers: cfg.stripDMEMarkers, + dmeMarkerValue: cfg.dmeMarkerValue, + }, + grpcServer: grpcServer, + grpcAddr: grpcLis.Addr().String(), + httpAddr: httpLis.Addr().String(), + } + + // Register the GNMIService with our server implementation + gpb.RegisterGNMIServer(grpcServer, server) + + // Enable reflection for easier testing + reflection.Register(grpcServer) + + // Setup HTTP server + mux := http.NewServeMux() + mux.HandleFunc("/v1/state", server.handleState) + mux.HandleFunc("/v1/clear", server.handleClear) + server.httpServer = &http.Server{Handler: mux} + + // Start HTTP server in a goroutine + go func() { + log.Printf("Starting HTTP server on %s", server.httpAddr) + if err := server.httpServer.Serve(httpLis); err != nil && err != http.ErrServerClosed { + log.Printf("HTTP server error: %v", err) + } + }() + + // Start gRPC server in a goroutine + go func() { + log.Printf("Starting gRPC server on %s", server.grpcAddr) + if err := grpcServer.Serve(grpcLis); err != nil { + log.Printf("gRPC server error: %v", err) + } + }() + + return server, server.grpcAddr, server.httpAddr, nil +} + +// GRPCAddr returns the gRPC server address +func (s *Server) GRPCAddr() string { + return s.grpcAddr +} + +// HTTPAddr returns the HTTP server address +func (s *Server) HTTPAddr() string { + return s.httpAddr +} + +// GetState returns the current JSON state +func (s *Server) GetState() ([]byte, error) { + s.State.RLock() + defer s.State.RUnlock() + if len(s.State.Buf) == 0 { + return []byte("{}"), nil + } + var buf bytes.Buffer + if err := json.Compact(&buf, s.State.Buf); err != nil { + return nil, fmt.Errorf("failed to compact JSON: %w", err) + } + return buf.Bytes(), nil +} + +// ClearState clears all accumulated state +func (s *Server) ClearState() { + s.State.Lock() + defer s.State.Unlock() + s.State.Buf = nil +} + +// Close gracefully shuts down the server +func (s *Server) Close() error { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + var errs []error + if s.httpServer != nil { + if err := s.httpServer.Shutdown(ctx); err != nil { + errs = append(errs, fmt.Errorf("HTTP shutdown: %w", err)) + } + } + if s.grpcServer != nil { + s.grpcServer.GracefulStop() + } + if len(errs) > 0 { + return errs[0] + } + return nil +} + +func (s *Server) Capabilities(_ context.Context, _ *gpb.CapabilityRequest) (*gpb.CapabilityResponse, error) { + return &gpb.CapabilityResponse{SupportedEncodings: []gpb.Encoding{gpb.Encoding_JSON}}, nil +} + +func (s *Server) Get(_ context.Context, req *gpb.GetRequest) (*gpb.GetResponse, error) { + notifications := make([]*gpb.Notification, 0, len(req.GetPath())) + for _, path := range req.GetPath() { + if len(path.GetElem()) == 0 { + return nil, status.Error(codes.InvalidArgument, "root path is not allowed") + } + log.Printf("Getting path: %v", path) + notifications = append(notifications, &gpb.Notification{ + Timestamp: time.Now().UnixNano(), + Update: []*gpb.Update{ + { + Path: path, + Val: &gpb.TypedValue{ + Value: &gpb.TypedValue_JsonVal{ + JsonVal: s.State.Get(path), + }, + }, + }, + }, + }) + } + return &gpb.GetResponse{ + Notification: notifications, + }, nil +} + +func (s *Server) Set(_ context.Context, req *gpb.SetRequest) (*gpb.SetResponse, error) { + log.Printf("Received Set request: %v", req) + res := make([]*gpb.UpdateResult, 0, len(req.GetDelete())+len(req.GetUpdate())) + for _, del := range req.GetDelete() { + log.Printf("Deleting path: %v", del) + res = append(res, &gpb.UpdateResult{ + Timestamp: time.Now().UnixNano(), + Path: del, + Op: gpb.UpdateResult_DELETE, + }) + s.State.Del(del) + } + for _, replace := range req.GetReplace() { + log.Printf("Replacing path: %v with value: %q", replace.GetPath(), replace.GetVal().GetJsonVal()) + res = append(res, &gpb.UpdateResult{ + Timestamp: time.Now().UnixNano(), + Path: replace.Path, + Op: gpb.UpdateResult_REPLACE, + }) + // Delete the existing value at the path and set the new value. + s.State.Del(replace.GetPath()) + s.State.Set(replace.GetPath(), replace.GetVal().GetJsonVal()) + } + for _, update := range req.GetUpdate() { + log.Printf("Updating path: %v with value: %q", update.GetPath(), update.GetVal().GetJsonVal()) + res = append(res, &gpb.UpdateResult{ + Timestamp: time.Now().UnixNano(), + Path: update.Path, + Op: gpb.UpdateResult_UPDATE, + }) + // The value will automatically be merged into the existing state. + s.State.Set(update.GetPath(), update.GetVal().GetJsonVal()) + } + // TODO: Handle UnionReplace + return &gpb.SetResponse{ + Response: res, + Timestamp: time.Now().UnixNano(), + }, nil +} + +func (s *Server) Subscribe(stream grpc.BidiStreamingServer[gpb.SubscribeRequest, gpb.SubscribeResponse]) error { + req, err := stream.Recv() + switch { + case err == io.EOF: + return nil + case err != nil: + return err + case req.GetSubscribe() == nil: + return status.Errorf(codes.InvalidArgument, "the subscribe request must contain a subscription definition") + } + + switch req.GetRequest().(type) { + case *gpb.SubscribeRequest_Poll: + return status.Errorf(codes.InvalidArgument, "invalid request type: %T", req.GetRequest()) + case *gpb.SubscribeRequest_Subscribe: + } + + switch mode := req.GetSubscribe().GetMode(); mode { + case gpb.SubscriptionList_ONCE: + log.Printf("Received Subscribe request with ONCE mode") + + paths := make([]*gpb.Path, 0, len(req.GetSubscribe().GetSubscription())) + for _, r := range req.GetSubscribe().GetSubscription() { + paths = append(paths, r.GetPath()) + } + + res, err := s.Get(stream.Context(), &gpb.GetRequest{ + Prefix: req.GetSubscribe().GetPrefix(), + Path: paths, + Encoding: req.GetSubscribe().GetEncoding(), + UseModels: req.GetSubscribe().GetUseModels(), + Extension: req.GetExtension(), + }) + if err != nil { + return err + } + + for _, notification := range res.GetNotification() { + if err := stream.Send(&gpb.SubscribeResponse{ + Response: &gpb.SubscribeResponse_Update{ + Update: notification, + }, + }); err != nil { + return status.Errorf(codes.Internal, "failed to send response: %v", err) + } + } + + case gpb.SubscriptionList_STREAM: + return status.Errorf(codes.Unimplemented, "subscribe method Stream not implemented") + case gpb.SubscriptionList_POLL: + return status.Errorf(codes.Unimplemented, "subscribe method Poll not implemented") + default: + return status.Errorf(codes.InvalidArgument, "unknown subscribe request mode: %v", mode) + } + + return nil +} + +// handleState handles HTTP requests to the /v1/state endpoint +// GET: returns current state as JSON +// POST: preloads nested JSON into state +// DELETE: clears all state +// Supports X-HTTP-Method-Override header for clients that can't send DELETE. +func (s *Server) handleState(w http.ResponseWriter, r *http.Request) { + method := r.Method + if override := r.Header.Get("X-HTTP-Method-Override"); override != "" { + method = override + } + switch method { + case http.MethodGet: + state, err := s.GetState() + if err != nil { + log.Printf("Failed to get state: %v", err) + w.WriteHeader(http.StatusInternalServerError) + w.Write([]byte("Internal Server Error")) + return + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + w.Write(state) + case http.MethodPost: + body, err := io.ReadAll(r.Body) + if err != nil { + log.Printf("Failed to read body: %v", err) + http.Error(w, "Failed to read body", http.StatusBadRequest) + return + } + if len(body) == 0 { + w.WriteHeader(http.StatusNoContent) + return + } + if !gjson.ValidBytes(body) { + http.Error(w, "invalid JSON", http.StatusBadRequest) + return + } + // Use Set with empty path to merge JSON into root of state + s.State.Set(&gpb.Path{}, body) + log.Printf("Merged state from JSON") + w.WriteHeader(http.StatusNoContent) + case http.MethodDelete: + s.ClearState() + w.WriteHeader(http.StatusNoContent) + default: + w.WriteHeader(http.StatusMethodNotAllowed) + } +} + +// handleClear handles POST /v1/clear to clear all state. +func (s *Server) handleClear(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + s.ClearState() + w.WriteHeader(http.StatusNoContent) +} + +// mergeJSON merges src JSON into dst JSON at the root level. +// Keys in src overwrite keys in dst. +func mergeJSON(dst, src []byte) []byte { + srcParsed := gjson.ParseBytes(src) + if !srcParsed.IsObject() { + return src + } + result := dst + srcParsed.ForEach(func(key, value gjson.Result) bool { + result, _ = sjson.SetRawBytes(result, key.String(), []byte(value.Raw)) + return true + }) + return result +} + +// State represents a JSON body that can be manipulated using [sjson] syntax. +type State struct { + sync.RWMutex + + Buf []byte + + // NX-OS behavior options + stripDMEMarkers bool + dmeMarkerValue string +} + +// stripMarkerFields removes fields with the DME marker value from JSON recursively. +// This emulates NX-OS behavior where these markers mean "unset this field" +// rather than "store this literal string". +func (s *State) stripMarkerFields(data []byte) []byte { + if !s.stripDMEMarkers || s.dmeMarkerValue == "" { + return data + } + return s.stripMarkersRecursive(data) +} + +// stripMarkersRecursive walks the JSON structure and removes marker fields at all levels. +func (s *State) stripMarkersRecursive(data []byte) []byte { + parsed := gjson.ParseBytes(data) + if !parsed.IsObject() && !parsed.IsArray() { + return data + } + + if parsed.IsArray() { + // Process each array element + var results []string + parsed.ForEach(func(_, value gjson.Result) bool { + processed := s.stripMarkersRecursive([]byte(value.Raw)) + results = append(results, string(processed)) + return true + }) + return []byte("[" + strings.Join(results, ",") + "]") + } + + // It's an object - process fields + var toDelete []string + parsed.ForEach(func(key, value gjson.Result) bool { + keyStr := key.String() + if value.Type == gjson.String && value.String() == s.dmeMarkerValue { + toDelete = append(toDelete, keyStr) + } else if value.IsObject() || value.IsArray() { + // Recurse into nested structures + processed := s.stripMarkersRecursive([]byte(value.Raw)) + data, _ = sjson.SetRawBytes(data, keyStr, processed) + } + return true + }) + for _, key := range toDelete { + data, _ = sjson.DeleteBytes(data, key) + } + return data +} + +func (s *State) Get(path *gpb.Path) []byte { + s.RLock() + defer s.RUnlock() + var sb strings.Builder + for _, elem := range path.GetElem() { + if elem.GetName() == "" { + continue + } + if sb.Len() > 0 { + sb.WriteByte('|') + } + sb.WriteString(elem.GetName()) + if len(elem.GetKey()) == 0 { + continue + } + for k, v := range elem.GetKey() { + sb.WriteByte('|') + sb.WriteString(`#(`) + sb.WriteString(k) + sb.WriteString(`=="`) + sb.WriteString(v) + sb.WriteString(`")#`) + } + } + res := gjson.GetBytes(s.Buf, sb.String()) + if !res.Exists() || (res.IsArray() && len(res.Array()) == 0) { + // Return empty bytes for non-existent paths. This triggers gnmiext's + // ErrNil handling (len(b) == 0), matching real NX-OS behavior which + // returns empty TypedValue for paths that don't exist yet. + return []byte{} + } + return []byte(res.Raw) +} + +func (s *State) Set(path *gpb.Path, raw []byte) { + s.Lock() + defer s.Unlock() + + // Strip DME marker fields if NX-OS behavior is enabled + raw = s.stripMarkerFields(raw) + + elems := path.GetElem() + + // Handle empty path - merge raw into state at root level + if len(elems) == 0 { + if len(s.Buf) == 0 { + s.Buf = raw + } else { + s.Buf = mergeJSON(s.Buf, raw) + } + return + } + + var sb strings.Builder + + for i, elem := range elems { + if elem.GetName() == "" { + continue + } + if sb.Len() > 0 { + sb.WriteByte('.') + } + sb.WriteString(elem.GetName()) + + if len(elem.GetKey()) == 0 { + continue + } + + // Find existing array index or append + var idx int + gjson.GetBytes(s.Buf, sb.String()).ForEach(func(_, r gjson.Result) bool { + for k, v := range elem.GetKey() { + if r.Get(k).String() != v { + idx++ + return true + } + } + return false + }) + sb.WriteByte('.') + sb.WriteString(strconv.Itoa(idx)) + + // Inject keys into this list element if it's not the final element + // (for the final element, keys go into raw below) + if i < len(elems)-1 { + currentPath := sb.String() + current := gjson.GetBytes(s.Buf, currentPath) + if !current.Exists() || current.Raw == "null" { + // Create the element with its keys + keyObj := make(map[string]string) + for k, v := range elem.GetKey() { + keyObj[k] = v + } + keyJSON, _ := json.Marshal(keyObj) + s.Buf, _ = sjson.SetRawBytes(s.Buf, currentPath, keyJSON) + } else { + // Element exists, ensure keys are set + for k, v := range elem.GetKey() { + if !gjson.GetBytes(s.Buf, currentPath+"."+k).Exists() { + s.Buf, _ = sjson.SetBytes(s.Buf, currentPath+"."+k, v) + } + } + } + } + } + + // For the final element, inject its keys (from the last keyed element) into raw + lastElem := elems[len(elems)-1] + for k, v := range lastElem.GetKey() { + if !gjson.GetBytes(raw, k).Exists() { + raw, _ = sjson.SetBytes(raw, k, v) + } + } + + s.Buf, _ = sjson.SetRawBytes(s.Buf, sb.String(), raw) //nolint:errcheck +} + +func (s *State) Del(path *gpb.Path) { + s.Lock() + defer s.Unlock() + var sb strings.Builder + for _, elem := range path.GetElem() { + if elem.GetName() == "" { + continue + } + if sb.Len() > 0 { + sb.WriteByte('.') + } + sb.WriteString(elem.GetName()) + if len(elem.GetKey()) == 0 { + continue + } + var ( + idx int + found bool + ) + gjson.GetBytes(s.Buf, sb.String()).ForEach(func(_, r gjson.Result) bool { + for k, v := range elem.GetKey() { + if r.Get(k).String() != v { + idx++ + return true + } + } + found = true + return false + }) + if !found { + return + } + sb.WriteByte('.') + sb.WriteString(strconv.Itoa(idx)) + } + + s.Buf, _ = sjson.DeleteBytes(s.Buf, sb.String()) //nolint:errcheck +}