HYPERFLEET-1132 - feat: enable parallel E2E execution via Ginkgo

.bingo/Variables.mk

@@ -29,3 +29,9 @@ $(GOLANGCI_LINT): $(BINGO_DIR)/golangci-lint.mod
 	@echo "(re)installing $(GOBIN)/golangci-lint-v2.7.0"
 	@cd $(BINGO_DIR) && GOWORK=off GOOS=$(GOHOSTOS) GOARCH=$(GOHOSTARCH) GOARM=$(GOHOSTARM) $(GO) build -mod=mod -modfile=golangci-lint.mod -o=$(GOBIN)/golangci-lint-v2.7.0 "github.com/golangci/golangci-lint/v2/cmd/golangci-lint"
 
+GINKGO := $(GOBIN)/ginkgo-v2.27.2
+$(GINKGO): $(BINGO_DIR)/ginkgo.mod
+	@# Install binary/ries using Go 1.14+ build command. This is using bwplotka/bingo-controlled, separate go module with pinned dependencies.
+	@echo "(re)installing $(GOBIN)/ginkgo-v2.27.2"
+	@cd $(BINGO_DIR) && GOWORK=off GOOS=$(GOHOSTOS) GOARCH=$(GOHOSTARCH) GOARM=$(GOHOSTARM) $(GO) build -mod=mod -modfile=ginkgo.mod -o=$(GOBIN)/ginkgo-v2.27.2 "github.com/onsi/ginkgo/v2/ginkgo"
+

.bingo/ginkgo.mod

@@ -0,0 +1,5 @@
+module _ // Auto generated by https://github.com/bwplotka/bingo. DO NOT EDIT
+
+go 1.25.3
+
+require github.com/onsi/ginkgo/v2 v2.27.2 // cmd/ginkgo

.bingo/ginkgo.sum

@@ -0,0 +1,24 @@
+github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0=
+github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM=
+github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI=
+github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8=
+github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 h1:BHT72Gu3keYf3ZEu2J0b1vyeLSOYI8bm5wbJM/8yDe8=
+github.com/google/pprof v0.0.0-20250403155104-27863c87afa6/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA=
+github.com/google/pprof v0.0.0-20260402051712-545e8a4df936 h1:EwtI+Al+DeppwYX2oXJCETMO23COyaKGP6fHVpkpWpg=
+github.com/google/pprof v0.0.0-20260402051712-545e8a4df936/go.mod h1:MxpfABSjhmINe3F1It9d+8exIHFvUqtLIRCdOGNXqiI=
+github.com/onsi/ginkgo/v2 v2.27.2 h1:LzwLj0b89qtIy6SSASkzlNvX6WktqurSHwkk2ipF/Ns=
+github.com/onsi/ginkgo/v2 v2.27.2/go.mod h1:ArE1D/XhNXBXCBkKOLkbsb2c81dQHCRcF5zwn/ykDRo=
+github.com/onsi/ginkgo/v2 v2.32.0 h1:Hw7s2pVrQo/8Yz5N77qdnpHaoc+c6cC9WIV1Jce+J6E=
+github.com/onsi/ginkgo/v2 v2.32.0/go.mod h1:+aXOY+vzZ5mu2iI2HpTZUPmM//oQfsNFX6gU9kNcA44=
+golang.org/x/mod v0.27.0 h1:kb+q2PyFnEADO2IEF935ehFUXlWiNjJWtRNgBLSfbxQ=
+golang.org/x/mod v0.27.0/go.mod h1:rWI627Fq0DEoudcK+MBkNkCe0EetEaDSwJJkCcjpazc=
+golang.org/x/mod v0.35.0 h1:Ww1D637e6Pg+Zb2KrWfHQUnH2dQRLBQyAtpr/haaJeM=
+golang.org/x/mod v0.35.0/go.mod h1:+GwiRhIInF8wPm+4AoT6L0FA1QWAad3OMdTRx4tFYlU=
+golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw=
+golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
+golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
+golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
+golang.org/x/tools v0.36.0 h1:kWS0uv/zsvHEle1LbV5LE8QujrxB3wfQyxHfhOk0Qkg=
+golang.org/x/tools v0.36.0/go.mod h1:WBDiHKJK8YgLHlcQPYQzNCkUxUypCaa5ZegCVutKm+s=
+golang.org/x/tools v0.44.0 h1:UP4ajHPIcuMjT1GqzDWRlalUEoY+uzoZKnhOjbIPD2c=
+golang.org/x/tools v0.44.0/go.mod h1:KA0AfVErSdxRZIsOVipbv3rQhVXTnlU6UhKxHd1seDI=

.bingo/variables.env

@@ -9,5 +9,5 @@ fi
 
 
 GOLANGCI_LINT="${GOBIN}/golangci-lint-v2.7.0"
-
+GINKGO="${GOBIN}/ginkgo-v2.27.2"
 

AGENTS.md

@@ -50,7 +50,7 @@ Pre-flight order: `make check` then `make build`.
 
 ### File naming and structure
 
-- **IMPORTANT:** Test files use `.go` extension, NOT `_test.go`. E2E tests are compiled into the binary, not run via `go test`.
+- **IMPORTANT:** Test files use `.go` extension, NOT `_test.go`. E2E tests are compiled into the binary, not run via `go test`. The single exception is `e2e/e2e_suite_test.go` — the ginkgo CLI entry point for parallel execution (`make e2e-ginkgo`); never add spec code there.
 - Location: `e2e/{suite}/descriptive-name.go` (package matches directory name)
 - Test name format: `[Suite: component][category] Description` (e.g., `[Suite: cluster][baseline] Cluster Resource Type Lifecycle`). Known categories: `baseline`, `update`, `delete`, `concurrent`, `negative`, `perf`.
 - Test suites auto-register via blank import in `e2e/e2e.go`
@@ -156,7 +156,7 @@ When `expectedIdentity` is empty, audit assertions are skipped.
 
 ### DON'T
 
-- Use `_test.go` suffix for E2E test files
+- Use `_test.go` suffix for E2E test files (sole exception: `e2e/e2e_suite_test.go`, the ginkgo CLI entry point)
 - Hardcode timeout durations — use `h.Cfg.Timeouts.*`
 - Skip cleanup (`DeferCleanup`)
 - Use `ginkgo.By()` inside `Eventually` closures

Dockerfile

@@ -21,7 +21,14 @@ COPY . .
 ARG GIT_COMMIT=unknown
 RUN make build GIT_COMMIT=${GIT_COMMIT}
 
-RUN chmod +x /build/bin/hyperfleet-e2e
+# Build ginkgo CLI (version-locked via .bingo/ginkgo.mod)
+RUN cd .bingo && GOWORK=off go build -mod=mod -modfile=ginkgo.mod \
+    -o /build/bin/ginkgo "github.com/onsi/ginkgo/v2/ginkgo"
+
+# Compile the E2E test binary for ginkgo CLI parallel execution
+RUN CGO_ENABLED=0 go test -c -o /build/bin/e2e.test ./e2e
+
+RUN chmod +x /build/bin/hyperfleet-e2e /build/bin/ginkgo /build/bin/e2e.test
 
 FROM registry.ci.openshift.org/ci/hyperfleet-credential-provider:latest AS hyperfleet-credential-provider
 
@@ -68,8 +75,10 @@ WORKDIR /e2e
 # Copy the hyperfleet-credential-provider binary
 COPY --from=hyperfleet-credential-provider /app/hyperfleet-credential-provider /usr/local/bin/
 
-# Copy binary from builder (make build outputs to bin/)
+# Copy binaries from builder (main binary + ginkgo CLI + test binary)
 COPY --from=builder /build/bin/hyperfleet-e2e /usr/local/bin/
+COPY --from=builder /build/bin/ginkgo /usr/local/bin/
+COPY --from=builder /build/bin/e2e.test /usr/local/bin/
 
 # Copy test payloads and fixtures
 COPY --from=builder /build/testdata /e2e/testdata

Makefile

@@ -84,11 +84,20 @@ test-coverage: test ## Run tests and generate HTML coverage report
 e2e: build ## Run all E2E tests
 	TESTDATA_DIR=$(PWD)/testdata ./$(BINARY_NAME) test
 
+PROCS ?= 4
+
 .PHONY: e2e-ci
-e2e-ci: build ## Run E2E tests with CI configuration
+e2e-ci: $(GINKGO) ## Run E2E tests with CI configuration (parallel via ginkgo CLI)
 	mkdir -p $(OUTPUT_DIR)
-	TESTDATA_DIR=$(PWD)/testdata ./$(BINARY_NAME) test --flake-attempts=$(FLAKE_ATTEMPTS) --junit-report $(OUTPUT_DIR)/junit.xml
-
+	TESTDATA_DIR=$(PWD)/testdata $(GINKGO) \
+		--procs=$(PROCS) \
+		--flake-attempts=$(FLAKE_ATTEMPTS) \
+		--junit-report=junit.xml \
+		--output-dir=$(OUTPUT_DIR) \
+		$(if $(GINKGO_LABEL_FILTER),--label-filter="$(GINKGO_LABEL_FILTER)") \
+		$(if $(GINKGO_FOCUS),--focus="$(GINKGO_FOCUS)") \
+		$(if $(GINKGO_SKIP),--skip="$(GINKGO_SKIP)") \
+		./e2e
 .PHONY: list-tests
 list-tests: build ## List E2E tests by tier without executing (dry-run)
 	@echo "=== tier0 ==="

docs/development.md

@@ -300,6 +300,39 @@ Available pollers: see `pkg/helper/pollers.go`. Available matchers: see `pkg/hel
 - Don't skip cleanup
 - Don't create `WaitFor*` wrapper functions that hide `Eventually` — use pollers + matchers instead
 
+## Parallel Safety
+
+The suite is designed to support Ginkgo multi-process parallelism (`--procs=N`). All new tests MUST be parallel-safe unless explicitly marked `ginkgo.Serial`.
+
+### Rules for parallel-safe tests
+
+- **Unique resource names.** Use payload templates (`.Random`, `.UUID`) for all API resources. Never use fixed names.
+- **Per-spec cleanup.** Register `ginkgo.DeferCleanup` right after resource creation. Never rely on `AfterSuite` as the primary cleanup path.
+- **No shared mutable state.** Each spec gets its own `helper.New()` instance. Never store state in package-level variables that other specs read.
+- **Temp adapter names must be unique per spec.** Two specs deploying the same adapter name would race on one Helm release.
+- **Never use `os.Setenv` for values that vary per spec.** Pass values via Helm `--set` or struct fields instead.
+
+### When to use `ginkgo.Serial`
+
+Add `ginkgo.Serial` to the `Describe` decorator (not inside an `It`) and include a comment explaining why:
+
+```go
+var _ = ginkgo.Describe("...",
+    ginkgo.Serial, // Serial: scales down sentinel replicas, blocking all concurrent specs
+    ginkgo.Label(labels.Tier1, labels.Disruptive),
+    func() { ... },
+)
+```
+
+Valid reasons to mark a spec `Serial`:
+
+- **Mutates shared infrastructure** (e.g., scales deployments, deletes shared resources)
+- **Deploys temporary adapters** that subscribe to all events, causing cross-talk with concurrent specs
+
+Specs already marked `Serial`: sentinel scale-down, force-delete, stuck-deletion, crash-recovery, maestro-unavailability, adapter-failover, adapter-failure, maestro negative scenarios.
+
+Performance specs (`labels.Performance`) are a separate category - they carry no tier label and run in their own dedicated CI job (`--label-filter="perf"`) on a quiet system. They are not marked `Serial` because they never run alongside functional tests.
+
 ## Adding New Tests
 
 ### 1. Create Test File

e2e/adapter/maestro_unavailability.go

@@ -13,7 +13,7 @@ import (
 )
 
 var _ = ginkgo.Describe("[Suite: adapter][maestro-transport][negative] Adapter can handle Maestro server unavailability gracefully",
-	ginkgo.Serial,
+	ginkgo.Serial, // Serial: scales down maestro, blocking all concurrent adapter specs
 	ginkgo.Label(labels.Tier2, labels.Negative, labels.Disruptive),
 	func() {
 		var h *helper.Helper

e2e/cluster/adapter_failure.go

@@ -13,6 +13,7 @@ import (
 )
 
 var _ = ginkgo.Describe("[Suite: cluster][negative] Cluster Can Reflect Adapter Failure in Top-Level Status",
+	ginkgo.Serial, // Serial: deploys temp adapter subscribing to all events, causes cross-talk
 	ginkgo.Label(labels.Tier1, labels.Negative),
 	func() {
 		var (

e2e/cluster/crash_recovery.go

@@ -13,7 +13,7 @@ import (
 )
 
 var _ = ginkgo.Describe("[Suite: cluster][negative] Cluster Can Reach Correct Status After Adapter Crash and Recovery",
-	ginkgo.Serial,
+	ginkgo.Serial, // Serial: kills and restarts adapter pod, disrupts concurrent specs
 	ginkgo.Label(labels.Tier2, labels.Negative),
 	func() {
 		var (

e2e/cluster/delete_edge_cases.go

@@ -14,6 +14,7 @@ import (
 )
 
 var _ = ginkgo.Describe("[Suite: cluster][delete] Re-DELETE Idempotency and API Boundary Tests",
+	ginkgo.Serial, // Serial: scales down sentinel replicas, blocking all concurrent specs
 	ginkgo.Label(labels.Tier1),
 	func() {
 		var h *helper.Helper

e2e/cluster/stuck_deletion.go

@@ -14,7 +14,7 @@ import (
 )
 
 var _ = ginkgo.Describe("[Suite: cluster][negative] Stuck Deletion -- Adapter Unable to Finalize Prevents Hard-Delete",
-	ginkgo.Serial,
+	ginkgo.Serial, // Serial: deploys stuck adapter that blocks deletion of all clusters
 	ginkgo.Label(labels.Tier2, labels.Negative),
 	func() {
 		var (

e2e/e2e_suite_test.go

@@ -0,0 +1,54 @@
+// e2e_suite_test.go is the ginkgo CLI entry point for parallel E2E execution.
+//
+// This file intentionally uses _test.go so the ginkgo CLI can discover and
+// compile it as a test binary. All E2E spec files remain plain .go (compiled
+// into the main binary via blank imports in e2e.go). This file is never part
+// of the production binary - go build ignores _test.go files.
+//
+// The blank imports in e2e.go (same package) register all spec suites, and
+// importing pkg/e2e registers the BeforeSuite/SynchronizedAfterSuite hooks.
+//
+// Usage:
+//
+//	ginkgo --procs=4 --label-filter=tier0 ./e2e
+package e2e
+
+import (
+	"log"
+	"os"
+	"testing"
+
+	"github.com/onsi/ginkgo/v2"
+	"github.com/onsi/gomega"
+
+	"github.com/openshift-hyperfleet/hyperfleet-e2e/cmd/hyperfleet-e2e/common"
+	"github.com/openshift-hyperfleet/hyperfleet-e2e/pkg/config"
+
+	// Import pkg/e2e to register BeforeSuite and SynchronizedAfterSuite hooks.
+	// The package name collides with this package, so we use an alias.
+	pkge2e "github.com/openshift-hyperfleet/hyperfleet-e2e/pkg/e2e"
+)
+
+func TestMain(m *testing.M) {
+	// Bootstrap configuration from env vars and config file.
+	// The ginkgo CLI handles all test-execution flags (--label-filter,
+	// --flake-attempts, --timeout, --junit-report, --procs) directly,
+	// so we only load HyperFleet-specific config here.
+	if err := common.LoadConfig(""); err != nil {
+		log.Fatalf("Failed to load config: %v", err)
+	}
+
+	cfg, err := config.Load()
+	if err != nil {
+		log.Fatalf("Configuration validation failed: %v", err)
+	}
+
+	pkge2e.SetSuiteConfig(cfg)
+
+	os.Exit(m.Run())
+}
+
+func TestE2E(t *testing.T) {
+	gomega.RegisterFailHandler(ginkgo.Fail)
+	ginkgo.RunSpecs(t, "HyperFleet E2E Suite")
+}

e2e/nodepool/creation.go

@@ -13,14 +13,14 @@ import (
 )
 
 var _ = ginkgo.Describe("[Suite: nodepool][baseline] NodePool Resource Type Lifecycle",
-	ginkgo.Label(labels.Tier0),
+	ginkgo.Ordered, ginkgo.Label(labels.Tier0),
 	func() {
 		var h *helper.Helper
 		var clusterID string
 		var nodepoolID string
 		var nodepoolName string
 
-		ginkgo.BeforeEach(func(ctx context.Context) {
+		ginkgo.BeforeAll(func(ctx context.Context) {
 			h = helper.New()
 
 			// Get or create cluster for nodepool tests
@@ -208,7 +208,7 @@ var _ = ginkgo.Describe("[Suite: nodepool][baseline] NodePool Resource Type Life
 				})
 		})
 
-		ginkgo.AfterEach(func(ctx context.Context) {
+		ginkgo.AfterAll(func(ctx context.Context) {
 			if h == nil || clusterID == "" {
 				return
 			}

pkg/e2e/suite.go

@@ -69,8 +69,18 @@ var _ = ginkgo.BeforeSuite(func(ctx ginkgo.SpecContext) {
 	logger.Info("starting hyperfleet-e2e test suite - each test creates temporary resources")
 })
 
-var _ = ginkgo.AfterSuite(func() {
-	helper.CleanupResources()
-	helper.ClearSuiteConfig()
-	logger.Info("test suite completed")
-})
+var _ = ginkgo.SynchronizedAfterSuite(
+	// Per-process: sweep Pub/Sub resources. Safe to call from every process
+	// since each tracks its own AdapterDeploymentList in memory.
+	func() {
+		helper.CleanupPubSubResources()
+	},
+	// Process 1 only, after all processes finish: sweep Helm releases and labeled K8s
+	// resources. Running this once prevents sweeping resources that belong to specs still
+	// executing on other processes.
+	func() {
+		helper.CleanupKubeResources()
+		helper.ClearSuiteConfig()
+		logger.Info("test suite completed")
+	},
+)

pkg/helper/cleanup.go

@@ -17,13 +17,11 @@ import (
 	"github.com/openshift-hyperfleet/hyperfleet-e2e/pkg/logger"
 )
 
-var (
-	AppliedManifestWorksGVR = &schema.GroupVersionResource{
-		Group:    "work.open-cluster-management.io",
-		Version:  "v1",
-		Resource: "appliedmanifestworks",
-	}
-)
+var AppliedManifestWorksGVR = &schema.GroupVersionResource{
+	Group:    "work.open-cluster-management.io",
+	Version:  "v1",
+	Resource: "appliedmanifestworks",
+}
 
 type CleanupHelper struct {
 	cfg                      *config.Config
@@ -63,12 +61,36 @@ func NewCleanupHelper() (*CleanupHelper, error) {
 	return &CleanupHelper{k8sClient: k8sClient, dynamicClient: dynamicClient, labelSelectorListOptions: labelSelectorListOptions, cfg: cfg, adapterDeploymentList: adapterDeploymentList}, nil
 }
 
-// CleanupResources is the entry point for the end-of-suite cleanup mechanism that does a final sweep
-// Using the label selector = e2e.hyperfleet.io/run-id=<run-id> that is set when the tests are initiated
-func CleanupResources() {
+// CleanupPubSubResources sweeps Pub/Sub topics and subscriptions created by adapters
+// deployed during this process's test execution. Safe to call from every Ginkgo process
+// since each process tracks its own AdapterDeploymentList in memory.
+func CleanupPubSubResources() {
+	c, err := NewCleanupHelper()
+	if err != nil {
+		logger.Error("failed to create cleanup helper for Pub/Sub sweep", "error", err)
+		return
+	}
+
+	if c.cfg.BrokerType != "googlepubsub" {
+		return
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
+	defer cancel()
+
+	if err := c.SweepPubsubTestAdapterResources(ctx); err != nil {
+		logger.Error("failed to cleanup Pub/Sub test resources", "error", err)
+	}
+}
+
+// CleanupKubeResources sweeps Helm releases and K8s resources labeled with the suite
+// run-id. In parallel mode this MUST run exactly once, after all processes finish
+// (via SynchronizedAfterSuite's final closure), to avoid sweeping resources belonging
+// to specs still running on other processes.
+func CleanupKubeResources() {
 	c, err := NewCleanupHelper()
 	if err != nil {
-		logger.Error("failed to create cleanup helper", "error", err)
+		logger.Error("failed to create cleanup helper for K8s sweep", "error", err)
 		return
 	}
 
@@ -99,13 +121,6 @@ func CleanupResources() {
 		logger.Error("failed to cleanup test resources", "error", err)
 	}
 
-	// Step 3: Sweep Pub/Sub test adapter resources
-	if c.cfg.BrokerType == "googlepubsub" {
-		if err := c.SweepPubsubTestAdapterResources(ctx); err != nil {
-			logger.Error("failed to cleanup Pub/Sub test resources", "error", err)
-		}
-	}
-
 	logger.Info("test resources cleaned up")
 }
 
@@ -175,7 +190,6 @@ func (c *CleanupHelper) SweepLabeledResources(ctx context.Context) error {
 		logger.Info("resources still being deleted", "remaining", remaining)
 		return false, nil
 	})
-
 	if err != nil {
 		return fmt.Errorf("not all resources were deleted in time: %w", err)
 	}