feat(parity): implement sharding for parallel CI jobs and add tests for partitioning

Author: aledbf

.github/workflows/go-cli.yml

@@ -138,15 +138,25 @@ jobs:
   # ~45 min, so scripts/parity-affected.sh maps the diff to a
   # PARITY_COMMAND allowlist (or "all"/"none"). The daily parity-runtime-full job
   # is the backstop that always runs the whole thing.
-  parity-runtime:
+  parity-runtime-shard:
     if: github.event_name == 'push' || github.event_name == 'pull_request'
     runs-on: ubuntu-latest
     needs: lint-and-test
+    strategy:
+      # Split the affected runtime cases across independent runners (each with its
+      # own docker daemon). fail-fast off so one shard's failure still reports the
+      # others. The harness selects this shard's slice via PARITY_SHARD_INDEX.
+      fail-fast: false
+      matrix:
+        shard: [0, 1]
     env:
       PARITY_RUNTIME_TIMEOUT: "10m"
-      # 2-core hosted runners can't sustain 2 concurrent heavy docker builds; run
-      # serially for gate determinism (override with PARITY_PARALLEL for beefy runners).
-      PARITY_PARALLEL: "1"
+      # Two shards, each running two cases at a time: the per-case isolation
+      # (unique id-labels / COMPOSE_PROJECT_NAME / BUILDX_BUILDER) makes -parallel 2
+      # safe on a 2-core runner now that each runner only owns half the cases.
+      PARITY_PARALLEL: "2"
+      PARITY_SHARD_INDEX: ${{ matrix.shard }}
+      PARITY_SHARD_TOTAL: "2"
     steps:
       - uses: actions/checkout@v7
         with:
@@ -193,12 +203,28 @@ jobs:
       - uses: actions/upload-artifact@v7
         if: always() && steps.affected.outputs.run == 'true'
         with:
-          name: parity-runtime-v0.88.0
+          name: parity-runtime-v0.88.0-shard-${{ matrix.shard }}
           path: artifacts/
           if-no-files-found: error
       - if: always() && steps.affected.outputs.run == 'true'
         run: task clean
 
+  # Stable required-check name aggregating the sharded runtime lane, so sharding
+  # does not change the check name branch protection depends on.
+  parity-runtime:
+    if: always() && (github.event_name == 'push' || github.event_name == 'pull_request')
+    runs-on: ubuntu-latest
+    needs: parity-runtime-shard
+    steps:
+      - name: Gate on all runtime shards
+        run: |
+          result="${{ needs.parity-runtime-shard.result }}"
+          echo "parity-runtime-shard aggregate result: $result"
+          case "$result" in
+            success|skipped) exit 0 ;;
+            *) echo "one or more runtime shards failed"; exit 1 ;;
+          esac
+
   # Full runtime parity matrix + publish parity. Runs once a day (schedule) and
   # on manual dispatch — the exhaustive backstop for the scoped push/PR job above.
   parity-runtime-full:
@@ -207,7 +233,10 @@ jobs:
     needs: lint-and-test
     env:
       PARITY_RUNTIME_TIMEOUT: "10m"
-      PARITY_PARALLEL: "1"
+      # Exhaustive daily run on a dedicated (non-gating) job: raise parallelism to
+      # 3 now that per-case isolation makes concurrent builds safe. Not sharded
+      # here because this lane also measures coverage (single covdata merge).
+      PARITY_PARALLEL: "3"
     steps:
       - uses: actions/checkout@v7
         with:

internal/cli/parity_matrix_helpers_test.go

@@ -167,3 +167,34 @@ func TestExtractCLIResultEnv_EmbeddedJSON(t *testing.T) {
 		t.Fatalf("IMAGE_NAME = %q", got["IMAGE_NAME"])
 	}
 }
+
+// TestInShardPartitions proves the shard split is a proper partition: with N
+// shards, every case is claimed by exactly one shard and the union is the whole
+// set (no case dropped, none run twice).
+func TestInShardPartitions(t *testing.T) {
+	const cases = 189
+	for _, total := range []int{1, 2, 3, 4, 7} {
+		claimed := make([]int, cases)
+		for shard := 0; shard < total; shard++ {
+			for i := 0; i < cases; i++ {
+				if inShard(i, shard, total) {
+					claimed[i]++
+				}
+			}
+		}
+		for i, c := range claimed {
+			if c != 1 {
+				t.Errorf("total=%d: case %d claimed by %d shards, want exactly 1", total, i, c)
+			}
+		}
+	}
+}
+
+// TestInShardNoShardingRunsEverything proves total<=1 disables sharding.
+func TestInShardNoShardingRunsEverything(t *testing.T) {
+	for i := 0; i < 10; i++ {
+		if !inShard(i, 0, 1) {
+			t.Errorf("inShard(%d,0,1) = false, want every case to run when unsharded", i)
+		}
+	}
+}

internal/cli/parity_matrix_test.go

@@ -28,6 +28,7 @@ import (
 	"path/filepath"
 	"regexp"
 	"sort"
+	"strconv"
 	"strings"
 	"sync"
 	"testing"
@@ -243,6 +244,14 @@ func TestParityMatrix(t *testing.T) {
 
 	dockerAvailable := isDockerAvailable()
 
+	// Sharding: split the selected cases round-robin across parallel CI jobs, so
+	// N runners (with independent docker daemons) share the wall-clock. Inert when
+	// PARITY_SHARD_TOTAL is unset or <= 1. Cases in other shards stay not-selected,
+	// which the strict gate ignores (it only fails on inconclusive cases), so each
+	// shard gates its own subset and the union covers everything exactly once.
+	shardIndex, shardTotal := parityShard(t)
+	selected := 0
+
 	for _, tc := range matrix.InitialCases {
 		tc := tc
 		if !matchesFilter(tc) {
@@ -253,6 +262,13 @@ func TestParityMatrix(t *testing.T) {
 		if tc.Lane == "runtime" && os.Getenv("PARITY_LANE") == "" {
 			continue
 		}
+		// Round-robin over the selected set (stable matrix order); consecutive
+		// cases land on different shards, which spreads the heavy build.* cases.
+		caseIndex := selected
+		selected++
+		if !inShard(caseIndex, shardIndex, shardTotal) {
+			continue
+		}
 
 		t.Run(tc.ID, func(t *testing.T) {
 			outcome := parityMatched
@@ -1075,6 +1091,36 @@ func envOr(key, fallback string) string {
 	return fallback
 }
 
+// inShard reports whether the case at caseIndex (its position in the selected
+// stream) belongs to shard index of total. total <= 1 means no sharding (every
+// case runs). The modulo partitions the stream into disjoint shards whose union
+// is the whole set.
+func inShard(caseIndex, index, total int) bool {
+	return total <= 1 || caseIndex%total == index
+}
+
+// parityShard reads PARITY_SHARD_INDEX / PARITY_SHARD_TOTAL for CI sharding.
+// Total defaults to 1 (no sharding). An invalid pair fails the test so a
+// misconfigured matrix is loud rather than silently skipping cases.
+func parityShard(t *testing.T) (index, total int) {
+	total = 1
+	if v := os.Getenv("PARITY_SHARD_TOTAL"); v != "" {
+		n, err := strconv.Atoi(v)
+		if err != nil || n < 1 {
+			t.Fatalf("PARITY_SHARD_TOTAL=%q: must be a positive integer", v)
+		}
+		total = n
+	}
+	if v := os.Getenv("PARITY_SHARD_INDEX"); v != "" {
+		n, err := strconv.Atoi(v)
+		if err != nil || n < 0 || n >= total {
+			t.Fatalf("PARITY_SHARD_INDEX=%q: must be in [0,%d)", v, total)
+		}
+		index = n
+	}
+	return index, total
+}
+
 func isDockerAvailable() bool {
 	cmd := exec.Command("docker", "info")
 	cmd.Stdout = nil