From 72f6b2ca8cfbe4edeb5b16d7337e612441809afe Mon Sep 17 00:00:00 2001
From: Buck Doyle <buck.doyle@cardstack.com>
Date: Fri, 21 Aug 2026 16:54:19 -0400
Subject: [PATCH] ci: Run the host suite on 12 shards instead of 20
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Twenty concurrent jobs per host run is a large share of the runner pool,
and the queue shows it: shards have been observed waiting up to 242s for a
runner while the pool works through the backlog.

Fewer, longer shards is the trade. Setup is ~195s per shard and largely
irreducible — four experiments on CS-12583 established it is bound by the
runner's cores rather than by latency, so parallelising or pre-computing it
moves the cost rather than removing it. Each added shard therefore buys a
shrinking slice of test time for a fixed ~3 minutes of machine time and one
more slot held.

Against the ~8,665s of test work in run 32523921998: 20 shards models to
~10.5 minutes and 3.49 machine-hours, 12 to ~15.3 minutes and 3.06. The
wall-clock figure overstates the real cost, since it assumes every shard
starts immediately, which is the assumption the queueing disproves.

The count lives in three places that have to move together — the matrix
list, `shardTotal`, and `--shard-count` on the timings generator, whose
drift gate predicts the slowest shard. A comment now says so.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
---
 .github/workflows/ci-host.yaml | 44 +++++++++++++++-------------------
 1 file changed, 19 insertions(+), 25 deletions(-)

diff --git a/.github/workflows/ci-host.yaml b/.github/workflows/ci-host.yaml
index 3ee46ca3cc2..4d9302762fc 100644
--- a/.github/workflows/ci-host.yaml
+++ b/.github/workflows/ci-host.yaml
@@ -459,31 +459,25 @@ jobs:
     needs: [test-web-assets, check-percy, check-index-cache]
     strategy:
       fail-fast: false
+      # Three places carry the shard count and must move together: this list,
+      # `shardTotal` below, and `--shard-count` on the timings generator, which
+      # decides whether a rebalance is worth committing by predicting the
+      # slowest shard.
+      #
+      # Fewer, longer shards rather than more, shorter ones. Setup is ~195s per
+      # shard and largely irreducible (it is bound by the runner's cores, not
+      # by latency — see the four experiments on CS-12583), so each added shard
+      # buys a shrinking slice of test time for a fixed ~3 minutes of machine
+      # time and one more slot held. With ~8,665s of test work: 20 shards is
+      # ~10.5 minutes and 3.49 machine-hours, 12 is ~15.3 minutes and 3.06.
+      #
+      # The modelled wall-clock cost overstates the real one, because it
+      # assumes every shard starts immediately. They do not: shards have been
+      # observed queueing up to 242s waiting for a runner, and 20 concurrent
+      # jobs per host run is a large share of the pool.
       matrix:
-        shardIndex:
-          [
-            1,
-            2,
-            3,
-            4,
-            5,
-            6,
-            7,
-            8,
-            9,
-            10,
-            11,
-            12,
-            13,
-            14,
-            15,
-            16,
-            17,
-            18,
-            19,
-            20,
-          ]
-        shardTotal: [20]
+        shardIndex: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]
+        shardTotal: [12]
     concurrency:
       group: boxel-host-test${{ github.head_ref || github.run_id }}-shard${{ matrix.shardIndex }}
       cancel-in-progress: true
@@ -1243,7 +1237,7 @@ jobs:
             # job loudly rather than committing a degraded weights file.
             if [ -f merged-junit-report/host.xml ]; then
               node packages/host/scripts/generate-test-module-timings.mjs merged-junit-report/host.xml \
-                --min-drift-seconds 60 --shard-count 20
+                --min-drift-seconds 60 --shard-count 12
             else
               echo "No merged junit report available — leaving shard timings unchanged."
             fi