fix(ci): make GitHub-hosted fallback actually build and test (#5814)

* fix(ci): make GitHub-hosted fallback actually build and test

* fix(ci): drop unused build-args passthrough so the heap ceiling can't be overridden

* chore(ci): trim comments to the non-obvious constraints

* fix(ci): build the app image on a larger runner in GitHub mode

Author: TheodoreSpeaks

.github/actions/cache-mount/action.yml

@@ -3,20 +3,18 @@ description: Mount a build cache directory using Blacksmith sticky disks, or the
 
 inputs:
   provider:
-    description: Empty or 'blacksmith' selects sticky disks. Any other value selects actions/cache. Must stay byte-identical to the runs-on predicate in the calling workflow.
+    description: Empty or 'blacksmith' selects sticky disks; anything else selects actions/cache.
     required: false
     default: ''
   key:
-    description: Cache key. Shared by both backends, so callers keep one key regardless of provider.
+    description: Cache key, shared by both backends.
     required: true
   path:
-    description: Filesystem path the cache is mounted at.
+    description: Path to mount.
     required: true
 
-# The two conditions are exact complements of the runs-on expression the caller
-# uses. They must be inverted together: a job placed on ubuntu-latest that then
-# took the sticky-disk branch would hard-fail, since stickydisk hot-mounts an
-# ext4 volume from Blacksmith's storage agents and cannot run anywhere else.
+# Predicates must stay the exact complement of the caller's runs-on — stickydisk
+# only runs on Blacksmith.
 runs:
   using: composite
   steps:
@@ -27,10 +25,8 @@ runs:
         key: ${{ inputs.key }}
         path: ${{ inputs.path }}
 
-    # Sticky disks always re-commit the mounted path, so the cache rolls forward
-    # every run. actions/cache skips its save step on an exact key hit, which
-    # would freeze the cache at whatever the first run wrote — the run-scoped
-    # suffix plus a prefix restore-key reproduces the rolling behaviour.
+    # run_id suffix + prefix restore-key: actions/cache skips save on an exact hit,
+    # which would freeze the cache at the first run's contents.
     - name: Restore and save cache
       if: inputs.provider != '' && inputs.provider != 'blacksmith'
       uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5

.github/actions/docker-build/action.yml

@@ -3,7 +3,7 @@ description: Set up a buildx builder and build/push an image, using Blacksmith's
 
 inputs:
   provider:
-    description: Empty or 'blacksmith' selects Blacksmith's builder. Any other value selects the upstream docker/* actions. Must stay byte-identical to the runs-on predicate in the calling workflow.
+    description: Empty or 'blacksmith' selects Blacksmith's builder; anything else selects the docker/* actions.
     required: false
     default: ''
   context:
@@ -20,10 +20,8 @@ inputs:
     description: Comma-separated list of tags to push.
     required: true
 
-# Registry logins must already have run in the calling job — both backends read
-# the same ~/.docker/config.json. provenance/sbom stay off everywhere: the
-# attestation manifests they add break `imagetools create` retagging in the
-# promote-images and create-ghcr-manifests jobs.
+# Registry logins must precede this action. provenance/sbom stay off: attestation
+# manifests break `imagetools create` retagging in promote-images.
 runs:
   using: composite
   steps:
@@ -47,10 +45,7 @@ runs:
       if: inputs.provider != '' && inputs.provider != 'blacksmith'
       uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4
 
-    # No layer cache here. Blacksmith caches layers on the builder itself;
-    # the GitHub equivalent (cache-to: type=gha) shares the same 10 GB repo
-    # cache quota as the bun/node_modules/turbo mounts, and four images of
-    # layers would evict them. Fallback builds are cold by design.
+    # No cache-to: type=gha — it shares the 10 GB repo quota with the cache mounts.
     - name: Build and push (GitHub)
       if: inputs.provider != '' && inputs.provider != 'blacksmith'
       uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7

.github/workflows/ci.yml

@@ -1,32 +1,19 @@
 name: CI
 
-# Runner provider toggle. Every job here and in the reusable workflows reads the
-# CI_PROVIDER repository variable:
+# Runner provider toggle, read from the CI_PROVIDER repo variable:
 #
 #   gh variable set CI_PROVIDER --body github   # fall back to GitHub-hosted
 #   gh variable delete CI_PROVIDER              # back to Blacksmith (default)
 #
-# It is a repo variable rather than a committed value on purpose: during a
-# Blacksmith outage there is no working CI to merge a switchover commit through,
-# and the variable takes effect on the next run with nothing to land.
+# A repo variable, not a committed value: during a Blacksmith outage there is no
+# working CI to merge a switchover through. Only unset/'blacksmith' selects
+# Blacksmith; anything unrecognized selects GitHub so a typo can't queue jobs
+# against the provider you're escaping. Every runs-on and both composite actions
+# share this predicate and must change together.
 #
-# Only unset or the exact value 'blacksmith' selects Blacksmith; everything else
-# selects GitHub-hosted. The unrecognized case resolves toward GitHub on purpose,
-# because that is the provider that is always available: a typo'd or whitespace-
-# padded break-glass value then yields slow-but-working CI instead of jobs queued
-# forever against the provider you were trying to escape. `==` is case-insensitive
-# in Actions expressions, so 'GitHub' and 'Blacksmith' both work. Every runs-on
-# expression and both composite actions use this same predicate — they must be
-# changed together, since a job on ubuntu-latest that took a Blacksmith-only
-# branch would hard-fail.
-#
-# GitHub mode is a break-glass fallback, not a peer. Public-repo ubuntu-latest is
-# 4 vCPU with no free 8-core tier, the 10 GB repo cache quota will thrash across
-# the bun/node_modules/turbo mounts, and image builds start from cold layers — so
-# expect slower runs, not different results. Blacksmith-only actions are reached
-# exclusively through .github/actions/cache-mount and .github/actions/docker-build,
-# which branch internally; a bare useblacksmith/* step added anywhere else will
-# hard-fail in GitHub mode rather than silently degrade.
+# GitHub mode is break-glass, not a peer — cold layers, slower runs. The app image
+# is the one job on a paid larger runner: next build needs ~32 GB and OOM-kills
+# (exit 137) on the free 16 GB runners at any heap ceiling.
 
 on:
   push:
@@ -109,7 +96,7 @@ jobs:
     name: Build Dev ECR
     needs: [detect-version, migrate-dev]
     if: github.event_name == 'push' && github.ref == 'refs/heads/dev'
-    runs-on: ${{ (vars.CI_PROVIDER == '' || vars.CI_PROVIDER == 'blacksmith') && 'blacksmith-8vcpu-ubuntu-2404' || 'ubuntu-latest' }}
+    runs-on: ${{ (vars.CI_PROVIDER == '' || vars.CI_PROVIDER == 'blacksmith') && 'blacksmith-8vcpu-ubuntu-2404' || matrix.gh_runner }}
     timeout-minutes: 30
     permissions:
       contents: read
@@ -118,14 +105,20 @@ jobs:
       fail-fast: false
       matrix:
         include:
+          # Only the app image needs the paid 8-core/32 GB runner: next build
+          # exhausts the free 16 GB one (exit 137). The others build in <5 min.
           - dockerfile: ./docker/app.Dockerfile
             ecr_repo_secret: ECR_APP
+            gh_runner: linux-x64-8-core
           - dockerfile: ./docker/db.Dockerfile
             ecr_repo_secret: ECR_MIGRATIONS
+            gh_runner: ubuntu-latest
           - dockerfile: ./docker/realtime.Dockerfile
             ecr_repo_secret: ECR_REALTIME
+            gh_runner: ubuntu-latest
           - dockerfile: ./docker/pii.Dockerfile
             ecr_repo_secret: ECR_PII
+            gh_runner: ubuntu-latest
     steps:
       - name: Checkout code
         uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
@@ -214,7 +207,7 @@ jobs:
     if: >-
       github.event_name == 'push' &&
       (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/staging')
-    runs-on: ${{ (vars.CI_PROVIDER == '' || vars.CI_PROVIDER == 'blacksmith') && 'blacksmith-8vcpu-ubuntu-2404' || 'ubuntu-latest' }}
+    runs-on: ${{ (vars.CI_PROVIDER == '' || vars.CI_PROVIDER == 'blacksmith') && 'blacksmith-8vcpu-ubuntu-2404' || matrix.gh_runner }}
     timeout-minutes: 30
     permissions:
       contents: read
@@ -227,15 +220,19 @@ jobs:
           - dockerfile: ./docker/app.Dockerfile
             ghcr_image: ghcr.io/simstudioai/simstudio
             ecr_repo_secret: ECR_APP
+            gh_runner: linux-x64-8-core
           - dockerfile: ./docker/db.Dockerfile
             ghcr_image: ghcr.io/simstudioai/migrations
             ecr_repo_secret: ECR_MIGRATIONS
+            gh_runner: ubuntu-latest
           - dockerfile: ./docker/realtime.Dockerfile
             ghcr_image: ghcr.io/simstudioai/realtime
             ecr_repo_secret: ECR_REALTIME
+            gh_runner: ubuntu-latest
           - dockerfile: ./docker/pii.Dockerfile
             ghcr_image: ghcr.io/simstudioai/pii
             ecr_repo_secret: ECR_PII
+            gh_runner: ubuntu-latest
     steps:
       - name: Checkout code
         uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
@@ -378,7 +375,7 @@ jobs:
   # never moves a documented tag.
   build-ghcr-arm64:
     name: Build ARM64 (GHCR Only)
-    runs-on: ${{ (vars.CI_PROVIDER == '' || vars.CI_PROVIDER == 'blacksmith') && 'blacksmith-8vcpu-ubuntu-2404-arm' || 'ubuntu-24.04-arm' }}
+    runs-on: ${{ (vars.CI_PROVIDER == '' || vars.CI_PROVIDER == 'blacksmith') && 'blacksmith-8vcpu-ubuntu-2404-arm' || matrix.gh_runner }}
     timeout-minutes: 30
     if: github.event_name == 'push' && github.ref == 'refs/heads/main'
     permissions:
@@ -390,12 +387,16 @@ jobs:
         include:
           - dockerfile: ./docker/app.Dockerfile
             image: ghcr.io/simstudioai/simstudio
+            gh_runner: linux-arm64-8-core
           - dockerfile: ./docker/db.Dockerfile
             image: ghcr.io/simstudioai/migrations
+            gh_runner: ubuntu-24.04-arm
           - dockerfile: ./docker/realtime.Dockerfile
             image: ghcr.io/simstudioai/realtime
+            gh_runner: ubuntu-24.04-arm
           - dockerfile: ./docker/pii.Dockerfile
             image: ghcr.io/simstudioai/pii
+            gh_runner: ubuntu-24.04-arm
 
     steps:
       - name: Checkout code

.github/workflows/test-build.yml

@@ -30,7 +30,7 @@ jobs:
       # Cache keys are scoped by event name, and fork PRs get their own
       # namespace on top: untrusted fork runs must never share a cache with
       # push runs (whose caches feed production image builds) or with trusted
-      # internal-PR runs. The scoping holds on both backends.
+      # internal-PR runs.
       - name: Mount Bun cache
         uses: ./.github/actions/cache-mount
         with:
@@ -153,6 +153,11 @@ jobs:
       - name: Type-check realtime server
         run: bunx turbo run type-check --filter=@sim/realtime
 
+      # cloud-review-tools.test.ts runs the real helper on the runner, which shells
+      # out to rg. Blacksmith's image ships it, GitHub's doesn't.
+      - name: Install ripgrep
+        run: command -v rg || (sudo apt-get update && sudo apt-get install -y ripgrep)
+
       - name: Run tests with coverage
         env:
           NODE_OPTIONS: '--no-warnings --max-old-space-size=8192'
@@ -188,8 +193,7 @@ jobs:
   # with test-build (identical content from the same lockfile — LWW loss is
   # harmless), but the Turbo cache gets its own key: with a shared key, only
   # the last committer's new entries survive each run, so the test and build
-  # Turbo entries would evict each other nondeterministically. The same split
-  # is correct on actions/cache, which is likewise last-writer-wins per key.
+  # Turbo entries would evict each other nondeterministically.
   build:
     name: Build App
     runs-on: ${{ (vars.CI_PROVIDER == '' || vars.CI_PROVIDER == 'blacksmith') && 'blacksmith-8vcpu-ubuntu-2404' || 'ubuntu-latest' }}