fix(ci): serialize staging-coder.ddev.com integration tests without cross-cancelling other runs

.github/workflows/drupal-contrib-integration-test.yml

@@ -26,6 +26,21 @@ name: Drupal Contrib Integration Tests
 env:
   FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true
 
+# Ref-scoped so a new commit on the same PR (or the same push-to-main queue)
+# cancels its own stale run, but never touches another PR's or main's run.
+#
+# Serialization against other workflows/refs that provision real workspaces on
+# the shared staging-coder.ddev.com box is NOT done via a shared job-level
+# `concurrency:` group -- a single PR push fires several box-provisioning jobs
+# across this file, integration-test.yml, and drupal-integration-test.yml at
+# nearly the same instant, and GitHub's concurrency groups only keep one job
+# running plus one pending; every other simultaneous contender gets silently
+# CANCELLED, not queued. Instead, each box-touching job below (including
+# contrib-plain-gh and contrib-issue-fork-gh, which otherwise run in parallel
+# within a single workflow run) calls scripts/ci-wait-for-staging-box.sh right
+# before creating its workspace, which polls the Coder server's actual
+# workspace count and waits its turn -- see that script's header comment for
+# detail.
 concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}
   cancel-in-progress: true
@@ -51,6 +66,9 @@ jobs:
     # To re-enable: restore `if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.owner.login == github.repository_owner }}`
     if: false
     runs-on: [self-hosted, sysbox]
+    # Long enough to cover queued time behind other box-provisioning jobs
+    # (see scripts/ci-wait-for-staging-box.sh) plus this job's own runtime.
+    timeout-minutes: 75
     strategy:
       matrix:
         include:
@@ -210,6 +228,9 @@ jobs:
     # To re-enable: restore `if: ${{ vars.CONTRIB_TEST_ISSUE_FORK != '' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.owner.login == github.repository_owner) }}`
     if: false
     runs-on: [self-hosted, sysbox]
+    # Long enough to cover queued time behind other box-provisioning jobs
+    # (see scripts/ci-wait-for-staging-box.sh) plus this job's own runtime.
+    timeout-minutes: 75
     defaults:
       run:
         shell: bash -euo pipefail {0}
@@ -386,6 +407,9 @@ jobs:
     name: Contrib ${{ matrix.project }} D${{ matrix.drupal_version }} (plain, GH)
     if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.owner.login == github.repository_owner }}
     runs-on: ubuntu-latest
+    # Long enough to cover queued time behind other box-provisioning jobs
+    # (see scripts/ci-wait-for-staging-box.sh) plus this job's own runtime.
+    timeout-minutes: 75
     strategy:
       # PRs run a single module (smoke); push-to-main and nightly run the full
       # set. Cuts per-PR workspace load on staging. See ci-reap.yml.
@@ -434,6 +458,9 @@ jobs:
       - name: Reap predecessor workspaces (cancelled prior runs of this cell)
         run: ./scripts/ci-reap-family.sh "gc-${{ matrix.project }}-d${{ matrix.drupal_version }}-" "${{ env.WORKSPACE_NAME }}"
 
+      - name: Wait for staging box to be free
+        run: ./scripts/ci-wait-for-staging-box.sh
+
       - name: Create workspace
         run: |
           coder create ${{ env.WORKSPACE_NAME }} \
@@ -525,6 +552,9 @@ jobs:
     name: Contrib ${{ vars.CONTRIB_TEST_PROJECT || 'token' }} issue fork (GH)
     if: ${{ vars.CONTRIB_TEST_ISSUE_FORK != '' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.owner.login == github.repository_owner) }}
     runs-on: ubuntu-latest
+    # Long enough to cover queued time behind other box-provisioning jobs
+    # (see scripts/ci-wait-for-staging-box.sh) plus this job's own runtime.
+    timeout-minutes: 75
     defaults:
       run:
         shell: bash -euo pipefail {0}
@@ -587,6 +617,9 @@ jobs:
       - name: Reap predecessor workspaces (cancelled prior runs of this cell)
         run: ./scripts/ci-reap-family.sh "gc-fork-" "${{ env.WORKSPACE_NAME }}"
 
+      - name: Wait for staging box to be free
+        run: ./scripts/ci-wait-for-staging-box.sh
+
       - name: Create workspace
         run: |
           coder create ${{ env.WORKSPACE_NAME }} \

.github/workflows/drupal-integration-test.yml

@@ -31,6 +31,21 @@ name: Drupal Integration Tests
 env:
   FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true
 
+# Ref-scoped so a new commit on the same PR (or the same push-to-main queue)
+# cancels its own stale run, but never touches another PR's or main's run.
+#
+# Serialization against other workflows/refs that provision real workspaces on
+# the shared staging-coder.ddev.com box is NOT done via a shared job-level
+# `concurrency:` group -- a single PR push fires several box-provisioning jobs
+# across this file, integration-test.yml, and
+# drupal-contrib-integration-test.yml at nearly the same instant, and GitHub's
+# concurrency groups only keep one job running plus one pending; every other
+# simultaneous contender gets silently CANCELLED, not queued. Instead, each
+# box-touching job below (including drupal-plain-gh and drupal-issue-fork-gh,
+# which otherwise run in parallel within a single workflow run) calls
+# scripts/ci-wait-for-staging-box.sh right before creating its workspace,
+# which polls the Coder server's actual workspace count and waits its turn --
+# see that script's header comment for detail.
 concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}
   cancel-in-progress: true
@@ -56,6 +71,9 @@ jobs:
     # To re-enable: restore `if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.owner.login == github.repository_owner }}`
     if: false
     runs-on: [self-hosted, sysbox]
+    # Long enough to cover queued time behind other box-provisioning jobs
+    # (see scripts/ci-wait-for-staging-box.sh) plus this job's own runtime.
+    timeout-minutes: 75
     strategy:
       matrix:
         drupal_version: ["12", "11"]
@@ -198,6 +216,9 @@ jobs:
     name: Drupal ${{ matrix.drupal_version }} (plain, GH)
     if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.owner.login == github.repository_owner }}
     runs-on: ubuntu-latest
+    # Long enough to cover queued time behind other box-provisioning jobs
+    # (see scripts/ci-wait-for-staging-box.sh) plus this job's own runtime.
+    timeout-minutes: 75
     strategy:
       # PRs run a single Drupal version (smoke); push-to-main and nightly run the
       # full matrix. Cuts per-PR workspace load on staging. See ci-reap.yml.
@@ -244,6 +265,9 @@ jobs:
       - name: Reap predecessor workspaces (cancelled prior runs of this cell)
         run: ./scripts/ci-reap-family.sh "gd-${{ matrix.drupal_version }}-" "${{ env.WORKSPACE_NAME }}"
 
+      - name: Wait for staging box to be free
+        run: ./scripts/ci-wait-for-staging-box.sh
+
       - name: Create workspace
         run: |
           coder create ${{ env.WORKSPACE_NAME }} \
@@ -276,6 +300,9 @@ jobs:
     # To re-enable: restore `if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.owner.login == github.repository_owner }}`
     if: false
     runs-on: [self-hosted, sysbox]
+    # Long enough to cover queued time behind other box-provisioning jobs
+    # (see scripts/ci-wait-for-staging-box.sh) plus this job's own runtime.
+    timeout-minutes: 75
     defaults:
       run:
         shell: bash -euo pipefail {0}
@@ -436,6 +463,9 @@ jobs:
     name: Drupal issue fork ${{ vars.DRUPAL_TEST_ISSUE_FORK || '3585397' }} (GH)
     if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.owner.login == github.repository_owner }}
     runs-on: ubuntu-latest
+    # Long enough to cover queued time behind other box-provisioning jobs
+    # (see scripts/ci-wait-for-staging-box.sh) plus this job's own runtime.
+    timeout-minutes: 75
     defaults:
       run:
         shell: bash -euo pipefail {0}
@@ -495,6 +525,9 @@ jobs:
       - name: Reap predecessor workspaces (cancelled prior runs of this cell)
         run: ./scripts/ci-reap-family.sh "gd-fork-" "${{ env.WORKSPACE_NAME }}"
 
+      - name: Wait for staging box to be free
+        run: ./scripts/ci-wait-for-staging-box.sh
+
       - name: Create workspace
         run: |
           coder create ${{ env.WORKSPACE_NAME }} \

.github/workflows/integration-test.yml

@@ -61,8 +61,21 @@ on:
         required: false
         default: false
 
+# Ref-scoped so a new commit on the same PR (or the same push-to-main queue)
+# cancels its own stale run, but never touches another PR's or main's run.
+#
+# Serialization against other workflows/refs that provision real workspaces on
+# the shared staging-coder.ddev.com box is NOT done via a shared job-level
+# `concurrency:` group -- a single PR push fires several box-provisioning jobs
+# across this file, drupal-integration-test.yml, and
+# drupal-contrib-integration-test.yml at nearly the same instant, and GitHub's
+# concurrency groups only keep one job running plus one pending; every other
+# simultaneous contender gets silently CANCELLED, not queued. Instead, each
+# box-touching job below runs scripts/ci-wait-for-staging-box.sh right before
+# creating its workspace, which polls the Coder server's actual workspace
+# count and waits its turn -- see that script's header comment for detail.
 concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
+  group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}
   cancel-in-progress: true
 
 jobs:
@@ -72,6 +85,9 @@ jobs:
     # To re-enable: restore `if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.owner.login == github.repository_owner }}`
     if: false
     runs-on: [self-hosted, sysbox]
+    # Long enough to cover queued time behind other box-provisioning jobs
+    # (see scripts/ci-wait-for-staging-box.sh) plus this job's own runtime.
+    timeout-minutes: 75
     strategy:
       matrix:
         include:
@@ -284,6 +300,9 @@ jobs:
     name: Integration test GH (${{ matrix.template }})
     if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.owner.login == github.repository_owner }}
     runs-on: ubuntu-latest
+    # Long enough to cover queued time behind other box-provisioning jobs
+    # (see scripts/ci-wait-for-staging-box.sh) plus this job's own runtime.
+    timeout-minutes: 75
     strategy:
       matrix:
         include:
@@ -341,6 +360,9 @@ jobs:
       - name: Reap predecessor workspaces (cancelled prior runs of this cell)
         run: ./scripts/ci-reap-family.sh "gh-${{ matrix.ws_name }}-" "${{ env.WORKSPACE_NAME }}"
 
+      - name: Wait for staging box to be free
+        run: ./scripts/ci-wait-for-staging-box.sh
+
       - name: Create workspace
         if: ${{ matrix.template != 'freeform' }}
         run: |

scripts/ci-wait-for-staging-box.sh

@@ -0,0 +1,57 @@
+#!/usr/bin/env bash
+# Wait until no other CI workspace exists on the shared staging box before this
+# job creates its own workspace.
+#
+# Why this exists:
+#   GitHub Actions' `concurrency:` keyword only keeps one job running plus one
+#   pending per group -- any additional job that tries to join while one is
+#   already pending gets CANCELLED outright, not queued behind it. A single PR
+#   push (or push to main) triggers up to five box-provisioning jobs across
+#   integration-test.yml, drupal-integration-test.yml, and
+#   drupal-contrib-integration-test.yml at nearly the same instant, so sharing
+#   one concurrency group silently dropped most of them instead of running
+#   them in turn. Polling the box's actual ci-bot workspace count avoids that:
+#   every contender waits and none are cancelled.
+#
+# Usage:
+#   ci-wait-for-staging-box.sh
+#
+# Environment:
+#   CI_OWNER          Coder username that owns CI workspaces (default: ci-bot)
+#   MAX_WAIT_SECONDS  Give up and fail after this long (default: 2700 = 45m)
+#   POLL_SECONDS      Delay between checks (default: 30)
+
+set -uo pipefail
+
+CI_OWNER="${CI_OWNER:-ci-bot}"
+MAX_WAIT_SECONDS="${MAX_WAIT_SECONDS:-2700}"
+POLL_SECONDS="${POLL_SECONDS:-30}"
+
+# Jitter so jobs that all started in the same instant don't all sample the
+# workspace count (and all see "clear") at the exact same moment.
+sleep "$((RANDOM % 15))"
+
+waited=0
+while true; do
+  if ! workspaces_json=$(coder list --all --output json 2>/dev/null); then
+    echo "WARN: 'coder list' failed; proceeding without the box-busy check" >&2
+    exit 0
+  fi
+
+  count=$(echo "$workspaces_json" | jq -r --arg owner "$CI_OWNER" \
+    '[.[] | select(.owner_name==$owner)] | length')
+
+  if [[ "$count" -eq 0 ]]; then
+    echo "Staging box is free (0 ci-bot workspaces) -- proceeding"
+    exit 0
+  fi
+
+  if [[ "$waited" -ge "$MAX_WAIT_SECONDS" ]]; then
+    echo "ERROR: staging box still busy ($count ci-bot workspace(s)) after ${MAX_WAIT_SECONDS}s -- giving up" >&2
+    exit 1
+  fi
+
+  echo "Staging box busy ($count ci-bot workspace(s)); waiting ${POLL_SECONDS}s (${waited}s/${MAX_WAIT_SECONDS}s so far)..."
+  sleep "$POLL_SECONDS"
+  waited=$((waited + POLL_SECONDS))
+done