From 696eeac97b5acc6c979898a1e3b891c8f922273f Mon Sep 17 00:00:00 2001 From: Randy Fay Date: Mon, 27 Jul 2026 09:45:18 -0600 Subject: [PATCH 1/3] fix(ci): serialize staging-coder.ddev.com integration tests across all triggers integration-test.yml, drupal-integration-test.yml, and drupal-contrib-integration-test.yml each provision real Coder workspaces (nested Docker/Sysbox builds) on the single shared staging-coder.ddev.com box, but their concurrency groups were scoped per-ref (github.ref / github.head_ref). A push to main and any open PR -- or two different PRs -- land in different groups, so GitHub Actions runs them at the same time, and the box's finite CPU/RAM/Docker capacity gets starved. Observed symptom: flaky mid-build "SSH connection ended unexpectedly" during ddev start, followed by cleanup failing with "a workspace build is already active" -- resource contention, not a real test failure. Switch all three to one shared literal concurrency group so any run that provisions real staging workspaces queues behind another, regardless of which branch, PR, or push triggered it. Co-Authored-By: Claude Sonnet 5 --- .github/workflows/drupal-contrib-integration-test.yml | 10 ++++++++-- .github/workflows/drupal-integration-test.yml | 10 ++++++++-- .github/workflows/integration-test.yml | 10 ++++++++-- 3 files changed, 24 insertions(+), 6 deletions(-) diff --git a/.github/workflows/drupal-contrib-integration-test.yml b/.github/workflows/drupal-contrib-integration-test.yml index a7065be..32972ef 100644 --- a/.github/workflows/drupal-contrib-integration-test.yml +++ b/.github/workflows/drupal-contrib-integration-test.yml @@ -26,9 +26,15 @@ name: Drupal Contrib Integration Tests env: FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true +# Shared literal (not ref-scoped) so this queues behind any other workflow +# that also provisions real workspaces on the same staging-coder.ddev.com +# box (integration-test.yml, drupal-integration-test.yml) -- see the +# matching comment in those files. Running two of these at once starves the +# box's Docker/Sysbox capacity and causes flaky mid-build SSH drops, not +# genuine test failures. concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.ref }} - cancel-in-progress: true + group: staging-coder-integration + cancel-in-progress: false on: schedule: diff --git a/.github/workflows/drupal-integration-test.yml b/.github/workflows/drupal-integration-test.yml index 3b12d48..b726e5e 100644 --- a/.github/workflows/drupal-integration-test.yml +++ b/.github/workflows/drupal-integration-test.yml @@ -31,9 +31,15 @@ name: Drupal Integration Tests env: FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true +# Shared literal (not ref-scoped) so this queues behind any other workflow +# that also provisions real workspaces on the same staging-coder.ddev.com +# box (integration-test.yml, drupal-contrib-integration-test.yml) -- see +# the matching comment in those files. Running two of these at once starves +# the box's Docker/Sysbox capacity and causes flaky mid-build SSH drops, not +# genuine test failures. concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.ref }} - cancel-in-progress: true + group: staging-coder-integration + cancel-in-progress: false on: schedule: diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index b6a8504..c72a4f9 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -61,9 +61,15 @@ on: required: false default: false +# Shared literal (not ref-scoped) so this queues behind any other workflow +# that also provisions real workspaces on the same staging-coder.ddev.com +# box (drupal-integration-test.yml, drupal-contrib-integration-test.yml) -- +# see the matching comment in those files. Running two of these at once +# starves the box's Docker/Sysbox capacity and causes flaky mid-build SSH +# drops, not genuine test failures. concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true + group: staging-coder-integration + cancel-in-progress: false jobs: integration-test: From 54ff4f4e508bd4b349011f30c7d6019dfa681fce Mon Sep 17 00:00:00 2001 From: Randy Fay Date: Mon, 27 Jul 2026 10:46:58 -0600 Subject: [PATCH 2/3] fix(ci): restore per-PR cancellation, serialize the shared staging box via job-level concurrency 696eeac fixed cross-ref resource contention on staging-coder.ddev.com by collapsing all three integration-test workflows into one literal concurrency group with cancel-in-progress: false -- but that also disabled the pre-existing behavior where a new commit on an open PR cancels that PR's own stale run, and it let a PR push queue behind, but never cancel, an in-progress run on main or another PR. It also missed that drupal-plain-gh/drupal-issue-fork-gh and contrib-plain-gh/contrib-issue-fork-gh run in parallel within a single workflow run with no dependency between them, so they could still starve the box even after that fix. Split concurrency into two tiers instead: - Workflow-level: ref-scoped, cancel-in-progress: true -- a new commit on a PR (or push to main) cancels only that ref's own stale run, never another ref's. - Job-level, on every job that provisions a real workspace on staging-coder.ddev.com: literal shared group across all three workflows, cancel-in-progress: false -- serializes real box usage globally without ever preempting someone else's in-flight build. Job-level groups match within a run as well as across runs, so this also serializes the previously-unaddressed parallel GH-hosted job pairs against each other. Also add timeout-minutes: 30 to those jobs -- with cancel-in-progress false on a shared group, a hung run would otherwise block every future trigger indefinitely. Co-Authored-By: Claude Sonnet 5 --- .../drupal-contrib-integration-test.yml | 38 +++++++++++++++---- .github/workflows/drupal-integration-test.yml | 38 +++++++++++++++---- .github/workflows/integration-test.yml | 28 ++++++++++---- 3 files changed, 80 insertions(+), 24 deletions(-) diff --git a/.github/workflows/drupal-contrib-integration-test.yml b/.github/workflows/drupal-contrib-integration-test.yml index 32972ef..0859b29 100644 --- a/.github/workflows/drupal-contrib-integration-test.yml +++ b/.github/workflows/drupal-contrib-integration-test.yml @@ -26,15 +26,18 @@ name: Drupal Contrib Integration Tests env: FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true -# Shared literal (not ref-scoped) so this queues behind any other workflow -# that also provisions real workspaces on the same staging-coder.ddev.com -# box (integration-test.yml, drupal-integration-test.yml) -- see the -# matching comment in those files. Running two of these at once starves the -# box's Docker/Sysbox capacity and causes flaky mid-build SSH drops, not -# genuine test failures. +# Ref-scoped so a new commit on the same PR (or the same push-to-main queue) +# cancels its own stale run, but never touches another PR's or main's run. +# Serialization against other workflows/refs that provision real workspaces +# on the shared staging-coder.ddev.com box happens via the job-level +# `concurrency:` on each box-touching job below (group: staging-coder-integration, +# cancel-in-progress: false) -- see the matching comment there and in +# integration-test.yml / drupal-integration-test.yml. That same job-level group +# also serializes contrib-plain-gh against contrib-issue-fork-gh, which +# otherwise run in parallel within a single workflow run. concurrency: - group: staging-coder-integration - cancel-in-progress: false + group: ${{ github.workflow }}-${{ github.head_ref || github.ref }} + cancel-in-progress: true on: schedule: @@ -57,6 +60,13 @@ jobs: # To re-enable: restore `if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.owner.login == github.repository_owner }}` if: false runs-on: [self-hosted, sysbox] + timeout-minutes: 30 + # Same literal group as every other job that provisions real workspaces on + # staging-coder.ddev.com, across all three integration-test workflows -- + # serializes actual box usage without cancelling anyone else's in-flight run. + concurrency: + group: staging-coder-integration + cancel-in-progress: false strategy: matrix: include: @@ -216,6 +226,10 @@ jobs: # To re-enable: restore `if: ${{ vars.CONTRIB_TEST_ISSUE_FORK != '' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.owner.login == github.repository_owner) }}` if: false runs-on: [self-hosted, sysbox] + timeout-minutes: 30 + concurrency: + group: staging-coder-integration + cancel-in-progress: false defaults: run: shell: bash -euo pipefail {0} @@ -392,6 +406,10 @@ jobs: name: Contrib ${{ matrix.project }} D${{ matrix.drupal_version }} (plain, GH) if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.owner.login == github.repository_owner }} runs-on: ubuntu-latest + timeout-minutes: 30 + concurrency: + group: staging-coder-integration + cancel-in-progress: false strategy: # PRs run a single module (smoke); push-to-main and nightly run the full # set. Cuts per-PR workspace load on staging. See ci-reap.yml. @@ -531,6 +549,10 @@ jobs: name: Contrib ${{ vars.CONTRIB_TEST_PROJECT || 'token' }} issue fork (GH) if: ${{ vars.CONTRIB_TEST_ISSUE_FORK != '' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.owner.login == github.repository_owner) }} runs-on: ubuntu-latest + timeout-minutes: 30 + concurrency: + group: staging-coder-integration + cancel-in-progress: false defaults: run: shell: bash -euo pipefail {0} diff --git a/.github/workflows/drupal-integration-test.yml b/.github/workflows/drupal-integration-test.yml index b726e5e..fdd875d 100644 --- a/.github/workflows/drupal-integration-test.yml +++ b/.github/workflows/drupal-integration-test.yml @@ -31,15 +31,18 @@ name: Drupal Integration Tests env: FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true -# Shared literal (not ref-scoped) so this queues behind any other workflow -# that also provisions real workspaces on the same staging-coder.ddev.com -# box (integration-test.yml, drupal-contrib-integration-test.yml) -- see -# the matching comment in those files. Running two of these at once starves -# the box's Docker/Sysbox capacity and causes flaky mid-build SSH drops, not -# genuine test failures. +# Ref-scoped so a new commit on the same PR (or the same push-to-main queue) +# cancels its own stale run, but never touches another PR's or main's run. +# Serialization against other workflows/refs that provision real workspaces +# on the shared staging-coder.ddev.com box happens via the job-level +# `concurrency:` on each box-touching job below (group: staging-coder-integration, +# cancel-in-progress: false) -- see the matching comment there and in +# integration-test.yml / drupal-contrib-integration-test.yml. That same job-level +# group also serializes drupal-plain-gh against drupal-issue-fork-gh, which +# otherwise run in parallel within a single workflow run. concurrency: - group: staging-coder-integration - cancel-in-progress: false + group: ${{ github.workflow }}-${{ github.head_ref || github.ref }} + cancel-in-progress: true on: schedule: @@ -62,6 +65,13 @@ jobs: # To re-enable: restore `if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.owner.login == github.repository_owner }}` if: false runs-on: [self-hosted, sysbox] + timeout-minutes: 30 + # Same literal group as every other job that provisions real workspaces on + # staging-coder.ddev.com, across all three integration-test workflows -- + # serializes actual box usage without cancelling anyone else's in-flight run. + concurrency: + group: staging-coder-integration + cancel-in-progress: false strategy: matrix: drupal_version: ["12", "11"] @@ -204,6 +214,10 @@ jobs: name: Drupal ${{ matrix.drupal_version }} (plain, GH) if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.owner.login == github.repository_owner }} runs-on: ubuntu-latest + timeout-minutes: 30 + concurrency: + group: staging-coder-integration + cancel-in-progress: false strategy: # PRs run a single Drupal version (smoke); push-to-main and nightly run the # full matrix. Cuts per-PR workspace load on staging. See ci-reap.yml. @@ -282,6 +296,10 @@ jobs: # To re-enable: restore `if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.owner.login == github.repository_owner }}` if: false runs-on: [self-hosted, sysbox] + timeout-minutes: 30 + concurrency: + group: staging-coder-integration + cancel-in-progress: false defaults: run: shell: bash -euo pipefail {0} @@ -442,6 +460,10 @@ jobs: name: Drupal issue fork ${{ vars.DRUPAL_TEST_ISSUE_FORK || '3585397' }} (GH) if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.owner.login == github.repository_owner }} runs-on: ubuntu-latest + timeout-minutes: 30 + concurrency: + group: staging-coder-integration + cancel-in-progress: false defaults: run: shell: bash -euo pipefail {0} diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index c72a4f9..1d220e4 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -61,15 +61,16 @@ on: required: false default: false -# Shared literal (not ref-scoped) so this queues behind any other workflow -# that also provisions real workspaces on the same staging-coder.ddev.com -# box (drupal-integration-test.yml, drupal-contrib-integration-test.yml) -- -# see the matching comment in those files. Running two of these at once -# starves the box's Docker/Sysbox capacity and causes flaky mid-build SSH -# drops, not genuine test failures. +# Ref-scoped so a new commit on the same PR (or the same push-to-main queue) +# cancels its own stale run, but never touches another PR's or main's run. +# Serialization against other workflows/refs that provision real workspaces +# on the shared staging-coder.ddev.com box happens via the job-level +# `concurrency:` on each box-touching job below (group: staging-coder-integration, +# cancel-in-progress: false) -- see the matching comment there and in +# drupal-integration-test.yml / drupal-contrib-integration-test.yml. concurrency: - group: staging-coder-integration - cancel-in-progress: false + group: ${{ github.workflow }}-${{ github.head_ref || github.ref }} + cancel-in-progress: true jobs: integration-test: @@ -78,6 +79,13 @@ jobs: # To re-enable: restore `if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.owner.login == github.repository_owner }}` if: false runs-on: [self-hosted, sysbox] + timeout-minutes: 30 + # Same literal group as every other job that provisions real workspaces on + # staging-coder.ddev.com, across all three integration-test workflows -- + # serializes actual box usage without cancelling anyone else's in-flight run. + concurrency: + group: staging-coder-integration + cancel-in-progress: false strategy: matrix: include: @@ -290,6 +298,10 @@ jobs: name: Integration test GH (${{ matrix.template }}) if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.owner.login == github.repository_owner }} runs-on: ubuntu-latest + timeout-minutes: 30 + concurrency: + group: staging-coder-integration + cancel-in-progress: false strategy: matrix: include: From 0b1b9c134529398d8dd565876b858006dedfcf12 Mon Sep 17 00:00:00 2001 From: Randy Fay Date: Mon, 27 Jul 2026 11:28:43 -0600 Subject: [PATCH 3/3] fix(ci): replace job-level concurrency with a real wait-your-turn poll The job-level concurrency group added in 54ff4f4 was the wrong tool: a single PR push fires five box-provisioning jobs across the three integration-test workflows at nearly the same instant, and GitHub Actions concurrency groups only keep one job running plus one pending -- every other simultaneous contender gets silently CANCELLED, not queued. On PR #188 this dropped 3 of 5 templates outright (0 steps run, no red flag beyond a generic "cancelled" status) while only the first and the last-arriving job actually executed. Drop the job-level concurrency: group entirely. Each box-touching job now calls the new scripts/ci-wait-for-staging-box.sh right before creating its workspace: it polls the Coder server's actual ci-bot workspace count and waits its turn, so every job eventually runs instead of some being dropped. Bump timeout-minutes to 75 on those jobs to cover worst-case queued time (up to ~4 other jobs ahead, each several minutes) plus the job's own runtime. Co-Authored-By: Claude Sonnet 5 --- .../drupal-contrib-integration-test.yml | 57 ++++++++++--------- .github/workflows/drupal-integration-test.yml | 57 ++++++++++--------- .github/workflows/integration-test.yml | 36 ++++++------ scripts/ci-wait-for-staging-box.sh | 57 +++++++++++++++++++ 4 files changed, 139 insertions(+), 68 deletions(-) create mode 100755 scripts/ci-wait-for-staging-box.sh diff --git a/.github/workflows/drupal-contrib-integration-test.yml b/.github/workflows/drupal-contrib-integration-test.yml index 0859b29..039266d 100644 --- a/.github/workflows/drupal-contrib-integration-test.yml +++ b/.github/workflows/drupal-contrib-integration-test.yml @@ -28,13 +28,19 @@ env: # Ref-scoped so a new commit on the same PR (or the same push-to-main queue) # cancels its own stale run, but never touches another PR's or main's run. -# Serialization against other workflows/refs that provision real workspaces -# on the shared staging-coder.ddev.com box happens via the job-level -# `concurrency:` on each box-touching job below (group: staging-coder-integration, -# cancel-in-progress: false) -- see the matching comment there and in -# integration-test.yml / drupal-integration-test.yml. That same job-level group -# also serializes contrib-plain-gh against contrib-issue-fork-gh, which -# otherwise run in parallel within a single workflow run. +# +# Serialization against other workflows/refs that provision real workspaces on +# the shared staging-coder.ddev.com box is NOT done via a shared job-level +# `concurrency:` group -- a single PR push fires several box-provisioning jobs +# across this file, integration-test.yml, and drupal-integration-test.yml at +# nearly the same instant, and GitHub's concurrency groups only keep one job +# running plus one pending; every other simultaneous contender gets silently +# CANCELLED, not queued. Instead, each box-touching job below (including +# contrib-plain-gh and contrib-issue-fork-gh, which otherwise run in parallel +# within a single workflow run) calls scripts/ci-wait-for-staging-box.sh right +# before creating its workspace, which polls the Coder server's actual +# workspace count and waits its turn -- see that script's header comment for +# detail. concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.ref }} cancel-in-progress: true @@ -60,13 +66,9 @@ jobs: # To re-enable: restore `if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.owner.login == github.repository_owner }}` if: false runs-on: [self-hosted, sysbox] - timeout-minutes: 30 - # Same literal group as every other job that provisions real workspaces on - # staging-coder.ddev.com, across all three integration-test workflows -- - # serializes actual box usage without cancelling anyone else's in-flight run. - concurrency: - group: staging-coder-integration - cancel-in-progress: false + # Long enough to cover queued time behind other box-provisioning jobs + # (see scripts/ci-wait-for-staging-box.sh) plus this job's own runtime. + timeout-minutes: 75 strategy: matrix: include: @@ -226,10 +228,9 @@ jobs: # To re-enable: restore `if: ${{ vars.CONTRIB_TEST_ISSUE_FORK != '' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.owner.login == github.repository_owner) }}` if: false runs-on: [self-hosted, sysbox] - timeout-minutes: 30 - concurrency: - group: staging-coder-integration - cancel-in-progress: false + # Long enough to cover queued time behind other box-provisioning jobs + # (see scripts/ci-wait-for-staging-box.sh) plus this job's own runtime. + timeout-minutes: 75 defaults: run: shell: bash -euo pipefail {0} @@ -406,10 +407,9 @@ jobs: name: Contrib ${{ matrix.project }} D${{ matrix.drupal_version }} (plain, GH) if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.owner.login == github.repository_owner }} runs-on: ubuntu-latest - timeout-minutes: 30 - concurrency: - group: staging-coder-integration - cancel-in-progress: false + # Long enough to cover queued time behind other box-provisioning jobs + # (see scripts/ci-wait-for-staging-box.sh) plus this job's own runtime. + timeout-minutes: 75 strategy: # PRs run a single module (smoke); push-to-main and nightly run the full # set. Cuts per-PR workspace load on staging. See ci-reap.yml. @@ -458,6 +458,9 @@ jobs: - name: Reap predecessor workspaces (cancelled prior runs of this cell) run: ./scripts/ci-reap-family.sh "gc-${{ matrix.project }}-d${{ matrix.drupal_version }}-" "${{ env.WORKSPACE_NAME }}" + - name: Wait for staging box to be free + run: ./scripts/ci-wait-for-staging-box.sh + - name: Create workspace run: | coder create ${{ env.WORKSPACE_NAME }} \ @@ -549,10 +552,9 @@ jobs: name: Contrib ${{ vars.CONTRIB_TEST_PROJECT || 'token' }} issue fork (GH) if: ${{ vars.CONTRIB_TEST_ISSUE_FORK != '' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.owner.login == github.repository_owner) }} runs-on: ubuntu-latest - timeout-minutes: 30 - concurrency: - group: staging-coder-integration - cancel-in-progress: false + # Long enough to cover queued time behind other box-provisioning jobs + # (see scripts/ci-wait-for-staging-box.sh) plus this job's own runtime. + timeout-minutes: 75 defaults: run: shell: bash -euo pipefail {0} @@ -615,6 +617,9 @@ jobs: - name: Reap predecessor workspaces (cancelled prior runs of this cell) run: ./scripts/ci-reap-family.sh "gc-fork-" "${{ env.WORKSPACE_NAME }}" + - name: Wait for staging box to be free + run: ./scripts/ci-wait-for-staging-box.sh + - name: Create workspace run: | coder create ${{ env.WORKSPACE_NAME }} \ diff --git a/.github/workflows/drupal-integration-test.yml b/.github/workflows/drupal-integration-test.yml index fdd875d..c12ee0e 100644 --- a/.github/workflows/drupal-integration-test.yml +++ b/.github/workflows/drupal-integration-test.yml @@ -33,13 +33,19 @@ env: # Ref-scoped so a new commit on the same PR (or the same push-to-main queue) # cancels its own stale run, but never touches another PR's or main's run. -# Serialization against other workflows/refs that provision real workspaces -# on the shared staging-coder.ddev.com box happens via the job-level -# `concurrency:` on each box-touching job below (group: staging-coder-integration, -# cancel-in-progress: false) -- see the matching comment there and in -# integration-test.yml / drupal-contrib-integration-test.yml. That same job-level -# group also serializes drupal-plain-gh against drupal-issue-fork-gh, which -# otherwise run in parallel within a single workflow run. +# +# Serialization against other workflows/refs that provision real workspaces on +# the shared staging-coder.ddev.com box is NOT done via a shared job-level +# `concurrency:` group -- a single PR push fires several box-provisioning jobs +# across this file, integration-test.yml, and +# drupal-contrib-integration-test.yml at nearly the same instant, and GitHub's +# concurrency groups only keep one job running plus one pending; every other +# simultaneous contender gets silently CANCELLED, not queued. Instead, each +# box-touching job below (including drupal-plain-gh and drupal-issue-fork-gh, +# which otherwise run in parallel within a single workflow run) calls +# scripts/ci-wait-for-staging-box.sh right before creating its workspace, +# which polls the Coder server's actual workspace count and waits its turn -- +# see that script's header comment for detail. concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.ref }} cancel-in-progress: true @@ -65,13 +71,9 @@ jobs: # To re-enable: restore `if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.owner.login == github.repository_owner }}` if: false runs-on: [self-hosted, sysbox] - timeout-minutes: 30 - # Same literal group as every other job that provisions real workspaces on - # staging-coder.ddev.com, across all three integration-test workflows -- - # serializes actual box usage without cancelling anyone else's in-flight run. - concurrency: - group: staging-coder-integration - cancel-in-progress: false + # Long enough to cover queued time behind other box-provisioning jobs + # (see scripts/ci-wait-for-staging-box.sh) plus this job's own runtime. + timeout-minutes: 75 strategy: matrix: drupal_version: ["12", "11"] @@ -214,10 +216,9 @@ jobs: name: Drupal ${{ matrix.drupal_version }} (plain, GH) if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.owner.login == github.repository_owner }} runs-on: ubuntu-latest - timeout-minutes: 30 - concurrency: - group: staging-coder-integration - cancel-in-progress: false + # Long enough to cover queued time behind other box-provisioning jobs + # (see scripts/ci-wait-for-staging-box.sh) plus this job's own runtime. + timeout-minutes: 75 strategy: # PRs run a single Drupal version (smoke); push-to-main and nightly run the # full matrix. Cuts per-PR workspace load on staging. See ci-reap.yml. @@ -264,6 +265,9 @@ jobs: - name: Reap predecessor workspaces (cancelled prior runs of this cell) run: ./scripts/ci-reap-family.sh "gd-${{ matrix.drupal_version }}-" "${{ env.WORKSPACE_NAME }}" + - name: Wait for staging box to be free + run: ./scripts/ci-wait-for-staging-box.sh + - name: Create workspace run: | coder create ${{ env.WORKSPACE_NAME }} \ @@ -296,10 +300,9 @@ jobs: # To re-enable: restore `if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.owner.login == github.repository_owner }}` if: false runs-on: [self-hosted, sysbox] - timeout-minutes: 30 - concurrency: - group: staging-coder-integration - cancel-in-progress: false + # Long enough to cover queued time behind other box-provisioning jobs + # (see scripts/ci-wait-for-staging-box.sh) plus this job's own runtime. + timeout-minutes: 75 defaults: run: shell: bash -euo pipefail {0} @@ -460,10 +463,9 @@ jobs: name: Drupal issue fork ${{ vars.DRUPAL_TEST_ISSUE_FORK || '3585397' }} (GH) if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.owner.login == github.repository_owner }} runs-on: ubuntu-latest - timeout-minutes: 30 - concurrency: - group: staging-coder-integration - cancel-in-progress: false + # Long enough to cover queued time behind other box-provisioning jobs + # (see scripts/ci-wait-for-staging-box.sh) plus this job's own runtime. + timeout-minutes: 75 defaults: run: shell: bash -euo pipefail {0} @@ -523,6 +525,9 @@ jobs: - name: Reap predecessor workspaces (cancelled prior runs of this cell) run: ./scripts/ci-reap-family.sh "gd-fork-" "${{ env.WORKSPACE_NAME }}" + - name: Wait for staging box to be free + run: ./scripts/ci-wait-for-staging-box.sh + - name: Create workspace run: | coder create ${{ env.WORKSPACE_NAME }} \ diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index 1d220e4..2a19a20 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -63,11 +63,17 @@ on: # Ref-scoped so a new commit on the same PR (or the same push-to-main queue) # cancels its own stale run, but never touches another PR's or main's run. -# Serialization against other workflows/refs that provision real workspaces -# on the shared staging-coder.ddev.com box happens via the job-level -# `concurrency:` on each box-touching job below (group: staging-coder-integration, -# cancel-in-progress: false) -- see the matching comment there and in -# drupal-integration-test.yml / drupal-contrib-integration-test.yml. +# +# Serialization against other workflows/refs that provision real workspaces on +# the shared staging-coder.ddev.com box is NOT done via a shared job-level +# `concurrency:` group -- a single PR push fires several box-provisioning jobs +# across this file, drupal-integration-test.yml, and +# drupal-contrib-integration-test.yml at nearly the same instant, and GitHub's +# concurrency groups only keep one job running plus one pending; every other +# simultaneous contender gets silently CANCELLED, not queued. Instead, each +# box-touching job below runs scripts/ci-wait-for-staging-box.sh right before +# creating its workspace, which polls the Coder server's actual workspace +# count and waits its turn -- see that script's header comment for detail. concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.ref }} cancel-in-progress: true @@ -79,13 +85,9 @@ jobs: # To re-enable: restore `if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.owner.login == github.repository_owner }}` if: false runs-on: [self-hosted, sysbox] - timeout-minutes: 30 - # Same literal group as every other job that provisions real workspaces on - # staging-coder.ddev.com, across all three integration-test workflows -- - # serializes actual box usage without cancelling anyone else's in-flight run. - concurrency: - group: staging-coder-integration - cancel-in-progress: false + # Long enough to cover queued time behind other box-provisioning jobs + # (see scripts/ci-wait-for-staging-box.sh) plus this job's own runtime. + timeout-minutes: 75 strategy: matrix: include: @@ -298,10 +300,9 @@ jobs: name: Integration test GH (${{ matrix.template }}) if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.owner.login == github.repository_owner }} runs-on: ubuntu-latest - timeout-minutes: 30 - concurrency: - group: staging-coder-integration - cancel-in-progress: false + # Long enough to cover queued time behind other box-provisioning jobs + # (see scripts/ci-wait-for-staging-box.sh) plus this job's own runtime. + timeout-minutes: 75 strategy: matrix: include: @@ -359,6 +360,9 @@ jobs: - name: Reap predecessor workspaces (cancelled prior runs of this cell) run: ./scripts/ci-reap-family.sh "gh-${{ matrix.ws_name }}-" "${{ env.WORKSPACE_NAME }}" + - name: Wait for staging box to be free + run: ./scripts/ci-wait-for-staging-box.sh + - name: Create workspace if: ${{ matrix.template != 'freeform' }} run: | diff --git a/scripts/ci-wait-for-staging-box.sh b/scripts/ci-wait-for-staging-box.sh new file mode 100755 index 0000000..f50417c --- /dev/null +++ b/scripts/ci-wait-for-staging-box.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash +# Wait until no other CI workspace exists on the shared staging box before this +# job creates its own workspace. +# +# Why this exists: +# GitHub Actions' `concurrency:` keyword only keeps one job running plus one +# pending per group -- any additional job that tries to join while one is +# already pending gets CANCELLED outright, not queued behind it. A single PR +# push (or push to main) triggers up to five box-provisioning jobs across +# integration-test.yml, drupal-integration-test.yml, and +# drupal-contrib-integration-test.yml at nearly the same instant, so sharing +# one concurrency group silently dropped most of them instead of running +# them in turn. Polling the box's actual ci-bot workspace count avoids that: +# every contender waits and none are cancelled. +# +# Usage: +# ci-wait-for-staging-box.sh +# +# Environment: +# CI_OWNER Coder username that owns CI workspaces (default: ci-bot) +# MAX_WAIT_SECONDS Give up and fail after this long (default: 2700 = 45m) +# POLL_SECONDS Delay between checks (default: 30) + +set -uo pipefail + +CI_OWNER="${CI_OWNER:-ci-bot}" +MAX_WAIT_SECONDS="${MAX_WAIT_SECONDS:-2700}" +POLL_SECONDS="${POLL_SECONDS:-30}" + +# Jitter so jobs that all started in the same instant don't all sample the +# workspace count (and all see "clear") at the exact same moment. +sleep "$((RANDOM % 15))" + +waited=0 +while true; do + if ! workspaces_json=$(coder list --all --output json 2>/dev/null); then + echo "WARN: 'coder list' failed; proceeding without the box-busy check" >&2 + exit 0 + fi + + count=$(echo "$workspaces_json" | jq -r --arg owner "$CI_OWNER" \ + '[.[] | select(.owner_name==$owner)] | length') + + if [[ "$count" -eq 0 ]]; then + echo "Staging box is free (0 ci-bot workspaces) -- proceeding" + exit 0 + fi + + if [[ "$waited" -ge "$MAX_WAIT_SECONDS" ]]; then + echo "ERROR: staging box still busy ($count ci-bot workspace(s)) after ${MAX_WAIT_SECONDS}s -- giving up" >&2 + exit 1 + fi + + echo "Staging box busy ($count ci-bot workspace(s)); waiting ${POLL_SECONDS}s (${waited}s/${MAX_WAIT_SECONDS}s so far)..." + sleep "$POLL_SECONDS" + waited=$((waited + POLL_SECONDS)) +done