diff --git a/.github/workflows/drupal-contrib-integration-test.yml b/.github/workflows/drupal-contrib-integration-test.yml index a7065be..039266d 100644 --- a/.github/workflows/drupal-contrib-integration-test.yml +++ b/.github/workflows/drupal-contrib-integration-test.yml @@ -26,6 +26,21 @@ name: Drupal Contrib Integration Tests env: FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true +# Ref-scoped so a new commit on the same PR (or the same push-to-main queue) +# cancels its own stale run, but never touches another PR's or main's run. +# +# Serialization against other workflows/refs that provision real workspaces on +# the shared staging-coder.ddev.com box is NOT done via a shared job-level +# `concurrency:` group -- a single PR push fires several box-provisioning jobs +# across this file, integration-test.yml, and drupal-integration-test.yml at +# nearly the same instant, and GitHub's concurrency groups only keep one job +# running plus one pending; every other simultaneous contender gets silently +# CANCELLED, not queued. Instead, each box-touching job below (including +# contrib-plain-gh and contrib-issue-fork-gh, which otherwise run in parallel +# within a single workflow run) calls scripts/ci-wait-for-staging-box.sh right +# before creating its workspace, which polls the Coder server's actual +# workspace count and waits its turn -- see that script's header comment for +# detail. concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.ref }} cancel-in-progress: true @@ -51,6 +66,9 @@ jobs: # To re-enable: restore `if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.owner.login == github.repository_owner }}` if: false runs-on: [self-hosted, sysbox] + # Long enough to cover queued time behind other box-provisioning jobs + # (see scripts/ci-wait-for-staging-box.sh) plus this job's own runtime. + timeout-minutes: 75 strategy: matrix: include: @@ -210,6 +228,9 @@ jobs: # To re-enable: restore `if: ${{ vars.CONTRIB_TEST_ISSUE_FORK != '' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.owner.login == github.repository_owner) }}` if: false runs-on: [self-hosted, sysbox] + # Long enough to cover queued time behind other box-provisioning jobs + # (see scripts/ci-wait-for-staging-box.sh) plus this job's own runtime. + timeout-minutes: 75 defaults: run: shell: bash -euo pipefail {0} @@ -386,6 +407,9 @@ jobs: name: Contrib ${{ matrix.project }} D${{ matrix.drupal_version }} (plain, GH) if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.owner.login == github.repository_owner }} runs-on: ubuntu-latest + # Long enough to cover queued time behind other box-provisioning jobs + # (see scripts/ci-wait-for-staging-box.sh) plus this job's own runtime. + timeout-minutes: 75 strategy: # PRs run a single module (smoke); push-to-main and nightly run the full # set. Cuts per-PR workspace load on staging. See ci-reap.yml. @@ -434,6 +458,9 @@ jobs: - name: Reap predecessor workspaces (cancelled prior runs of this cell) run: ./scripts/ci-reap-family.sh "gc-${{ matrix.project }}-d${{ matrix.drupal_version }}-" "${{ env.WORKSPACE_NAME }}" + - name: Wait for staging box to be free + run: ./scripts/ci-wait-for-staging-box.sh + - name: Create workspace run: | coder create ${{ env.WORKSPACE_NAME }} \ @@ -525,6 +552,9 @@ jobs: name: Contrib ${{ vars.CONTRIB_TEST_PROJECT || 'token' }} issue fork (GH) if: ${{ vars.CONTRIB_TEST_ISSUE_FORK != '' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.owner.login == github.repository_owner) }} runs-on: ubuntu-latest + # Long enough to cover queued time behind other box-provisioning jobs + # (see scripts/ci-wait-for-staging-box.sh) plus this job's own runtime. + timeout-minutes: 75 defaults: run: shell: bash -euo pipefail {0} @@ -587,6 +617,9 @@ jobs: - name: Reap predecessor workspaces (cancelled prior runs of this cell) run: ./scripts/ci-reap-family.sh "gc-fork-" "${{ env.WORKSPACE_NAME }}" + - name: Wait for staging box to be free + run: ./scripts/ci-wait-for-staging-box.sh + - name: Create workspace run: | coder create ${{ env.WORKSPACE_NAME }} \ diff --git a/.github/workflows/drupal-integration-test.yml b/.github/workflows/drupal-integration-test.yml index 3b12d48..c12ee0e 100644 --- a/.github/workflows/drupal-integration-test.yml +++ b/.github/workflows/drupal-integration-test.yml @@ -31,6 +31,21 @@ name: Drupal Integration Tests env: FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true +# Ref-scoped so a new commit on the same PR (or the same push-to-main queue) +# cancels its own stale run, but never touches another PR's or main's run. +# +# Serialization against other workflows/refs that provision real workspaces on +# the shared staging-coder.ddev.com box is NOT done via a shared job-level +# `concurrency:` group -- a single PR push fires several box-provisioning jobs +# across this file, integration-test.yml, and +# drupal-contrib-integration-test.yml at nearly the same instant, and GitHub's +# concurrency groups only keep one job running plus one pending; every other +# simultaneous contender gets silently CANCELLED, not queued. Instead, each +# box-touching job below (including drupal-plain-gh and drupal-issue-fork-gh, +# which otherwise run in parallel within a single workflow run) calls +# scripts/ci-wait-for-staging-box.sh right before creating its workspace, +# which polls the Coder server's actual workspace count and waits its turn -- +# see that script's header comment for detail. concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.ref }} cancel-in-progress: true @@ -56,6 +71,9 @@ jobs: # To re-enable: restore `if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.owner.login == github.repository_owner }}` if: false runs-on: [self-hosted, sysbox] + # Long enough to cover queued time behind other box-provisioning jobs + # (see scripts/ci-wait-for-staging-box.sh) plus this job's own runtime. + timeout-minutes: 75 strategy: matrix: drupal_version: ["12", "11"] @@ -198,6 +216,9 @@ jobs: name: Drupal ${{ matrix.drupal_version }} (plain, GH) if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.owner.login == github.repository_owner }} runs-on: ubuntu-latest + # Long enough to cover queued time behind other box-provisioning jobs + # (see scripts/ci-wait-for-staging-box.sh) plus this job's own runtime. + timeout-minutes: 75 strategy: # PRs run a single Drupal version (smoke); push-to-main and nightly run the # full matrix. Cuts per-PR workspace load on staging. See ci-reap.yml. @@ -244,6 +265,9 @@ jobs: - name: Reap predecessor workspaces (cancelled prior runs of this cell) run: ./scripts/ci-reap-family.sh "gd-${{ matrix.drupal_version }}-" "${{ env.WORKSPACE_NAME }}" + - name: Wait for staging box to be free + run: ./scripts/ci-wait-for-staging-box.sh + - name: Create workspace run: | coder create ${{ env.WORKSPACE_NAME }} \ @@ -276,6 +300,9 @@ jobs: # To re-enable: restore `if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.owner.login == github.repository_owner }}` if: false runs-on: [self-hosted, sysbox] + # Long enough to cover queued time behind other box-provisioning jobs + # (see scripts/ci-wait-for-staging-box.sh) plus this job's own runtime. + timeout-minutes: 75 defaults: run: shell: bash -euo pipefail {0} @@ -436,6 +463,9 @@ jobs: name: Drupal issue fork ${{ vars.DRUPAL_TEST_ISSUE_FORK || '3585397' }} (GH) if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.owner.login == github.repository_owner }} runs-on: ubuntu-latest + # Long enough to cover queued time behind other box-provisioning jobs + # (see scripts/ci-wait-for-staging-box.sh) plus this job's own runtime. + timeout-minutes: 75 defaults: run: shell: bash -euo pipefail {0} @@ -495,6 +525,9 @@ jobs: - name: Reap predecessor workspaces (cancelled prior runs of this cell) run: ./scripts/ci-reap-family.sh "gd-fork-" "${{ env.WORKSPACE_NAME }}" + - name: Wait for staging box to be free + run: ./scripts/ci-wait-for-staging-box.sh + - name: Create workspace run: | coder create ${{ env.WORKSPACE_NAME }} \ diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index b6a8504..2a19a20 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -61,8 +61,21 @@ on: required: false default: false +# Ref-scoped so a new commit on the same PR (or the same push-to-main queue) +# cancels its own stale run, but never touches another PR's or main's run. +# +# Serialization against other workflows/refs that provision real workspaces on +# the shared staging-coder.ddev.com box is NOT done via a shared job-level +# `concurrency:` group -- a single PR push fires several box-provisioning jobs +# across this file, drupal-integration-test.yml, and +# drupal-contrib-integration-test.yml at nearly the same instant, and GitHub's +# concurrency groups only keep one job running plus one pending; every other +# simultaneous contender gets silently CANCELLED, not queued. Instead, each +# box-touching job below runs scripts/ci-wait-for-staging-box.sh right before +# creating its workspace, which polls the Coder server's actual workspace +# count and waits its turn -- see that script's header comment for detail. concurrency: - group: ${{ github.workflow }}-${{ github.ref }} + group: ${{ github.workflow }}-${{ github.head_ref || github.ref }} cancel-in-progress: true jobs: @@ -72,6 +85,9 @@ jobs: # To re-enable: restore `if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.owner.login == github.repository_owner }}` if: false runs-on: [self-hosted, sysbox] + # Long enough to cover queued time behind other box-provisioning jobs + # (see scripts/ci-wait-for-staging-box.sh) plus this job's own runtime. + timeout-minutes: 75 strategy: matrix: include: @@ -284,6 +300,9 @@ jobs: name: Integration test GH (${{ matrix.template }}) if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.owner.login == github.repository_owner }} runs-on: ubuntu-latest + # Long enough to cover queued time behind other box-provisioning jobs + # (see scripts/ci-wait-for-staging-box.sh) plus this job's own runtime. + timeout-minutes: 75 strategy: matrix: include: @@ -341,6 +360,9 @@ jobs: - name: Reap predecessor workspaces (cancelled prior runs of this cell) run: ./scripts/ci-reap-family.sh "gh-${{ matrix.ws_name }}-" "${{ env.WORKSPACE_NAME }}" + - name: Wait for staging box to be free + run: ./scripts/ci-wait-for-staging-box.sh + - name: Create workspace if: ${{ matrix.template != 'freeform' }} run: | diff --git a/scripts/ci-wait-for-staging-box.sh b/scripts/ci-wait-for-staging-box.sh new file mode 100755 index 0000000..f50417c --- /dev/null +++ b/scripts/ci-wait-for-staging-box.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash +# Wait until no other CI workspace exists on the shared staging box before this +# job creates its own workspace. +# +# Why this exists: +# GitHub Actions' `concurrency:` keyword only keeps one job running plus one +# pending per group -- any additional job that tries to join while one is +# already pending gets CANCELLED outright, not queued behind it. A single PR +# push (or push to main) triggers up to five box-provisioning jobs across +# integration-test.yml, drupal-integration-test.yml, and +# drupal-contrib-integration-test.yml at nearly the same instant, so sharing +# one concurrency group silently dropped most of them instead of running +# them in turn. Polling the box's actual ci-bot workspace count avoids that: +# every contender waits and none are cancelled. +# +# Usage: +# ci-wait-for-staging-box.sh +# +# Environment: +# CI_OWNER Coder username that owns CI workspaces (default: ci-bot) +# MAX_WAIT_SECONDS Give up and fail after this long (default: 2700 = 45m) +# POLL_SECONDS Delay between checks (default: 30) + +set -uo pipefail + +CI_OWNER="${CI_OWNER:-ci-bot}" +MAX_WAIT_SECONDS="${MAX_WAIT_SECONDS:-2700}" +POLL_SECONDS="${POLL_SECONDS:-30}" + +# Jitter so jobs that all started in the same instant don't all sample the +# workspace count (and all see "clear") at the exact same moment. +sleep "$((RANDOM % 15))" + +waited=0 +while true; do + if ! workspaces_json=$(coder list --all --output json 2>/dev/null); then + echo "WARN: 'coder list' failed; proceeding without the box-busy check" >&2 + exit 0 + fi + + count=$(echo "$workspaces_json" | jq -r --arg owner "$CI_OWNER" \ + '[.[] | select(.owner_name==$owner)] | length') + + if [[ "$count" -eq 0 ]]; then + echo "Staging box is free (0 ci-bot workspaces) -- proceeding" + exit 0 + fi + + if [[ "$waited" -ge "$MAX_WAIT_SECONDS" ]]; then + echo "ERROR: staging box still busy ($count ci-bot workspace(s)) after ${MAX_WAIT_SECONDS}s -- giving up" >&2 + exit 1 + fi + + echo "Staging box busy ($count ci-bot workspace(s)); waiting ${POLL_SECONDS}s (${waited}s/${MAX_WAIT_SECONDS}s so far)..." + sleep "$POLL_SECONDS" + waited=$((waited + POLL_SECONDS)) +done