From dcb4fd0c7c37a0854d1c328e27eab0b397992665 Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Mon, 3 Aug 2026 09:21:50 -0600 Subject: [PATCH] ci: bound the build jobs and the package fetches they hang on (#351) The build (PG 15, x86_64) job hung indefinitely on its package install step on two consecutive PRs, #347 for 40 minutes and #350 for 18, against a normal runtime of about 70 seconds. Every other build passed both times, including PG 15 on aarch64 -- same major, same compiler, different runner -- and the step that wedged is package installation, before any project code compiles. Three things were missing, and all three are needed. The build and build-beta jobs had no timeout-minutes at all; only suites did. So a stalled mirror produced a check that stayed pending forever rather than failing. That is worse than a failure: the PR shows "1 pending" and the reviewer either waits or merges on an incomplete gate. It came up twice in a row here, and a gate that is routinely bypassed stops being a gate. curl was invoked as "curl -fsSL" with no connect or total timeout, so it waits forever on a stalled connection rather than failing and retrying. It now bounds both, and retries with backoff. apt-get had no retry configuration and no outer bound. It now sets Acquire::Retries and runs under timeout, so a wedged mirror costs a bounded amount of the job budget instead of all of it. The same unbounded curl and apt appear in nightly.yml, whose jobs do have timeouts, so a hang there fails after up to two hours rather than never. Hardened identically, since it is the same defect with a slower fuse. docs.yml had no job timeouts either; added. Every job across the three workflows is now bounded, and every network fetch is retried and time-limited. Worth recording for whoever hits this next: when the hung run is cancelled, GitHub may retry the job as a new attempt, and the PR check then shows green from that retry while the cancelled attempt sits at conclusion=cancelled with Build: skipped. On #350 the cancel produced a green-looking check over a job that compiled nothing. Check the retried job's steps, not the check mark. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_011miCFRSatixeNRw3w5yNq8 --- .github/workflows/ci.yml | 32 ++++++++++++++++++++++++++------ .github/workflows/docs.yml | 2 ++ .github/workflows/nightly.yml | 24 +++++++++++++++--------- 3 files changed, 43 insertions(+), 15 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9d1d55c..03f0e0c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -46,6 +46,11 @@ jobs: build: name: build (PG ${{ matrix.pg }}, ${{ matrix.runner == 'ubuntu-24.04-arm' && 'aarch64' || 'x86_64' }}) runs-on: ${{ matrix.runner }} + # This job normally finishes in about 70 seconds. Without a bound, a stalled + # package mirror leaves it running forever and the PR sits at "1 pending" + # rather than failing, which pushes a reviewer toward merging on an + # incomplete gate (#351). A gate that is routinely bypassed is not a gate. + timeout-minutes: 15 strategy: fail-fast: false # Two dimensions, so this is a cross product: every major on both @@ -63,13 +68,20 @@ jobs: run: | set -euo pipefail sudo install -d /usr/share/postgresql-common/pgdg - sudo curl -fsSL -o /usr/share/postgresql-common/pgdg/apt.postgresql.org.asc \ + # Bounded and retried: plain "curl -fsSL" waits indefinitely on a + # stalled connection, which is how this step came to hang for 40 + # minutes on an otherwise healthy runner (#351). + sudo curl -fsSL --connect-timeout 15 --max-time 120 \ + --retry 5 --retry-delay 5 --retry-all-errors \ + -o /usr/share/postgresql-common/pgdg/apt.postgresql.org.asc \ https://www.postgresql.org/media/keys/ACCC4CF8.asc echo "deb [signed-by=/usr/share/postgresql-common/pgdg/apt.postgresql.org.asc] \ https://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" \ | sudo tee /etc/apt/sources.list.d/pgdg.list >/dev/null - sudo apt-get update - sudo apt-get install -y --no-install-recommends \ + # Acquire retries cover a flaky mirror; the outer timeout keeps a + # wedged one from consuming the whole job budget. + sudo timeout 300 apt-get -o Acquire::Retries=5 update + sudo timeout 600 apt-get -o Acquire::Retries=5 install -y --no-install-recommends \ postgresql-${{ matrix.pg }} postgresql-server-dev-${{ matrix.pg }} \ liblz4-dev libzstd-dev zlib1g-dev @@ -119,6 +131,7 @@ jobs: build-beta: name: build (PG 19 beta, from source) runs-on: ubuntu-latest + timeout-minutes: 20 env: PG_BETA: 19beta2 steps: @@ -206,13 +219,20 @@ jobs: run: | set -euo pipefail sudo install -d /usr/share/postgresql-common/pgdg - sudo curl -fsSL -o /usr/share/postgresql-common/pgdg/apt.postgresql.org.asc \ + # Bounded and retried: plain "curl -fsSL" waits indefinitely on a + # stalled connection, which is how this step came to hang for 40 + # minutes on an otherwise healthy runner (#351). + sudo curl -fsSL --connect-timeout 15 --max-time 120 \ + --retry 5 --retry-delay 5 --retry-all-errors \ + -o /usr/share/postgresql-common/pgdg/apt.postgresql.org.asc \ https://www.postgresql.org/media/keys/ACCC4CF8.asc echo "deb [signed-by=/usr/share/postgresql-common/pgdg/apt.postgresql.org.asc] \ https://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" \ | sudo tee /etc/apt/sources.list.d/pgdg.list >/dev/null - sudo apt-get update - sudo apt-get install -y --no-install-recommends \ + # Acquire retries cover a flaky mirror; the outer timeout keeps a + # wedged one from consuming the whole job budget. + sudo timeout 300 apt-get -o Acquire::Retries=5 update + sudo timeout 600 apt-get -o Acquire::Retries=5 install -y --no-install-recommends \ postgresql-${{ matrix.pg }} postgresql-server-dev-${{ matrix.pg }} \ liblz4-dev libzstd-dev zlib1g-dev python3-pip # The Arrow and Parquet suites use pyarrow as an independent reader and diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index c184d86..5cf6f6c 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -27,6 +27,7 @@ concurrency: jobs: build: runs-on: ubuntu-latest + timeout-minutes: 20 steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 @@ -42,6 +43,7 @@ jobs: deploy: needs: build runs-on: ubuntu-latest + timeout-minutes: 20 environment: name: github-pages url: ${{ steps.deployment.outputs.page_url }} diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index c1ec5ae..579d26b 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -65,13 +65,15 @@ jobs: run: | set -euo pipefail sudo install -d /usr/share/postgresql-common/pgdg - sudo curl -fsSL -o /usr/share/postgresql-common/pgdg/apt.postgresql.org.asc \ + sudo curl -fsSL --connect-timeout 15 --max-time 120 \ + --retry 5 --retry-delay 5 --retry-all-errors \ + -o /usr/share/postgresql-common/pgdg/apt.postgresql.org.asc \ https://www.postgresql.org/media/keys/ACCC4CF8.asc echo "deb [signed-by=/usr/share/postgresql-common/pgdg/apt.postgresql.org.asc] \ https://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" \ | sudo tee /etc/apt/sources.list.d/pgdg.list >/dev/null - sudo apt-get update - sudo apt-get install -y --no-install-recommends \ + sudo timeout 300 apt-get -o Acquire::Retries=5 update + sudo timeout 600 apt-get -o Acquire::Retries=5 install -y --no-install-recommends \ postgresql-${{ matrix.pg }} postgresql-server-dev-${{ matrix.pg }} \ liblz4-dev libzstd-dev zlib1g-dev python3-pip # pyarrow for root (the suites run under sudo); the silent-skip trap the @@ -122,8 +124,8 @@ jobs: - name: Install clang, build tools, codec headers, pyarrow; create the postgres user run: | set -euo pipefail - sudo apt-get update - sudo apt-get install -y --no-install-recommends \ + sudo timeout 300 apt-get -o Acquire::Retries=5 update + sudo timeout 600 apt-get -o Acquire::Retries=5 install -y --no-install-recommends \ clang llvm bison flex make perl \ liblz4-dev libzstd-dev zlib1g-dev python3-pip clang --version | head -1 @@ -148,7 +150,9 @@ jobs: if: steps.san-cache.outputs.cache-hit != 'true' run: | set -euo pipefail - curl -fsSL -o /tmp/pg-src.tar.bz2 \ + curl -fsSL --connect-timeout 15 --max-time 600 \ + --retry 5 --retry-delay 5 --retry-all-errors \ + -o /tmp/pg-src.tar.bz2 \ "https://ftp.postgresql.org/pub/source/v${PG_VERSION}/postgresql-${PG_VERSION}.tar.bz2" # build_san.sh honours PGC_PG_TARBALL (source) and PGC_SAN_SRC (scratch) # and installs to the prefix argument; point all three at CI-writable @@ -237,13 +241,15 @@ jobs: run: | set -euo pipefail sudo install -d /usr/share/postgresql-common/pgdg - sudo curl -fsSL -o /usr/share/postgresql-common/pgdg/apt.postgresql.org.asc \ + sudo curl -fsSL --connect-timeout 15 --max-time 120 \ + --retry 5 --retry-delay 5 --retry-all-errors \ + -o /usr/share/postgresql-common/pgdg/apt.postgresql.org.asc \ https://www.postgresql.org/media/keys/ACCC4CF8.asc echo "deb [signed-by=/usr/share/postgresql-common/pgdg/apt.postgresql.org.asc] \ https://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" \ | sudo tee /etc/apt/sources.list.d/pgdg.list >/dev/null - sudo apt-get update - sudo apt-get install -y --no-install-recommends \ + sudo timeout 300 apt-get -o Acquire::Retries=5 update + sudo timeout 600 apt-get -o Acquire::Retries=5 install -y --no-install-recommends \ postgresql-18 postgresql-server-dev-18 \ liblz4-dev libzstd-dev zlib1g-dev python3-pip lcov # Installed with sudo because the suites run under sudo; a plain