From fc3e4e2605962fab16cbef739893906b705faf7e Mon Sep 17 00:00:00 2001 From: Jeremy Hatfield Date: Wed, 19 Aug 2026 13:02:34 +0000 Subject: [PATCH] fix(ci): tune apt timeouts to fail fast on stalled CI mirror connections The bash timeout/retry loop wrapping npx playwright install --with-deps (added previously to stop apt hangs from burning the job's full 60-minute timeout) still lost all 3 retry attempts to the same degraded connection in a recent run: apt wasn't erroring or fully stalled with zero progress, it was crawling at near-zero throughput on azure.archive.ubuntu.com -- one package's download sat idle for over 5 minutes before the next one even started, so each 10-minute attempt timed out against the same slow mirror with no actual retry benefit. Write an apt.conf.d drop-in before the first browser install in each job, setting Acquire::Retries/Acquire::http::Timeout/Acquire::https::Timeout to bounded values (3 retries, 15s timeout). This makes apt itself detect and abandon a stalled connection within seconds and retry against a fresh one, instead of sitting on one bad connection for the whole outer 10-minute budget -- so our existing 3-attempt outer loop actually gets a chance to succeed against a different connection attempt, rather than exhausting all 3 attempts on the same degraded path. --- .github/workflows/e2e-tests-split.yml | 48 +++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/.github/workflows/e2e-tests-split.yml b/.github/workflows/e2e-tests-split.yml index 5f42fe116..c26273e21 100644 --- a/.github/workflows/e2e-tests-split.yml +++ b/.github/workflows/e2e-tests-split.yml @@ -323,6 +323,14 @@ jobs: - name: Install dependencies run: npm ci --ignore-scripts + - name: Harden apt for flaky CI mirrors + run: | + sudo tee /etc/apt/apt.conf.d/99-charon-ci-reliability > /dev/null <<'APTCONF' + Acquire::Retries "3"; + Acquire::http::Timeout "15"; + Acquire::https::Timeout "15"; + APTCONF + - name: Install Playwright Chromium run: | set -uo pipefail @@ -565,6 +573,14 @@ jobs: - name: Install dependencies run: npm ci --ignore-scripts + - name: Harden apt for flaky CI mirrors + run: | + sudo tee /etc/apt/apt.conf.d/99-charon-ci-reliability > /dev/null <<'APTCONF' + Acquire::Retries "3"; + Acquire::http::Timeout "15"; + Acquire::https::Timeout "15"; + APTCONF + - name: Install Playwright Chromium (required by security-tests dependency) run: | set -uo pipefail @@ -825,6 +841,14 @@ jobs: - name: Install dependencies run: npm ci --ignore-scripts + - name: Harden apt for flaky CI mirrors + run: | + sudo tee /etc/apt/apt.conf.d/99-charon-ci-reliability > /dev/null <<'APTCONF' + Acquire::Retries "3"; + Acquire::http::Timeout "15"; + Acquire::https::Timeout "15"; + APTCONF + - name: Install Playwright Chromium (required by security-tests dependency) run: | set -uo pipefail @@ -1112,6 +1136,14 @@ jobs: - name: Install dependencies run: npm ci --ignore-scripts + - name: Harden apt for flaky CI mirrors + run: | + sudo tee /etc/apt/apt.conf.d/99-charon-ci-reliability > /dev/null <<'APTCONF' + Acquire::Retries "3"; + Acquire::http::Timeout "15"; + Acquire::https::Timeout "15"; + APTCONF + - name: Install Playwright Chromium run: | set -uo pipefail @@ -1357,6 +1389,14 @@ jobs: - name: Install dependencies run: npm ci --ignore-scripts + - name: Harden apt for flaky CI mirrors + run: | + sudo tee /etc/apt/apt.conf.d/99-charon-ci-reliability > /dev/null <<'APTCONF' + Acquire::Retries "3"; + Acquire::http::Timeout "15"; + Acquire::https::Timeout "15"; + APTCONF + - name: Install Playwright Chromium (required by security-tests dependency) run: | set -uo pipefail @@ -1617,6 +1657,14 @@ jobs: - name: Install dependencies run: npm ci --ignore-scripts + - name: Harden apt for flaky CI mirrors + run: | + sudo tee /etc/apt/apt.conf.d/99-charon-ci-reliability > /dev/null <<'APTCONF' + Acquire::Retries "3"; + Acquire::http::Timeout "15"; + Acquire::https::Timeout "15"; + APTCONF + - name: Install Playwright Chromium (required by security-tests dependency) run: | set -uo pipefail