From 6cc17796711d3908fb498a4a1b7b771244788512 Mon Sep 17 00:00:00 2001 From: Jeremy Hatfield Date: Wed, 19 Aug 2026 17:06:52 +0000 Subject: [PATCH] fix(ci): clean up orphaned apt/dpkg processes between Playwright install retries The outer `timeout 10m` wrapping npx playwright install --with-deps kills the top-level node process when it fires, but the underlying apt-get runs as root via sudo in a separate process tree that timeout doesn't reach. When an attempt genuinely times out (a slow mirror, not a full stall), the orphaned apt-get process keeps running in the background holding the dpkg lock -- so the very next retry attempt fails immediately with "Could not get lock /var/lib/dpkg/lock-frontend. It is held by process (apt-get)", burning all remaining retries in seconds with no chance of success. Same underlying class of bug as the nick-fields/retry EPERM issue fixed previously: killing a process that has escalated privilege via sudo doesn't reliably reach its children. Before each retry, kill any lingering apt/apt-get/dpkg processes and clear stale lock files so the next attempt starts clean instead of immediately failing on a lock held by the previous attempt's orphan. Observed on PR #1262: both WebKit shards' first install attempt hit the full 10-minute timeout (mirror slowness, not a hang), then both retries failed in ~17s each with the dpkg lock error above. --- .github/workflows/e2e-tests-split.yml | 40 +++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/.github/workflows/e2e-tests-split.yml b/.github/workflows/e2e-tests-split.yml index c26273e21..485fd56dd 100644 --- a/.github/workflows/e2e-tests-split.yml +++ b/.github/workflows/e2e-tests-split.yml @@ -339,6 +339,10 @@ jobs: timeout 10m npx playwright install --with-deps chromium && break if [ "$attempt" -lt 3 ]; then echo "Attempt ${attempt}/3 failed; retrying in 15s..." >&2 + echo "Cleaning up any orphaned apt/dpkg processes and stale locks..." >&2 + sudo pkill -9 -f '(^|/)(apt|apt-get|dpkg)( |$)' 2>/dev/null || true + sudo rm -f /var/lib/dpkg/lock-frontend /var/lib/dpkg/lock /var/cache/apt/archives/lock 2>/dev/null || true + sudo dpkg --configure -a > /dev/null 2>&1 || true sleep 15 else echo "ERROR: Playwright chromium install failed after 3 attempts" >&2 @@ -589,6 +593,10 @@ jobs: timeout 10m npx playwright install --with-deps chromium && break if [ "$attempt" -lt 3 ]; then echo "Attempt ${attempt}/3 failed; retrying in 15s..." >&2 + echo "Cleaning up any orphaned apt/dpkg processes and stale locks..." >&2 + sudo pkill -9 -f '(^|/)(apt|apt-get|dpkg)( |$)' 2>/dev/null || true + sudo rm -f /var/lib/dpkg/lock-frontend /var/lib/dpkg/lock /var/cache/apt/archives/lock 2>/dev/null || true + sudo dpkg --configure -a > /dev/null 2>&1 || true sleep 15 else echo "ERROR: Playwright chromium install failed after 3 attempts" >&2 @@ -604,6 +612,10 @@ jobs: timeout 10m npx playwright install --with-deps firefox && break if [ "$attempt" -lt 3 ]; then echo "Attempt ${attempt}/3 failed; retrying in 15s..." >&2 + echo "Cleaning up any orphaned apt/dpkg processes and stale locks..." >&2 + sudo pkill -9 -f '(^|/)(apt|apt-get|dpkg)( |$)' 2>/dev/null || true + sudo rm -f /var/lib/dpkg/lock-frontend /var/lib/dpkg/lock /var/cache/apt/archives/lock 2>/dev/null || true + sudo dpkg --configure -a > /dev/null 2>&1 || true sleep 15 else echo "ERROR: Playwright firefox install failed after 3 attempts" >&2 @@ -857,6 +869,10 @@ jobs: timeout 10m npx playwright install --with-deps chromium && break if [ "$attempt" -lt 3 ]; then echo "Attempt ${attempt}/3 failed; retrying in 15s..." >&2 + echo "Cleaning up any orphaned apt/dpkg processes and stale locks..." >&2 + sudo pkill -9 -f '(^|/)(apt|apt-get|dpkg)( |$)' 2>/dev/null || true + sudo rm -f /var/lib/dpkg/lock-frontend /var/lib/dpkg/lock /var/cache/apt/archives/lock 2>/dev/null || true + sudo dpkg --configure -a > /dev/null 2>&1 || true sleep 15 else echo "ERROR: Playwright chromium install failed after 3 attempts" >&2 @@ -872,6 +888,10 @@ jobs: timeout 10m npx playwright install --with-deps webkit && break if [ "$attempt" -lt 3 ]; then echo "Attempt ${attempt}/3 failed; retrying in 15s..." >&2 + echo "Cleaning up any orphaned apt/dpkg processes and stale locks..." >&2 + sudo pkill -9 -f '(^|/)(apt|apt-get|dpkg)( |$)' 2>/dev/null || true + sudo rm -f /var/lib/dpkg/lock-frontend /var/lib/dpkg/lock /var/cache/apt/archives/lock 2>/dev/null || true + sudo dpkg --configure -a > /dev/null 2>&1 || true sleep 15 else echo "ERROR: Playwright webkit install failed after 3 attempts" >&2 @@ -1152,6 +1172,10 @@ jobs: timeout 10m npx playwright install --with-deps chromium && break if [ "$attempt" -lt 3 ]; then echo "Attempt ${attempt}/3 failed; retrying in 15s..." >&2 + echo "Cleaning up any orphaned apt/dpkg processes and stale locks..." >&2 + sudo pkill -9 -f '(^|/)(apt|apt-get|dpkg)( |$)' 2>/dev/null || true + sudo rm -f /var/lib/dpkg/lock-frontend /var/lib/dpkg/lock /var/cache/apt/archives/lock 2>/dev/null || true + sudo dpkg --configure -a > /dev/null 2>&1 || true sleep 15 else echo "ERROR: Playwright chromium install failed after 3 attempts" >&2 @@ -1405,6 +1429,10 @@ jobs: timeout 10m npx playwright install --with-deps chromium && break if [ "$attempt" -lt 3 ]; then echo "Attempt ${attempt}/3 failed; retrying in 15s..." >&2 + echo "Cleaning up any orphaned apt/dpkg processes and stale locks..." >&2 + sudo pkill -9 -f '(^|/)(apt|apt-get|dpkg)( |$)' 2>/dev/null || true + sudo rm -f /var/lib/dpkg/lock-frontend /var/lib/dpkg/lock /var/cache/apt/archives/lock 2>/dev/null || true + sudo dpkg --configure -a > /dev/null 2>&1 || true sleep 15 else echo "ERROR: Playwright chromium install failed after 3 attempts" >&2 @@ -1420,6 +1448,10 @@ jobs: timeout 10m npx playwright install --with-deps firefox && break if [ "$attempt" -lt 3 ]; then echo "Attempt ${attempt}/3 failed; retrying in 15s..." >&2 + echo "Cleaning up any orphaned apt/dpkg processes and stale locks..." >&2 + sudo pkill -9 -f '(^|/)(apt|apt-get|dpkg)( |$)' 2>/dev/null || true + sudo rm -f /var/lib/dpkg/lock-frontend /var/lib/dpkg/lock /var/cache/apt/archives/lock 2>/dev/null || true + sudo dpkg --configure -a > /dev/null 2>&1 || true sleep 15 else echo "ERROR: Playwright firefox install failed after 3 attempts" >&2 @@ -1673,6 +1705,10 @@ jobs: timeout 10m npx playwright install --with-deps chromium && break if [ "$attempt" -lt 3 ]; then echo "Attempt ${attempt}/3 failed; retrying in 15s..." >&2 + echo "Cleaning up any orphaned apt/dpkg processes and stale locks..." >&2 + sudo pkill -9 -f '(^|/)(apt|apt-get|dpkg)( |$)' 2>/dev/null || true + sudo rm -f /var/lib/dpkg/lock-frontend /var/lib/dpkg/lock /var/cache/apt/archives/lock 2>/dev/null || true + sudo dpkg --configure -a > /dev/null 2>&1 || true sleep 15 else echo "ERROR: Playwright chromium install failed after 3 attempts" >&2 @@ -1688,6 +1724,10 @@ jobs: timeout 10m npx playwright install --with-deps webkit && break if [ "$attempt" -lt 3 ]; then echo "Attempt ${attempt}/3 failed; retrying in 15s..." >&2 + echo "Cleaning up any orphaned apt/dpkg processes and stale locks..." >&2 + sudo pkill -9 -f '(^|/)(apt|apt-get|dpkg)( |$)' 2>/dev/null || true + sudo rm -f /var/lib/dpkg/lock-frontend /var/lib/dpkg/lock /var/cache/apt/archives/lock 2>/dev/null || true + sudo dpkg --configure -a > /dev/null 2>&1 || true sleep 15 else echo "ERROR: Playwright webkit install failed after 3 attempts" >&2