diff --git a/Dockerfile b/Dockerfile index 531acdd..54647d8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -78,9 +78,12 @@ ENV ANDROID_SDK_ROOT=$ANDROID_ROOT/sdk ENV ANDROID_HOME=$ANDROID_ROOT/sdk LABEL maintainer="ernstjason1@gmail.com" -# Post-job cleanup hook to prevent unbounded cache growth +# Job hooks: JOB_STARTED is the poison sweep (covers OOM / killed runner); +# JOB_COMPLETED is the tidy-up. Neither wipes package / Gradle caches. +COPY pre-job.sh /usr/local/bin/pre-job.sh COPY cleanup.sh /usr/local/bin/cleanup.sh -RUN chmod +x /usr/local/bin/cleanup.sh +RUN chmod +x /usr/local/bin/pre-job.sh /usr/local/bin/cleanup.sh +ENV ACTIONS_RUNNER_HOOK_JOB_STARTED=/usr/local/bin/pre-job.sh ENV ACTIONS_RUNNER_HOOK_JOB_COMPLETED=/usr/local/bin/cleanup.sh # Preflight entrypoint: validates the persisted runner registration against diff --git a/README.md b/README.md index 58194e4..727228e 100644 --- a/README.md +++ b/README.md @@ -17,8 +17,55 @@ Linux hosts run natively without emulation. To run the container, check out the environment variables from the base image: https://github.com/myoung34/docker-github-actions-runner#environment-variables -There is also an example docker-compose file which uses .env file to set the -variables. +[`docker-compose.yml`](docker-compose.yml) is the X64 beelink layout — three +persistent runners, each with its own work **and** cache volumes — not a +side file. After pulling a new image on the host: + +``` +docker compose pull && docker compose up -d +``` + +Existing named `runner-work*` volumes keep their data; new cache volumes +start empty and warm on first jobs. Do not share one bun/gradle volume +across the three containers: concurrent jobs would corrupt the same tree. + +## X64 cache volumes + +This compose is for the **X64** class (`beelink`, `docker`, `large`). System +labels (`Linux`, `X64`) are still applied by the runner binary; set +`LABELS` in `.env` to `beelink,docker,large` (do not add a `build` label). +`EPHEMERAL` stays `false` so the volumes survive between jobs. + +Each runner gets its own: + +| Mount | Volume (runner / runner-2 / runner-3) | Why | +|-------|----------------------------------------|-----| +| `/tmp/runner/work` | `runner-work*` | Job workspace. Cleanup spares `$RUNNER_WORKDIR`. | +| `/root/.bun` | `runner-bun*` | Bun install cache | +| `/root/.npm` | `runner-npm*` | npm cache | +| `/root/.cache/pnpm` | `runner-pnpm*` | pnpm store | +| `/root/.gradle` | `runner-gradle*` | Gradle modules, build-cache, transforms | +| `/root/.cargo` | `runner-cargo*` | Cargo registry | + +Budget ~20 GB per runner (enforced by the post-job hook with LRU, not a +wipe), ~60 GB per beelink. ARM64 / Pi hosts should **not** mount these +cache volumes — those runners isolate to `RUNNER_TEMP`. + +Do **not** tmpfs `/tmp`. `RUNNER_WORKDIR` is `/tmp/runner/work`; a tmpfs +or a blanket `find /tmp -delete` hides or wipes the work volume. + +### Workflows must use the volume homes + +Named volumes are unused if the job relocates the caches. On these X64 +self-hosted runners, workflows must **not** isolate `GRADLE_USER_HOME` or +`BUN_INSTALL_CACHE_DIR` (or `npm_config_cache` / `CARGO_HOME`) to +`$RUNNER_TEMP`. That is the `setup-stack` local-home mode in +`dodi-smart/.github` — `deps-verify` stays isolated; `pr-checks` uses the +default homes so the volumes are actually read. + +The other half of cache poison is GitHub `actions/cache` with +`restore-keys` re-importing a partial bun tarball. Use an exact key, or +do not upload the bun install cache at all. ## Image tags diff --git a/cleanup.sh b/cleanup.sh old mode 100644 new mode 100755 index c071226..3a88827 --- a/cleanup.sh +++ b/cleanup.sh @@ -1,28 +1,104 @@ #!/bin/bash # Post-job cleanup script for GitHub Actions runner. # Invoked automatically via ACTIONS_RUNNER_HOOK_JOB_COMPLETED after each job. -# Removes rebuildable caches that grow unbounded between jobs. +# +# Persist package and Gradle/Cargo caches on the named volumes. Cap each +# runner's gradle-home and bun-home at ~20 GB with LRU eviction — do not +# wholesale wipe. JOB_STARTED (pre-job.sh) is the poison sweep for a killed +# runner process; this hook is the tidy-up when the job finishes normally. # # Intentionally not using set -e: individual cleanup failures should not # prevent other cleanups from running. +# +# KEEP: /root/.bun, /root/.npm, /root/.cache/pnpm, /root/.cache/yarn, +# Gradle modules-2 / build-cache-* / transforms-*, Cargo registry. +# DELETE: /tmp contents except $RUNNER_WORKDIR (and $RUNNER_TEMP if under +# /tmp), Gradle daemon/journals/locks, runner _diag logs. + +CACHE_CAP_BYTES="${CACHE_CAP_BYTES:-21474836480}" # 20 GiB echo "[cleanup] Running post-job cleanup..." BEFORE=$(df -h / | awk 'NR==2 {print $4}') -# Temp files from builds -find /tmp -mindepth 1 -delete 2>/dev/null || true +# True when $1 is $2, a descendant of $2, or an ancestor of $2. +_spared() { + local path="$1" + local keep="$2" + [ -z "$keep" ] && return 1 + case "$path" in + "$keep"|"$keep"/*) return 0 ;; + esac + case "$keep" in + "$path"/*) return 0 ;; + esac + return 1 +} -# Gradle build caches (rebuilt each job from source) -rm -rf /root/.gradle/caches/build-cache-* 2>/dev/null || true -rm -rf /root/.gradle/caches/transforms-* 2>/dev/null || true -rm -rf /root/.gradle/caches/journal-* 2>/dev/null || true +# RUNNER_WORKDIR is /tmp/runner/work on the beelinks. A blanket +# `find /tmp -delete` wipes the work volume. Spare the workdir (and +# RUNNER_TEMP if it sits under /tmp), plus their ancestors under /tmp. +clean_tmp() { + local workdir="${RUNNER_WORKDIR:-/tmp/runner/work}" + local temp="${RUNNER_TEMP:-}" + workdir="${workdir%/}" + temp="${temp%/}" + + if [ -n "$temp" ] && [ "$temp" != "/tmp" ] && [ "${temp#/tmp/}" = "$temp" ]; then + temp="" + fi + if [ "$temp" = "/tmp" ]; then + temp="" + fi -# Gradle daemon logs and state + local path + while IFS= read -r -d '' path; do + _spared "$path" "$workdir" && continue + _spared "$path" "$temp" && continue + rm -rf "$path" 2>/dev/null || true + done < <(find /tmp -mindepth 1 -depth -print0 2>/dev/null) +} + +# LRU-evict oldest regular files until $1 is at or under CACHE_CAP_BYTES. +# Does not wholesale wipe the tree. +lru_cap() { + local dir="$1" + [ -d "$dir" ] || return 0 + local size + size=$(du -sb "$dir" 2>/dev/null | awk '{print $1}') + [ -n "$size" ] || return 0 + if [ "$size" -le "$CACHE_CAP_BYTES" ]; then + return 0 + fi + local excess=$((size - CACHE_CAP_BYTES)) + local freed=0 + echo "[cleanup] $dir is ${size} bytes (cap ${CACHE_CAP_BYTES}); LRU-evicting oldest files" + local rec fsize path + while IFS= read -r -d '' rec; do + [ "$freed" -ge "$excess" ] && break + rec="${rec#* }" + fsize="${rec%% *}" + path="${rec#* }" + [ -n "$path" ] && [ -n "$fsize" ] || continue + rm -f "$path" 2>/dev/null || true + freed=$((freed + fsize)) + done < <(find "$dir" -type f -printf '%T@\t%s\t%p\0' 2>/dev/null | sort -z -n) + find "$dir" -type d -empty -delete 2>/dev/null || true +} + +clean_tmp + +# Gradle daemon / journals / locks only. Keep modules-2, build-cache-*, +# transforms-*. rm -rf /root/.gradle/daemon/ 2>/dev/null || true +rm -rf /root/.gradle/caches/journal-* 2>/dev/null || true +find /root/.gradle -name '*.lock' -delete 2>/dev/null || true # Runner diagnostic logs find /actions-runner/_diag -name '*.log' -delete 2>/dev/null || true find /runner-data/_diag -name '*.log' -delete 2>/dev/null || true +lru_cap /root/.gradle +lru_cap /root/.bun + AFTER=$(df -h / | awk 'NR==2 {print $4}') echo "[cleanup] Post-job cleanup complete. Disk free: ${BEFORE} -> ${AFTER}" diff --git a/docker-compose.yml b/docker-compose.yml index e232c6c..0194ee8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,16 +1,75 @@ +# Beelink X64 layout: three persistent runners, each with its own work and +# cache volumes. Do not share bun/gradle/npm/cargo volumes across services — +# concurrent jobs would corrupt the same tree. YAML merge of volumes: +# replaces the list, so docker.sock is repeated on each service. +x-runner: &runner + image: ghcr.io/dodi-smart/github-runner + restart: unless-stopped + environment: + APP_ID: ${APP_ID} + APP_PRIVATE_KEY: ${APP_PRIVATE_KEY} + APP_LOGIN: ${APP_LOGIN} + RUNNER_NAME_PREFIX: ${RUNNER_NAME_PREFIX:-runner} + RUNNER_WORKDIR: /tmp/runner/work + RUNNER_SCOPE: org + ORG_NAME: ${APP_LOGIN} + LABELS: ${LABELS:-self-hosted,Linux,X64,beelink,docker,large} + RUNNER_GROUP: ${RUNNER_GROUP:-Default} + DISABLE_AUTO_UPDATE: ${DISABLE_AUTO_UPDATE:-false} + EPHEMERAL: "false" + volumes: + - /var/run/docker.sock:/var/run/docker.sock + security_opt: + - label:disable + services: - gha: - build: - context: . - # if you don't want to build and instead just use the pre-built image - # comment out the build block above and uncomment below - # image: ghcr.io/dodi-smart/github-runner - container_name: gha - env_file: - - .env - restart: always - # uncomment below for interactive debugging, this will override the normal entrypoint and prevent - # gh actions runner from starting up, but is good for debugging path issues, etc. -# entrypoint: /bin/bash -# tty: true -# stdin_open: true + runner: + <<: *runner + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - runner-work:/tmp/runner/work + - runner-bun:/root/.bun + - runner-npm:/root/.npm + - runner-pnpm:/root/.cache/pnpm + - runner-gradle:/root/.gradle + - runner-cargo:/root/.cargo + runner-2: + <<: *runner + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - runner-work-2:/tmp/runner/work + - runner-bun-2:/root/.bun + - runner-npm-2:/root/.npm + - runner-pnpm-2:/root/.cache/pnpm + - runner-gradle-2:/root/.gradle + - runner-cargo-2:/root/.cargo + runner-3: + <<: *runner + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - runner-work-3:/tmp/runner/work + - runner-bun-3:/root/.bun + - runner-npm-3:/root/.npm + - runner-pnpm-3:/root/.cache/pnpm + - runner-gradle-3:/root/.gradle + - runner-cargo-3:/root/.cargo + +volumes: + runner-work: {} + runner-work-2: {} + runner-work-3: {} + runner-bun: {} + runner-bun-2: {} + runner-bun-3: {} + runner-npm: {} + runner-npm-2: {} + runner-npm-3: {} + runner-pnpm: {} + runner-pnpm-2: {} + runner-pnpm-3: {} + runner-gradle: {} + runner-gradle-2: {} + runner-gradle-3: {} + runner-cargo: {} + runner-cargo-2: {} + runner-cargo-3: {} diff --git a/pre-job.sh b/pre-job.sh new file mode 100755 index 0000000..b1384de --- /dev/null +++ b/pre-job.sh @@ -0,0 +1,31 @@ +#!/bin/bash +# Pre-job hook for GitHub Actions runner. +# Invoked via ACTIONS_RUNNER_HOOK_JOB_STARTED before each job. +# +# JOB_COMPLETED does not run when the runner process dies (OOM). This sweep +# covers that case: delete incomplete/poison entries only. Do not rm -rf +# cache trees — named volumes are the warm cache. +# +# Intentionally not using set -e: individual prune failures should not +# prevent the job from starting. +# +# NOTE: this is only half the fix. Workflows that restore ~/.bun/install/cache +# via actions/cache with restore-keys can re-import a poisoned entry from the +# GitHub cache, which outlives this container. Those workflows should use an +# exact cache key (no restore-keys) so a corrupt entry cannot be carried +# forward across lockfile changes. + +echo "[pre-job] Surgical prune of incomplete cache entries..." + +# Bun: a killed job leaves partial tarballs (*.tmp / *.part). The next job +# then fails with "Fail extracting tarball for ..." and it looks like a +# dependency problem rather than a runner problem. +find /root/.bun/install/cache \( -name '*.tmp' -o -name '*.part' \) -delete 2>/dev/null || true + +# Gradle: leftover lock / journal / daemon state from a killed compile. +# Keep modules-2, build-cache-*, and transforms-*. +rm -f /root/.gradle/caches/modules-2/modules-2.lock 2>/dev/null || true +rm -rf /root/.gradle/caches/journal-* 2>/dev/null || true +rm -rf /root/.gradle/daemon/ 2>/dev/null || true + +echo "[pre-job] Prune complete."