From 6761be79d5aa7a2aeebbba5411e16b3f0dc62f49 Mon Sep 17 00:00:00 2001 From: Jeff Larson Date: Mon, 27 Jul 2026 17:27:44 -0700 Subject: [PATCH] build(docker): move both image builds' sccache from Redis to R2 (JEF-584) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Dockerfile and agent/Dockerfile baked `ENV SCCACHE_REDIS=redis://sccache-redis.dev...` and build inside the BuildKit pod, which never sees the runner pod's env — so the JEF-564 runner-side cutover left this half of the cache on redis. That split the shared cache and is one of the things blocking redis from being retired. Adopts the pattern murmurify landed first in JEF-589 rather than inventing a second one: - The R2 config + bucket-scoped token arrive as BuildKit build SECRETS, never ENV or a build-arg. An `ENV AWS_SECRET_ACCESS_KEY=…` or an ARG consumed in a layer persists in `docker history` for every image we push to ghcr — that would publish the token. A build secret is mounted for one RUN, lands in no layer, and is not part of the layer cache key. - Both workflows use `secret-envs`, reading straight from the RUNNER POD's env where the `sccache-r2` Secret is injected via envFrom. - scripts/start-sccache-docker.sh (copied from murmurify, with the fallback dir parameterised as SCCACHE_LOCAL_CACHE_DIR) probes, retries, then degrades. agent/Dockerfile overrides it to /app/agent/target/.sccache-local, which is where THAT image's BuildKit target cache mount lives. NO LONGER A HARD GATE — deliberately. Redis was an in-cluster Service reachable with no credentials, so "can't reach it" really did mean "this builder is misconfigured". R2 is a remote bucket behind a rotatable token: a blip or an expired key would now fail a build for a reason unrelated to the code, while degrading costs one cold compile. SCCACHE_DISABLE therefore becomes belt-and-braces rather than load-bearing — scripts/e2e.sh's ubuntu-latest build would now pass without it, but it still skips the sccache path on a builder with no reason to touch the shared cache. agent/Dockerfile also needed `COPY scripts/start-sccache-docker.sh` — unlike the main image (COPY . .) it copies only agent/ + behavior/. Verified: --selftest passes all 5 fixtures in both copies, the SCCACHE_LOCAL_CACHE_DIR override takes effect, `docker buildx build --call=check` reports no warnings for BOTH Dockerfiles, no comment line survives inside a RUN continuation (it would be a shell comment swallowing the joined command, not a Dockerfile comment), and scripts/ is not excluded by .dockerignore. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_013cYVqzH7Xfwea7fAozdQK7 --- .github/workflows/agent.yml | 16 +++ .github/workflows/rust.yml | 16 +++ Dockerfile | 28 +++-- agent/Dockerfile | 22 +++- scripts/start-sccache-docker.sh | 200 ++++++++++++++++++++++++++++++++ 5 files changed, 270 insertions(+), 12 deletions(-) create mode 100755 scripts/start-sccache-docker.sh diff --git a/.github/workflows/agent.yml b/.github/workflows/agent.yml index 31965fba..70249ab0 100644 --- a/.github/workflows/agent.yml +++ b/.github/workflows/agent.yml @@ -114,6 +114,22 @@ jobs: push: ${{ github.event_name != 'pull_request' && startsWith(github.ref, 'refs/tags/v') }} tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} + # sccache's R2 backend for the in-image cargo build (JEF-584, ADR-0020). + # `secret-envs` (key=envname) reads these straight out of the RUNNER POD's env, + # where the `sccache-r2` Secret is injected via envFrom (cluster repo: + # charts/actions/runners/values-protector.yaml) — the repo has no Actions secret + # for R2, and this needs no lift step or on-disk secret file. Secrets, not + # build-args: a build-arg persists in `docker history` on every pushed image. + # Unset (e.g. the Secret is missing) -> the mounts are empty and + # scripts/start-sccache-docker.sh degrades to a local disk cache, not a failure. + secret-envs: | + AWS_ACCESS_KEY_ID=AWS_ACCESS_KEY_ID + AWS_SECRET_ACCESS_KEY=AWS_SECRET_ACCESS_KEY + SCCACHE_BUCKET=SCCACHE_BUCKET + SCCACHE_ENDPOINT=SCCACHE_ENDPOINT + SCCACHE_REGION=SCCACHE_REGION + SCCACHE_S3_KEY_PREFIX=SCCACHE_S3_KEY_PREFIX + SCCACHE_S3_USE_SSL=SCCACHE_S3_USE_SSL cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-cache # Exporting cache pushes to the cache repo, which needs the registry login that # is gated off for PRs — so only export on non-PR events (otherwise PR builds diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index bcce0d66..6eb30b2e 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -236,6 +236,22 @@ jobs: push: ${{ github.event_name != 'pull_request' }} tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} + # sccache's R2 backend for the in-image cargo build (JEF-584, ADR-0020). + # `secret-envs` (key=envname) reads these straight out of the RUNNER POD's env, + # where the `sccache-r2` Secret is injected via envFrom (cluster repo: + # charts/actions/runners/values-protector.yaml) — the repo has no Actions secret + # for R2, and this needs no lift step or on-disk secret file. Secrets, not + # build-args: a build-arg persists in `docker history` on every pushed image. + # Unset (e.g. the Secret is missing) -> the mounts are empty and + # scripts/start-sccache-docker.sh degrades to a local disk cache, not a failure. + secret-envs: | + AWS_ACCESS_KEY_ID=AWS_ACCESS_KEY_ID + AWS_SECRET_ACCESS_KEY=AWS_SECRET_ACCESS_KEY + SCCACHE_BUCKET=SCCACHE_BUCKET + SCCACHE_ENDPOINT=SCCACHE_ENDPOINT + SCCACHE_REGION=SCCACHE_REGION + SCCACHE_S3_KEY_PREFIX=SCCACHE_S3_KEY_PREFIX + SCCACHE_S3_USE_SSL=SCCACHE_S3_USE_SSL cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-cache cache-to: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-cache,mode=max # SLSA build provenance + SBOM attestations, matching the other images. diff --git a/Dockerfile b/Dockerfile index 7a4db4c0..94d73ba5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -52,13 +52,16 @@ RUN set -eux; ver=0.16.0; \ case "$(uname -m)" in x86_64) a=x86_64 ;; aarch64) a=aarch64 ;; *) echo "unsupported arch $(uname -m)" >&2; exit 1 ;; esac; \ wget -qO- "https://github.com/mozilla/sccache/releases/download/v${ver}/sccache-v${ver}-${a}-unknown-linux-musl.tar.gz" \ | tar -xz -C /usr/local/bin --strip-components=1 "sccache-v${ver}-${a}-unknown-linux-musl/sccache" -ENV RUSTC_WRAPPER=sccache CARGO_INCREMENTAL=0 \ - SCCACHE_REDIS=redis://sccache-redis.dev.svc.cluster.local:6379 -# Opt-out for builds that CANNOT reach the in-cluster redis. The github-hosted e2e -# (scripts/e2e.sh, runs-on: ubuntu-latest) does a plain `docker build` of this Dockerfile and can -# never reach sccache-redis.dev, so the hard gate would always trip there. Default empty => sccache -# stays a HARD GATE for the real deploy build on the meshed BuildKit; the e2e passes -# `--build-arg SCCACHE_DISABLE=1` to build plain (uncached) instead of failing. +ENV RUSTC_WRAPPER=sccache CARGO_INCREMENTAL=0 +# sccache backend = the shared Cloudflare R2 bucket (cluster repo: charts/sccache, +# ADR-0020, JEF-584), replacing the in-cluster Redis this used to hardcode. Config + +# bucket-scoped token arrive as BuildKit build SECRETS below — never ENV or a build-arg, +# both of which persist in `docker history` on every image we push to ghcr. +# +# SCCACHE_DISABLE is now belt-and-braces rather than load-bearing: the backend is no +# longer a hard gate (scripts/start-sccache-docker.sh degrades to a local disk cache), +# so the github-hosted e2e would pass without it. scripts/e2e.sh still sets it to skip +# the sccache path entirely on a builder that has no reason to touch the shared cache. ARG SCCACHE_DISABLE="" # Build application @@ -74,12 +77,19 @@ COPY --from=web /web/dist/dashboard.js engine/web/dist/dashboard.js RUN --mount=type=cache,target=/app/target,id=protector-target-v2,sharing=locked \ --mount=type=cache,target=/usr/local/cargo/git/db \ --mount=type=cache,target=/usr/local/cargo/registry \ + --mount=type=secret,id=AWS_ACCESS_KEY_ID \ + --mount=type=secret,id=AWS_SECRET_ACCESS_KEY \ + --mount=type=secret,id=SCCACHE_BUCKET \ + --mount=type=secret,id=SCCACHE_ENDPOINT \ + --mount=type=secret,id=SCCACHE_REGION \ + --mount=type=secret,id=SCCACHE_S3_KEY_PREFIX \ + --mount=type=secret,id=SCCACHE_S3_USE_SSL \ set -e; \ if [ -n "$SCCACHE_DISABLE" ]; then \ - echo "sccache disabled (SCCACHE_DISABLE set) — plain build, no redis"; unset RUSTC_WRAPPER; \ + echo "sccache disabled (SCCACHE_DISABLE set) — plain build, no shared cache"; unset RUSTC_WRAPPER; \ else \ export SCCACHE_SERVER_PORT=$(awk 'BEGIN{srand(); print int(20000+rand()*40000)}'); \ - timeout 10 sccache --start-server; \ + sh scripts/start-sccache-docker.sh; \ fi; \ cargo build --release; \ cp /app/target/release/protector ./protector; \ diff --git a/agent/Dockerfile b/agent/Dockerfile index 469ef77e..991bc457 100644 --- a/agent/Dockerfile +++ b/agent/Dockerfile @@ -32,8 +32,7 @@ RUN set -eux; ver=0.16.0; \ case "$(uname -m)" in x86_64) a=x86_64 ;; aarch64) a=aarch64 ;; *) echo "unsupported arch $(uname -m)" >&2; exit 1 ;; esac; \ wget -qO- "https://github.com/mozilla/sccache/releases/download/v${ver}/sccache-v${ver}-${a}-unknown-linux-musl.tar.gz" \ | tar -xz -C /usr/local/bin --strip-components=1 "sccache-v${ver}-${a}-unknown-linux-musl/sccache" -ENV RUSTC_WRAPPER=sccache CARGO_INCREMENTAL=0 \ - SCCACHE_REDIS=redis://sccache-redis.dev.svc.cluster.local:6379 +ENV RUSTC_WRAPPER=sccache CARGO_INCREMENTAL=0 # Prebuilt bpf-linker (sha256-pinned, per-arch) + nightly/rust-src for the bpf target's # build-std. Prebuilt so the image build stays light (no LLVM source build). # ARCH-AWARE: with native multi-arch builders (JEF-295/#161) this builder stage runs on @@ -59,17 +58,34 @@ RUN rustup toolchain install nightly --component rust-src --profile minimal # The agent workspace (protector-agent + common + the excluded ebpf crate) and the shared # behavior crate it depends on. build.rs compiles agent/protector-agent-ebpf during the # userspace build. +COPY scripts/start-sccache-docker.sh /app/scripts/start-sccache-docker.sh COPY agent/ agent/ COPY behavior/ behavior/ WORKDIR /app/agent # Workspace target is /app/agent/target; the excluded ebpf crate keeps its own. +# sccache backend = the shared Cloudflare R2 bucket (charts/sccache, ADR-0020, JEF-584). +# Config + bucket-scoped token arrive as build SECRETS, never ENV/build-arg (those persist +# in `docker history`). Fail-soft: scripts/start-sccache-docker.sh probes, retries, then +# degrades to a local disk cache rather than failing the build. +# +# SCCACHE_LOCAL_CACHE_DIR override: WORKDIR is /app/agent here and this image's BuildKit +# target cache mount is /app/agent/target, so point the degraded-build local cache inside +# it rather than at the script's /app/target default (no cache mount there in this image). RUN --mount=type=cache,target=/app/agent/target,sharing=locked \ --mount=type=cache,target=/app/agent/protector-agent-ebpf/target,sharing=locked \ --mount=type=cache,target=/usr/local/cargo/git/db \ --mount=type=cache,target=/usr/local/cargo/registry \ + --mount=type=secret,id=AWS_ACCESS_KEY_ID \ + --mount=type=secret,id=AWS_SECRET_ACCESS_KEY \ + --mount=type=secret,id=SCCACHE_BUCKET \ + --mount=type=secret,id=SCCACHE_ENDPOINT \ + --mount=type=secret,id=SCCACHE_REGION \ + --mount=type=secret,id=SCCACHE_S3_KEY_PREFIX \ + --mount=type=secret,id=SCCACHE_S3_USE_SSL \ set -e; \ export SCCACHE_SERVER_PORT=$(awk 'BEGIN{srand(); print int(20000+rand()*40000)}'); \ - timeout 10 sccache --start-server; \ + SCCACHE_LOCAL_CACHE_DIR=/app/agent/target/.sccache-local \ + sh /app/scripts/start-sccache-docker.sh; \ cargo build --release -p protector-agent --features ebpf; \ cp target/release/protector-agent /app/protector-agent-bin; \ sccache --show-stats diff --git a/scripts/start-sccache-docker.sh b/scripts/start-sccache-docker.sh new file mode 100755 index 00000000..b2abcdac --- /dev/null +++ b/scripts/start-sccache-docker.sh @@ -0,0 +1,200 @@ +#!/bin/sh +# Fail-soft sccache backend selection INSIDE the Rust image builds (JEF-584). +# The in-build twin of .github/scripts/start-sccache.sh — same shape, different +# transport for the config: the CI script reads the runner pod's env directly, +# this one reads BuildKit build secrets mounted at /run/secrets by the RUN that +# calls it. +# +# Adopted verbatim (bar the overridable LOCAL_CACHE_DIR below) from the murmurify +# repo's scripts/start-sccache-docker.sh, which landed this pattern first in +# JEF-589 — same reasoning, same fixtures. Keep them in step when either changes. +# +# Usage: +# sh scripts/start-sccache-docker.sh # start a server, never fail +# sh scripts/start-sccache-docker.sh --selftest # run the built-in fixtures +# +# Exit status: ALWAYS 0 for the start path — that is the whole point (see below). +# +# WHY SECRETS AND NOT ENV/BUILD-ARGS. `ENV AWS_SECRET_ACCESS_KEY=…` or an ARG +# consumed in a layer persists in `docker history` for every image we push to +# ghcr — that would publish the R2 token. A build secret is mounted for exactly +# one RUN, lands in no layer, and (unlike a build-arg) is not part of the layer +# cache key, so rotating the R2 config never churns the registry layer cache. +# +# WHY THIS IS FAIL-SOFT, WHERE THE REDIS PREDECESSOR WAS A HARD GATE. That +# backend was an in-cluster Service reachable with no credentials, so "can't +# reach it" really did mean "this builder is misconfigured" and failing the +# build was the honest signal. R2 is a remote bucket behind a rotatable token: a +# transient blip, an expired key or a missing `sccache-r2` Secret would now fail +# a build for a reason that has nothing to do with the code. On a release tag +# that costs a VERSION NUMBER (published tags are immutable — a failed release +# burns v0.X.Y), where degrading costs one cold compile, i.e. exactly what a +# cache miss already costs. So: probe, retry, degrade. +# +# sccache's S3 backend is EAGER, exactly like the redis one it replaced: +# `sccache --start-server` FAILS outright when the bucket is unreachable, and +# the first rustc-through-sccache call then dies with "sccache: Timed out +# waiting for server startup". Measured against sccache 0.16.0: +# +# S3 configured, endpoint unreachable -> --start-server FAILS +# empty SCCACHE_BUCKET + SCCACHE_DIR set -> "Cache location: Local disk" +# +# Only the sccache SERVER holds the backend config, so this script exporting +# nothing back to its caller is fine: the later `cargo build` and +# `sccache --show-stats` just talk to whichever server we left running on +# SCCACHE_SERVER_PORT (exported by the caller before invoking us). +set -u + +# Overridable only so --selftest can point at fixture files; the Dockerfiles +# never set it, and BuildKit always mounts secrets under /run/secrets. +SECRET_DIR="${SCCACHE_SECRET_DIR:-/run/secrets}" + +# Non-empty contents of a mounted build secret, or nothing. `required=false` is +# the Dockerfile default, so an unprovided secret is simply an absent/empty file +# (local `docker build` / docker-compose, which pass no secrets at all). +secret() { + _f="${SECRET_DIR}/$1" + [ -s "$_f" ] || return 0 + tr -d '\r\n' <"$_f" +} + +# Where a degraded build puts its local disk cache. Overridable per-Dockerfile +# (this repo builds two images whose BuildKit target cache mounts differ), so +# point it INSIDE a cache mount wherever one exists and a degraded build still +# warms something for the next degraded build. GITHUB_ENV has no equivalent here +# — we just start the server ourselves with the fallback config. An empty +# SCCACHE_BUCKET reads as "unconfigured"; SCCACHE_DIR alone is not enough, a +# non-empty bucket still wins. +LOCAL_CACHE_DIR="${SCCACHE_LOCAL_CACHE_DIR:-/app/target/.sccache-local}" + +fall_back_to_local_disk() { + sccache --stop-server >/dev/null 2>&1 || true + env -u AWS_ACCESS_KEY_ID -u AWS_SECRET_ACCESS_KEY \ + SCCACHE_BUCKET= SCCACHE_DIR="${LOCAL_CACHE_DIR}" \ + sccache --start-server >/dev/null 2>&1 || true +} + +start_sccache() { + # Make the backend switch deterministic if an earlier client call spawned one. + sccache --stop-server >/dev/null 2>&1 || true + + SCCACHE_BUCKET=$(secret SCCACHE_BUCKET) + if [ -z "${SCCACHE_BUCKET}" ]; then + echo "sccache: no R2 config in the build secrets — using a local disk cache" >&2 + fall_back_to_local_disk + return 0 + fi + + AWS_ACCESS_KEY_ID=$(secret AWS_ACCESS_KEY_ID) + AWS_SECRET_ACCESS_KEY=$(secret AWS_SECRET_ACCESS_KEY) + SCCACHE_ENDPOINT=$(secret SCCACHE_ENDPOINT) + SCCACHE_REGION=$(secret SCCACHE_REGION) + SCCACHE_S3_KEY_PREFIX=$(secret SCCACHE_S3_KEY_PREFIX) + SCCACHE_S3_USE_SSL=$(secret SCCACHE_S3_USE_SSL) + export SCCACHE_BUCKET AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY \ + SCCACHE_ENDPOINT SCCACHE_REGION SCCACHE_S3_KEY_PREFIX SCCACHE_S3_USE_SSL + + for attempt in 1 2 3; do + # The 10s guard bounds a genuinely-unreachable bucket (a normal start is + # ~0.4s); the retries ride out a transient R2 blip. + if timeout 10 sccache --start-server >/dev/null 2>&1; then + echo "sccache: R2 backend up (bucket=${SCCACHE_BUCKET}, attempt ${attempt})" >&2 + return 0 + fi + echo "sccache: R2 backend not ready (attempt ${attempt}/3); retrying in 3s" >&2 + sccache --stop-server >/dev/null 2>&1 || true + sleep 3 + done + + echo "sccache: R2 backend unreachable — falling back to local disk cache" >&2 + fall_back_to_local_disk + return 0 +} + +# ── Self-test ──────────────────────────────────────────────────────────────── +# Drives the three states a build can be in against stub `sccache`/`timeout` +# binaries, so a regression in the fail-soft logic is caught by CI in seconds +# instead of by a burned release tag. The stub records the backend env of every +# `--start-server` it is asked to run. +selftest() { + _tmp="$(mktemp -d)" + trap 'rm -rf "${_tmp}"' EXIT + mkdir -p "${_tmp}/bin" "${_tmp}/secrets" + + cat >"${_tmp}/bin/sccache" <<'STUB' +#!/bin/sh +if [ "${1:-}" = "--start-server" ]; then + echo "start bucket=[${SCCACHE_BUCKET:-}] dir=[${SCCACHE_DIR:-}] key=[${AWS_ACCESS_KEY_ID:-}]" \ + >>"${STUB_LOG}" + [ "${STUB_START_FAILS:-0}" = "1" ] && exit 1 +fi +exit 0 +STUB + # `timeout N cmd …` — drop the duration, run the command. Shadows the real + # coreutils binary so the fixtures behave the same on Linux and macOS. + cat >"${_tmp}/bin/timeout" <<'STUB' +#!/bin/sh +shift +exec "$@" +STUB + chmod +x "${_tmp}/bin/sccache" "${_tmp}/bin/timeout" + PATH="${_tmp}/bin:${PATH}" + export PATH STUB_LOG + SECRET_DIR="${_tmp}/secrets" + + _fails=0 + expect() { # expect