build(docker): move both image builds' sccache from Redis to R2 (JEF-584)

.github/workflows/agent.yml

@@ -114,6 +114,22 @@ jobs:
           push: ${{ github.event_name != 'pull_request' && startsWith(github.ref, 'refs/tags/v') }}
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
+          # sccache's R2 backend for the in-image cargo build (JEF-584, ADR-0020).
+          # `secret-envs` (key=envname) reads these straight out of the RUNNER POD's env,
+          # where the `sccache-r2` Secret is injected via envFrom (cluster repo:
+          # charts/actions/runners/values-protector.yaml) — the repo has no Actions secret
+          # for R2, and this needs no lift step or on-disk secret file. Secrets, not
+          # build-args: a build-arg persists in `docker history` on every pushed image.
+          # Unset (e.g. the Secret is missing) -> the mounts are empty and
+          # scripts/start-sccache-docker.sh degrades to a local disk cache, not a failure.
+          secret-envs: |
+            AWS_ACCESS_KEY_ID=AWS_ACCESS_KEY_ID
+            AWS_SECRET_ACCESS_KEY=AWS_SECRET_ACCESS_KEY
+            SCCACHE_BUCKET=SCCACHE_BUCKET
+            SCCACHE_ENDPOINT=SCCACHE_ENDPOINT
+            SCCACHE_REGION=SCCACHE_REGION
+            SCCACHE_S3_KEY_PREFIX=SCCACHE_S3_KEY_PREFIX
+            SCCACHE_S3_USE_SSL=SCCACHE_S3_USE_SSL
           cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-cache
           # Exporting cache pushes to the cache repo, which needs the registry login that
           # is gated off for PRs — so only export on non-PR events (otherwise PR builds

.github/workflows/rust.yml

@@ -236,6 +236,22 @@ jobs:
           push: ${{ github.event_name != 'pull_request' }}
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
+          # sccache's R2 backend for the in-image cargo build (JEF-584, ADR-0020).
+          # `secret-envs` (key=envname) reads these straight out of the RUNNER POD's env,
+          # where the `sccache-r2` Secret is injected via envFrom (cluster repo:
+          # charts/actions/runners/values-protector.yaml) — the repo has no Actions secret
+          # for R2, and this needs no lift step or on-disk secret file. Secrets, not
+          # build-args: a build-arg persists in `docker history` on every pushed image.
+          # Unset (e.g. the Secret is missing) -> the mounts are empty and
+          # scripts/start-sccache-docker.sh degrades to a local disk cache, not a failure.
+          secret-envs: |
+            AWS_ACCESS_KEY_ID=AWS_ACCESS_KEY_ID
+            AWS_SECRET_ACCESS_KEY=AWS_SECRET_ACCESS_KEY
+            SCCACHE_BUCKET=SCCACHE_BUCKET
+            SCCACHE_ENDPOINT=SCCACHE_ENDPOINT
+            SCCACHE_REGION=SCCACHE_REGION
+            SCCACHE_S3_KEY_PREFIX=SCCACHE_S3_KEY_PREFIX
+            SCCACHE_S3_USE_SSL=SCCACHE_S3_USE_SSL
           cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-cache
           cache-to: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-cache,mode=max
           # SLSA build provenance + SBOM attestations, matching the other images.

Dockerfile

@@ -52,13 +52,16 @@ RUN set -eux; ver=0.16.0; \
     case "$(uname -m)" in x86_64) a=x86_64 ;; aarch64) a=aarch64 ;; *) echo "unsupported arch $(uname -m)" >&2; exit 1 ;; esac; \
     wget -qO- "https://github.com/mozilla/sccache/releases/download/v${ver}/sccache-v${ver}-${a}-unknown-linux-musl.tar.gz" \
       | tar -xz -C /usr/local/bin --strip-components=1 "sccache-v${ver}-${a}-unknown-linux-musl/sccache"
-ENV RUSTC_WRAPPER=sccache CARGO_INCREMENTAL=0 \
-    SCCACHE_REDIS=redis://sccache-redis.dev.svc.cluster.local:6379
-# Opt-out for builds that CANNOT reach the in-cluster redis. The github-hosted e2e
-# (scripts/e2e.sh, runs-on: ubuntu-latest) does a plain `docker build` of this Dockerfile and can
-# never reach sccache-redis.dev, so the hard gate would always trip there. Default empty => sccache
-# stays a HARD GATE for the real deploy build on the meshed BuildKit; the e2e passes
-# `--build-arg SCCACHE_DISABLE=1` to build plain (uncached) instead of failing.
+ENV RUSTC_WRAPPER=sccache CARGO_INCREMENTAL=0
+# sccache backend = the shared Cloudflare R2 bucket (cluster repo: charts/sccache,
+# ADR-0020, JEF-584), replacing the in-cluster Redis this used to hardcode. Config +
+# bucket-scoped token arrive as BuildKit build SECRETS below — never ENV or a build-arg,
+# both of which persist in `docker history` on every image we push to ghcr.
+#
+# SCCACHE_DISABLE is now belt-and-braces rather than load-bearing: the backend is no
+# longer a hard gate (scripts/start-sccache-docker.sh degrades to a local disk cache),
+# so the github-hosted e2e would pass without it. scripts/e2e.sh still sets it to skip
+# the sccache path entirely on a builder that has no reason to touch the shared cache.
 ARG SCCACHE_DISABLE=""
 
 # Build application
@@ -74,12 +77,19 @@ COPY --from=web /web/dist/dashboard.js engine/web/dist/dashboard.js
 RUN --mount=type=cache,target=/app/target,id=protector-target-v2,sharing=locked \
     --mount=type=cache,target=/usr/local/cargo/git/db \
     --mount=type=cache,target=/usr/local/cargo/registry \
+    --mount=type=secret,id=AWS_ACCESS_KEY_ID \
+    --mount=type=secret,id=AWS_SECRET_ACCESS_KEY \
+    --mount=type=secret,id=SCCACHE_BUCKET \
+    --mount=type=secret,id=SCCACHE_ENDPOINT \
+    --mount=type=secret,id=SCCACHE_REGION \
+    --mount=type=secret,id=SCCACHE_S3_KEY_PREFIX \
+    --mount=type=secret,id=SCCACHE_S3_USE_SSL \
     set -e; \
     if [ -n "$SCCACHE_DISABLE" ]; then \
-      echo "sccache disabled (SCCACHE_DISABLE set) — plain build, no redis"; unset RUSTC_WRAPPER; \
+      echo "sccache disabled (SCCACHE_DISABLE set) — plain build, no shared cache"; unset RUSTC_WRAPPER; \
     else \
       export SCCACHE_SERVER_PORT=$(awk 'BEGIN{srand(); print int(20000+rand()*40000)}'); \
-      timeout 10 sccache --start-server; \
+      sh scripts/start-sccache-docker.sh; \
     fi; \
     cargo build --release; \
     cp /app/target/release/protector ./protector; \

agent/Dockerfile

@@ -32,8 +32,7 @@ RUN set -eux; ver=0.16.0; \
     case "$(uname -m)" in x86_64) a=x86_64 ;; aarch64) a=aarch64 ;; *) echo "unsupported arch $(uname -m)" >&2; exit 1 ;; esac; \
     wget -qO- "https://github.com/mozilla/sccache/releases/download/v${ver}/sccache-v${ver}-${a}-unknown-linux-musl.tar.gz" \
       | tar -xz -C /usr/local/bin --strip-components=1 "sccache-v${ver}-${a}-unknown-linux-musl/sccache"
-ENV RUSTC_WRAPPER=sccache CARGO_INCREMENTAL=0 \
-    SCCACHE_REDIS=redis://sccache-redis.dev.svc.cluster.local:6379
+ENV RUSTC_WRAPPER=sccache CARGO_INCREMENTAL=0
 # Prebuilt bpf-linker (sha256-pinned, per-arch) + nightly/rust-src for the bpf target's
 # build-std. Prebuilt so the image build stays light (no LLVM source build).
 # ARCH-AWARE: with native multi-arch builders (JEF-295/#161) this builder stage runs on
@@ -59,17 +58,34 @@ RUN rustup toolchain install nightly --component rust-src --profile minimal
 # The agent workspace (protector-agent + common + the excluded ebpf crate) and the shared
 # behavior crate it depends on. build.rs compiles agent/protector-agent-ebpf during the
 # userspace build.
+COPY scripts/start-sccache-docker.sh /app/scripts/start-sccache-docker.sh
 COPY agent/ agent/
 COPY behavior/ behavior/
 WORKDIR /app/agent
 # Workspace target is /app/agent/target; the excluded ebpf crate keeps its own.
+# sccache backend = the shared Cloudflare R2 bucket (charts/sccache, ADR-0020, JEF-584).
+# Config + bucket-scoped token arrive as build SECRETS, never ENV/build-arg (those persist
+# in `docker history`). Fail-soft: scripts/start-sccache-docker.sh probes, retries, then
+# degrades to a local disk cache rather than failing the build.
+#
+# SCCACHE_LOCAL_CACHE_DIR override: WORKDIR is /app/agent here and this image's BuildKit
+# target cache mount is /app/agent/target, so point the degraded-build local cache inside
+# it rather than at the script's /app/target default (no cache mount there in this image).
 RUN --mount=type=cache,target=/app/agent/target,sharing=locked \
     --mount=type=cache,target=/app/agent/protector-agent-ebpf/target,sharing=locked \
     --mount=type=cache,target=/usr/local/cargo/git/db \
     --mount=type=cache,target=/usr/local/cargo/registry \
+    --mount=type=secret,id=AWS_ACCESS_KEY_ID \
+    --mount=type=secret,id=AWS_SECRET_ACCESS_KEY \
+    --mount=type=secret,id=SCCACHE_BUCKET \
+    --mount=type=secret,id=SCCACHE_ENDPOINT \
+    --mount=type=secret,id=SCCACHE_REGION \
+    --mount=type=secret,id=SCCACHE_S3_KEY_PREFIX \
+    --mount=type=secret,id=SCCACHE_S3_USE_SSL \
     set -e; \
     export SCCACHE_SERVER_PORT=$(awk 'BEGIN{srand(); print int(20000+rand()*40000)}'); \
-    timeout 10 sccache --start-server; \
+    SCCACHE_LOCAL_CACHE_DIR=/app/agent/target/.sccache-local \
+      sh /app/scripts/start-sccache-docker.sh; \
     cargo build --release -p protector-agent --features ebpf; \
     cp target/release/protector-agent /app/protector-agent-bin; \
     sccache --show-stats

scripts/start-sccache-docker.sh

@@ -0,0 +1,200 @@
+#!/bin/sh
+# Fail-soft sccache backend selection INSIDE the Rust image builds (JEF-584).
+# The in-build twin of .github/scripts/start-sccache.sh — same shape, different
+# transport for the config: the CI script reads the runner pod's env directly,
+# this one reads BuildKit build secrets mounted at /run/secrets by the RUN that
+# calls it.
+#
+# Adopted verbatim (bar the overridable LOCAL_CACHE_DIR below) from the murmurify
+# repo's scripts/start-sccache-docker.sh, which landed this pattern first in
+# JEF-589 — same reasoning, same fixtures. Keep them in step when either changes.
+#
+# Usage:
+#   sh scripts/start-sccache-docker.sh              # start a server, never fail
+#   sh scripts/start-sccache-docker.sh --selftest   # run the built-in fixtures
+#
+# Exit status: ALWAYS 0 for the start path — that is the whole point (see below).
+#
+# WHY SECRETS AND NOT ENV/BUILD-ARGS. `ENV AWS_SECRET_ACCESS_KEY=…` or an ARG
+# consumed in a layer persists in `docker history` for every image we push to
+# ghcr — that would publish the R2 token. A build secret is mounted for exactly
+# one RUN, lands in no layer, and (unlike a build-arg) is not part of the layer
+# cache key, so rotating the R2 config never churns the registry layer cache.
+#
+# WHY THIS IS FAIL-SOFT, WHERE THE REDIS PREDECESSOR WAS A HARD GATE. That
+# backend was an in-cluster Service reachable with no credentials, so "can't
+# reach it" really did mean "this builder is misconfigured" and failing the
+# build was the honest signal. R2 is a remote bucket behind a rotatable token: a
+# transient blip, an expired key or a missing `sccache-r2` Secret would now fail
+# a build for a reason that has nothing to do with the code. On a release tag
+# that costs a VERSION NUMBER (published tags are immutable — a failed release
+# burns v0.X.Y), where degrading costs one cold compile, i.e. exactly what a
+# cache miss already costs. So: probe, retry, degrade.
+#
+# sccache's S3 backend is EAGER, exactly like the redis one it replaced:
+# `sccache --start-server` FAILS outright when the bucket is unreachable, and
+# the first rustc-through-sccache call then dies with "sccache: Timed out
+# waiting for server startup". Measured against sccache 0.16.0:
+#
+#   S3 configured, endpoint unreachable    -> --start-server FAILS
+#   empty SCCACHE_BUCKET + SCCACHE_DIR set -> "Cache location: Local disk"
+#
+# Only the sccache SERVER holds the backend config, so this script exporting
+# nothing back to its caller is fine: the later `cargo build` and
+# `sccache --show-stats` just talk to whichever server we left running on
+# SCCACHE_SERVER_PORT (exported by the caller before invoking us).
+set -u
+
+# Overridable only so --selftest can point at fixture files; the Dockerfiles
+# never set it, and BuildKit always mounts secrets under /run/secrets.
+SECRET_DIR="${SCCACHE_SECRET_DIR:-/run/secrets}"
+
+# Non-empty contents of a mounted build secret, or nothing. `required=false` is
+# the Dockerfile default, so an unprovided secret is simply an absent/empty file
+# (local `docker build` / docker-compose, which pass no secrets at all).
+secret() {
+    _f="${SECRET_DIR}/$1"
+    [ -s "$_f" ] || return 0
+    tr -d '\r\n' <"$_f"
+}
+
+# Where a degraded build puts its local disk cache. Overridable per-Dockerfile
+# (this repo builds two images whose BuildKit target cache mounts differ), so
+# point it INSIDE a cache mount wherever one exists and a degraded build still
+# warms something for the next degraded build. GITHUB_ENV has no equivalent here
+# — we just start the server ourselves with the fallback config. An empty
+# SCCACHE_BUCKET reads as "unconfigured"; SCCACHE_DIR alone is not enough, a
+# non-empty bucket still wins.
+LOCAL_CACHE_DIR="${SCCACHE_LOCAL_CACHE_DIR:-/app/target/.sccache-local}"
+
+fall_back_to_local_disk() {
+    sccache --stop-server >/dev/null 2>&1 || true
+    env -u AWS_ACCESS_KEY_ID -u AWS_SECRET_ACCESS_KEY \
+        SCCACHE_BUCKET= SCCACHE_DIR="${LOCAL_CACHE_DIR}" \
+        sccache --start-server >/dev/null 2>&1 || true
+}
+
+start_sccache() {
+    # Make the backend switch deterministic if an earlier client call spawned one.
+    sccache --stop-server >/dev/null 2>&1 || true
+
+    SCCACHE_BUCKET=$(secret SCCACHE_BUCKET)
+    if [ -z "${SCCACHE_BUCKET}" ]; then
+        echo "sccache: no R2 config in the build secrets — using a local disk cache" >&2
+        fall_back_to_local_disk
+        return 0
+    fi
+
+    AWS_ACCESS_KEY_ID=$(secret AWS_ACCESS_KEY_ID)
+    AWS_SECRET_ACCESS_KEY=$(secret AWS_SECRET_ACCESS_KEY)
+    SCCACHE_ENDPOINT=$(secret SCCACHE_ENDPOINT)
+    SCCACHE_REGION=$(secret SCCACHE_REGION)
+    SCCACHE_S3_KEY_PREFIX=$(secret SCCACHE_S3_KEY_PREFIX)
+    SCCACHE_S3_USE_SSL=$(secret SCCACHE_S3_USE_SSL)
+    export SCCACHE_BUCKET AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY \
+        SCCACHE_ENDPOINT SCCACHE_REGION SCCACHE_S3_KEY_PREFIX SCCACHE_S3_USE_SSL
+
+    for attempt in 1 2 3; do
+        # The 10s guard bounds a genuinely-unreachable bucket (a normal start is
+        # ~0.4s); the retries ride out a transient R2 blip.
+        if timeout 10 sccache --start-server >/dev/null 2>&1; then
+            echo "sccache: R2 backend up (bucket=${SCCACHE_BUCKET}, attempt ${attempt})" >&2
+            return 0
+        fi
+        echo "sccache: R2 backend not ready (attempt ${attempt}/3); retrying in 3s" >&2
+        sccache --stop-server >/dev/null 2>&1 || true
+        sleep 3
+    done
+
+    echo "sccache: R2 backend unreachable — falling back to local disk cache" >&2
+    fall_back_to_local_disk
+    return 0
+}
+
+# ── Self-test ────────────────────────────────────────────────────────────────
+# Drives the three states a build can be in against stub `sccache`/`timeout`
+# binaries, so a regression in the fail-soft logic is caught by CI in seconds
+# instead of by a burned release tag. The stub records the backend env of every
+# `--start-server` it is asked to run.
+selftest() {
+    _tmp="$(mktemp -d)"
+    trap 'rm -rf "${_tmp}"' EXIT
+    mkdir -p "${_tmp}/bin" "${_tmp}/secrets"
+
+    cat >"${_tmp}/bin/sccache" <<'STUB'
+#!/bin/sh
+if [ "${1:-}" = "--start-server" ]; then
+  echo "start bucket=[${SCCACHE_BUCKET:-}] dir=[${SCCACHE_DIR:-}] key=[${AWS_ACCESS_KEY_ID:-}]" \
+    >>"${STUB_LOG}"
+  [ "${STUB_START_FAILS:-0}" = "1" ] && exit 1
+fi
+exit 0
+STUB
+    # `timeout N cmd …` — drop the duration, run the command. Shadows the real
+    # coreutils binary so the fixtures behave the same on Linux and macOS.
+    cat >"${_tmp}/bin/timeout" <<'STUB'
+#!/bin/sh
+shift
+exec "$@"
+STUB
+    chmod +x "${_tmp}/bin/sccache" "${_tmp}/bin/timeout"
+    PATH="${_tmp}/bin:${PATH}"
+    export PATH STUB_LOG
+    SECRET_DIR="${_tmp}/secrets"
+
+    _fails=0
+    expect() { # expect <label> <grep-pattern>
+        if grep -q "$2" "${STUB_LOG}"; then
+            echo "  ok   $1"
+        else
+            echo "  FAIL $1 — no line matching /$2/ in:" >&2
+            sed 's/^/       /' "${STUB_LOG}" >&2
+            _fails=$((_fails + 1))
+        fi
+    }
+
+    # 1. No secrets at all (plain `docker build`, docker-compose): local disk,
+    #    and NOT a hard failure.
+    STUB_LOG="${_tmp}/log1"; : >"${STUB_LOG}"
+    STUB_START_FAILS=0 start_sccache >/dev/null 2>&1 || _fails=$((_fails + 1))
+    expect "no config -> local disk" "bucket=\[\] dir=\[${LOCAL_CACHE_DIR}\]"
+
+    # 2. Config present and the bucket answers: S3 backend, credential passed
+    #    through, no fallback.
+    printf 'a-bucket' >"${_tmp}/secrets/SCCACHE_BUCKET"
+    printf 'a-key-id' >"${_tmp}/secrets/AWS_ACCESS_KEY_ID"
+    STUB_LOG="${_tmp}/log2"; : >"${STUB_LOG}"
+    STUB_START_FAILS=0 start_sccache >/dev/null 2>&1 || _fails=$((_fails + 1))
+    expect "R2 reachable -> S3 backend" 'bucket=\[a-bucket\] dir=\[\] key=\[a-key-id\]'
+    if [ "$(grep -c 'start ' "${STUB_LOG}")" != "1" ]; then
+        echo "  FAIL R2 reachable -> exactly one start (no fallback)" >&2
+        _fails=$((_fails + 1))
+    else
+        echo "  ok   R2 reachable -> exactly one start (no fallback)"
+    fi
+
+    # 3. Config present, bucket unreachable: retried, then degraded to local
+    #    disk WITHOUT the credential — and still exit 0. This is the case that
+    #    used to fail the build (and a release tag with it).
+    STUB_LOG="${_tmp}/log3"; : >"${STUB_LOG}"
+    STUB_START_FAILS=1 start_sccache >/dev/null 2>&1 || _fails=$((_fails + 1))
+    expect "R2 down -> local disk fallback" "bucket=\[\] dir=\[${LOCAL_CACHE_DIR}\] key=\[\]"
+    if [ "$(grep -c 'bucket=\[a-bucket\]' "${STUB_LOG}")" != "3" ]; then
+        echo "  FAIL R2 down -> 3 attempts before degrading" >&2
+        _fails=$((_fails + 1))
+    else
+        echo "  ok   R2 down -> 3 attempts before degrading"
+    fi
+
+    if [ "${_fails}" -ne 0 ]; then
+        echo "selftest: ${_fails} failure(s)" >&2
+        return 1
+    fi
+    echo "selftest: all fixtures passed"
+}
+
+if [ "${1:-}" = "--selftest" ]; then
+    selftest
+else
+    start_sccache
+fi