diff --git a/.github/workflows/agent.yml b/.github/workflows/agent.yml index 31965fba..70249ab0 100644 --- a/.github/workflows/agent.yml +++ b/.github/workflows/agent.yml @@ -114,6 +114,22 @@ jobs: push: ${{ github.event_name != 'pull_request' && startsWith(github.ref, 'refs/tags/v') }} tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} + # sccache's R2 backend for the in-image cargo build (JEF-584, ADR-0020). + # `secret-envs` (key=envname) reads these straight out of the RUNNER POD's env, + # where the `sccache-r2` Secret is injected via envFrom (cluster repo: + # charts/actions/runners/values-protector.yaml) — the repo has no Actions secret + # for R2, and this needs no lift step or on-disk secret file. Secrets, not + # build-args: a build-arg persists in `docker history` on every pushed image. + # Unset (e.g. the Secret is missing) -> the mounts are empty and + # scripts/start-sccache-docker.sh degrades to a local disk cache, not a failure. + secret-envs: | + AWS_ACCESS_KEY_ID=AWS_ACCESS_KEY_ID + AWS_SECRET_ACCESS_KEY=AWS_SECRET_ACCESS_KEY + SCCACHE_BUCKET=SCCACHE_BUCKET + SCCACHE_ENDPOINT=SCCACHE_ENDPOINT + SCCACHE_REGION=SCCACHE_REGION + SCCACHE_S3_KEY_PREFIX=SCCACHE_S3_KEY_PREFIX + SCCACHE_S3_USE_SSL=SCCACHE_S3_USE_SSL cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-cache # Exporting cache pushes to the cache repo, which needs the registry login that # is gated off for PRs — so only export on non-PR events (otherwise PR builds diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index bcce0d66..6eb30b2e 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -236,6 +236,22 @@ jobs: push: ${{ github.event_name != 'pull_request' }} tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} + # sccache's R2 backend for the in-image cargo build (JEF-584, ADR-0020). + # `secret-envs` (key=envname) reads these straight out of the RUNNER POD's env, + # where the `sccache-r2` Secret is injected via envFrom (cluster repo: + # charts/actions/runners/values-protector.yaml) — the repo has no Actions secret + # for R2, and this needs no lift step or on-disk secret file. Secrets, not + # build-args: a build-arg persists in `docker history` on every pushed image. + # Unset (e.g. the Secret is missing) -> the mounts are empty and + # scripts/start-sccache-docker.sh degrades to a local disk cache, not a failure. + secret-envs: | + AWS_ACCESS_KEY_ID=AWS_ACCESS_KEY_ID + AWS_SECRET_ACCESS_KEY=AWS_SECRET_ACCESS_KEY + SCCACHE_BUCKET=SCCACHE_BUCKET + SCCACHE_ENDPOINT=SCCACHE_ENDPOINT + SCCACHE_REGION=SCCACHE_REGION + SCCACHE_S3_KEY_PREFIX=SCCACHE_S3_KEY_PREFIX + SCCACHE_S3_USE_SSL=SCCACHE_S3_USE_SSL cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-cache cache-to: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-cache,mode=max # SLSA build provenance + SBOM attestations, matching the other images. diff --git a/Dockerfile b/Dockerfile index 7a4db4c0..94d73ba5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -52,13 +52,16 @@ RUN set -eux; ver=0.16.0; \ case "$(uname -m)" in x86_64) a=x86_64 ;; aarch64) a=aarch64 ;; *) echo "unsupported arch $(uname -m)" >&2; exit 1 ;; esac; \ wget -qO- "https://github.com/mozilla/sccache/releases/download/v${ver}/sccache-v${ver}-${a}-unknown-linux-musl.tar.gz" \ | tar -xz -C /usr/local/bin --strip-components=1 "sccache-v${ver}-${a}-unknown-linux-musl/sccache" -ENV RUSTC_WRAPPER=sccache CARGO_INCREMENTAL=0 \ - SCCACHE_REDIS=redis://sccache-redis.dev.svc.cluster.local:6379 -# Opt-out for builds that CANNOT reach the in-cluster redis. The github-hosted e2e -# (scripts/e2e.sh, runs-on: ubuntu-latest) does a plain `docker build` of this Dockerfile and can -# never reach sccache-redis.dev, so the hard gate would always trip there. Default empty => sccache -# stays a HARD GATE for the real deploy build on the meshed BuildKit; the e2e passes -# `--build-arg SCCACHE_DISABLE=1` to build plain (uncached) instead of failing. +ENV RUSTC_WRAPPER=sccache CARGO_INCREMENTAL=0 +# sccache backend = the shared Cloudflare R2 bucket (cluster repo: charts/sccache, +# ADR-0020, JEF-584), replacing the in-cluster Redis this used to hardcode. Config + +# bucket-scoped token arrive as BuildKit build SECRETS below — never ENV or a build-arg, +# both of which persist in `docker history` on every image we push to ghcr. +# +# SCCACHE_DISABLE is now belt-and-braces rather than load-bearing: the backend is no +# longer a hard gate (scripts/start-sccache-docker.sh degrades to a local disk cache), +# so the github-hosted e2e would pass without it. scripts/e2e.sh still sets it to skip +# the sccache path entirely on a builder that has no reason to touch the shared cache. ARG SCCACHE_DISABLE="" # Build application @@ -74,12 +77,19 @@ COPY --from=web /web/dist/dashboard.js engine/web/dist/dashboard.js RUN --mount=type=cache,target=/app/target,id=protector-target-v2,sharing=locked \ --mount=type=cache,target=/usr/local/cargo/git/db \ --mount=type=cache,target=/usr/local/cargo/registry \ + --mount=type=secret,id=AWS_ACCESS_KEY_ID \ + --mount=type=secret,id=AWS_SECRET_ACCESS_KEY \ + --mount=type=secret,id=SCCACHE_BUCKET \ + --mount=type=secret,id=SCCACHE_ENDPOINT \ + --mount=type=secret,id=SCCACHE_REGION \ + --mount=type=secret,id=SCCACHE_S3_KEY_PREFIX \ + --mount=type=secret,id=SCCACHE_S3_USE_SSL \ set -e; \ if [ -n "$SCCACHE_DISABLE" ]; then \ - echo "sccache disabled (SCCACHE_DISABLE set) — plain build, no redis"; unset RUSTC_WRAPPER; \ + echo "sccache disabled (SCCACHE_DISABLE set) — plain build, no shared cache"; unset RUSTC_WRAPPER; \ else \ export SCCACHE_SERVER_PORT=$(awk 'BEGIN{srand(); print int(20000+rand()*40000)}'); \ - timeout 10 sccache --start-server; \ + sh scripts/start-sccache-docker.sh; \ fi; \ cargo build --release; \ cp /app/target/release/protector ./protector; \ diff --git a/agent/Dockerfile b/agent/Dockerfile index 469ef77e..991bc457 100644 --- a/agent/Dockerfile +++ b/agent/Dockerfile @@ -32,8 +32,7 @@ RUN set -eux; ver=0.16.0; \ case "$(uname -m)" in x86_64) a=x86_64 ;; aarch64) a=aarch64 ;; *) echo "unsupported arch $(uname -m)" >&2; exit 1 ;; esac; \ wget -qO- "https://github.com/mozilla/sccache/releases/download/v${ver}/sccache-v${ver}-${a}-unknown-linux-musl.tar.gz" \ | tar -xz -C /usr/local/bin --strip-components=1 "sccache-v${ver}-${a}-unknown-linux-musl/sccache" -ENV RUSTC_WRAPPER=sccache CARGO_INCREMENTAL=0 \ - SCCACHE_REDIS=redis://sccache-redis.dev.svc.cluster.local:6379 +ENV RUSTC_WRAPPER=sccache CARGO_INCREMENTAL=0 # Prebuilt bpf-linker (sha256-pinned, per-arch) + nightly/rust-src for the bpf target's # build-std. Prebuilt so the image build stays light (no LLVM source build). # ARCH-AWARE: with native multi-arch builders (JEF-295/#161) this builder stage runs on @@ -59,17 +58,34 @@ RUN rustup toolchain install nightly --component rust-src --profile minimal # The agent workspace (protector-agent + common + the excluded ebpf crate) and the shared # behavior crate it depends on. build.rs compiles agent/protector-agent-ebpf during the # userspace build. +COPY scripts/start-sccache-docker.sh /app/scripts/start-sccache-docker.sh COPY agent/ agent/ COPY behavior/ behavior/ WORKDIR /app/agent # Workspace target is /app/agent/target; the excluded ebpf crate keeps its own. +# sccache backend = the shared Cloudflare R2 bucket (charts/sccache, ADR-0020, JEF-584). +# Config + bucket-scoped token arrive as build SECRETS, never ENV/build-arg (those persist +# in `docker history`). Fail-soft: scripts/start-sccache-docker.sh probes, retries, then +# degrades to a local disk cache rather than failing the build. +# +# SCCACHE_LOCAL_CACHE_DIR override: WORKDIR is /app/agent here and this image's BuildKit +# target cache mount is /app/agent/target, so point the degraded-build local cache inside +# it rather than at the script's /app/target default (no cache mount there in this image). RUN --mount=type=cache,target=/app/agent/target,sharing=locked \ --mount=type=cache,target=/app/agent/protector-agent-ebpf/target,sharing=locked \ --mount=type=cache,target=/usr/local/cargo/git/db \ --mount=type=cache,target=/usr/local/cargo/registry \ + --mount=type=secret,id=AWS_ACCESS_KEY_ID \ + --mount=type=secret,id=AWS_SECRET_ACCESS_KEY \ + --mount=type=secret,id=SCCACHE_BUCKET \ + --mount=type=secret,id=SCCACHE_ENDPOINT \ + --mount=type=secret,id=SCCACHE_REGION \ + --mount=type=secret,id=SCCACHE_S3_KEY_PREFIX \ + --mount=type=secret,id=SCCACHE_S3_USE_SSL \ set -e; \ export SCCACHE_SERVER_PORT=$(awk 'BEGIN{srand(); print int(20000+rand()*40000)}'); \ - timeout 10 sccache --start-server; \ + SCCACHE_LOCAL_CACHE_DIR=/app/agent/target/.sccache-local \ + sh /app/scripts/start-sccache-docker.sh; \ cargo build --release -p protector-agent --features ebpf; \ cp target/release/protector-agent /app/protector-agent-bin; \ sccache --show-stats diff --git a/scripts/start-sccache-docker.sh b/scripts/start-sccache-docker.sh new file mode 100755 index 00000000..b2abcdac --- /dev/null +++ b/scripts/start-sccache-docker.sh @@ -0,0 +1,200 @@ +#!/bin/sh +# Fail-soft sccache backend selection INSIDE the Rust image builds (JEF-584). +# The in-build twin of .github/scripts/start-sccache.sh — same shape, different +# transport for the config: the CI script reads the runner pod's env directly, +# this one reads BuildKit build secrets mounted at /run/secrets by the RUN that +# calls it. +# +# Adopted verbatim (bar the overridable LOCAL_CACHE_DIR below) from the murmurify +# repo's scripts/start-sccache-docker.sh, which landed this pattern first in +# JEF-589 — same reasoning, same fixtures. Keep them in step when either changes. +# +# Usage: +# sh scripts/start-sccache-docker.sh # start a server, never fail +# sh scripts/start-sccache-docker.sh --selftest # run the built-in fixtures +# +# Exit status: ALWAYS 0 for the start path — that is the whole point (see below). +# +# WHY SECRETS AND NOT ENV/BUILD-ARGS. `ENV AWS_SECRET_ACCESS_KEY=…` or an ARG +# consumed in a layer persists in `docker history` for every image we push to +# ghcr — that would publish the R2 token. A build secret is mounted for exactly +# one RUN, lands in no layer, and (unlike a build-arg) is not part of the layer +# cache key, so rotating the R2 config never churns the registry layer cache. +# +# WHY THIS IS FAIL-SOFT, WHERE THE REDIS PREDECESSOR WAS A HARD GATE. That +# backend was an in-cluster Service reachable with no credentials, so "can't +# reach it" really did mean "this builder is misconfigured" and failing the +# build was the honest signal. R2 is a remote bucket behind a rotatable token: a +# transient blip, an expired key or a missing `sccache-r2` Secret would now fail +# a build for a reason that has nothing to do with the code. On a release tag +# that costs a VERSION NUMBER (published tags are immutable — a failed release +# burns v0.X.Y), where degrading costs one cold compile, i.e. exactly what a +# cache miss already costs. So: probe, retry, degrade. +# +# sccache's S3 backend is EAGER, exactly like the redis one it replaced: +# `sccache --start-server` FAILS outright when the bucket is unreachable, and +# the first rustc-through-sccache call then dies with "sccache: Timed out +# waiting for server startup". Measured against sccache 0.16.0: +# +# S3 configured, endpoint unreachable -> --start-server FAILS +# empty SCCACHE_BUCKET + SCCACHE_DIR set -> "Cache location: Local disk" +# +# Only the sccache SERVER holds the backend config, so this script exporting +# nothing back to its caller is fine: the later `cargo build` and +# `sccache --show-stats` just talk to whichever server we left running on +# SCCACHE_SERVER_PORT (exported by the caller before invoking us). +set -u + +# Overridable only so --selftest can point at fixture files; the Dockerfiles +# never set it, and BuildKit always mounts secrets under /run/secrets. +SECRET_DIR="${SCCACHE_SECRET_DIR:-/run/secrets}" + +# Non-empty contents of a mounted build secret, or nothing. `required=false` is +# the Dockerfile default, so an unprovided secret is simply an absent/empty file +# (local `docker build` / docker-compose, which pass no secrets at all). +secret() { + _f="${SECRET_DIR}/$1" + [ -s "$_f" ] || return 0 + tr -d '\r\n' <"$_f" +} + +# Where a degraded build puts its local disk cache. Overridable per-Dockerfile +# (this repo builds two images whose BuildKit target cache mounts differ), so +# point it INSIDE a cache mount wherever one exists and a degraded build still +# warms something for the next degraded build. GITHUB_ENV has no equivalent here +# — we just start the server ourselves with the fallback config. An empty +# SCCACHE_BUCKET reads as "unconfigured"; SCCACHE_DIR alone is not enough, a +# non-empty bucket still wins. +LOCAL_CACHE_DIR="${SCCACHE_LOCAL_CACHE_DIR:-/app/target/.sccache-local}" + +fall_back_to_local_disk() { + sccache --stop-server >/dev/null 2>&1 || true + env -u AWS_ACCESS_KEY_ID -u AWS_SECRET_ACCESS_KEY \ + SCCACHE_BUCKET= SCCACHE_DIR="${LOCAL_CACHE_DIR}" \ + sccache --start-server >/dev/null 2>&1 || true +} + +start_sccache() { + # Make the backend switch deterministic if an earlier client call spawned one. + sccache --stop-server >/dev/null 2>&1 || true + + SCCACHE_BUCKET=$(secret SCCACHE_BUCKET) + if [ -z "${SCCACHE_BUCKET}" ]; then + echo "sccache: no R2 config in the build secrets — using a local disk cache" >&2 + fall_back_to_local_disk + return 0 + fi + + AWS_ACCESS_KEY_ID=$(secret AWS_ACCESS_KEY_ID) + AWS_SECRET_ACCESS_KEY=$(secret AWS_SECRET_ACCESS_KEY) + SCCACHE_ENDPOINT=$(secret SCCACHE_ENDPOINT) + SCCACHE_REGION=$(secret SCCACHE_REGION) + SCCACHE_S3_KEY_PREFIX=$(secret SCCACHE_S3_KEY_PREFIX) + SCCACHE_S3_USE_SSL=$(secret SCCACHE_S3_USE_SSL) + export SCCACHE_BUCKET AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY \ + SCCACHE_ENDPOINT SCCACHE_REGION SCCACHE_S3_KEY_PREFIX SCCACHE_S3_USE_SSL + + for attempt in 1 2 3; do + # The 10s guard bounds a genuinely-unreachable bucket (a normal start is + # ~0.4s); the retries ride out a transient R2 blip. + if timeout 10 sccache --start-server >/dev/null 2>&1; then + echo "sccache: R2 backend up (bucket=${SCCACHE_BUCKET}, attempt ${attempt})" >&2 + return 0 + fi + echo "sccache: R2 backend not ready (attempt ${attempt}/3); retrying in 3s" >&2 + sccache --stop-server >/dev/null 2>&1 || true + sleep 3 + done + + echo "sccache: R2 backend unreachable — falling back to local disk cache" >&2 + fall_back_to_local_disk + return 0 +} + +# ── Self-test ──────────────────────────────────────────────────────────────── +# Drives the three states a build can be in against stub `sccache`/`timeout` +# binaries, so a regression in the fail-soft logic is caught by CI in seconds +# instead of by a burned release tag. The stub records the backend env of every +# `--start-server` it is asked to run. +selftest() { + _tmp="$(mktemp -d)" + trap 'rm -rf "${_tmp}"' EXIT + mkdir -p "${_tmp}/bin" "${_tmp}/secrets" + + cat >"${_tmp}/bin/sccache" <<'STUB' +#!/bin/sh +if [ "${1:-}" = "--start-server" ]; then + echo "start bucket=[${SCCACHE_BUCKET:-}] dir=[${SCCACHE_DIR:-}] key=[${AWS_ACCESS_KEY_ID:-}]" \ + >>"${STUB_LOG}" + [ "${STUB_START_FAILS:-0}" = "1" ] && exit 1 +fi +exit 0 +STUB + # `timeout N cmd …` — drop the duration, run the command. Shadows the real + # coreutils binary so the fixtures behave the same on Linux and macOS. + cat >"${_tmp}/bin/timeout" <<'STUB' +#!/bin/sh +shift +exec "$@" +STUB + chmod +x "${_tmp}/bin/sccache" "${_tmp}/bin/timeout" + PATH="${_tmp}/bin:${PATH}" + export PATH STUB_LOG + SECRET_DIR="${_tmp}/secrets" + + _fails=0 + expect() { # expect