diff --git a/demo-rate-limiter/evidence.md b/demo-rate-limiter/evidence.md index 90826da..c0808e2 100644 --- a/demo-rate-limiter/evidence.md +++ b/demo-rate-limiter/evidence.md @@ -89,7 +89,11 @@ Status legend: pass / fail / unverified / n-a. failure branch was proven able to fire with one-off controls: a planted `time.sleep` fixture (failed as required), a chmod-000 unreadable file (failed closed), a nonexistent scan path (failed closed); fixtures removed - after. During the fold-in the secret scan caught its own pattern literal in + after. Those one-off controls are now standing: `tools/test_gauntlet_checks.sh` + runs as the gauntlet's first layer and asserts all three outcomes against the + real `must_not_match` sourced from `tools/must_not_match.sh`, so a regression + in the helper fails the run rather than passing vacuously. During the fold-in + the secret scan caught its own pattern literal in the script — a true positive, resolved by bracketing letters in the pattern (`s[e]cret`), not by excluding the file. This repo's own history includes a fail-open checker: `tools/mutants.py` originally counted any nonzero pytest diff --git a/demo-rate-limiter/tools/gauntlet.sh b/demo-rate-limiter/tools/gauntlet.sh index 2303afb..14e4be0 100755 --- a/demo-rate-limiter/tools/gauntlet.sh +++ b/demo-rate-limiter/tools/gauntlet.sh @@ -5,16 +5,10 @@ cd "$(dirname "$0")/.." rm -f .coverage coverage.xml # stale artifacts from previous runs PY=.venv/bin -# Must-find-nothing grep, fail closed: rc 1 (no matches) is the only pass; -# rc 0 = forbidden pattern present, rc >= 2 = the check itself broke. -must_not_match() { - pattern=$1; shift - if grep -rniE "$pattern" "$@"; then - echo "FAIL: forbidden pattern present: $pattern"; return 1 - elif [ $? -ne 1 ]; then - echo "FAIL: scan itself broke (fail closed): $pattern"; return 1 - fi -} +. tools/must_not_match.sh + +echo "=== checker self-test ===" +sh tools/test_gauntlet_checks.sh echo "=== tests + coverage ===" "$PY/pytest" -q --cov=ratelimiter --cov-report=term-missing diff --git a/demo-rate-limiter/tools/must_not_match.sh b/demo-rate-limiter/tools/must_not_match.sh new file mode 100644 index 0000000..be10c67 --- /dev/null +++ b/demo-rate-limiter/tools/must_not_match.sh @@ -0,0 +1,11 @@ +# Must-find-nothing grep, fail closed: rc 1 (no matches) is the only pass; +# rc 0 = forbidden pattern present, rc >= 2 = the check itself broke. +# Sourced by tools/gauntlet.sh; exercised by tools/test_gauntlet_checks.sh. +must_not_match() { + pattern=$1; shift + if grep -rniE "$pattern" "$@"; then + echo "FAIL: forbidden pattern present: $pattern"; return 1 + elif [ $? -ne 1 ]; then + echo "FAIL: scan itself broke (fail closed): $pattern"; return 1 + fi +} diff --git a/demo-rate-limiter/tools/test_gauntlet_checks.sh b/demo-rate-limiter/tools/test_gauntlet_checks.sh new file mode 100755 index 0000000..1209572 --- /dev/null +++ b/demo-rate-limiter/tools/test_gauntlet_checks.sh @@ -0,0 +1,49 @@ +#!/bin/sh +# Regression check for the fail-closed must-not scan. +# +# It sources the real helper rather than copying it: a copy would keep passing +# after must_not_match itself regressed, which is the exact failure mode this +# check exists to prevent. +cd "$(dirname "$0")/.." +. tools/must_not_match.sh + +failures=0 + +expect() { + want=$1; got=$2; what=$3 + if [ "$got" -eq "$want" ]; then + echo " ok: $what (rc=$got)" + else + echo " NOT OK: $what (want rc $want, got rc $got)" + failures=$((failures + 1)) + fi +} + +work=$(mktemp -d) +trap 'rm -rf "$work"' EXIT INT TERM +mkdir "$work/tree" +printf 'nothing to see here\n' > "$work/tree/clean.txt" + +# 1. Forbidden pattern present -> must fail. +printf 'FORBIDDEN marker\n' > "$work/tree/dirty.txt" +must_not_match 'forbidden' "$work/tree" >/dev/null 2>&1 +expect 1 $? "forbidden pattern present fails" + +# 2. Nothing to find -> must pass. Guards against a check that can never pass. +rm "$work/tree/dirty.txt" +must_not_match 'forbidden' "$work/tree" >/dev/null 2>&1 +expect 0 $? "clean tree passes" + +# 3. The scan itself breaks -> must fail. This is the fail-open guard: grep +# exits >= 2 here, which must never be read as "no matches". A nonexistent +# path is used rather than an unreadable (chmod 000) file: root can still +# read a chmod-000 file, so that variant would need a skip branch to stay +# portable, and a silent skip is the thing this check exists to prevent. +must_not_match 'forbidden' "$work/tree/no-such-path" >/dev/null 2>&1 +expect 1 $? "broken scan fails closed" + +if [ "$failures" -ne 0 ]; then + echo "FAIL: $failures fail-closed expectation(s) violated" + exit 1 +fi +echo "checker self-test clean"