Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 12 additions & 2 deletions hydra-gates/scripts/lib/test_gate_base_ref_delivery_channel.sh
Original file line number Diff line number Diff line change
Expand Up @@ -103,10 +103,20 @@ trap 'rm -rf "${WORK}"' EXIT
# EXTRACT ON THE VERDICT SHAPE, NEVER BY EXCLUDING WHAT LOOKS LIKE A PASS: a
# FAIL line whose remedial prose contains the word "pass" was silently dropped
# by a `grep -v PASS` elsewhere in this programme on 2026-08-12.
#
# ⚠️ WARNING BELONGS IN THIS SET (.github#477, added late).
# When gate-19 was demoted to advisory, WARNING was added to the package's two
# OTHER verdict parsers and not to this one. Nothing went red: a verdict word
# this pattern does not know is simply DROPPED, so gate-19 vanished from both
# channels' sets and the comparison went on matching — over a set that no
# longer contained the gate this whole suite was built to watch. A parser that
# narrows silently is the same defect as two parsers that disagree loudly, and
# only the loud one has ever been caught. This set is asserted identical to the
# other two by ARM P3 of test_gate_discarded_counts_and_empty_deltas.sh.
_verdict_set() {
printf '%s\n' "$1" \
| grep -E '^\[gate-[0-9]+\] [a-z0-9-]+: (PASS|FAIL|NOT APPLICABLE|SKIPPED)' \
| sed -E 's/^\[gate-([0-9]+)\] [a-z0-9-]+: (PASS|FAIL — [0-9]+|FAIL|NOT APPLICABLE|SKIPPED).*/\1|\2/' \
| grep -E '^\[gate-[0-9]+\] [a-z0-9-]+: (PASS|FAIL|WARNING|NOT APPLICABLE|SKIPPED)' \
| sed -E 's/^\[gate-([0-9]+)\] [a-z0-9-]+: (PASS|FAIL — [0-9]+|FAIL|WARNING — [0-9]+|WARNING|NOT APPLICABLE|SKIPPED).*/\1|\2/' \
| sort -t'|' -k1,1n -u
}

Expand Down
133 changes: 125 additions & 8 deletions hydra-gates/scripts/lib/test_gate_discarded_counts_and_empty_deltas.sh
Original file line number Diff line number Diff line change
Expand Up @@ -702,25 +702,142 @@ case "${_pv2}" in
_bad "gf_verdict returned gate-53's PRE-EXISTING advisory instead of the verdict. Got: ${_pv2}" ;;
esac

# ARM P3 — THE TWO PARSERS MUST STAY IN STEP. `gf_verdict` is used by the
# ARM P3 — EVERY VERDICT PARSER MUST STAY IN STEP. `gf_verdict` is used by the
# repo-shaped suites; the acceptance matrix has its own copy, and that copy is
# the one that filtered NOTHING. A comment saying "keep these in step" is not a
# mechanism; this is.
#
# ⚠️ THE FIRST MECHANISM WAS THE WRONG SHAPE, AND IT SAID SO OUT LOUD.
# It asserted that the literal string `(PASS|FAIL|NOT APPLICABLE|SKIPPED)`
# appeared in BOTH files. That is not "the parsers agree" — it is "the verdict
# vocabulary is frozen at these four words". `.github#477` added WARNING to
# both files, IN STEP, exactly as their comments require, and this assertion
# went red anyway; it was one of the two that reddened `.github@main` from
# 13:15 on 2026-08-16. Worse, its own repair path was to retype the literal in
# the test, which nobody can check against the files it is supposed to police.
#
# So the property is asserted directly instead: EXTRACT the alternation each
# parser actually uses and require the two to be byte-identical. Vocabulary may
# grow — it just cannot grow in one file only. The shape half of the property
# is kept as a floor: whatever the alternation contains, it must still contain
# the four core verdict forms, so nobody can satisfy "identical" by gutting
# both parsers back to a denylist.
# ⚠️ THERE ARE THREE PARSERS, NOT TWO. `.github#477` taught two of them the new
# WARNING form and left `_verdict_set` in test_gate_base_ref_delivery_channel.sh
# behind — and THAT one failed silently, because a verdict word the pattern does
# not know is simply dropped: gate-19 disappeared from both channels' verdict
# sets and the comparison went on matching over a set that no longer contained
# the gate that suite exists to watch. A parser that narrows quietly is worse
# than two that disagree loudly. All three are compared here, by name.
_PARSER_FILES="
gate_fixture_support.sh
test_gate_acceptance_matrix.sh
test_gate_base_ref_delivery_channel.sh
"

# `\((PASS|FAIL)…\)` — the parenthesised alternation a verdict matcher uses.
# Anchored on PASS|FAIL so it cannot latch onto some unrelated group.
_verdict_alternation() { # <file> -> the alternation, or empty
grep -oE '\(PASS\|FAIL[A-Z| ]*\)' "$1" | head -1
}

_alt_ref=""; _alt_ref_file=""; _alt_unreadable=""; _alt_drift=""
for _pf in ${_PARSER_FILES}; do
_alt="$(_verdict_alternation "${GF_PKG_ROOT}/scripts/lib/${_pf}")"
if [ -z "${_alt}" ]; then
_alt_unreadable="${_alt_unreadable}${_pf} "
continue
fi
if [ -z "${_alt_ref}" ]; then
_alt_ref="${_alt}"; _alt_ref_file="${_pf}"
elif [ "${_alt}" != "${_alt_ref}" ]; then
_alt_drift="${_alt_drift}${_pf} matches ${_alt}; "
fi
done

if [ -n "${_alt_unreadable}" ]; then
_bad "verdict parser(s) ${_alt_unreadable}no longer carry a recognisable \`(PASS|FAIL|…)\` alternation. A parser this suite cannot read is a parser it cannot police — and an unreadable parser is not an agreeing one."
elif [ -n "${_alt_drift}" ]; then
_bad "the verdict parsers have drifted apart: ${_alt_ref_file} matches ${_alt_ref}, but ${_alt_drift%; } — the same output would be graded differently by different suites, and a parser missing a word DROPS that gate rather than complaining"
else
_missing_form=""
for _form in 'PASS' 'FAIL' 'NOT APPLICABLE' 'SKIPPED'; do
printf '%s' "${_alt_ref}" | grep -qF -- "${_form}" \
|| _missing_form="${_missing_form}${_form}, "
done
if [ -n "${_missing_form}" ]; then
_bad "all verdict parsers agree, but the shared alternation ${_alt_ref} no longer covers ${_missing_form%, } — agreement satisfied by narrowing every parser is the .github#401 defect in a new coat"
else
_ok "all 3 verdict parsers match by the SAME shape — gate_fixture_support.sh, the acceptance matrix and the base-ref channel set agree on ${_alt_ref}"
fi
fi
_alt_support="${_alt_ref}"
_matrix="${GF_PKG_ROOT}/scripts/lib/test_gate_acceptance_matrix.sh"
_shape='(PASS|FAIL|NOT APPLICABLE|SKIPPED)'
if grep -qF -- "${_shape}" "${GF_PKG_ROOT}/scripts/lib/gate_fixture_support.sh" \
&& grep -qF -- "${_shape}" "${_matrix}"; then
_ok "both verdict parsers match by shape — gate_fixture_support.sh and the acceptance matrix agree"

# POSITIVE CONTROL for the assertion above. Without it, "extract and compare"
# is satisfiable by an extractor that returns empty for everything and calls
# that agreement. Drift a COPY of one parser by one word and require the same
# check to say NO.
_pdrift="${_tmp}/parser-drift"; mkdir -p "${_pdrift}"
sed 's/(PASS|FAIL/(PASS|FAIL|BOGUS/' "${_matrix}" > "${_pdrift}/matrix.sh"
_alt_drifted="$(_verdict_alternation "${_pdrift}/matrix.sh")"
if [ -n "${_alt_drifted}" ] && [ "${_alt_drifted}" != "${_alt_support}" ]; then
_ok "control: the drift check FAILS a parser changed on one side only (${_alt_drifted} vs ${_alt_support})"
else
_bad "control BROKEN: a one-sided parser change was not detected (drifted='${_alt_drifted:-none}', support='${_alt_support}') — the arm above proves nothing"
fi

# ---------------------------------------------------------------------------
# ARM P4 — WHICH GATES MAY BE ADVISORY IS BOUNDED BY NAME.
#
# `_warn` (added by .github#477) is one line away from being a soft-fail switch
# for any gate in the suite: a gate that reports a real finding and does not
# reach `_FAILED`. That is the invisible pass, in the package whose entire job
# is preventing them — and nothing else in this package would notice a second
# one landing.
#
# So the allowlist lives here, and demoting another gate means editing this
# test and stating why. This does not stop a demotion; it stops a SILENT one.
#
# ⚠️ Note this counts `_warn` CALL SITES, not the word WARNING. gate-18 prints
# a long-standing advisory line that is not a demotion of its verdict.
# ---------------------------------------------------------------------------
# gate -> the reason it is permitted to be non-blocking.
_ADVISORY_ALLOWED_GATES="19" # e2e-coverage — .github#477, temporary, owned.

_runner_src="${GF_PKG_ROOT}/scripts/run-hydra-gates.sh"
_warn_gates="$(grep -oE '^[[:space:]]*_warn[[:space:]]+[0-9]+' "${_runner_src}" \
| grep -oE '[0-9]+$' | sort -un | tr '\n' ' ')"
_warn_gates="${_warn_gates% }"
if [ "${_warn_gates}" = "${_ADVISORY_ALLOWED_GATES}" ]; then
_ok "the advisory (_warn) gates are exactly the allowlisted ones: ${_ADVISORY_ALLOWED_GATES:-none}"
else
_bad "the set of gates demoted to advisory has changed without this test being told: found '${_warn_gates:-none}', allowed '${_ADVISORY_ALLOWED_GATES:-none}'. A gate that reports a real finding and does not block is a soft-fail; if the demotion is intended, add it here WITH its tracking issue."
fi

# POSITIVE CONTROL — the allowlist check must be able to say NO. Plant a second
# `_warn` call site in a COPY of the runner and require the same extraction to
# come back with a set the allowlist rejects.
_wdrift="${_tmp}/warn-drift"; mkdir -p "${_wdrift}"
{ cat "${_runner_src}"; printf ' _warn 7 "no-admin-idor" "pretend"\n'; } > "${_wdrift}/runner.sh"
_warn_drifted="$(grep -oE '^[[:space:]]*_warn[[:space:]]+[0-9]+' "${_wdrift}/runner.sh" \
| grep -oE '[0-9]+$' | sort -un | tr '\n' ' ')"
_warn_drifted="${_warn_drifted% }"
if [ "${_warn_drifted}" != "${_ADVISORY_ALLOWED_GATES}" ] && [ -n "${_warn_drifted}" ]; then
_ok "control: a second _warn call site is detected (${_warn_drifted}) — the allowlist can refuse"
else
_bad "the two verdict parsers have drifted apart: one matches a verdict by shape and the other does not, so the same output would be graded differently by two suites"
_bad "control BROKEN: planting a _warn on gate-7 produced '${_warn_drifted:-none}', which the allowlist would accept — ARM P4 proves nothing"
fi

echo
echo "== summary =="
echo " assertions: ${_asserts}"
echo " failures: ${_failures}"
if [ "${_asserts}" -lt 34 ]; then
echo "FAIL — only ${_asserts} assertions ran; this suite declares 34+. A short run is not a green run."
# 34 -> 37: ARM P3 gained its positive control, and ARM P4 (the advisory-gate
# allowlist) plus its control are new. The floor tracks the suite, otherwise a
# whole arm can be deleted and the run still reads green.
if [ "${_asserts}" -lt 37 ]; then
echo "FAIL — only ${_asserts} assertions ran; this suite declares 37+. A short run is not a green run."
exit 1
fi
[ "${_failures}" -eq 0 ] || exit 1
Expand Down
67 changes: 62 additions & 5 deletions hydra-gates/scripts/lib/test_gate_empty_scope_never_passes.sh
Original file line number Diff line number Diff line change
Expand Up @@ -133,16 +133,73 @@ _verdict() { grep -oE "^\[gate-$2\] [^:]+: [A-Z]+( [A-Z]+)*( \([a-z]+\))?" "$1"

# ---------------------------------------------------------------------------
# ARM 1 — the planted true positives are still caught, full-tree.
#
# ⚠️ WHAT THIS ARM ASSERTS, AND WHAT IT DELIBERATELY DOES NOT (.github#477)
# ------------------------------------------------------------------------
# This arm is the POSITIVE CONTROL for the fixture, not a statement about
# blocking policy. Everything below it — "an empty scope is NOT APPLICABLE" —
# proves nothing unless the gate demonstrably SEES the planted defect when the
# scope is open. So what has to hold is: the gate opened the scope, computed a
# real finding, and named a count.
#
# `.github#477` demoted gate-19 (e2e-coverage) from `_fail` to `_warn`: it
# still runs, still counts and still names every unannotated scenario, but the
# finding no longer stops a merge. That moved gate-19's verdict word from FAIL
# to WARNING and this arm — which matched the word rather than the property —
# went red on `.github@main` from 13:15 on 2026-08-16, alongside ARM P3 of
# test_gate_discarded_counts_and_empty_deltas.sh. Three sibling suites were
# taught the new word in the same commit (exit-code-semantics,
# base-ref-delivery-channel, the acceptance matrix + its expect.conf); these
# two were not, because they were not run. That is the hand-maintained-list
# decay this package's own runner exists to prevent — see
# tests/run-helper-suites.sh.
#
# So gate-19 is accepted at FAIL *or* WARNING, and NOT on the word alone: the
# verdict line must also carry a non-zero finding count, because a demotion's
# real failure mode is the finding quietly ceasing to be read. gate-25 is NOT
# demoted and keeps the strict FAIL — accepting WARNING for a gate nobody
# demoted would be exactly the invisible pass this file guards.
#
# Note what this arm still catches for gate-19 without the word FAIL: a gate
# that went blind prints PASS, and a gate that self-scoped to nothing prints
# NOT APPLICABLE. Neither is accepted here, and ARM 2 separately requires
# NOT APPLICABLE over the empty scope, so the two arms remain distinguishable.
#
# Which gates may be advisory at all is bounded by name — see ARM P4 of
# test_gate_discarded_counts_and_empty_deltas.sh, so a second demotion cannot
# land without reddening this package.
# ---------------------------------------------------------------------------
_full="${_tmp}/full.txt"
_run "${_full}"
for _g in 19 25; do
if grep -qE "^\[gate-${_g}\][^:]*: FAIL" "${_full}"; then
_ok "gate-${_g} still catches its planted true positive over the full tree"

# gate-25 — NOT demoted. Strict FAIL.
if grep -qE "^\[gate-25\][^:]*: FAIL" "${_full}"; then
_ok "gate-25 still catches its planted true positive over the full tree"
else
_bad "gate-25 did NOT catch its planted true positive — got: $(_verdict "${_full}" 25)"
fi

# gate-19 — advisory since .github#477. FAIL or WARNING, and it must NAME a count.
_g19_line="$(grep -E "^\[gate-19\][^:]*: (FAIL|WARNING)\b" "${_full}" | head -1)"
if [ -n "${_g19_line}" ]; then
if printf '%s' "${_g19_line}" | grep -qE '[1-9][0-9]* scenario'; then
_ok "gate-19 still catches its planted true positive over the full tree, and names the count — ${_g19_line#*: }"
else
_bad "gate-${_g} did NOT catch its planted true positive — got: $(_verdict "${_full}" "${_g}")"
_bad "gate-19 reached a FAIL/WARNING verdict but named no non-zero scenario count — a demotion whose finding stops being legible is the invisible pass. Got: ${_g19_line}"
fi
done
else
_bad "gate-19 did NOT catch its planted true positive — got: $(_verdict "${_full}" 19)"
fi

# …and the SUMMARY must say so. A demotion is only safe while the reader is
# told that a green verdict means "nothing BLOCKING failed", not "nothing was
# found". Without this, `_warn` degrades into a silent soft-fail.
if grep -qF 'reported ADVISORY findings' "${_full}" \
&& grep -qF "means 'nothing BLOCKING failed'" "${_full}"; then
_ok "the run's summary announces the advisory findings and says green != 'nothing was found'"
else
_bad "a gate reported an advisory finding and the summary did not say so — a demoted gate whose finding is not announced is a soft-fail nobody reads"
fi

# Full-tree must actually OPEN the manifests rather than diff-scope itself to
# nothing: 62/63 are clean in this fixture, so they must PASS, not SKIP.
Expand Down
Loading