diff --git a/hydra-gates/scripts/lib/test_gate_base_ref_delivery_channel.sh b/hydra-gates/scripts/lib/test_gate_base_ref_delivery_channel.sh index 9d7ab839..2b7f86ba 100755 --- a/hydra-gates/scripts/lib/test_gate_base_ref_delivery_channel.sh +++ b/hydra-gates/scripts/lib/test_gate_base_ref_delivery_channel.sh @@ -103,10 +103,20 @@ trap 'rm -rf "${WORK}"' EXIT # EXTRACT ON THE VERDICT SHAPE, NEVER BY EXCLUDING WHAT LOOKS LIKE A PASS: a # FAIL line whose remedial prose contains the word "pass" was silently dropped # by a `grep -v PASS` elsewhere in this programme on 2026-08-12. +# +# ⚠️ WARNING BELONGS IN THIS SET (.github#477, added late). +# When gate-19 was demoted to advisory, WARNING was added to the package's two +# OTHER verdict parsers and not to this one. Nothing went red: a verdict word +# this pattern does not know is simply DROPPED, so gate-19 vanished from both +# channels' sets and the comparison went on matching — over a set that no +# longer contained the gate this whole suite was built to watch. A parser that +# narrows silently is the same defect as two parsers that disagree loudly, and +# only the loud one has ever been caught. This set is asserted identical to the +# other two by ARM P3 of test_gate_discarded_counts_and_empty_deltas.sh. _verdict_set() { printf '%s\n' "$1" \ - | grep -E '^\[gate-[0-9]+\] [a-z0-9-]+: (PASS|FAIL|NOT APPLICABLE|SKIPPED)' \ - | sed -E 's/^\[gate-([0-9]+)\] [a-z0-9-]+: (PASS|FAIL — [0-9]+|FAIL|NOT APPLICABLE|SKIPPED).*/\1|\2/' \ + | grep -E '^\[gate-[0-9]+\] [a-z0-9-]+: (PASS|FAIL|WARNING|NOT APPLICABLE|SKIPPED)' \ + | sed -E 's/^\[gate-([0-9]+)\] [a-z0-9-]+: (PASS|FAIL — [0-9]+|FAIL|WARNING — [0-9]+|WARNING|NOT APPLICABLE|SKIPPED).*/\1|\2/' \ | sort -t'|' -k1,1n -u } diff --git a/hydra-gates/scripts/lib/test_gate_discarded_counts_and_empty_deltas.sh b/hydra-gates/scripts/lib/test_gate_discarded_counts_and_empty_deltas.sh index e67173eb..42ae54ec 100644 --- a/hydra-gates/scripts/lib/test_gate_discarded_counts_and_empty_deltas.sh +++ b/hydra-gates/scripts/lib/test_gate_discarded_counts_and_empty_deltas.sh @@ -702,25 +702,142 @@ case "${_pv2}" in _bad "gf_verdict returned gate-53's PRE-EXISTING advisory instead of the verdict. Got: ${_pv2}" ;; esac -# ARM P3 — THE TWO PARSERS MUST STAY IN STEP. `gf_verdict` is used by the +# ARM P3 — EVERY VERDICT PARSER MUST STAY IN STEP. `gf_verdict` is used by the # repo-shaped suites; the acceptance matrix has its own copy, and that copy is # the one that filtered NOTHING. A comment saying "keep these in step" is not a # mechanism; this is. +# +# ⚠️ THE FIRST MECHANISM WAS THE WRONG SHAPE, AND IT SAID SO OUT LOUD. +# It asserted that the literal string `(PASS|FAIL|NOT APPLICABLE|SKIPPED)` +# appeared in BOTH files. That is not "the parsers agree" — it is "the verdict +# vocabulary is frozen at these four words". `.github#477` added WARNING to +# both files, IN STEP, exactly as their comments require, and this assertion +# went red anyway; it was one of the two that reddened `.github@main` from +# 13:15 on 2026-08-16. Worse, its own repair path was to retype the literal in +# the test, which nobody can check against the files it is supposed to police. +# +# So the property is asserted directly instead: EXTRACT the alternation each +# parser actually uses and require the two to be byte-identical. Vocabulary may +# grow — it just cannot grow in one file only. The shape half of the property +# is kept as a floor: whatever the alternation contains, it must still contain +# the four core verdict forms, so nobody can satisfy "identical" by gutting +# both parsers back to a denylist. +# ⚠️ THERE ARE THREE PARSERS, NOT TWO. `.github#477` taught two of them the new +# WARNING form and left `_verdict_set` in test_gate_base_ref_delivery_channel.sh +# behind — and THAT one failed silently, because a verdict word the pattern does +# not know is simply dropped: gate-19 disappeared from both channels' verdict +# sets and the comparison went on matching over a set that no longer contained +# the gate that suite exists to watch. A parser that narrows quietly is worse +# than two that disagree loudly. All three are compared here, by name. +_PARSER_FILES=" +gate_fixture_support.sh +test_gate_acceptance_matrix.sh +test_gate_base_ref_delivery_channel.sh +" + +# `\((PASS|FAIL)…\)` — the parenthesised alternation a verdict matcher uses. +# Anchored on PASS|FAIL so it cannot latch onto some unrelated group. +_verdict_alternation() { # -> the alternation, or empty + grep -oE '\(PASS\|FAIL[A-Z| ]*\)' "$1" | head -1 +} + +_alt_ref=""; _alt_ref_file=""; _alt_unreadable=""; _alt_drift="" +for _pf in ${_PARSER_FILES}; do + _alt="$(_verdict_alternation "${GF_PKG_ROOT}/scripts/lib/${_pf}")" + if [ -z "${_alt}" ]; then + _alt_unreadable="${_alt_unreadable}${_pf} " + continue + fi + if [ -z "${_alt_ref}" ]; then + _alt_ref="${_alt}"; _alt_ref_file="${_pf}" + elif [ "${_alt}" != "${_alt_ref}" ]; then + _alt_drift="${_alt_drift}${_pf} matches ${_alt}; " + fi +done + +if [ -n "${_alt_unreadable}" ]; then + _bad "verdict parser(s) ${_alt_unreadable}no longer carry a recognisable \`(PASS|FAIL|…)\` alternation. A parser this suite cannot read is a parser it cannot police — and an unreadable parser is not an agreeing one." +elif [ -n "${_alt_drift}" ]; then + _bad "the verdict parsers have drifted apart: ${_alt_ref_file} matches ${_alt_ref}, but ${_alt_drift%; } — the same output would be graded differently by different suites, and a parser missing a word DROPS that gate rather than complaining" +else + _missing_form="" + for _form in 'PASS' 'FAIL' 'NOT APPLICABLE' 'SKIPPED'; do + printf '%s' "${_alt_ref}" | grep -qF -- "${_form}" \ + || _missing_form="${_missing_form}${_form}, " + done + if [ -n "${_missing_form}" ]; then + _bad "all verdict parsers agree, but the shared alternation ${_alt_ref} no longer covers ${_missing_form%, } — agreement satisfied by narrowing every parser is the .github#401 defect in a new coat" + else + _ok "all 3 verdict parsers match by the SAME shape — gate_fixture_support.sh, the acceptance matrix and the base-ref channel set agree on ${_alt_ref}" + fi +fi +_alt_support="${_alt_ref}" _matrix="${GF_PKG_ROOT}/scripts/lib/test_gate_acceptance_matrix.sh" -_shape='(PASS|FAIL|NOT APPLICABLE|SKIPPED)' -if grep -qF -- "${_shape}" "${GF_PKG_ROOT}/scripts/lib/gate_fixture_support.sh" \ - && grep -qF -- "${_shape}" "${_matrix}"; then - _ok "both verdict parsers match by shape — gate_fixture_support.sh and the acceptance matrix agree" + +# POSITIVE CONTROL for the assertion above. Without it, "extract and compare" +# is satisfiable by an extractor that returns empty for everything and calls +# that agreement. Drift a COPY of one parser by one word and require the same +# check to say NO. +_pdrift="${_tmp}/parser-drift"; mkdir -p "${_pdrift}" +sed 's/(PASS|FAIL/(PASS|FAIL|BOGUS/' "${_matrix}" > "${_pdrift}/matrix.sh" +_alt_drifted="$(_verdict_alternation "${_pdrift}/matrix.sh")" +if [ -n "${_alt_drifted}" ] && [ "${_alt_drifted}" != "${_alt_support}" ]; then + _ok "control: the drift check FAILS a parser changed on one side only (${_alt_drifted} vs ${_alt_support})" +else + _bad "control BROKEN: a one-sided parser change was not detected (drifted='${_alt_drifted:-none}', support='${_alt_support}') — the arm above proves nothing" +fi + +# --------------------------------------------------------------------------- +# ARM P4 — WHICH GATES MAY BE ADVISORY IS BOUNDED BY NAME. +# +# `_warn` (added by .github#477) is one line away from being a soft-fail switch +# for any gate in the suite: a gate that reports a real finding and does not +# reach `_FAILED`. That is the invisible pass, in the package whose entire job +# is preventing them — and nothing else in this package would notice a second +# one landing. +# +# So the allowlist lives here, and demoting another gate means editing this +# test and stating why. This does not stop a demotion; it stops a SILENT one. +# +# ⚠️ Note this counts `_warn` CALL SITES, not the word WARNING. gate-18 prints +# a long-standing advisory line that is not a demotion of its verdict. +# --------------------------------------------------------------------------- +# gate -> the reason it is permitted to be non-blocking. +_ADVISORY_ALLOWED_GATES="19" # e2e-coverage — .github#477, temporary, owned. + +_runner_src="${GF_PKG_ROOT}/scripts/run-hydra-gates.sh" +_warn_gates="$(grep -oE '^[[:space:]]*_warn[[:space:]]+[0-9]+' "${_runner_src}" \ + | grep -oE '[0-9]+$' | sort -un | tr '\n' ' ')" +_warn_gates="${_warn_gates% }" +if [ "${_warn_gates}" = "${_ADVISORY_ALLOWED_GATES}" ]; then + _ok "the advisory (_warn) gates are exactly the allowlisted ones: ${_ADVISORY_ALLOWED_GATES:-none}" +else + _bad "the set of gates demoted to advisory has changed without this test being told: found '${_warn_gates:-none}', allowed '${_ADVISORY_ALLOWED_GATES:-none}'. A gate that reports a real finding and does not block is a soft-fail; if the demotion is intended, add it here WITH its tracking issue." +fi + +# POSITIVE CONTROL — the allowlist check must be able to say NO. Plant a second +# `_warn` call site in a COPY of the runner and require the same extraction to +# come back with a set the allowlist rejects. +_wdrift="${_tmp}/warn-drift"; mkdir -p "${_wdrift}" +{ cat "${_runner_src}"; printf ' _warn 7 "no-admin-idor" "pretend"\n'; } > "${_wdrift}/runner.sh" +_warn_drifted="$(grep -oE '^[[:space:]]*_warn[[:space:]]+[0-9]+' "${_wdrift}/runner.sh" \ + | grep -oE '[0-9]+$' | sort -un | tr '\n' ' ')" +_warn_drifted="${_warn_drifted% }" +if [ "${_warn_drifted}" != "${_ADVISORY_ALLOWED_GATES}" ] && [ -n "${_warn_drifted}" ]; then + _ok "control: a second _warn call site is detected (${_warn_drifted}) — the allowlist can refuse" else - _bad "the two verdict parsers have drifted apart: one matches a verdict by shape and the other does not, so the same output would be graded differently by two suites" + _bad "control BROKEN: planting a _warn on gate-7 produced '${_warn_drifted:-none}', which the allowlist would accept — ARM P4 proves nothing" fi echo echo "== summary ==" echo " assertions: ${_asserts}" echo " failures: ${_failures}" -if [ "${_asserts}" -lt 34 ]; then - echo "FAIL — only ${_asserts} assertions ran; this suite declares 34+. A short run is not a green run." +# 34 -> 37: ARM P3 gained its positive control, and ARM P4 (the advisory-gate +# allowlist) plus its control are new. The floor tracks the suite, otherwise a +# whole arm can be deleted and the run still reads green. +if [ "${_asserts}" -lt 37 ]; then + echo "FAIL — only ${_asserts} assertions ran; this suite declares 37+. A short run is not a green run." exit 1 fi [ "${_failures}" -eq 0 ] || exit 1 diff --git a/hydra-gates/scripts/lib/test_gate_empty_scope_never_passes.sh b/hydra-gates/scripts/lib/test_gate_empty_scope_never_passes.sh index 96d624b8..0f033081 100755 --- a/hydra-gates/scripts/lib/test_gate_empty_scope_never_passes.sh +++ b/hydra-gates/scripts/lib/test_gate_empty_scope_never_passes.sh @@ -133,16 +133,73 @@ _verdict() { grep -oE "^\[gate-$2\] [^:]+: [A-Z]+( [A-Z]+)*( \([a-z]+\))?" "$1" # --------------------------------------------------------------------------- # ARM 1 — the planted true positives are still caught, full-tree. +# +# ⚠️ WHAT THIS ARM ASSERTS, AND WHAT IT DELIBERATELY DOES NOT (.github#477) +# ------------------------------------------------------------------------ +# This arm is the POSITIVE CONTROL for the fixture, not a statement about +# blocking policy. Everything below it — "an empty scope is NOT APPLICABLE" — +# proves nothing unless the gate demonstrably SEES the planted defect when the +# scope is open. So what has to hold is: the gate opened the scope, computed a +# real finding, and named a count. +# +# `.github#477` demoted gate-19 (e2e-coverage) from `_fail` to `_warn`: it +# still runs, still counts and still names every unannotated scenario, but the +# finding no longer stops a merge. That moved gate-19's verdict word from FAIL +# to WARNING and this arm — which matched the word rather than the property — +# went red on `.github@main` from 13:15 on 2026-08-16, alongside ARM P3 of +# test_gate_discarded_counts_and_empty_deltas.sh. Three sibling suites were +# taught the new word in the same commit (exit-code-semantics, +# base-ref-delivery-channel, the acceptance matrix + its expect.conf); these +# two were not, because they were not run. That is the hand-maintained-list +# decay this package's own runner exists to prevent — see +# tests/run-helper-suites.sh. +# +# So gate-19 is accepted at FAIL *or* WARNING, and NOT on the word alone: the +# verdict line must also carry a non-zero finding count, because a demotion's +# real failure mode is the finding quietly ceasing to be read. gate-25 is NOT +# demoted and keeps the strict FAIL — accepting WARNING for a gate nobody +# demoted would be exactly the invisible pass this file guards. +# +# Note what this arm still catches for gate-19 without the word FAIL: a gate +# that went blind prints PASS, and a gate that self-scoped to nothing prints +# NOT APPLICABLE. Neither is accepted here, and ARM 2 separately requires +# NOT APPLICABLE over the empty scope, so the two arms remain distinguishable. +# +# Which gates may be advisory at all is bounded by name — see ARM P4 of +# test_gate_discarded_counts_and_empty_deltas.sh, so a second demotion cannot +# land without reddening this package. # --------------------------------------------------------------------------- _full="${_tmp}/full.txt" _run "${_full}" -for _g in 19 25; do - if grep -qE "^\[gate-${_g}\][^:]*: FAIL" "${_full}"; then - _ok "gate-${_g} still catches its planted true positive over the full tree" + +# gate-25 — NOT demoted. Strict FAIL. +if grep -qE "^\[gate-25\][^:]*: FAIL" "${_full}"; then + _ok "gate-25 still catches its planted true positive over the full tree" +else + _bad "gate-25 did NOT catch its planted true positive — got: $(_verdict "${_full}" 25)" +fi + +# gate-19 — advisory since .github#477. FAIL or WARNING, and it must NAME a count. +_g19_line="$(grep -E "^\[gate-19\][^:]*: (FAIL|WARNING)\b" "${_full}" | head -1)" +if [ -n "${_g19_line}" ]; then + if printf '%s' "${_g19_line}" | grep -qE '[1-9][0-9]* scenario'; then + _ok "gate-19 still catches its planted true positive over the full tree, and names the count — ${_g19_line#*: }" else - _bad "gate-${_g} did NOT catch its planted true positive — got: $(_verdict "${_full}" "${_g}")" + _bad "gate-19 reached a FAIL/WARNING verdict but named no non-zero scenario count — a demotion whose finding stops being legible is the invisible pass. Got: ${_g19_line}" fi -done +else + _bad "gate-19 did NOT catch its planted true positive — got: $(_verdict "${_full}" 19)" +fi + +# …and the SUMMARY must say so. A demotion is only safe while the reader is +# told that a green verdict means "nothing BLOCKING failed", not "nothing was +# found". Without this, `_warn` degrades into a silent soft-fail. +if grep -qF 'reported ADVISORY findings' "${_full}" \ + && grep -qF "means 'nothing BLOCKING failed'" "${_full}"; then + _ok "the run's summary announces the advisory findings and says green != 'nothing was found'" +else + _bad "a gate reported an advisory finding and the summary did not say so — a demoted gate whose finding is not announced is a soft-fail nobody reads" +fi # Full-tree must actually OPEN the manifests rather than diff-scope itself to # nothing: 62/63 are clean in this fixture, so they must PASS, not SKIP.