From 3770380c986547f216b4451e161926fece21a652 Mon Sep 17 00:00:00 2001 From: wshallwshall Date: Sat, 8 Aug 2026 18:49:46 -0500 Subject: [PATCH 1/4] fix(asvs): anchor DRIFT is not claim INVALIDATION -- stop failing the gate on it check_anchors failed on any evidence anchor whose token moved more than +/-ANCHOR_WINDOW lines, whether or not the claim was still true. That collapses two different assertions: "the evidence still exists" and "the evidence still sits where it sat". Only the first is what this gate is for. NOT A LOOSENED THRESHOLD. Execution reaches the moved-token branch only PAST the `occurrences > 1` guard, so the token provably occurs EXACTLY ONCE in the file. Its presence alone pins the evidence; the recorded line is a navigation aid. Where the line IS load-bearing -- an ambiguous token, where a re-anchor to the wrong occurrence cannot be detected -- it still hard-fails, unchanged. Still fatal, i.e. the whole signal: a token that is GONE (the claim may now be false), an AMBIGUOUS token, a missing evidence path, and every absence-claim failure. This narrows the gate to claim-invalidation, which ADR 0156 already describes as the intent ("Evidence anchors will break on refactors. That is the feature ... a re-score becomes re-verify the cells whose anchors moved"). WHY IT MATTERS HERE, THOUGH THIS REPO NEVER FEELS IT. No workflow in this repo runs scripts/asvs/scorecard.py -- the scorecard DATA and the gate that consumes it live in the private vault, which mirrors this file byte-for-byte. So the repo that causes the drift never runs the tool, and the repo that runs it is the one blocked by it. Measured downstream: on 2026-08-07, 130 of 146 failures were a single constant offset per file (+57 auth/service.py, +61 __main__.py, +47 store/base.py) and the count went 67 -> 146 in ten hours as this tree advanced. It recurred the next day (+102 on .github/workflows/ci.yml). Every one of those was a line number, not a claim. The cost is not just noise. A gate that is red every morning for a reason nobody must act on is a gate whose next REAL finding gets waved through -- and there was one underneath that day: a cell asserting a security control was ABSENT when it had since been built and merged. It sat in the same undifferentiated list as 130 line numbers. Drift is still REPORTED, as DRIFT plus a count, because letting the recorded line numbers rot silently is the failure mode on the other side. Adds 3 tests. Mutation-checked: with this change reverted, test_a_unique_token_that_drifted_is_advisory_not_fatal fails BEHAVIOURALLY (the drift lands in `problems`). The other two are controls -- ambiguous stays fatal, ordinary small movement stays silent -- and fail on the missing attribute, which is a weaker signal; they assert unchanged behaviour by design. Verified: 57 tests pass in tests/test_asvs_scorecard.py (was 54); ruff format and ruff check clean on both files. Co-Authored-By: Claude Opus 5 --- scripts/asvs/scorecard.py | 41 ++++++++++++++++++++-- tests/test_asvs_scorecard.py | 67 ++++++++++++++++++++++++++++++++++++ 2 files changed, 106 insertions(+), 2 deletions(-) diff --git a/scripts/asvs/scorecard.py b/scripts/asvs/scorecard.py index 2526fa01..d31923d8 100644 --- a/scripts/asvs/scorecard.py +++ b/scripts/asvs/scorecard.py @@ -176,6 +176,10 @@ class Findings: """What a verification pass found. Empty ``problems`` is the only pass condition.""" problems: list[str] = field(default_factory=list) + #: Reported, never fatal. A UNIQUE evidence token that merely MOVED is not a broken claim — see + #: :func:`check_anchors`. These still print, because letting them accumulate silently is how the + #: recorded line numbers rot; they just do not red the gate. + advisories: list[str] = field(default_factory=list) checked_anchors: int = 0 checked_absences: int = 0 skipped_anchors: int = 0 @@ -426,10 +430,35 @@ def check_anchors(cells: list[Cell], root: Path, findings: Findings) -> None: continue if a.expect in "\n".join(lines[lo:hi]): continue - where = " (found elsewhere in the file)" if a.expect in "\n".join(lines) else "" + if a.expect in "\n".join(lines): + # MOVED, not broken — and for a UNIQUE token the line number was never the proof. + # Execution only reaches here PAST the ``occurrences > 1`` guard above, so the token + # occurs exactly once in this file: its presence alone pins the evidence, and the line + # is a navigation aid. Failing on it asserts something different from what this gate + # exists to assert — that the claim is still true, not that it sits where it sat. + # + # This is not a theoretical tidy-up. The scorecard lives in a repo that no longer + # develops this tree, while this tree lands ~21 commits/day, so ONE insertion above a + # hot file re-breaks every anchor below it: on 2026-08-07, 130 of 146 failures were a + # single constant offset per file (+57 auth/service.py, +61 __main__.py, +47 + # store/base.py) and the count went 67 -> 146 in ten hours. It recurred the next day + # (+102 on .github/workflows/ci.yml). A gate that is red every morning for a reason + # nobody must act on is a gate whose next REAL finding gets waved through — and there + # was one underneath: a cell asserting a control was absent that had since been built. + # + # What still reds the gate is unchanged and is the whole signal: a token that is GONE + # (the claim may now be false), an AMBIGUOUS token (there the line IS load-bearing and + # a re-anchor to the wrong occurrence cannot be detected), a missing evidence path, + # and every absence-claim failure. That narrows the gate to claim-invalidation, which + # is what ADR 0156 already describes as the intent. + findings.advisories.append( + f"{c.id}: {a.path}:{a.line} moved — {a.expect!r} is still in the file (unique), " + f"but >±{ANCHOR_WINDOW} lines away. Re-anchor the line number" + ) + continue findings.problems.append( f"{c.id}: {a.path}:{a.line} no longer contains {a.expect!r} within " - f"±{ANCHOR_WINDOW} lines{where} — the evidence moved or the claim is now false" + f"±{ANCHOR_WINDOW} lines — the evidence moved or the claim is now false" ) @@ -950,6 +979,14 @@ def main(argv: list[str] | None = None) -> int: f"{n['unverified']} unverified); " f"verified {findings.checked_anchors} evidence anchors and {findings.checked_absences} absence claims" ) + for a in findings.advisories: + print(f" DRIFT {a}", file=sys.stderr) + if findings.advisories: + print( + f" {len(findings.advisories)} anchor(s) drifted: evidence still present and unique, line " + "numbers stale. Not fatal — re-anchor them, do not let them accumulate.", + file=sys.stderr, + ) for p in findings.problems: print(f" FAIL {p}", file=sys.stderr) diff --git a/tests/test_asvs_scorecard.py b/tests/test_asvs_scorecard.py index 58fd0bc3..3a7538f6 100644 --- a/tests/test_asvs_scorecard.py +++ b/tests/test_asvs_scorecard.py @@ -160,6 +160,73 @@ def test_anchor_goes_red_when_the_token_is_gone(tmp_path: Path) -> None: assert not f.ok and "no longer contains" in f.problems[0] +def test_a_unique_token_that_drifted_is_advisory_not_fatal(tmp_path: Path) -> None: + """DRIFT is not INVALIDATION. The token sits far below its recorded line but occurs exactly once. + + Execution only reaches the drift branch PAST the ``occurrences > 1`` guard, so uniqueness is what + pins the evidence and the line number is navigation. Failing here would assert that the claim sits + where it sat, which is a different proposition from the one this gate exists to check. + """ + (tmp_path / "messagefoundry").mkdir() + body = "\n".join(["filler"] * 300 + ["tls_cert_file = None"] + ["tail"] * 5) + (tmp_path / "messagefoundry" / "m.py").write_text(body, encoding="utf-8") + cells = [ + Cell( + id="1.1.1", + level=1, + verdict="pass", + evidence=(Anchor("messagefoundry/m.py", 5, "tls_cert_file"),), + ) + ] + f = Findings() + check_anchors(cells, tmp_path, f) + assert f.ok and f.problems == [] + assert len(f.advisories) == 1 and "moved" in f.advisories[0] + + +def test_an_ambiguous_token_stays_fatal_even_when_it_drifted(tmp_path: Path) -> None: + """Where the line IS load-bearing, drift must not soften it. + + With two occurrences a re-anchor to the WRONG one cannot be detected — the defect this module's + uniqueness rule exists to catch. Advisory treatment is reserved for the case where the token + itself certifies the evidence. + """ + (tmp_path / "messagefoundry").mkdir() + body = "\n".join(["filler"] * 200 + ["dupe_token"] + ["filler"] * 200 + ["dupe_token"]) + (tmp_path / "messagefoundry" / "m.py").write_text(body, encoding="utf-8") + cells = [ + Cell( + id="1.1.1", + level=1, + verdict="pass", + evidence=(Anchor("messagefoundry/m.py", 1, "dupe_token"),), + ) + ] + f = Findings() + check_anchors(cells, tmp_path, f) + assert not f.ok and "AMBIGUOUS" in f.problems[0] + assert f.advisories == [] + + +def test_ordinary_small_movement_is_neither_problem_nor_advisory(tmp_path: Path) -> None: + """The control. Without it, a test asserting "drift is advisory" would still pass if EVERY anchor + started reporting drift — the window would have stopped absorbing anything and nothing would say so.""" + (tmp_path / "messagefoundry").mkdir() + body = "\n".join(["filler"] * 20 + ["tls_cert_file = None"] + ["filler"] * 20) + (tmp_path / "messagefoundry" / "m.py").write_text(body, encoding="utf-8") + cells = [ + Cell( + id="1.1.1", + level=1, + verdict="pass", + evidence=(Anchor("messagefoundry/m.py", 15, "tls_cert_file"),), + ) + ] + f = Findings() + check_anchors(cells, tmp_path, f) + assert f.problems == [] and f.advisories == [] + + def test_anchor_goes_red_when_the_file_is_gone(tmp_path: Path) -> None: cells = [ Cell( From e3bac0eb75353df0a7a03ecf1b760a0a01a4a409 Mon Sep 17 00:00:00 2001 From: wshallwshall Date: Sun, 9 Aug 2026 00:07:28 -0500 Subject: [PATCH 2/4] fix(asvs): advise on ANY stale anchor line, not only past a window #295 as drafted merges into an EMPTY advisory list. Its advisory fired only when the token sat beyond +/-40, and measured against the current record NOTHING is beyond it: 725 of 1,980 anchors are stale-but-inside, and 0 are outside. So the change prevented the next recurrence while surfacing none of the accumulation, and its own closing instruction -- 're-anchor them, do not let them accumulate' -- was unactionable because the accumulation was invisible. The window is now gone from the decision path entirely. Uniqueness locates the evidence; the recorded line is reported output that an advisory corrects. GONE, AMBIGUOUS and missing-path stay fatal, unchanged -- those are claim invalidation, which is what this gate exists to assert. Measured on this branch against the live record: 725 of 1980 (36.6%) advisory, zero fatal, exit 0. Max offset in the corpus was 39 against a limit of 40, three days after a re-anchor pass reset the distribution by hand-retyping 130 integers. A tolerance 37% spent three days after a reset is a decaying budget, not a tolerance. THE LINE IS DERIVED FROM THE CHARACTER OFFSET, NOT BY SCANNING LINES. 42 live expect tokens span a newline, because the old check matched joined text and nothing forbade it. A per-line scan finds none of them and raises on the lookup. Now tested. Two tests changed contract and say so rather than being quietly deleted. test_ordinary_small_movement_is_neither_problem_nor_advisory asserted the silence that was the defect; it is renamed and inverted. Its guard-the-guard role is preserved in a new shape -- an exactly-located anchor must report NOTHING -- which catches the opposite failure, a resolver that advises on everything. Both were mutation-proven: an off-by-one in the derivation reds four tests including the new control. --- scripts/asvs/scorecard.py | 128 ++++++++++++++++++++--------------- tests/test_asvs_scorecard.py | 75 ++++++++++++++++++-- 2 files changed, 145 insertions(+), 58 deletions(-) diff --git a/scripts/asvs/scorecard.py b/scripts/asvs/scorecard.py index d31923d8..46e0991f 100644 --- a/scripts/asvs/scorecard.py +++ b/scripts/asvs/scorecard.py @@ -67,11 +67,25 @@ #: `needs-review` cells the two sets are identical, so no test and no CI run could tell them apart. EXAMINED_VERDICTS: Final[frozenset[str]] = DECIDED_VERDICTS | {"needs-review"} -#: How far from the recorded line the expected token may drift before the anchor is considered broken. -#: Anchors name a TOKEN rather than a bare line number precisely so that ordinary edits above a cell's -#: evidence do not thrash every anchor in the file; the window keeps the line number meaningful without -#: making it load-bearing. -ANCHOR_WINDOW: Final[int] = 40 +#: RETIRED 2026-08-09. There was an ``ANCHOR_WINDOW = 40`` here: a token found within 40 lines of its +#: recorded position passed silently, beyond it the anchor failed. It is gone rather than widened, +#: because measurement showed it could not do the job its docstring claimed. +#: +#: It could not disambiguate. ``check_anchors`` rejects a multi-occurrence token as AMBIGUOUS and +#: ``continue``s BEFORE any window test, so by the time a window could apply the token is unique, and a +#: unique token is located by searching for it. The window's own justification named the +#: ``UPDATE sessions SET revoked_at=`` pair, 19 lines apart — the exact case the uniqueness guard now +#: rejects first. +#: +#: What it actually did was decide, on an arbitrary threshold, which stale line numbers to keep quiet +#: about. Measured on 2026-08-09 against a green record: of 1,980 anchors, 731 (36.9%) resolved ONLY +#: because of the window, median offset 9, p90 30, MAX 39 — one line from the cliff, three days after a +#: re-anchor pass reset the whole distribution by hand-retyping 130 integers. A tolerance that is 37% +#: spent three days after a reset is not a tolerance; it is a decaying budget whose next expiry is one +#: insertion above a hot file away. +#: +#: So the line number left the decision path entirely: uniqueness locates the evidence, and the +#: recorded line is reported output that an advisory corrects. See :func:`check_anchors`. class ScorecardError(Exception): @@ -393,21 +407,33 @@ def check_completeness(cells: list[Cell], corpus: dict[str, int]) -> list[str]: def check_anchors(cells: list[Cell], root: Path, findings: Findings) -> None: """Open every evidence anchor and assert its token still resolves, and resolves UNAMBIGUOUSLY. - When the code moves, this reds a test — instead of the sentence rotting in place and the next - session funding work that is already done. - - **Uniqueness is not pedantry; it is what makes the resolution mean anything.** An ``expect`` that - occurs many times in its file resolves from almost anywhere: with ``await conn.rollback()`` - appearing 101 times in one module, *any* line number in that file lands within ±40 of some - occurrence, so the anchor cannot fail and certifies nothing. Two such anchors sat in this scorecard - as evidence for weeks. - - It also closes a defect in the REPAIR path rather than the detection path. When code moves, the - check correctly reports it — but a re-anchor to the nearest occurrence can silently install a - *stale-but-resolving* anchor that passes forever. That happened live: after ADR 0154 landed, - ``UPDATE sessions SET revoked_at=`` had two occurrences 19 lines apart — one the keep-N revoke, one - a different method entirely — each inside the other's window, so the check would have accepted the - wrong one. A repair is exactly where suspicion lapses, because the tool has just proved it works. + **Two outcomes, and they are not the same event.** A token that is GONE or AMBIGUOUS reds the gate: + the claim it supported may now be false, or cannot be checked. A token that is unique and present + but sits at a different line is an ADVISORY: the evidence is exactly where searching for it says it + is, and only the recorded number is stale. Conflating the two is what made this gate red every + morning for a reason nobody had to act on — and a gate in that state is one whose next real finding + gets waved through. There was one underneath: a cell asserting a control was absent that had since + been built. + + **Uniqueness is not pedantry; it is the whole locator.** An ``expect`` that occurs many times in its + file resolves from almost anywhere: with ``await conn.rollback()`` appearing 101 times in one + module, any line in that file sits near some occurrence, so the anchor cannot fail and certifies + nothing. Two such anchors sat in this scorecard as evidence for weeks. Because uniqueness now does + the locating alone, it is the strictest rule here. + + It also closes a defect in the REPAIR path rather than the detection path. A re-anchor to the + nearest occurrence can silently install a *stale-but-resolving* anchor that passes forever. That + happened live: after ADR 0154 landed, ``UPDATE sessions SET revoked_at=`` had two occurrences 19 + lines apart — one the keep-N revoke, one a different method entirely — and a positional check would + have accepted the wrong one. A repair is exactly where suspicion lapses, because the tool has just + proved it works. Rejecting the ambiguity outright is what makes the repair safe. + + **What this still cannot see, stated so nobody reads more into a green than is there.** ``expect`` + is matched as a substring of the file, so a statement that moves into a ``try``, into a different + function, or under a different condition still resolves. The anchor certifies *this token exists in + this file, once* — not *this control operates on the path the cell describes*. A cell can therefore + be green here and wrong: measured 2026-08-09 on 15.3.1, which was ``pass`` with every anchor + resolving while the control it named had a hole, found only by executing the code. """ for c in cells: for a in c.evidence: @@ -416,9 +442,6 @@ def check_anchors(cells: list[Cell], root: Path, findings: Findings) -> None: findings.problems.append(f"{c.id}: evidence path {a.path} does not exist") continue text = target.read_text(encoding="utf-8", errors="replace") - lines = text.splitlines() - lo = max(0, a.line - 1 - ANCHOR_WINDOW) - hi = min(len(lines), a.line + ANCHOR_WINDOW) findings.checked_anchors += 1 occurrences = text.count(a.expect) if occurrences > 1: @@ -428,38 +451,27 @@ def check_anchors(cells: list[Cell], root: Path, findings: Findings) -> None: "re-anchor cannot be checked. Cite a longer token that appears exactly once" ) continue - if a.expect in "\n".join(lines[lo:hi]): + if occurrences == 0: + findings.problems.append( + f"{c.id}: {a.path}:{a.line} no longer contains {a.expect!r} anywhere in the file " + "— the evidence is GONE, so the claim it supported may now be false" + ) continue - if a.expect in "\n".join(lines): - # MOVED, not broken — and for a UNIQUE token the line number was never the proof. - # Execution only reaches here PAST the ``occurrences > 1`` guard above, so the token - # occurs exactly once in this file: its presence alone pins the evidence, and the line - # is a navigation aid. Failing on it asserts something different from what this gate - # exists to assert — that the claim is still true, not that it sits where it sat. - # - # This is not a theoretical tidy-up. The scorecard lives in a repo that no longer - # develops this tree, while this tree lands ~21 commits/day, so ONE insertion above a - # hot file re-breaks every anchor below it: on 2026-08-07, 130 of 146 failures were a - # single constant offset per file (+57 auth/service.py, +61 __main__.py, +47 - # store/base.py) and the count went 67 -> 146 in ten hours. It recurred the next day - # (+102 on .github/workflows/ci.yml). A gate that is red every morning for a reason - # nobody must act on is a gate whose next REAL finding gets waved through — and there - # was one underneath: a cell asserting a control was absent that had since been built. - # - # What still reds the gate is unchanged and is the whole signal: a token that is GONE - # (the claim may now be false), an AMBIGUOUS token (there the line IS load-bearing and - # a re-anchor to the wrong occurrence cannot be detected), a missing evidence path, - # and every absence-claim failure. That narrows the gate to claim-invalidation, which - # is what ADR 0156 already describes as the intent. + # Unique, and therefore LOCATED: past the guard above, the token occurs exactly once in + # this file, so its presence alone pins the evidence and the line number proves nothing + # extra. Derive where it actually is and report any disagreement with the record. + # + # DERIVED FROM THE CHARACTER OFFSET, NOT BY SCANNING LINES. 42 of the ~1,980 ``expect`` + # tokens span a newline, because the old check matched against joined text and nothing + # forbade it. A per-line scan finds none of those and raises on the lookup; counting + # newlines before the match handles a multi-line token as naturally as a single-line one. + actual = text.count("\n", 0, text.index(a.expect)) + 1 + if actual != a.line: findings.advisories.append( - f"{c.id}: {a.path}:{a.line} moved — {a.expect!r} is still in the file (unique), " - f"but >±{ANCHOR_WINDOW} lines away. Re-anchor the line number" + f"{c.id}: {a.path} — {a.expect!r} is unique and present, recorded at line " + f"{a.line} but actually at {actual} (offset {actual - a.line:+d}). " + "Advisory: the line is a navigation aid, not the proof" ) - continue - findings.problems.append( - f"{c.id}: {a.path}:{a.line} no longer contains {a.expect!r} within " - f"±{ANCHOR_WINDOW} lines — the evidence moved or the claim is now false" - ) def check_absences(cells: list[Cell], root: Path, findings: Findings) -> None: @@ -982,9 +994,17 @@ def main(argv: list[str] | None = None) -> int: for a in findings.advisories: print(f" DRIFT {a}", file=sys.stderr) if findings.advisories: + pct = ( + 100.0 * len(findings.advisories) / findings.checked_anchors + if findings.checked_anchors + else 0.0 + ) print( - f" {len(findings.advisories)} anchor(s) drifted: evidence still present and unique, line " - "numbers stale. Not fatal — re-anchor them, do not let them accumulate.", + f" {len(findings.advisories)} of {findings.checked_anchors} anchors " + f"({pct:.1f}%) carry a stale line number: the evidence is present and unique, only the " + "recorded position is wrong. NOT fatal, and re-anchoring is bookkeeping rather than " + "assessment — but the percentage is the thing to watch, because it only ever grows " + "between re-anchor passes.", file=sys.stderr, ) for p in findings.problems: diff --git a/tests/test_asvs_scorecard.py b/tests/test_asvs_scorecard.py index 3a7538f6..17f9dba2 100644 --- a/tests/test_asvs_scorecard.py +++ b/tests/test_asvs_scorecard.py @@ -181,7 +181,12 @@ def test_a_unique_token_that_drifted_is_advisory_not_fatal(tmp_path: Path) -> No f = Findings() check_anchors(cells, tmp_path, f) assert f.ok and f.problems == [] - assert len(f.advisories) == 1 and "moved" in f.advisories[0] + assert len(f.advisories) == 1 + # The advisory carries BOTH numbers and the signed delta, because "it moved" is not actionable and + # "301 not 5" is. A re-anchor pass reads this line and needs no second lookup. + assert "recorded at line 5" in f.advisories[0] + assert "actually at 301" in f.advisories[0] + assert "+296" in f.advisories[0] def test_an_ambiguous_token_stays_fatal_even_when_it_drifted(tmp_path: Path) -> None: @@ -208,9 +213,18 @@ def test_an_ambiguous_token_stays_fatal_even_when_it_drifted(tmp_path: Path) -> assert f.advisories == [] -def test_ordinary_small_movement_is_neither_problem_nor_advisory(tmp_path: Path) -> None: - """The control. Without it, a test asserting "drift is advisory" would still pass if EVERY anchor - started reporting drift — the window would have stopped absorbing anything and nothing would say so.""" +def test_a_small_movement_is_advisory_too_not_silent(tmp_path: Path) -> None: + """CONTRACT CHANGE 2026-08-09, and this test previously asserted the opposite. + + It used to be named ``test_ordinary_small_movement_is_neither_problem_nor_advisory`` and pinned a + six-line offset as reporting NOTHING, because a 40-line window absorbed it. That silence was the + defect: measured against a green record, 725 of 1,980 anchors (36.6%) were inside the window and + therefore invisible, with the worst at 39 against a limit of 40. An advisory that fires only past + the window merges into an empty list and prevents the next recurrence while surfacing none of the + accumulation. + + So the rule is now: any nonzero offset is advisory. The window is gone from the decision path. + """ (tmp_path / "messagefoundry").mkdir() body = "\n".join(["filler"] * 20 + ["tls_cert_file = None"] + ["filler"] * 20) (tmp_path / "messagefoundry" / "m.py").write_text(body, encoding="utf-8") @@ -224,9 +238,62 @@ def test_ordinary_small_movement_is_neither_problem_nor_advisory(tmp_path: Path) ] f = Findings() check_anchors(cells, tmp_path, f) + assert f.problems == [] # still not fatal + assert len(f.advisories) == 1 and "+6" in f.advisories[0] + + +def test_an_exactly_located_anchor_reports_nothing(tmp_path: Path) -> None: + """The control, in the shape the new contract needs. + + Its predecessor guarded against "the window stopped absorbing anything and nothing said so". That + risk is retired with the window. The live risk now is the mirror image: a resolver that reports + drift for EVERY anchor — an off-by-one in the line derivation would do it — while the advisory + tests above still pass, because they only assert that an advisory appears. This asserts the zero. + """ + (tmp_path / "messagefoundry").mkdir() + body = "\n".join(["filler"] * 20 + ["tls_cert_file = None"] + ["filler"] * 20) + (tmp_path / "messagefoundry" / "m.py").write_text(body, encoding="utf-8") + cells = [ + Cell( + id="1.1.1", + level=1, + verdict="pass", + # 20 filler lines, so the token is line 21. Recorded exactly. + evidence=(Anchor("messagefoundry/m.py", 21, "tls_cert_file"),), + ) + ] + f = Findings() + check_anchors(cells, tmp_path, f) assert f.problems == [] and f.advisories == [] +def test_a_multi_line_expect_token_resolves_and_reports_its_start_line(tmp_path: Path) -> None: + """42 of the ~1,980 live ``expect`` tokens SPAN A NEWLINE. + + The old check matched against joined text, so nothing ever forbade a multi-line token and 42 + accumulated. Deriving the line by scanning ``splitlines()`` finds none of them and raises on the + lookup — measured, it bit the parallel session's first measurement script. Counting newlines before + the character offset handles a multi-line token as naturally as a single-line one, and the line it + reports is the token's FIRST line, which is what a reader navigating to it wants. + """ + (tmp_path / "messagefoundry").mkdir() + body = "\n".join(["filler"] * 10 + ["def f(", " x: int,", ") -> None:"] + ["tail"] * 5) + (tmp_path / "messagefoundry" / "m.py").write_text(body, encoding="utf-8") + cells = [ + Cell( + id="1.1.1", + level=1, + verdict="pass", + evidence=(Anchor("messagefoundry/m.py", 3, "def f(\n x: int,"),), + ) + ] + f = Findings() + check_anchors(cells, tmp_path, f) + assert f.problems == [] # it resolves; a line-scanning implementation would not find it at all + assert len(f.advisories) == 1 + assert "actually at 11" in f.advisories[0] # the token's FIRST line, not its last + + def test_anchor_goes_red_when_the_file_is_gone(tmp_path: Path) -> None: cells = [ Cell( From 68b425667ccab315fe3f0509eabb451b6e5371e0 Mon Sep 17 00:00:00 2001 From: wshallwshall Date: Sun, 9 Aug 2026 00:10:46 -0500 Subject: [PATCH 3/4] fix(asvs): refuse to propose a replacement anchor for a GONE token A GONE token has FOUR causes and only two are re-anchors: it moved beyond detection, it was renamed, THE GAP IT CERTIFIED WAS CLOSED, or its control was removed. A tool that helpfully suggests the nearest similar line collapses all four into the first, and case three is the dangerous one -- re-anchoring to the code that CLOSED a gap, while the residual still narrates the gap, yields an anchor that resolves forever while asserting the opposite of the truth. Worked example, measured this session and named in the code: 3.7.5 anchored pyproject.toml testpaths = ["tests"]. BACKLOG #1027 widened it, so the token vanished -- but the anchor certified an EXCLUSION that had just been CLOSED, because the guard's test sits inside the path #1027 added. Re-anchoring would have been silently wrong. It was retired instead. So the failure message now puts the retire-vs-rescore fork in front of the reader and says 'do not re-anchor by default', and a test enforces the refusal: the message must not fuzzy-suggest a target even when a plausible near-match sits on the very next line. Enforced rather than documented, because a convention is what decays -- the fixture reproduces 3.7.5's exact shape, which is when a suggestion would be most tempting and most wrong. Taxonomy contributed by the asvs-tracking-rework session, which adopted the retire-as-closed class after I hit it; this is its enforcement half. --- scripts/asvs/scorecard.py | 27 +++++++++++++++++++++++++- tests/test_asvs_scorecard.py | 37 ++++++++++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+), 1 deletion(-) diff --git a/scripts/asvs/scorecard.py b/scripts/asvs/scorecard.py index 46e0991f..1392deab 100644 --- a/scripts/asvs/scorecard.py +++ b/scripts/asvs/scorecard.py @@ -452,9 +452,34 @@ def check_anchors(cells: list[Cell], root: Path, findings: Findings) -> None: ) continue if occurrences == 0: + # DELIBERATE NON-AFFORDANCE: this branch does NOT propose a replacement anchor, and + # must not be "improved" to fuzzy-match a nearby similar line and suggest one. That + # single affordance is what manufactures silent corruption, because a GONE token has + # FOUR possible causes and only two of them are re-anchors: + # + # (a) moved beyond detection, control intact -> re-anchor (mechanical) + # (b) renamed or refactored, control intact -> re-anchor (judgment) + # (c) THE GAP THIS ANCHOR CERTIFIED WAS CLOSED -> RETIRE the anchor, rewrite the + # residual; the verdict may IMPROVE + # (d) the control was removed or weakened -> the claim is broken; RE-SCORE + # + # Worked example of (c), measured 2026-08-09: cell 3.7.5 anchored + # `pyproject.toml:311 testpaths = ["tests"]`. BACKLOG #1027 widened testpaths to + # include the web console package, so the token vanished -- but the anchor existed to + # certify an EXCLUSION (the bucket-drift guard does not run), and that exclusion had + # just been CLOSED, because the guard's test sits inside the path #1027 added. A + # re-anchor to the new line would have pointed the anchor at the code that closed the + # gap while the residual still narrated the gap: a stale-but-resolving anchor, + # green forever, asserting the opposite of the truth. It was retired instead. + # + # A human distinguishes (a)-(d) by reading the cell. A tool cannot, so this one says + # what it found and stops. Reporting candidate locations would be acceptable; + # recommending one is not. findings.problems.append( f"{c.id}: {a.path}:{a.line} no longer contains {a.expect!r} anywhere in the file " - "— the evidence is GONE, so the claim it supported may now be false" + "— the evidence is GONE. Re-read the cell before touching the anchor: the token " + "may have moved, been renamed, had the gap it certified CLOSED (retire it), or " + "had its control removed (re-score). Do not re-anchor by default" ) continue # Unique, and therefore LOCATED: past the guard above, the token occurs exactly once in diff --git a/tests/test_asvs_scorecard.py b/tests/test_asvs_scorecard.py index 17f9dba2..b81f5884 100644 --- a/tests/test_asvs_scorecard.py +++ b/tests/test_asvs_scorecard.py @@ -160,6 +160,43 @@ def test_anchor_goes_red_when_the_token_is_gone(tmp_path: Path) -> None: assert not f.ok and "no longer contains" in f.problems[0] +def test_a_gone_token_is_not_offered_a_replacement_anchor(tmp_path: Path) -> None: + """The refusal is ENFORCED, not a convention, because the convention is what decays. + + A GONE token has four possible causes and only two are re-anchors: it moved, it was renamed, THE + GAP IT CERTIFIED WAS CLOSED, or its control was removed. A tool that helpfully suggests the + nearest similar line collapses all four into the first, and the (c) case is the dangerous one -- + re-anchoring to the code that CLOSED a gap, while the residual still narrates the gap, yields an + anchor that resolves forever while asserting the opposite of the truth. + + Measured instance: 3.7.5 at engine `71dfc2ce`. The file below reproduces its shape -- the old + token is gone and a plausible near-match sits right there, which is exactly when a fuzzy + suggestion would be most tempting and most wrong. + """ + (tmp_path / "messagefoundry").mkdir() + (tmp_path / "messagefoundry" / "m.py").write_text( + 'testpaths = ["tests", "packaging/messagefoundry-webconsole/tests"]\n', encoding="utf-8" + ) + cells = [ + Cell( + id="3.7.5", + level=3, + verdict="partial", + evidence=(Anchor("messagefoundry/m.py", 1, 'testpaths = ["tests"]'),), + ) + ] + f = Findings() + check_anchors(cells, tmp_path, f) + assert not f.ok + msg = f.problems[0] + # It must NOT name a line to move to, nor tell the reader to re-anchor. + assert "did you mean" not in msg.lower() + assert "re-anchor to" not in msg.lower() + assert "Do not re-anchor by default" in msg + # And it must put the retire-vs-rescore fork in front of the reader. + assert "CLOSED" in msg and "re-score" in msg + + def test_a_unique_token_that_drifted_is_advisory_not_fatal(tmp_path: Path) -> None: """DRIFT is not INVALIDATION. The token sits far below its recorded line but occurs exactly once. From f4738ce3b8c2e375b38c64daa9d1514864f3f387 Mon Sep 17 00:00:00 2001 From: wshallwshall Date: Sun, 9 Aug 2026 00:57:08 -0500 Subject: [PATCH 4/4] fix(asvs): the summary said "verified" and the check cannot support it On the line that IS the record's rendered face. An anchor check proves the token is present and unique in its file. It does not prove the statement still executes under the control flow the cell reasoned about, and it cannot prove the cell's conclusion follows -- `expect` is matched as a SUBSTRING, so a statement that moved inside a try, into another function, or under a different condition still resolves. Measured instance: 15.3.1 sat at `pass` with every anchor resolving while the control it named had a hole, found only by EXECUTING the code. "verified" invites exactly the inference the tool cannot make, and it is the summary a reader quotes. Now: "resolved N evidence anchors (token present and unique -- NOT proof the control operates) and checked M absence claims". Contributed by the parallel asvs-tracking-rework session's substring analysis, whose framing is that an Anchor models a PREMISE and Cell.verdict models a CONCLUSION, with no object for the inference step between them -- so no check can attach there, and the summary must stop implying one did. --- scripts/asvs/scorecard.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/scripts/asvs/scorecard.py b/scripts/asvs/scorecard.py index 58d5dcc5..dc978efc 100644 --- a/scripts/asvs/scorecard.py +++ b/scripts/asvs/scorecard.py @@ -1051,7 +1051,17 @@ def main(argv: list[str] | None = None) -> int: f"scanned {len(cells)} cells " f"({n['pass']} pass / {n['partial']} partial / {n['fail']} fail / {n['na']} na / " f"{n['unverified']} unverified); " - f"verified {findings.checked_anchors} evidence anchors and {findings.checked_absences} absence claims" + # "verified" OVERCLAIMED, on the line that IS the record's rendered face. An anchor check + # proves the token is present and unique in the file. It does not prove the statement still + # executes under the control flow the cell reasoned about, and it cannot prove the cell's + # conclusion follows from it -- `expect` is matched as a substring, so a statement that moved + # inside a `try`, into another function, or under a different condition still resolves. + # Measured instance: 15.3.1 sat at `pass` with every anchor resolving while the control it + # named had a hole, found only by EXECUTING the code. A summary that says "verified" invites + # exactly the inference the tool cannot support. + f"resolved {findings.checked_anchors} evidence anchors " + f"(token present and unique -- NOT proof the control operates) " + f"and checked {findings.checked_absences} absence claims" ) for a in findings.advisories: print(f" DRIFT {a}", file=sys.stderr)