diff --git a/scripts/asvs/scorecard.py b/scripts/asvs/scorecard.py index ad4664e0..dc978efc 100644 --- a/scripts/asvs/scorecard.py +++ b/scripts/asvs/scorecard.py @@ -67,11 +67,25 @@ #: `needs-review` cells the two sets are identical, so no test and no CI run could tell them apart. EXAMINED_VERDICTS: Final[frozenset[str]] = DECIDED_VERDICTS | {"needs-review"} -#: How far from the recorded line the expected token may drift before the anchor is considered broken. -#: Anchors name a TOKEN rather than a bare line number precisely so that ordinary edits above a cell's -#: evidence do not thrash every anchor in the file; the window keeps the line number meaningful without -#: making it load-bearing. -ANCHOR_WINDOW: Final[int] = 40 +#: RETIRED 2026-08-09. There was an ``ANCHOR_WINDOW = 40`` here: a token found within 40 lines of its +#: recorded position passed silently, beyond it the anchor failed. It is gone rather than widened, +#: because measurement showed it could not do the job its docstring claimed. +#: +#: It could not disambiguate. ``check_anchors`` rejects a multi-occurrence token as AMBIGUOUS and +#: ``continue``s BEFORE any window test, so by the time a window could apply the token is unique, and a +#: unique token is located by searching for it. The window's own justification named the +#: ``UPDATE sessions SET revoked_at=`` pair, 19 lines apart — the exact case the uniqueness guard now +#: rejects first. +#: +#: What it actually did was decide, on an arbitrary threshold, which stale line numbers to keep quiet +#: about. Measured on 2026-08-09 against a green record: of 1,980 anchors, 731 (36.9%) resolved ONLY +#: because of the window, median offset 9, p90 30, MAX 39 — one line from the cliff, three days after a +#: re-anchor pass reset the whole distribution by hand-retyping 130 integers. A tolerance that is 37% +#: spent three days after a reset is not a tolerance; it is a decaying budget whose next expiry is one +#: insertion above a hot file away. +#: +#: So the line number left the decision path entirely: uniqueness locates the evidence, and the +#: recorded line is reported output that an advisory corrects. See :func:`check_anchors`. class ScorecardError(Exception): @@ -176,6 +190,10 @@ class Findings: """What a verification pass found. Empty ``problems`` is the only pass condition.""" problems: list[str] = field(default_factory=list) + #: Reported, never fatal. A UNIQUE evidence token that merely MOVED is not a broken claim — see + #: :func:`check_anchors`. These still print, because letting them accumulate silently is how the + #: recorded line numbers rot; they just do not red the gate. + advisories: list[str] = field(default_factory=list) checked_anchors: int = 0 checked_absences: int = 0 skipped_anchors: int = 0 @@ -389,21 +407,33 @@ def check_completeness(cells: list[Cell], corpus: dict[str, int]) -> list[str]: def check_anchors(cells: list[Cell], root: Path, findings: Findings) -> None: """Open every evidence anchor and assert its token still resolves, and resolves UNAMBIGUOUSLY. - When the code moves, this reds a test — instead of the sentence rotting in place and the next - session funding work that is already done. - - **Uniqueness is not pedantry; it is what makes the resolution mean anything.** An ``expect`` that - occurs many times in its file resolves from almost anywhere: with ``await conn.rollback()`` - appearing 101 times in one module, *any* line number in that file lands within ±40 of some - occurrence, so the anchor cannot fail and certifies nothing. Two such anchors sat in this scorecard - as evidence for weeks. - - It also closes a defect in the REPAIR path rather than the detection path. When code moves, the - check correctly reports it — but a re-anchor to the nearest occurrence can silently install a - *stale-but-resolving* anchor that passes forever. That happened live: after ADR 0154 landed, - ``UPDATE sessions SET revoked_at=`` had two occurrences 19 lines apart — one the keep-N revoke, one - a different method entirely — each inside the other's window, so the check would have accepted the - wrong one. A repair is exactly where suspicion lapses, because the tool has just proved it works. + **Two outcomes, and they are not the same event.** A token that is GONE or AMBIGUOUS reds the gate: + the claim it supported may now be false, or cannot be checked. A token that is unique and present + but sits at a different line is an ADVISORY: the evidence is exactly where searching for it says it + is, and only the recorded number is stale. Conflating the two is what made this gate red every + morning for a reason nobody had to act on — and a gate in that state is one whose next real finding + gets waved through. There was one underneath: a cell asserting a control was absent that had since + been built. + + **Uniqueness is not pedantry; it is the whole locator.** An ``expect`` that occurs many times in its + file resolves from almost anywhere: with ``await conn.rollback()`` appearing 101 times in one + module, any line in that file sits near some occurrence, so the anchor cannot fail and certifies + nothing. Two such anchors sat in this scorecard as evidence for weeks. Because uniqueness now does + the locating alone, it is the strictest rule here. + + It also closes a defect in the REPAIR path rather than the detection path. A re-anchor to the + nearest occurrence can silently install a *stale-but-resolving* anchor that passes forever. That + happened live: after ADR 0154 landed, ``UPDATE sessions SET revoked_at=`` had two occurrences 19 + lines apart — one the keep-N revoke, one a different method entirely — and a positional check would + have accepted the wrong one. A repair is exactly where suspicion lapses, because the tool has just + proved it works. Rejecting the ambiguity outright is what makes the repair safe. + + **What this still cannot see, stated so nobody reads more into a green than is there.** ``expect`` + is matched as a substring of the file, so a statement that moves into a ``try``, into a different + function, or under a different condition still resolves. The anchor certifies *this token exists in + this file, once* — not *this control operates on the path the cell describes*. A cell can therefore + be green here and wrong: measured 2026-08-09 on 15.3.1, which was ``pass`` with every anchor + resolving while the control it named had a hole, found only by executing the code. """ for c in cells: for a in c.evidence: @@ -412,9 +442,6 @@ def check_anchors(cells: list[Cell], root: Path, findings: Findings) -> None: findings.problems.append(f"{c.id}: evidence path {a.path} does not exist") continue text = target.read_text(encoding="utf-8", errors="replace") - lines = text.splitlines() - lo = max(0, a.line - 1 - ANCHOR_WINDOW) - hi = min(len(lines), a.line + ANCHOR_WINDOW) findings.checked_anchors += 1 occurrences = text.count(a.expect) if occurrences > 1: @@ -424,13 +451,52 @@ def check_anchors(cells: list[Cell], root: Path, findings: Findings) -> None: "re-anchor cannot be checked. Cite a longer token that appears exactly once" ) continue - if a.expect in "\n".join(lines[lo:hi]): + if occurrences == 0: + # DELIBERATE NON-AFFORDANCE: this branch does NOT propose a replacement anchor, and + # must not be "improved" to fuzzy-match a nearby similar line and suggest one. That + # single affordance is what manufactures silent corruption, because a GONE token has + # FOUR possible causes and only two of them are re-anchors: + # + # (a) moved beyond detection, control intact -> re-anchor (mechanical) + # (b) renamed or refactored, control intact -> re-anchor (judgment) + # (c) THE GAP THIS ANCHOR CERTIFIED WAS CLOSED -> RETIRE the anchor, rewrite the + # residual; the verdict may IMPROVE + # (d) the control was removed or weakened -> the claim is broken; RE-SCORE + # + # Worked example of (c), measured 2026-08-09: cell 3.7.5 anchored + # `pyproject.toml:311 testpaths = ["tests"]`. BACKLOG #1027 widened testpaths to + # include the web console package, so the token vanished -- but the anchor existed to + # certify an EXCLUSION (the bucket-drift guard does not run), and that exclusion had + # just been CLOSED, because the guard's test sits inside the path #1027 added. A + # re-anchor to the new line would have pointed the anchor at the code that closed the + # gap while the residual still narrated the gap: a stale-but-resolving anchor, + # green forever, asserting the opposite of the truth. It was retired instead. + # + # A human distinguishes (a)-(d) by reading the cell. A tool cannot, so this one says + # what it found and stops. Reporting candidate locations would be acceptable; + # recommending one is not. + findings.problems.append( + f"{c.id}: {a.path}:{a.line} no longer contains {a.expect!r} anywhere in the file " + "— the evidence is GONE. Re-read the cell before touching the anchor: the token " + "may have moved, been renamed, had the gap it certified CLOSED (retire it), or " + "had its control removed (re-score). Do not re-anchor by default" + ) continue - where = " (found elsewhere in the file)" if a.expect in "\n".join(lines) else "" - findings.problems.append( - f"{c.id}: {a.path}:{a.line} no longer contains {a.expect!r} within " - f"±{ANCHOR_WINDOW} lines{where} — the evidence moved or the claim is now false" - ) + # Unique, and therefore LOCATED: past the guard above, the token occurs exactly once in + # this file, so its presence alone pins the evidence and the line number proves nothing + # extra. Derive where it actually is and report any disagreement with the record. + # + # DERIVED FROM THE CHARACTER OFFSET, NOT BY SCANNING LINES. 42 of the ~1,980 ``expect`` + # tokens span a newline, because the old check matched against joined text and nothing + # forbade it. A per-line scan finds none of those and raises on the lookup; counting + # newlines before the match handles a multi-line token as naturally as a single-line one. + actual = text.count("\n", 0, text.index(a.expect)) + 1 + if actual != a.line: + findings.advisories.append( + f"{c.id}: {a.path} — {a.expect!r} is unique and present, recorded at line " + f"{a.line} but actually at {actual} (offset {actual - a.line:+d}). " + "Advisory: the line is a navigation aid, not the proof" + ) def check_absences(cells: list[Cell], root: Path, findings: Findings) -> None: @@ -985,8 +1051,34 @@ def main(argv: list[str] | None = None) -> int: f"scanned {len(cells)} cells " f"({n['pass']} pass / {n['partial']} partial / {n['fail']} fail / {n['na']} na / " f"{n['unverified']} unverified); " - f"verified {findings.checked_anchors} evidence anchors and {findings.checked_absences} absence claims" + # "verified" OVERCLAIMED, on the line that IS the record's rendered face. An anchor check + # proves the token is present and unique in the file. It does not prove the statement still + # executes under the control flow the cell reasoned about, and it cannot prove the cell's + # conclusion follows from it -- `expect` is matched as a substring, so a statement that moved + # inside a `try`, into another function, or under a different condition still resolves. + # Measured instance: 15.3.1 sat at `pass` with every anchor resolving while the control it + # named had a hole, found only by EXECUTING the code. A summary that says "verified" invites + # exactly the inference the tool cannot support. + f"resolved {findings.checked_anchors} evidence anchors " + f"(token present and unique -- NOT proof the control operates) " + f"and checked {findings.checked_absences} absence claims" ) + for a in findings.advisories: + print(f" DRIFT {a}", file=sys.stderr) + if findings.advisories: + pct = ( + 100.0 * len(findings.advisories) / findings.checked_anchors + if findings.checked_anchors + else 0.0 + ) + print( + f" {len(findings.advisories)} of {findings.checked_anchors} anchors " + f"({pct:.1f}%) carry a stale line number: the evidence is present and unique, only the " + "recorded position is wrong. NOT fatal, and re-anchoring is bookkeeping rather than " + "assessment — but the percentage is the thing to watch, because it only ever grows " + "between re-anchor passes.", + file=sys.stderr, + ) for p in findings.problems: print(f" FAIL {p}", file=sys.stderr) diff --git a/tests/test_asvs_scorecard.py b/tests/test_asvs_scorecard.py index 58fd0bc3..b81f5884 100644 --- a/tests/test_asvs_scorecard.py +++ b/tests/test_asvs_scorecard.py @@ -160,6 +160,177 @@ def test_anchor_goes_red_when_the_token_is_gone(tmp_path: Path) -> None: assert not f.ok and "no longer contains" in f.problems[0] +def test_a_gone_token_is_not_offered_a_replacement_anchor(tmp_path: Path) -> None: + """The refusal is ENFORCED, not a convention, because the convention is what decays. + + A GONE token has four possible causes and only two are re-anchors: it moved, it was renamed, THE + GAP IT CERTIFIED WAS CLOSED, or its control was removed. A tool that helpfully suggests the + nearest similar line collapses all four into the first, and the (c) case is the dangerous one -- + re-anchoring to the code that CLOSED a gap, while the residual still narrates the gap, yields an + anchor that resolves forever while asserting the opposite of the truth. + + Measured instance: 3.7.5 at engine `71dfc2ce`. The file below reproduces its shape -- the old + token is gone and a plausible near-match sits right there, which is exactly when a fuzzy + suggestion would be most tempting and most wrong. + """ + (tmp_path / "messagefoundry").mkdir() + (tmp_path / "messagefoundry" / "m.py").write_text( + 'testpaths = ["tests", "packaging/messagefoundry-webconsole/tests"]\n', encoding="utf-8" + ) + cells = [ + Cell( + id="3.7.5", + level=3, + verdict="partial", + evidence=(Anchor("messagefoundry/m.py", 1, 'testpaths = ["tests"]'),), + ) + ] + f = Findings() + check_anchors(cells, tmp_path, f) + assert not f.ok + msg = f.problems[0] + # It must NOT name a line to move to, nor tell the reader to re-anchor. + assert "did you mean" not in msg.lower() + assert "re-anchor to" not in msg.lower() + assert "Do not re-anchor by default" in msg + # And it must put the retire-vs-rescore fork in front of the reader. + assert "CLOSED" in msg and "re-score" in msg + + +def test_a_unique_token_that_drifted_is_advisory_not_fatal(tmp_path: Path) -> None: + """DRIFT is not INVALIDATION. The token sits far below its recorded line but occurs exactly once. + + Execution only reaches the drift branch PAST the ``occurrences > 1`` guard, so uniqueness is what + pins the evidence and the line number is navigation. Failing here would assert that the claim sits + where it sat, which is a different proposition from the one this gate exists to check. + """ + (tmp_path / "messagefoundry").mkdir() + body = "\n".join(["filler"] * 300 + ["tls_cert_file = None"] + ["tail"] * 5) + (tmp_path / "messagefoundry" / "m.py").write_text(body, encoding="utf-8") + cells = [ + Cell( + id="1.1.1", + level=1, + verdict="pass", + evidence=(Anchor("messagefoundry/m.py", 5, "tls_cert_file"),), + ) + ] + f = Findings() + check_anchors(cells, tmp_path, f) + assert f.ok and f.problems == [] + assert len(f.advisories) == 1 + # The advisory carries BOTH numbers and the signed delta, because "it moved" is not actionable and + # "301 not 5" is. A re-anchor pass reads this line and needs no second lookup. + assert "recorded at line 5" in f.advisories[0] + assert "actually at 301" in f.advisories[0] + assert "+296" in f.advisories[0] + + +def test_an_ambiguous_token_stays_fatal_even_when_it_drifted(tmp_path: Path) -> None: + """Where the line IS load-bearing, drift must not soften it. + + With two occurrences a re-anchor to the WRONG one cannot be detected — the defect this module's + uniqueness rule exists to catch. Advisory treatment is reserved for the case where the token + itself certifies the evidence. + """ + (tmp_path / "messagefoundry").mkdir() + body = "\n".join(["filler"] * 200 + ["dupe_token"] + ["filler"] * 200 + ["dupe_token"]) + (tmp_path / "messagefoundry" / "m.py").write_text(body, encoding="utf-8") + cells = [ + Cell( + id="1.1.1", + level=1, + verdict="pass", + evidence=(Anchor("messagefoundry/m.py", 1, "dupe_token"),), + ) + ] + f = Findings() + check_anchors(cells, tmp_path, f) + assert not f.ok and "AMBIGUOUS" in f.problems[0] + assert f.advisories == [] + + +def test_a_small_movement_is_advisory_too_not_silent(tmp_path: Path) -> None: + """CONTRACT CHANGE 2026-08-09, and this test previously asserted the opposite. + + It used to be named ``test_ordinary_small_movement_is_neither_problem_nor_advisory`` and pinned a + six-line offset as reporting NOTHING, because a 40-line window absorbed it. That silence was the + defect: measured against a green record, 725 of 1,980 anchors (36.6%) were inside the window and + therefore invisible, with the worst at 39 against a limit of 40. An advisory that fires only past + the window merges into an empty list and prevents the next recurrence while surfacing none of the + accumulation. + + So the rule is now: any nonzero offset is advisory. The window is gone from the decision path. + """ + (tmp_path / "messagefoundry").mkdir() + body = "\n".join(["filler"] * 20 + ["tls_cert_file = None"] + ["filler"] * 20) + (tmp_path / "messagefoundry" / "m.py").write_text(body, encoding="utf-8") + cells = [ + Cell( + id="1.1.1", + level=1, + verdict="pass", + evidence=(Anchor("messagefoundry/m.py", 15, "tls_cert_file"),), + ) + ] + f = Findings() + check_anchors(cells, tmp_path, f) + assert f.problems == [] # still not fatal + assert len(f.advisories) == 1 and "+6" in f.advisories[0] + + +def test_an_exactly_located_anchor_reports_nothing(tmp_path: Path) -> None: + """The control, in the shape the new contract needs. + + Its predecessor guarded against "the window stopped absorbing anything and nothing said so". That + risk is retired with the window. The live risk now is the mirror image: a resolver that reports + drift for EVERY anchor — an off-by-one in the line derivation would do it — while the advisory + tests above still pass, because they only assert that an advisory appears. This asserts the zero. + """ + (tmp_path / "messagefoundry").mkdir() + body = "\n".join(["filler"] * 20 + ["tls_cert_file = None"] + ["filler"] * 20) + (tmp_path / "messagefoundry" / "m.py").write_text(body, encoding="utf-8") + cells = [ + Cell( + id="1.1.1", + level=1, + verdict="pass", + # 20 filler lines, so the token is line 21. Recorded exactly. + evidence=(Anchor("messagefoundry/m.py", 21, "tls_cert_file"),), + ) + ] + f = Findings() + check_anchors(cells, tmp_path, f) + assert f.problems == [] and f.advisories == [] + + +def test_a_multi_line_expect_token_resolves_and_reports_its_start_line(tmp_path: Path) -> None: + """42 of the ~1,980 live ``expect`` tokens SPAN A NEWLINE. + + The old check matched against joined text, so nothing ever forbade a multi-line token and 42 + accumulated. Deriving the line by scanning ``splitlines()`` finds none of them and raises on the + lookup — measured, it bit the parallel session's first measurement script. Counting newlines before + the character offset handles a multi-line token as naturally as a single-line one, and the line it + reports is the token's FIRST line, which is what a reader navigating to it wants. + """ + (tmp_path / "messagefoundry").mkdir() + body = "\n".join(["filler"] * 10 + ["def f(", " x: int,", ") -> None:"] + ["tail"] * 5) + (tmp_path / "messagefoundry" / "m.py").write_text(body, encoding="utf-8") + cells = [ + Cell( + id="1.1.1", + level=1, + verdict="pass", + evidence=(Anchor("messagefoundry/m.py", 3, "def f(\n x: int,"),), + ) + ] + f = Findings() + check_anchors(cells, tmp_path, f) + assert f.problems == [] # it resolves; a line-scanning implementation would not find it at all + assert len(f.advisories) == 1 + assert "actually at 11" in f.advisories[0] # the token's FIRST line, not its last + + def test_anchor_goes_red_when_the_file_is_gone(tmp_path: Path) -> None: cells = [ Cell(