From f3b44018c428fd64485735dc7017caf8bd068596 Mon Sep 17 00:00:00 2001 From: vahid-ahmadi Date: Fri, 14 Aug 2026 13:10:23 +0100 Subject: [PATCH 1/5] Mode-3 case schema + first UK battery for the UKMOD/EUROMOD lane MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Groundwork for #41, designed as the single case-level schema mode 3 shares with the TAXSIM lane (#5) — country on the case and oracle on the result row keep it engine-agnostic. - sources/ukmod-cases/SCHEMA.md: the case-diff contract — engine-neutral household input vocabulary, result-row shape, the closed fail-loud classification set (match_exact / match_within_tolerance / pe_gap / oracle_difference / policy_scope_mismatch / rounding / unclassified), and per-variable-class tolerances (currency GBP 0.01/week = 0.52/year; booleans exact). - sources/ukmod-cases/battery/cases.json: 14 curated UK hypothetical households, inputs + expected focus only (no fabricated expected values): UC standard/housing, lone parent + LHA, two-child limit with the pre-April-2017 protection, London benefit cap, taper + work allowance, minimum income floor, Pension Credit single/couple, mixed-age couple routed to UC, Scottish bands + Scottish Child Payment, HICBC mid-taper, NI threshold edges with salary sacrifice, carer CA/UC interaction, and the GBP 100k personal-allowance taper. - scorecard_db/case_diffs.py: CaseSpec/CaseResult dataclasses (closed enums raise on unknown values, models.py doctrine), battery loader, and the classify() first-pass classifier (above tolerance defaults to unclassified — adjudication, never a flattering bucket). - tests/test_case_schema.py: battery validation, unique ids, inputs-only invariant, classifier and result-row edge cases. - data/lanes.json (+ app mirror): ukmod-cases registered -> cataloged; connector run pending the UKMOD environment. Co-Authored-By: Claude Fable 5 --- app/public/data/lanes.json | 8 +- data/lanes.json | 8 +- scorecard_db/__init__.py | 18 ++ scorecard_db/case_diffs.py | 317 +++++++++++++++++++++++++ sources/ukmod-cases/SCHEMA.md | 184 ++++++++++++++ sources/ukmod-cases/battery/cases.json | 291 +++++++++++++++++++++++ tests/test_case_schema.py | 235 ++++++++++++++++++ 7 files changed, 1053 insertions(+), 8 deletions(-) create mode 100644 scorecard_db/case_diffs.py create mode 100644 sources/ukmod-cases/SCHEMA.md create mode 100644 sources/ukmod-cases/battery/cases.json create mode 100644 tests/test_case_schema.py diff --git a/app/public/data/lanes.json b/app/public/data/lanes.json index d2ea06a..68dab29 100644 --- a/app/public/data/lanes.json +++ b/app/public/data/lanes.json @@ -4,7 +4,7 @@ "country": "US | UK \u2014 the model instance the lane feeds (issue #42)", "note": "Repo-committed status feed (issue #7); graduate to Supabase when lane count makes commits noisy. Every lane writes its transition here." }, - "updated": "2026-08-19", + "updated": "2026-08-20", "lanes": [ { "id": "urban-sotsn", @@ -265,10 +265,10 @@ "source": "UKMOD/EUROMOD", "area": "household cases via JRC connector", "mode": 3, - "stage": "registered", + "stage": "cataloged", "running": false, - "updated": "2026-08-01", - "note": "Connector runnable per axiom-oracles#264 (#5)" + "updated": "2026-08-14", + "note": "Case schema + 1st battery landed (#41); connector run pending UKMOD environment; schema shared with taxsim-cases (#5)" }, { "id": "cbo-baseline", diff --git a/data/lanes.json b/data/lanes.json index d2ea06a..68dab29 100644 --- a/data/lanes.json +++ b/data/lanes.json @@ -4,7 +4,7 @@ "country": "US | UK \u2014 the model instance the lane feeds (issue #42)", "note": "Repo-committed status feed (issue #7); graduate to Supabase when lane count makes commits noisy. Every lane writes its transition here." }, - "updated": "2026-08-19", + "updated": "2026-08-20", "lanes": [ { "id": "urban-sotsn", @@ -265,10 +265,10 @@ "source": "UKMOD/EUROMOD", "area": "household cases via JRC connector", "mode": 3, - "stage": "registered", + "stage": "cataloged", "running": false, - "updated": "2026-08-01", - "note": "Connector runnable per axiom-oracles#264 (#5)" + "updated": "2026-08-14", + "note": "Case schema + 1st battery landed (#41); connector run pending UKMOD environment; schema shared with taxsim-cases (#5)" }, { "id": "cbo-baseline", diff --git a/scorecard_db/__init__.py b/scorecard_db/__init__.py index 11d9483..3554b41 100644 --- a/scorecard_db/__init__.py +++ b/scorecard_db/__init__.py @@ -1,3 +1,13 @@ +from .case_diffs import ( + DEFAULT_TOLERANCES, + CaseResult, + CaseSpec, + DiffClassification, + Oracle, + VariableClass, + classify, + load_battery, +) from .db import ScorecardDB from .models import ( BASELINE, @@ -18,16 +28,24 @@ __all__ = [ "BASELINE", "CURRENT_LAW_DESCRIPTOR", + "DEFAULT_TOLERANCES", "STANDARD_CONDITIONS", "CalibrationRelationship", + "CaseResult", + "CaseSpec", "ComparisonStatus", "DiagnosisClass", + "DiffClassification", "ExternalScore", "Metric", + "Oracle", "PEResult", "ReformRef", "ScorecardDB", "TimeBasis", "UnitConcept", + "VariableClass", "baseline_key", + "classify", + "load_battery", ] diff --git a/scorecard_db/case_diffs.py b/scorecard_db/case_diffs.py new file mode 100644 index 0000000..e6f0ce3 --- /dev/null +++ b/scorecard_db/case_diffs.py @@ -0,0 +1,317 @@ +"""Mode-3 case-diff models: record-level comparisons against household +oracles (UKMOD/EUROMOD via the JRC connector, NBER TAXSIM). + +One schema for every mode-3 lane (#5, #41) — the contract lives at +sources/ukmod-cases/SCHEMA.md and is deliberately engine-agnostic: the +``country`` field on a case and the ``oracle`` field on a result are what +keep ``ukmod-cases`` and ``taxsim-cases`` on a single schema instead of two. + +Design, same doctrine as :mod:`scorecard_db.models`: + +1. **Closed vocabularies fail loudly.** Oracles, diff classifications, + variable classes, household/person input keys — all closed sets; unknown + values raise instead of passing through. +2. **Cases are inputs only.** A ``CaseSpec`` never embeds expected output + values; both sides of every comparison come from engine runs, so the + battery cannot smuggle in hand-computed truth. +3. **The classifier never flatters.** ``classify`` emits match buckets, the + two null-side scope buckets, or ``unclassified``; ``oracle_difference`` + and ``rounding`` exist only as adjudicated outcomes with a traceable + writeup — misses stay visible until someone explains them. +""" + +from __future__ import annotations + +import json +from dataclasses import dataclass, field +from enum import Enum +from pathlib import Path +from typing import Optional + + +class Oracle(str, Enum): + UKMOD = "ukmod" + TAXSIM = "taxsim" + + +class VariableClass(str, Enum): + CURRENCY = "currency" + BOOLEAN = "boolean" + + +class DiffClassification(str, Enum): + MATCH_EXACT = "match_exact" + MATCH_WITHIN_TOLERANCE = "match_within_tolerance" + PE_GAP = "pe_gap" + ORACLE_DIFFERENCE = "oracle_difference" + POLICY_SCOPE_MISMATCH = "policy_scope_mismatch" + ROUNDING = "rounding" + UNCLASSIFIED = "unclassified" + + +# Classifications the automatic classifier may emit; the rest +# (oracle_difference, rounding, and pe_gap/policy_scope_mismatch on +# numeric-vs-numeric rows) require adjudication with a writeup. +CLASSIFIER_EMITTED = frozenset( + { + DiffClassification.MATCH_EXACT, + DiffClassification.MATCH_WITHIN_TOLERANCE, + DiffClassification.PE_GAP, + DiffClassification.POLICY_SCOPE_MISMATCH, + DiffClassification.UNCLASSIFIED, + } +) + +# Currency tolerance: benefit rules are stated weekly to the penny and +# comparisons are annual, so 52 x GBP/USD 0.01. Booleans agree or they +# don't. Wider slack is never a tolerance bump — a documented oracle +# rounding rule adjudicates to ROUNDING instead (SCHEMA.md). +DEFAULT_TOLERANCES: dict[VariableClass, float] = { + VariableClass.CURRENCY: 0.52, + VariableClass.BOOLEAN: 0.0, +} + +VALID_COUNTRIES = frozenset({"UK", "US"}) + +VALID_TENURES = frozenset( + {"owned_outright", "owned_mortgage", "rented_social", "rented_private"} +) + +# The engine-agnostic household input vocabulary (SCHEMA.md). Each mode-3 +# connector owns the mapping to its engines' variables; the battery speaks +# only these keys. Extending the vocabulary means extending these sets AND +# documenting the key in SCHEMA.md. +PERSON_NUMERIC_KEYS = frozenset( + { + "employment_income", + "self_employment_income", + "pension_income", + "state_pension", + "savings_income", + "capital", + "employee_pension_contributions", + "hours_worked_per_week", + } +) +PERSON_BOOL_KEYS = frozenset( + {"salary_sacrifice", "is_disabled", "is_carer", "gainfully_self_employed"} +) +PERSON_KEYS = ( + frozenset({"age", "date_of_birth"}) | PERSON_NUMERIC_KEYS | PERSON_BOOL_KEYS +) + +HOUSEHOLD_KEYS = frozenset( + {"people", "benefit_units", "region", "tenure", "rent", "council_tax"} +) + + +def _validate_person(person_id: str, person: dict) -> None: + if not isinstance(person, dict): + raise ValueError(f"person {person_id!r} must be a mapping") + unknown = set(person) - PERSON_KEYS + if unknown: + raise ValueError(f"person {person_id!r} has unknown keys: {sorted(unknown)}") + age = person.get("age") + if not isinstance(age, int) or isinstance(age, bool) or age < 0: + raise ValueError(f"person {person_id!r} needs an integer age >= 0") + dob = person.get("date_of_birth") + if dob is not None and not ( + isinstance(dob, str) and len(dob) == 10 and dob[4] == "-" and dob[7] == "-" + ): + raise ValueError(f"person {person_id!r} date_of_birth must be YYYY-MM-DD") + for key in PERSON_NUMERIC_KEYS & set(person): + v = person[key] + if isinstance(v, bool) or not isinstance(v, (int, float)) or v < 0: + raise ValueError(f"person {person_id!r} {key} must be a number >= 0") + for key in PERSON_BOOL_KEYS & set(person): + if not isinstance(person[key], bool): + raise ValueError(f"person {person_id!r} {key} must be a boolean") + + +def _validate_household(household: dict) -> None: + if not isinstance(household, dict): + raise ValueError("household must be a mapping") + unknown = set(household) - HOUSEHOLD_KEYS + if unknown: + raise ValueError(f"household has unknown keys: {sorted(unknown)}") + people = household.get("people") + if not isinstance(people, dict) or not people: + raise ValueError("household.people must be a non-empty mapping") + for person_id, person in people.items(): + _validate_person(person_id, person) + units = household.get("benefit_units") + if not isinstance(units, list) or not units: + raise ValueError("household.benefit_units must be a non-empty list") + assigned: list[str] = [] + for unit in units: + if not isinstance(unit, dict) or set(unit) != {"adults", "children"}: + raise ValueError("each benefit unit needs exactly adults + children lists") + if not unit["adults"]: + raise ValueError("each benefit unit needs at least one adult") + assigned.extend(unit["adults"]) + assigned.extend(unit["children"]) + if sorted(assigned) != sorted(people): + raise ValueError( + "benefit units must assign every person exactly once " + f"(people={sorted(people)}, assigned={sorted(assigned)})" + ) + tenure = household.get("tenure") + if tenure is not None and tenure not in VALID_TENURES: + raise ValueError(f"unknown tenure: {tenure!r}") + for key in ("rent", "council_tax"): + v = household.get(key) + if v is not None and ( + isinstance(v, bool) or not isinstance(v, (int, float)) or v < 0 + ): + raise ValueError(f"household.{key} must be a number >= 0") + if household.get("rent", 0) and tenure in ("owned_outright", "owned_mortgage"): + raise ValueError("owner-occupier households carry no rent") + + +@dataclass +class CaseSpec: + """One curated hypothetical household: inputs + expected focus only.""" + + case_id: str + description: str + policy_year: int + country: str + household: dict + expected_focus: list + rationale: str = "" + + def __post_init__(self): + if not self.case_id or not isinstance(self.case_id, str): + raise ValueError("case_id must be a non-empty string") + if self.country not in VALID_COUNTRIES: + raise ValueError(f"unknown country: {self.country!r}") + if not self.case_id.startswith(self.country.lower() + "-"): + raise ValueError( + f"case_id {self.case_id!r} must be prefixed '{self.country.lower()}-'" + ) + if ( + not isinstance(self.policy_year, int) + or not 1990 <= self.policy_year <= 2100 + ): + raise ValueError(f"implausible policy_year: {self.policy_year!r}") + if not self.expected_focus or not all( + isinstance(v, str) and v for v in self.expected_focus + ): + raise ValueError("expected_focus must be a non-empty list of variables") + _validate_household(self.household) + + +@dataclass +class CaseResult: + """PE vs oracle for one case x output variable x run (history kept).""" + + case_id: str + variable: str + pe_value: Optional[float] + oracle_value: Optional[float] + oracle: Oracle + engine_version: str + oracle_version: str + computed_at: str # ISO timestamp, caller-supplied + classification: DiffClassification + abs_diff: Optional[float] = None + annotations: list = field(default_factory=list) + + def __post_init__(self): + self.oracle = Oracle(self.oracle) + self.classification = DiffClassification(self.classification) + if not self.case_id or not self.variable: + raise ValueError("case_id and variable are required") + both_numeric = self.pe_value is not None and self.oracle_value is not None + if both_numeric: + expected = abs(self.pe_value - self.oracle_value) + if self.abs_diff is None or abs(self.abs_diff - expected) > 1e-9: + raise ValueError( + f"abs_diff must equal |pe_value - oracle_value| " + f"(got {self.abs_diff}, expected {expected})" + ) + elif self.abs_diff is not None: + raise ValueError("abs_diff requires both values") + if self.pe_value is None and self.classification != DiffClassification.PE_GAP: + raise ValueError("null pe_value requires classification=pe_gap") + if ( + self.oracle_value is None + and self.pe_value is not None + and self.classification != DiffClassification.POLICY_SCOPE_MISMATCH + ): + raise ValueError( + "null oracle_value requires classification=policy_scope_mismatch" + ) + if self.classification == DiffClassification.MATCH_EXACT and ( + not both_numeric or self.abs_diff != 0.0 + ): + raise ValueError("match_exact requires identical numeric values") + + +def classify( + pe_value: Optional[float], + oracle_value: Optional[float], + variable_class: VariableClass, + tolerance_table: dict = DEFAULT_TOLERANCES, +) -> DiffClassification: + """Automatic first-pass classification of one (pe, oracle) pair. + + Emits only ``CLASSIFIER_EMITTED`` buckets: null sides map to the scope + buckets, exact/tolerance matches to the match buckets, and everything + above tolerance to ``unclassified`` — the adjudication queue, never a + silently flattering default. + """ + variable_class = VariableClass(variable_class) + if pe_value is None: + return DiffClassification.PE_GAP + if oracle_value is None: + return DiffClassification.POLICY_SCOPE_MISMATCH + if variable_class not in tolerance_table: + raise ValueError(f"no tolerance for variable class {variable_class.value!r}") + diff = abs(float(pe_value) - float(oracle_value)) + if diff == 0.0: + return DiffClassification.MATCH_EXACT + if diff <= tolerance_table[variable_class]: + return DiffClassification.MATCH_WITHIN_TOLERANCE + return DiffClassification.UNCLASSIFIED + + +BATTERY_KEYS = frozenset({"schema", "description", "cases"}) +CASE_KEYS = frozenset( + { + "case_id", + "description", + "policy_year", + "country", + "household", + "expected_focus", + "rationale", + } +) + + +def load_battery(path) -> list[CaseSpec]: + """Load and validate a case battery file; every defect raises.""" + raw = json.loads(Path(path).read_text()) + if not isinstance(raw, dict): + raise ValueError("battery must be a JSON object") + unknown = set(raw) - BATTERY_KEYS + if unknown: + raise ValueError(f"battery has unknown keys: {sorted(unknown)}") + if not isinstance(raw.get("cases"), list) or not raw["cases"]: + raise ValueError("battery.cases must be a non-empty list") + cases = [] + for entry in raw["cases"]: + if not isinstance(entry, dict): + raise ValueError("each case must be a JSON object") + unknown = set(entry) - CASE_KEYS + if unknown: + raise ValueError( + f"case {entry.get('case_id')!r} has unknown keys: {sorted(unknown)}" + ) + cases.append(CaseSpec(**entry)) + ids = [c.case_id for c in cases] + dupes = sorted({i for i in ids if ids.count(i) > 1}) + if dupes: + raise ValueError(f"duplicate case_ids: {dupes}") + return cases diff --git a/sources/ukmod-cases/SCHEMA.md b/sources/ukmod-cases/SCHEMA.md new file mode 100644 index 0000000..61517a6 --- /dev/null +++ b/sources/ukmod-cases/SCHEMA.md @@ -0,0 +1,184 @@ +# Mode-3 case-diff schema (shared: UKMOD/EUROMOD and TAXSIM lanes) + +The scorecard's third claim class: **record-level case diffs**. Instead of a +published aggregate, the external model is run as an *oracle* on curated +hypothetical households, and PolicyEngine's per-variable outputs are compared +case by case. One schema serves every mode-3 lane — this document is the +single contract for both `ukmod-cases` (#41) and `taxsim-cases` (#5); the +`country` and `oracle` fields are what keep it engine-agnostic. Do not fork a +second schema for a new oracle: extend the closed vocabularies here, fail-loud +style, and note the extension. + +Code: `scorecard_db/case_diffs.py` (dataclasses, battery loader, classifier). +Battery: `sources/ukmod-cases/battery/cases.json`. Tests: +`tests/test_case_schema.py`. + +## Case (input) — `CaseSpec` + +A case is a fully specified hypothetical household plus the variables it is +designed to exercise. **Inputs only** — a case never embeds expected output +values; both sides of every comparison come from engine runs (PE and the +oracle), so the battery cannot smuggle in hand-computed "truth". + +```json +{ + "case_id": "uk-uc-single-unemployed", + "description": "Single unemployed adult, social rent, on UC", + "policy_year": 2026, + "country": "UK", + "household": { ... }, + "expected_focus": ["universal_credit", "income_tax"], + "rationale": "Why this case is in the battery, and what edge it pins" +} +``` + +- `case_id` — globally unique slug, prefixed with the country + (`uk-…`, `us-…`); stable forever once results reference it. +- `policy_year` — the tax-benefit year the case is evaluated in (for the UK, + the fiscal year starting 6 April of that calendar year). +- `country` — closed set, currently `{"UK", "US"}`; extend when a new + mode-3 lane lands (EUROMOD EU countries, state calculators). +- `expected_focus` — the output variables (canonical names, below) the case + was designed to exercise. The runner computes and diffs *all* mapped + variables; `expected_focus` drives coverage accounting (every battery must + exercise each focus area at least once) and diff triage order. +- `rationale` — human audit trail: why these numbers, what boundary they sit + on. + +### Household spec (engine-agnostic) + +The household is described in a small, closed, engine-neutral input +vocabulary. Each lane's connector owns the mapping from this vocabulary to +its engines' variables (PE UK / UKMOD policy spine; PE US / TAXSIM v35 +columns); the mapping table lives with the connector, never in the battery. +Unknown keys are a hard error — adding an input concept means extending the +vocabulary in `case_diffs.py` and documenting it here. + +```json +{ + "people": { + "adult_1": {"age": 35, "employment_income": 12000}, + "child_1": {"age": 8} + }, + "benefit_units": [ + {"adults": ["adult_1"], "children": ["child_1"]} + ], + "region": "LONDON", + "tenure": "rented_private", + "rent": 13000 +} +``` + +Conventions: + +- **All monetary amounts are annual, in the country's currency** (GBP for + UK, USD for US). Weekly-quoted UK amounts enter as weekly × 52 (round, + auditable numbers preferred — e.g. £250/week rent = 13000). +- Every person appears in exactly one benefit unit. A benefit unit is the + assessment unit (UK: single/couple + dependent children; US: the tax + unit). Multi-benefit-unit households are allowed (e.g. a non-dependant + adult) but every person must be assigned. +- Omitted person keys default to 0 / false. Only non-defaults are written, + so each case reads as exactly its rationale. + +Person keys (closed set): + +| key | type | meaning | +|---|---|---| +| `age` | int, required | age at the start of the policy year | +| `date_of_birth` | `YYYY-MM-DD` | only when the exact date is load-bearing (two-child-limit protection, state pension age) | +| `employment_income` | number ≥ 0 | gross annual employee earnings | +| `self_employment_income` | number ≥ 0 | annual trading profit | +| `pension_income` | number ≥ 0 | private/occupational pension in payment | +| `state_pension` | number ≥ 0 | annual state pension in payment | +| `savings_income` | number ≥ 0 | annual interest income | +| `capital` | number ≥ 0 | liquid capital / savings stock | +| `employee_pension_contributions` | number ≥ 0 | annual employee pension contributions; `salary_sacrifice` true means they reduce gross pay for tax and NI | +| `salary_sacrifice` | bool | pension contributions are via salary sacrifice | +| `is_disabled` | bool | disabled for benefit purposes (drives disability elements/premia; the connector maps to each engine's disability concept and records the mapping) | +| `is_carer` | bool | provides ≥ 35 hours/week care for a disabled person (Carer's Allowance / UC carer element eligibility) | +| `gainfully_self_employed` | bool | in gainful self-employment for UC (minimum income floor applies after the start-up period) | +| `hours_worked_per_week` | number ≥ 0 | contracted weekly hours | + +Household keys (closed set): `people`, `benefit_units`, `region`, `tenure` +(`owned_outright` | `owned_mortgage` | `rented_social` | `rented_private`), +`rent` (annual), `council_tax` (annual, optional). `region` uses the target +country's own geography vocabulary (UK: ITL-1 slugs like `LONDON`, +`SCOTLAND`; US: state codes) — it is what routes devolved policy (Scottish +income tax, Scottish Child Payment; US state taxes) and location-dependent +amounts (LHA rates, benefit-cap tier). + +## Result row — `CaseResult` + +One row per case × output variable × run. History is preserved: re-running +on a new engine or oracle version appends, never overwrites (same doctrine +as `pe_results`). + +```json +{ + "case_id": "uk-uc-single-unemployed", + "variable": "universal_credit", + "pe_value": 4796.48, + "oracle_value": 4796.52, + "oracle": "ukmod", + "engine_version": "policyengine-uk 2.x.y", + "oracle_version": "UKMOD B2026.08 / EUROMOD I7.0+", + "computed_at": "2026-08-14T12:00:00Z", + "abs_diff": 0.04, + "classification": "match_within_tolerance" +} +``` + +- `variable` — canonical output name in PE's vocabulary for the case's + country (`universal_credit`, `income_tax`, `national_insurance`, + `child_benefit`, `pension_credit`, …). The connector's mapping table + pairs it with the oracle's variable (UKMOD `bsauc_s`, etc.) and records + any construction (e.g. summing UKMOD monthly output × 12). +- `oracle` — closed set `{"ukmod", "taxsim"}`; grows with #5. +- `pe_value` / `oracle_value` — annual amounts (booleans as 0/1). `null` + means that side cannot produce the variable: `pe_value: null` ⇒ + `pe_gap`; `oracle_value: null` ⇒ `policy_scope_mismatch` (the oracle + does not model it — e.g. TAXSIM has no benefits; UKMOD's UK model omits + some devolved payments depending on version). +- `abs_diff` — `|pe_value − oracle_value|` when both sides are numeric, + else `null`. Stored, not derived at read time, so the miss table is + self-contained. + +## Classification (closed set — fail loud) + +| classification | meaning | assigned by | +|---|---|---| +| `match_exact` | values identical | classifier | +| `match_within_tolerance` | nonzero diff ≤ the variable class's tolerance | classifier | +| `pe_gap` | PE cannot produce the variable, or adjudication found PE wrong | classifier (null side) / adjudication | +| `oracle_difference` | adjudication found the oracle wrong (upstream report filed) | adjudication | +| `policy_scope_mismatch` | the two engines model different policy scope for this variable (documented) | classifier (null side) / adjudication | +| `rounding` | diff fully explained by the oracle's documented rounding rules (e.g. UKMOD monthly rounding × 12) | adjudication | +| `unclassified` | diff above tolerance, not yet adjudicated | classifier (default) | + +The classifier (`case_diffs.classify`) only ever emits `match_exact`, +`match_within_tolerance`, `pe_gap` / `policy_scope_mismatch` (null sides), +or `unclassified`. Every `unclassified` row is a work item: adjudication — +the diagnosis stage, human-or-agent, with a traceable writeup — moves it to +`pe_gap`, `oracle_difference`, `policy_scope_mismatch`, or `rounding`. +Nothing defaults to a flattering bucket; misses stay visible, exactly as in +modes 1–2. Unknown classification strings raise. + +## Tolerances (per variable class) + +| variable class | tolerance | rationale | +|---|---|---| +| `currency` | £0.01/week ⇒ **0.52/year** (same rule in USD for TAXSIM) | benefit rules are stated weekly to the penny; comparisons are annual, so 52 × 0.01. A wider tolerance is allowed only with a documented oracle rounding rule, and then the adjudicated class is `rounding`, not a silent tolerance bump. | +| `boolean` | exact (0) | eligibility flags either agree or they don't | + +The tolerance table is data (`DEFAULT_TOLERANCES`), passed to `classify` +explicitly; a variable class missing from the table raises rather than +guessing. + +## What this lane still needs (out of scope here) + +The JRC connector run (runnable per axiom-oracles#264, needs the UKMOD +environment): map the vocabulary to UKMOD input variables, execute the +battery on both engines, append `CaseResult` rows, publish the miss table, +and advance the lane. Until then the battery is inputs + focus only, by +design. diff --git a/sources/ukmod-cases/battery/cases.json b/sources/ukmod-cases/battery/cases.json new file mode 100644 index 0000000..2b62570 --- /dev/null +++ b/sources/ukmod-cases/battery/cases.json @@ -0,0 +1,291 @@ +{ + "schema": "sources/ukmod-cases/SCHEMA.md", + "description": "Initial UK mode-3 case battery for the UKMOD/EUROMOD lane (#41): standard EUROMOD-style hypothetical households plus the edge cases PE UK carries regression coverage for. Inputs + expected focus only — expected output values come from engine runs (PE and the oracle), never from this file. All monetary amounts are annual GBP; weekly-quoted amounts enter as weekly x 52.", + "cases": [ + { + "case_id": "uk-uc-single-unemployed", + "description": "Single unemployed adult on UC, social rent", + "policy_year": 2026, + "country": "UK", + "household": { + "people": { + "adult_1": {"age": 30} + }, + "benefit_units": [ + {"adults": ["adult_1"], "children": []} + ], + "region": "NORTH_EAST", + "tenure": "rented_social", + "rent": 6240 + }, + "expected_focus": ["universal_credit", "income_tax", "national_insurance"], + "rationale": "The simplest UC case: standard allowance (single 25+) plus social-rent housing element with no deductions. Rent GBP 120/week = 6,240/year. Pins the standard allowance and housing element before any taper, cap, or element interacts." + }, + { + "case_id": "uk-uc-lone-parent-two-children-rent", + "description": "Workless lone parent, two children, private rent", + "policy_year": 2026, + "country": "UK", + "household": { + "people": { + "adult_1": {"age": 32}, + "child_1": {"age": 8}, + "child_2": {"age": 4} + }, + "benefit_units": [ + {"adults": ["adult_1"], "children": ["child_1", "child_2"]} + ], + "region": "YORKSHIRE", + "tenure": "rented_private", + "rent": 9360 + }, + "expected_focus": ["universal_credit", "child_benefit"], + "rationale": "UC child elements (two children, both post-April-2017 so no protected rate) plus the private-rent housing element via LHA (2-bed rate, Yorkshire BRMA) and Child Benefit for two. Rent GBP 180/week = 9,360/year, below the LHA cap in most Yorkshire BRMAs so the LHA-vs-actual-rent minimum is exercised without binding." + }, + { + "case_id": "uk-uc-two-child-limit-protected-eldest", + "description": "Workless couple, three children, eldest born pre-April-2017", + "policy_year": 2026, + "country": "UK", + "household": { + "people": { + "adult_1": {"age": 36}, + "adult_2": {"age": 34}, + "child_1": {"age": 9, "date_of_birth": "2016-06-01"}, + "child_2": {"age": 7, "date_of_birth": "2019-03-01"}, + "child_3": {"age": 3, "date_of_birth": "2022-09-01"} + }, + "benefit_units": [ + {"adults": ["adult_1", "adult_2"], "children": ["child_1", "child_2", "child_3"]} + ], + "region": "WEST_MIDLANDS", + "tenure": "rented_social", + "rent": 8320 + }, + "expected_focus": ["universal_credit", "child_benefit"], + "rationale": "Two-child limit binding with the pre-April-2017 protection: child_1 (born 2016-06-01) is exempt from the limit, child_2 takes the second element, child_3 gets none. Also pins that Child Benefit is unaffected by the limit (paid for all three). DOBs are explicit because the exemption keys off date of birth, not age." + }, + { + "case_id": "uk-benefit-cap-london", + "description": "Workless couple, two children, high London private rent", + "policy_year": 2026, + "country": "UK", + "household": { + "people": { + "adult_1": {"age": 33}, + "adult_2": {"age": 31}, + "child_1": {"age": 6}, + "child_2": {"age": 3} + }, + "benefit_units": [ + {"adults": ["adult_1", "adult_2"], "children": ["child_1", "child_2"]} + ], + "region": "LONDON", + "tenure": "rented_private", + "rent": 20800 + }, + "expected_focus": ["benefit_cap", "universal_credit", "child_benefit"], + "rationale": "Benefit cap binding: rent GBP 400/week = 20,800/year drives pre-cap UC (standard + 2 child elements + housing) above the Greater London couple cap, so the cap reduction is nonzero. Exercises the London cap tier and the cap-applies-to-UC-not-Child-Benefit accounting." + }, + { + "case_id": "uk-uc-taper-part-time-earner", + "description": "Lone parent working part time, UC taper and work allowance", + "policy_year": 2026, + "country": "UK", + "household": { + "people": { + "adult_1": {"age": 29, "employment_income": 12000, "hours_worked_per_week": 20}, + "child_1": {"age": 5} + }, + "benefit_units": [ + {"adults": ["adult_1"], "children": ["child_1"]} + ], + "region": "NORTH_WEST", + "tenure": "rented_private", + "rent": 7800 + }, + "expected_focus": ["universal_credit", "income_tax", "national_insurance", "child_benefit"], + "rationale": "The 55% UC earnings taper above the lower (with-housing) work allowance. Earnings GBP 12,000 sit below the personal allowance (income tax GBP 0) but above the NI primary threshold pro-rata, so net-earnings-for-UC differs from gross — the taper base (net of tax and NI) is the thing under test. Rent GBP 150/week = 7,800/year." + }, + { + "case_id": "uk-uc-minimum-income-floor", + "description": "Gainfully self-employed single adult below the MIF", + "policy_year": 2026, + "country": "UK", + "household": { + "people": { + "adult_1": {"age": 40, "self_employment_income": 6000, "gainfully_self_employed": true} + }, + "benefit_units": [ + {"adults": ["adult_1"], "children": []} + ], + "region": "EAST_MIDLANDS", + "tenure": "rented_private", + "rent": 7280 + }, + "expected_focus": ["universal_credit"], + "rationale": "Minimum income floor: actual trading profit GBP 6,000/year is far below the assumed floor (35 hours x NLW net of notional tax/NI), so UC is tapered on the floor, not on actual income. Past the 12-month start-up period by assumption (gainfully_self_employed = true). Pins the floor construction, a known cross-model divergence spot." + }, + { + "case_id": "uk-pension-credit-single", + "description": "Single pensioner, state pension only, guarantee credit", + "policy_year": 2026, + "country": "UK", + "household": { + "people": { + "adult_1": {"age": 70, "state_pension": 9000} + }, + "benefit_units": [ + {"adults": ["adult_1"], "children": []} + ], + "region": "SOUTH_WEST", + "tenure": "owned_outright", + "rent": 0 + }, + "expected_focus": ["pension_credit", "income_tax"], + "rationale": "Guarantee credit tops a GBP 9,000 state pension up to the single-person standard minimum guarantee; income below the personal allowance so income tax is zero. Deliberately below the full new state pension so the top-up is strictly positive." + }, + { + "case_id": "uk-pension-credit-couple", + "description": "Pensioner couple, mixed state and private pension income", + "policy_year": 2026, + "country": "UK", + "household": { + "people": { + "adult_1": {"age": 72, "state_pension": 9500, "pension_income": 1000}, + "adult_2": {"age": 70, "state_pension": 4500} + }, + "benefit_units": [ + {"adults": ["adult_1", "adult_2"], "children": []} + ], + "region": "WALES", + "tenure": "owned_outright", + "rent": 0 + }, + "expected_focus": ["pension_credit", "income_tax"], + "rationale": "Couple guarantee credit with joint income assessment: GBP 15,000 combined (two state pensions of 9,500 and 4,500 plus a 1,000 private pension) against the couple standard minimum guarantee. Pins joint aggregation and that private pension income counts in full." + }, + { + "case_id": "uk-mixed-age-couple-uc", + "description": "Mixed-age couple routed to UC, not Pension Credit", + "policy_year": 2026, + "country": "UK", + "household": { + "people": { + "adult_1": {"age": 68, "state_pension": 9500}, + "adult_2": {"age": 60} + }, + "benefit_units": [ + {"adults": ["adult_1", "adult_2"], "children": []} + ], + "region": "NORTH_EAST", + "tenure": "rented_social", + "rent": 6760 + }, + "expected_focus": ["universal_credit", "pension_credit"], + "rationale": "Since May 2019 a mixed-age couple (one over, one under state pension age) claims UC, not Pension Credit. Expect pension_credit = 0 and UC assessed with the state pension counted as unearned income (GBP-for-GBP deduction). A known routing rule models get wrong in both directions. Rent GBP 130/week = 6,760/year." + }, + { + "case_id": "uk-scotland-earner-scp", + "description": "Scottish couple, one earner, two young children, on UC", + "policy_year": 2026, + "country": "UK", + "household": { + "people": { + "adult_1": {"age": 35, "employment_income": 22000, "hours_worked_per_week": 35}, + "adult_2": {"age": 33}, + "child_1": {"age": 7}, + "child_2": {"age": 2} + }, + "benefit_units": [ + {"adults": ["adult_1", "adult_2"], "children": ["child_1", "child_2"]} + ], + "region": "SCOTLAND", + "tenure": "rented_social", + "rent": 8840 + }, + "expected_focus": ["income_tax", "universal_credit", "scottish_child_payment", "child_benefit"], + "rationale": "Devolution double-check: GBP 22,000 earnings span the Scottish starter, basic, and intermediate bands (different liability than rUK at the same gross), while UC entitlement plus two under-16s triggers Scottish Child Payment for both children. Exercises whether the oracle models devolved Scottish payments at all — a candidate policy_scope_mismatch, which is exactly why it is in the battery." + }, + { + "case_id": "uk-hicbc-partial-taper", + "description": "Higher-rate earner at GBP 70,000 with Child Benefit (HICBC mid-taper)", + "policy_year": 2026, + "country": "UK", + "household": { + "people": { + "adult_1": {"age": 41, "employment_income": 70000}, + "adult_2": {"age": 39}, + "child_1": {"age": 10}, + "child_2": {"age": 7} + }, + "benefit_units": [ + {"adults": ["adult_1", "adult_2"], "children": ["child_1", "child_2"]} + ], + "region": "SOUTH_EAST", + "tenure": "owned_mortgage", + "rent": 0 + }, + "expected_focus": ["income_tax", "child_benefit"], + "rationale": "High Income Child Benefit Charge at the midpoint of the post-2024 GBP 60,000-80,000 taper: adjusted net income 70,000 claws back exactly half the two-child Child Benefit through the earner's income tax. Round midpoint chosen so the expected clawback fraction (1/2) is auditable by inspection." + }, + { + "case_id": "uk-ni-thresholds-salary-sacrifice", + "description": "Couple pinned at the NI primary threshold and UEL, with salary sacrifice", + "policy_year": 2026, + "country": "UK", + "household": { + "people": { + "adult_1": {"age": 45, "employment_income": 12570}, + "adult_2": {"age": 44, "employment_income": 50270, "employee_pension_contributions": 2000, "salary_sacrifice": true} + }, + "benefit_units": [ + {"adults": ["adult_1", "adult_2"], "children": []} + ], + "region": "EAST_OF_ENGLAND", + "tenure": "owned_mortgage", + "rent": 0 + }, + "expected_focus": ["national_insurance", "income_tax"], + "rationale": "Threshold edges: adult_1 sits exactly at GBP 12,570 (personal allowance = NI primary threshold; both liabilities should be exactly zero). Adult_2 sits at the GBP 50,270 UEL with a GBP 2,000 salary sacrifice, so NI-able and taxable pay drop to 48,270 — pinning that sacrifice reduces both tax and Class 1 NI, and that the main-vs-additional NI rate boundary is applied to post-sacrifice pay." + }, + { + "case_id": "uk-carer-ca-uc", + "description": "Single carer on Carer's Allowance and UC with carer element", + "policy_year": 2026, + "country": "UK", + "household": { + "people": { + "adult_1": {"age": 45, "is_carer": true} + }, + "benefit_units": [ + {"adults": ["adult_1"], "children": []} + ], + "region": "NORTH_WEST", + "tenure": "rented_social", + "rent": 6240 + }, + "expected_focus": ["carers_allowance", "universal_credit"], + "rationale": "Carer's Allowance plus UC carer element interaction: CA is unearned income deducted from UC GBP-for-GBP, while the carer element is added — the net gain should equal the carer element alone. The cared-for person (receiving a qualifying disability benefit) lives outside the household, per the is_carer flag's definition. A classic double-counting trap between models." + }, + { + "case_id": "uk-pa-taper-110k", + "description": "Single earner at GBP 110,000, personal allowance taper", + "policy_year": 2026, + "country": "UK", + "household": { + "people": { + "adult_1": {"age": 48, "employment_income": 110000} + }, + "benefit_units": [ + {"adults": ["adult_1"], "children": []} + ], + "region": "LONDON", + "tenure": "owned_mortgage", + "rent": 0 + }, + "expected_focus": ["income_tax", "national_insurance"], + "rationale": "The GBP 100,000 cliff: at 110,000 the personal allowance is tapered by GBP 1 per 2 over 100,000 (5,000 of allowance lost), producing the 60% effective marginal band. Round 10,000 excess makes the expected allowance (12,570 - 5,000 = 7,570) auditable by inspection." + } + ] +} diff --git a/tests/test_case_schema.py b/tests/test_case_schema.py new file mode 100644 index 0000000..dc86a49 --- /dev/null +++ b/tests/test_case_schema.py @@ -0,0 +1,235 @@ +"""The mode-3 case-diff schema (#41, shared with taxsim-cases per #5): +the UK battery validates, ids are unique and country-prefixed, cases stay +inputs-only, and the classifier's closed vocabulary fails loudly.""" + +import json +from pathlib import Path + +import pytest + +from scorecard_db.case_diffs import ( + DEFAULT_TOLERANCES, + CaseResult, + CaseSpec, + DiffClassification, + Oracle, + VariableClass, + classify, + load_battery, +) + +BATTERY = Path(__file__).parent.parent / "sources/ukmod-cases/battery/cases.json" + + +def case_kwargs(**kw): + base = dict( + case_id="uk-test-case", + description="test", + policy_year=2026, + country="UK", + household={ + "people": {"adult_1": {"age": 30}}, + "benefit_units": [{"adults": ["adult_1"], "children": []}], + "tenure": "rented_social", + "rent": 6240, + }, + expected_focus=["universal_credit"], + ) + base.update(kw) + return base + + +class TestBattery: + def test_loads_and_validates(self): + cases = load_battery(BATTERY) + assert 12 <= len(cases) <= 16 + assert all(isinstance(c, CaseSpec) for c in cases) + + def test_unique_ids(self): + cases = load_battery(BATTERY) + ids = [c.case_id for c in cases] + assert len(ids) == len(set(ids)) + + def test_uk_battery_shape(self): + for c in load_battery(BATTERY): + assert c.country == "UK" + assert c.case_id.startswith("uk-") + assert c.policy_year == 2026 + assert c.rationale # every case explains itself + + def test_battery_is_inputs_only(self): + # No case embeds expected output values (SCHEMA.md doctrine). + raw = json.loads(BATTERY.read_text()) + for entry in raw["cases"]: + assert set(entry) <= { + "case_id", + "description", + "policy_year", + "country", + "household", + "expected_focus", + "rationale", + } + + def test_focus_coverage(self): + focus = {v for c in load_battery(BATTERY) for v in c.expected_focus} + assert { + "universal_credit", + "income_tax", + "national_insurance", + "child_benefit", + "pension_credit", + "benefit_cap", + "scottish_child_payment", + "carers_allowance", + } <= focus + + def test_duplicate_ids_rejected(self, tmp_path): + raw = json.loads(BATTERY.read_text()) + raw["cases"].append(dict(raw["cases"][0])) + p = tmp_path / "cases.json" + p.write_text(json.dumps(raw)) + with pytest.raises(ValueError, match="duplicate case_ids"): + load_battery(p) + + def test_unknown_case_key_rejected(self, tmp_path): + raw = json.loads(BATTERY.read_text()) + raw["cases"][0]["expected_value"] = 123.0 + p = tmp_path / "cases.json" + p.write_text(json.dumps(raw)) + with pytest.raises(ValueError, match="unknown keys"): + load_battery(p) + + +class TestCaseSpec: + def test_unknown_country_rejected(self): + with pytest.raises(ValueError, match="unknown country"): + CaseSpec(**case_kwargs(country="FR")) + + def test_country_prefix_enforced(self): + with pytest.raises(ValueError, match="prefixed"): + CaseSpec(**case_kwargs(case_id="single-unemployed")) + + def test_unknown_person_key_rejected(self): + kw = case_kwargs() + kw["household"]["people"]["adult_1"]["wages"] = 1000 + with pytest.raises(ValueError, match="unknown keys"): + CaseSpec(**kw) + + def test_unassigned_person_rejected(self): + kw = case_kwargs() + kw["household"]["people"]["adult_2"] = {"age": 40} + with pytest.raises(ValueError, match="exactly once"): + CaseSpec(**kw) + + def test_owner_with_rent_rejected(self): + kw = case_kwargs() + kw["household"]["tenure"] = "owned_outright" + with pytest.raises(ValueError, match="no rent"): + CaseSpec(**kw) + + def test_missing_age_rejected(self): + kw = case_kwargs() + kw["household"]["people"]["adult_1"] = {"employment_income": 1000} + with pytest.raises(ValueError, match="age"): + CaseSpec(**kw) + + def test_empty_focus_rejected(self): + with pytest.raises(ValueError, match="expected_focus"): + CaseSpec(**case_kwargs(expected_focus=[])) + + +class TestClassify: + def test_exact(self): + assert classify(100.0, 100.0, VariableClass.CURRENCY) == ( + DiffClassification.MATCH_EXACT + ) + + def test_within_tolerance(self): + # 52 x GBP 0.01 = 0.52/year currency tolerance. + assert classify(100.0, 100.52, VariableClass.CURRENCY) == ( + DiffClassification.MATCH_WITHIN_TOLERANCE + ) + + def test_above_tolerance_defaults_unclassified(self): + assert classify(100.0, 100.53, VariableClass.CURRENCY) == ( + DiffClassification.UNCLASSIFIED + ) + + def test_bool_exact_only(self): + assert classify(1.0, 1.0, VariableClass.BOOLEAN) == ( + DiffClassification.MATCH_EXACT + ) + assert classify(1.0, 0.0, VariableClass.BOOLEAN) == ( + DiffClassification.UNCLASSIFIED + ) + + def test_null_sides(self): + assert classify(None, 5.0, VariableClass.CURRENCY) == ( + DiffClassification.PE_GAP + ) + assert classify(5.0, None, VariableClass.CURRENCY) == ( + DiffClassification.POLICY_SCOPE_MISMATCH + ) + + def test_unknown_variable_class_rejected(self): + with pytest.raises(ValueError): + classify(1.0, 1.0, "percentage") + + def test_missing_tolerance_rejected(self): + with pytest.raises(ValueError, match="no tolerance"): + classify(1.0, 2.0, VariableClass.CURRENCY, tolerance_table={}) + + def test_default_tolerances_closed(self): + assert set(DEFAULT_TOLERANCES) == set(VariableClass) + + +class TestCaseResult: + def result_kwargs(self, **kw): + base = dict( + case_id="uk-uc-single-unemployed", + variable="universal_credit", + pe_value=4796.48, + oracle_value=4796.52, + oracle="ukmod", + engine_version="policyengine-uk 2.0.0", + oracle_version="UKMOD B2026.08", + computed_at="2026-08-14T12:00:00Z", + classification="match_within_tolerance", + abs_diff=0.04, + ) + base.update(kw) + return base + + def test_valid_row(self): + r = CaseResult(**self.result_kwargs()) + assert r.oracle is Oracle.UKMOD + assert r.classification is DiffClassification.MATCH_WITHIN_TOLERANCE + + def test_unknown_oracle_rejected(self): + with pytest.raises(ValueError): + CaseResult(**self.result_kwargs(oracle="euromod-lite")) + + def test_unknown_classification_rejected(self): + with pytest.raises(ValueError): + CaseResult(**self.result_kwargs(classification="close_enough")) + + def test_abs_diff_must_be_consistent(self): + with pytest.raises(ValueError, match="abs_diff"): + CaseResult(**self.result_kwargs(abs_diff=1.0)) + + def test_null_pe_value_requires_pe_gap(self): + with pytest.raises(ValueError, match="pe_gap"): + CaseResult(**self.result_kwargs(pe_value=None, abs_diff=None)) + r = CaseResult( + **self.result_kwargs(pe_value=None, abs_diff=None, classification="pe_gap") + ) + assert r.classification is DiffClassification.PE_GAP + + def test_null_oracle_value_requires_scope_mismatch(self): + with pytest.raises(ValueError, match="policy_scope_mismatch"): + CaseResult(**self.result_kwargs(oracle_value=None, abs_diff=None)) + + def test_match_exact_requires_identity(self): + with pytest.raises(ValueError, match="match_exact"): + CaseResult(**self.result_kwargs(classification="match_exact")) From 5ba3c4f34635c8995b990146b6ed74736bfd54c2 Mon Sep 17 00:00:00 2001 From: r Date: Mon, 17 Aug 2026 10:04:16 +0100 Subject: [PATCH 2/5] Enforce adjudication contract, close schema list gaps, redesign two-child-limit cases MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three review blockers from the dual-gate pass: 1. Adjudicated misses need their paperwork: CaseResult now requires the judged-against tolerance on match_within_tolerance rows (0 < abs_diff <= tolerance, tolerance stored so table changes never re-bless old rows) and a non-empty annotations writeup on the adjudicated-only classes oracle_difference / rounding (ADJUDICATED_ONLY); tolerance is forbidden elsewhere and annotations must be a list of non-empty strings. 2. Closed-schema list types: benefit-unit adults/children must be lists of person-id strings and expected_focus must be a list — bare strings no longer validate by iterating as characters. 3. The two-child-limit case is split into an honest pair: a policy-year-2025 multiple-birth case where the exemption sits on the third child and strictly changes entitlement (the limit was abolished 6 April 2026, so only a pre-abolition year can exercise it), plus a 2026 abolition counterpart of the same family where an engine still applying the limit under-pays one element. SCHEMA.md documents the tightened contract; tests extended to 37 in test_case_schema.py (suite 164 passed / 4 skipped). Co-Authored-By: Claude Fable 5 --- scorecard_db/__init__.py | 2 + scorecard_db/case_diffs.py | 53 ++++++++++++++- sources/ukmod-cases/SCHEMA.md | 14 +++- sources/ukmod-cases/battery/cases.json | 35 ++++++++-- tests/test_case_schema.py | 92 +++++++++++++++++++++++++- 5 files changed, 185 insertions(+), 11 deletions(-) diff --git a/scorecard_db/__init__.py b/scorecard_db/__init__.py index 3554b41..f7f5896 100644 --- a/scorecard_db/__init__.py +++ b/scorecard_db/__init__.py @@ -1,4 +1,5 @@ from .case_diffs import ( + ADJUDICATED_ONLY, DEFAULT_TOLERANCES, CaseResult, CaseSpec, @@ -26,6 +27,7 @@ ) __all__ = [ + "ADJUDICATED_ONLY", "BASELINE", "CURRENT_LAW_DESCRIPTOR", "DEFAULT_TOLERANCES", diff --git a/scorecard_db/case_diffs.py b/scorecard_db/case_diffs.py index e6f0ce3..e369b01 100644 --- a/scorecard_db/case_diffs.py +++ b/scorecard_db/case_diffs.py @@ -62,6 +62,16 @@ class DiffClassification(str, Enum): } ) +# Adjudicated-only outcomes: a row may carry one of these only with a +# traceable writeup in ``annotations`` (SCHEMA.md — "misses stay visible +# until someone explains them"). +ADJUDICATED_ONLY = frozenset( + { + DiffClassification.ORACLE_DIFFERENCE, + DiffClassification.ROUNDING, + } +) + # Currency tolerance: benefit rules are stated weekly to the penny and # comparisons are annual, so 52 x GBP/USD 0.01. Booleans agree or they # don't. Wider slack is never a tolerance bump — a documented oracle @@ -146,6 +156,14 @@ def _validate_household(household: dict) -> None: for unit in units: if not isinstance(unit, dict) or set(unit) != {"adults", "children"}: raise ValueError("each benefit unit needs exactly adults + children lists") + for role in ("adults", "children"): + members = unit[role] + if not isinstance(members, list) or not all( + isinstance(m, str) and m for m in members + ): + raise ValueError( + f"benefit unit {role} must be a list of person ids, got {members!r}" + ) if not unit["adults"]: raise ValueError("each benefit unit needs at least one adult") assigned.extend(unit["adults"]) @@ -194,8 +212,10 @@ def __post_init__(self): or not 1990 <= self.policy_year <= 2100 ): raise ValueError(f"implausible policy_year: {self.policy_year!r}") - if not self.expected_focus or not all( - isinstance(v, str) and v for v in self.expected_focus + if ( + not isinstance(self.expected_focus, list) + or not self.expected_focus + or not all(isinstance(v, str) and v for v in self.expected_focus) ): raise ValueError("expected_focus must be a non-empty list of variables") _validate_household(self.household) @@ -215,6 +235,7 @@ class CaseResult: computed_at: str # ISO timestamp, caller-supplied classification: DiffClassification abs_diff: Optional[float] = None + tolerance: Optional[float] = None annotations: list = field(default_factory=list) def __post_init__(self): @@ -222,6 +243,10 @@ def __post_init__(self): self.classification = DiffClassification(self.classification) if not self.case_id or not self.variable: raise ValueError("case_id and variable are required") + if not isinstance(self.annotations, list) or not all( + isinstance(a, str) and a.strip() for a in self.annotations + ): + raise ValueError("annotations must be a list of non-empty strings") both_numeric = self.pe_value is not None and self.oracle_value is not None if both_numeric: expected = abs(self.pe_value - self.oracle_value) @@ -246,6 +271,30 @@ def __post_init__(self): not both_numeric or self.abs_diff != 0.0 ): raise ValueError("match_exact requires identical numeric values") + if self.classification == DiffClassification.MATCH_WITHIN_TOLERANCE: + if ( + isinstance(self.tolerance, bool) + or not isinstance(self.tolerance, (int, float)) + or self.tolerance <= 0 + ): + raise ValueError( + "match_within_tolerance requires the numeric tolerance " + "the row was judged against (tolerance > 0)" + ) + if not both_numeric or not 0 < self.abs_diff <= self.tolerance: + raise ValueError( + "match_within_tolerance requires 0 < abs_diff <= tolerance " + f"(abs_diff={self.abs_diff}, tolerance={self.tolerance})" + ) + elif self.tolerance is not None: + raise ValueError( + "tolerance is recorded only on match_within_tolerance rows" + ) + if self.classification in ADJUDICATED_ONLY and not self.annotations: + raise ValueError( + f"{self.classification.value} is an adjudicated outcome and " + "requires a writeup in annotations (SCHEMA.md)" + ) def classify( diff --git a/sources/ukmod-cases/SCHEMA.md b/sources/ukmod-cases/SCHEMA.md index 61517a6..6426539 100644 --- a/sources/ukmod-cases/SCHEMA.md +++ b/sources/ukmod-cases/SCHEMA.md @@ -125,6 +125,7 @@ as `pe_results`). "oracle_version": "UKMOD B2026.08 / EUROMOD I7.0+", "computed_at": "2026-08-14T12:00:00Z", "abs_diff": 0.04, + "tolerance": 0.52, "classification": "match_within_tolerance" } ``` @@ -143,6 +144,14 @@ as `pe_results`). - `abs_diff` — `|pe_value − oracle_value|` when both sides are numeric, else `null`. Stored, not derived at read time, so the miss table is self-contained. +- `tolerance` — required on (and only on) `match_within_tolerance` rows: + the numeric tolerance the row was judged against, satisfying + `0 < abs_diff ≤ tolerance`. Stored so a tolerance-table change can never + silently re-bless old rows. +- `annotations` — a list of non-empty strings. Required (non-empty) on the + adjudicated-only classifications `oracle_difference` and `rounding`: the + traceable writeup naming the oracle defect / documented rounding rule. + A row claiming either without a writeup fails validation. ## Classification (closed set — fail loud) @@ -162,7 +171,10 @@ or `unclassified`. Every `unclassified` row is a work item: adjudication — the diagnosis stage, human-or-agent, with a traceable writeup — moves it to `pe_gap`, `oracle_difference`, `policy_scope_mismatch`, or `rounding`. Nothing defaults to a flattering bucket; misses stay visible, exactly as in -modes 1–2. Unknown classification strings raise. +modes 1–2. Unknown classification strings raise. The writeup requirement is +enforced at validation: `oracle_difference` and `rounding` rows raise +without a non-empty `annotations` writeup, and `match_within_tolerance` +rows raise without the `tolerance` they were judged against. ## Tolerances (per variable class) diff --git a/sources/ukmod-cases/battery/cases.json b/sources/ukmod-cases/battery/cases.json index 2b62570..1afbadc 100644 --- a/sources/ukmod-cases/battery/cases.json +++ b/sources/ukmod-cases/battery/cases.json @@ -43,16 +43,39 @@ "rationale": "UC child elements (two children, both post-April-2017 so no protected rate) plus the private-rent housing element via LHA (2-bed rate, Yorkshire BRMA) and Child Benefit for two. Rent GBP 180/week = 9,360/year, below the LHA cap in most Yorkshire BRMAs so the LHA-vs-actual-rent minimum is exercised without binding." }, { - "case_id": "uk-uc-two-child-limit-protected-eldest", - "description": "Workless couple, three children, eldest born pre-April-2017", - "policy_year": 2026, + "case_id": "uk-uc-two-child-limit-multiple-birth", + "description": "Workless couple, three children incl. twins, limit in force (2025)", + "policy_year": 2025, "country": "UK", "household": { "people": { "adult_1": {"age": 36}, "adult_2": {"age": 34}, - "child_1": {"age": 9, "date_of_birth": "2016-06-01"}, - "child_2": {"age": 7, "date_of_birth": "2019-03-01"}, + "child_1": {"age": 6, "date_of_birth": "2018-06-01"}, + "child_2": {"age": 2, "date_of_birth": "2022-09-01"}, + "child_3": {"age": 2, "date_of_birth": "2022-09-01"} + }, + "benefit_units": [ + {"adults": ["adult_1", "adult_2"], "children": ["child_1", "child_2", "child_3"]} + ], + "region": "WEST_MIDLANDS", + "tenure": "rented_social", + "rent": 8320 + }, + "expected_focus": ["universal_credit", "child_benefit"], + "rationale": "Two-child limit with the multiple-birth exception, pinned to policy year 2025 because the limit was abolished from 6 April 2026 (AB2025) and only a pre-abolition year can exercise it. All three children are post-April-2017, so the limit binds on ordinal position alone: child_1 takes the first element, child_2 (first of the twins) the second, and child_3 - the second child of the multiple birth - is exempt and takes a third element the limit would otherwise deny. The exemption sits on a third-or-later child, so it strictly changes the entitlement (3 elements vs 2). Also pins that Child Benefit ignores the limit (paid for all three). Shared DOBs are explicit because the exception keys off the birth event." + }, + { + "case_id": "uk-uc-two-child-limit-abolished", + "description": "Same three-child family evaluated post-abolition (2026)", + "policy_year": 2026, + "country": "UK", + "household": { + "people": { + "adult_1": {"age": 37}, + "adult_2": {"age": 35}, + "child_1": {"age": 7, "date_of_birth": "2018-06-01"}, + "child_2": {"age": 3, "date_of_birth": "2022-09-01"}, "child_3": {"age": 3, "date_of_birth": "2022-09-01"} }, "benefit_units": [ @@ -63,7 +86,7 @@ "rent": 8320 }, "expected_focus": ["universal_credit", "child_benefit"], - "rationale": "Two-child limit binding with the pre-April-2017 protection: child_1 (born 2016-06-01) is exempt from the limit, child_2 takes the second element, child_3 gets none. Also pins that Child Benefit is unaffected by the limit (paid for all three). DOBs are explicit because the exemption keys off date of birth, not age." + "rationale": "The abolition counterpart to uk-uc-two-child-limit-multiple-birth: the same family one year on, in the first year the two-child limit no longer exists (removed from 6 April 2026, AB2025). All three post-April-2017 children take child elements with no exception needed; an engine still applying the limit under-pays exactly one element. Paired with the 2025 case, the year-on-year delta isolates the abolition itself." }, { "case_id": "uk-benefit-cap-london", diff --git a/tests/test_case_schema.py b/tests/test_case_schema.py index dc86a49..6f5e695 100644 --- a/tests/test_case_schema.py +++ b/tests/test_case_schema.py @@ -54,9 +54,34 @@ def test_uk_battery_shape(self): for c in load_battery(BATTERY): assert c.country == "UK" assert c.case_id.startswith("uk-") - assert c.policy_year == 2026 + # 2025 is deliberate: the two-child limit ended 6 April 2026, + # so only a pre-abolition year can exercise its exemptions. + assert c.policy_year in (2025, 2026) assert c.rationale # every case explains itself + def test_two_child_limit_pair(self): + # The exemption case must sit in a year the limit exists, and its + # abolition counterpart in the first year it does not. + by_id = {c.case_id: c for c in load_battery(BATTERY)} + binding = by_id["uk-uc-two-child-limit-multiple-birth"] + abolished = by_id["uk-uc-two-child-limit-abolished"] + assert binding.policy_year == 2025 + assert abolished.policy_year == 2026 + for case in (binding, abolished): + children = case.household["benefit_units"][0]["children"] + assert len(children) == 3 + # Every child post-April-2017, so no pre-2017 protection can + # stand in for the exemption / abolition under test. + for child in children: + dob = case.household["people"][child]["date_of_birth"] + assert dob > "2017-04-06" + # The multiple birth spans a third-or-later child: shared DOBs on + # the 2nd and 3rd children, so the exception changes entitlement. + people = binding.household["people"] + assert ( + people["child_2"]["date_of_birth"] == (people["child_3"]["date_of_birth"]) + ) + def test_battery_is_inputs_only(self): # No case embeds expected output values (SCHEMA.md doctrine). raw = json.loads(BATTERY.read_text()) @@ -138,6 +163,22 @@ def test_empty_focus_rejected(self): with pytest.raises(ValueError, match="expected_focus"): CaseSpec(**case_kwargs(expected_focus=[])) + def test_string_focus_rejected(self): + # A bare string iterates as characters; the schema demands a list. + with pytest.raises(ValueError, match="expected_focus"): + CaseSpec(**case_kwargs(expected_focus="universal_credit")) + + def test_string_unit_members_rejected(self): + kw = case_kwargs() + kw["household"]["benefit_units"] = [{"adults": "adult_1", "children": []}] + with pytest.raises(ValueError, match="list of person ids"): + CaseSpec(**kw) + kw["household"]["benefit_units"] = [ + {"adults": ["adult_1"], "children": "child_1"} + ] + with pytest.raises(ValueError, match="list of person ids"): + CaseSpec(**kw) + class TestClassify: def test_exact(self): @@ -197,6 +238,7 @@ def result_kwargs(self, **kw): computed_at="2026-08-14T12:00:00Z", classification="match_within_tolerance", abs_diff=0.04, + tolerance=0.52, ) base.update(kw) return base @@ -222,7 +264,9 @@ def test_null_pe_value_requires_pe_gap(self): with pytest.raises(ValueError, match="pe_gap"): CaseResult(**self.result_kwargs(pe_value=None, abs_diff=None)) r = CaseResult( - **self.result_kwargs(pe_value=None, abs_diff=None, classification="pe_gap") + **self.result_kwargs( + pe_value=None, abs_diff=None, tolerance=None, classification="pe_gap" + ) ) assert r.classification is DiffClassification.PE_GAP @@ -233,3 +277,47 @@ def test_null_oracle_value_requires_scope_mismatch(self): def test_match_exact_requires_identity(self): with pytest.raises(ValueError, match="match_exact"): CaseResult(**self.result_kwargs(classification="match_exact")) + + def test_within_tolerance_requires_tolerance(self): + # An empty-annotation, no-tolerance row must not validate. + with pytest.raises(ValueError, match="tolerance"): + CaseResult(**self.result_kwargs(tolerance=None)) + + def test_within_tolerance_bounds_enforced(self): + with pytest.raises(ValueError, match="abs_diff <= tolerance"): + CaseResult(**self.result_kwargs(oracle_value=4797.48, abs_diff=1.0)) + with pytest.raises(ValueError, match="tolerance > 0"): + CaseResult(**self.result_kwargs(tolerance=0.0)) + with pytest.raises(ValueError, match="abs_diff <= tolerance"): + # A zero diff is match_exact, never match_within_tolerance. + CaseResult(**self.result_kwargs(oracle_value=4796.48, abs_diff=0.0)) + + def test_tolerance_only_on_within_tolerance_rows(self): + with pytest.raises(ValueError, match="only on match_within_tolerance"): + CaseResult( + **self.result_kwargs( + pe_value=None, abs_diff=None, classification="pe_gap" + ) + ) + + def test_adjudicated_classes_require_writeup(self): + for cls in ("oracle_difference", "rounding"): + with pytest.raises(ValueError, match="adjudicated"): + CaseResult(**self.result_kwargs(classification=cls, tolerance=None)) + r = CaseResult( + **self.result_kwargs( + classification=cls, + tolerance=None, + annotations=[ + "UKMOD rounds monthly amounts to the penny; " + "x12 explains the 0.04 (writeup ref)" + ], + ) + ) + assert r.classification is DiffClassification(cls) + + def test_annotations_must_be_nonempty_strings(self): + with pytest.raises(ValueError, match="annotations"): + CaseResult(**self.result_kwargs(annotations="a writeup")) + with pytest.raises(ValueError, match="annotations"): + CaseResult(**self.result_kwargs(annotations=[""])) From 8adbc93db6994e532187feaf966e81844b3e5217 Mon Sep 17 00:00:00 2001 From: r Date: Wed, 19 Aug 2026 14:07:21 +0100 Subject: [PATCH 3/5] Close the classify->CaseResult seam, persist variable_class, add schema_version, real-date DOB validation Review items from the 2026-08-18 pass: - CaseResult.from_classification() is the one wiring path from classify to a stored row: derives abs_diff, threads the exact tolerance classify judged against, persists variable_class. Six end-to-end seam tests. - variable_class persisted on CaseResult (closed vocabulary) so stored rows are auditable without inferring the rule from the variable name. - SCHEMA_VERSION = 1 on the battery file and every result row; a mismatched or missing version raises, so breaking changes migrate explicitly. - date_of_birth must be a real calendar date (2026-13-40 now fails). Suite: 176 passed, 4 skipped. Co-Authored-By: Claude Fable 5 --- scorecard_db/case_diffs.py | 89 +++++++++++++++++++++-- sources/ukmod-cases/SCHEMA.md | 17 ++++- sources/ukmod-cases/battery/cases.json | 1 + tests/test_case_schema.py | 98 ++++++++++++++++++++++++++ 4 files changed, 198 insertions(+), 7 deletions(-) diff --git a/scorecard_db/case_diffs.py b/scorecard_db/case_diffs.py index e369b01..dd1e2f4 100644 --- a/scorecard_db/case_diffs.py +++ b/scorecard_db/case_diffs.py @@ -24,10 +24,16 @@ import json from dataclasses import dataclass, field +from datetime import date from enum import Enum from pathlib import Path from typing import Optional +# Contract version for the battery file and result rows. Bump on any +# breaking change to the vocabularies or row shape so stored artifacts +# can be migrated explicitly instead of silently reinterpreted. +SCHEMA_VERSION = 1 + class Oracle(str, Enum): UKMOD = "ukmod" @@ -125,10 +131,18 @@ def _validate_person(person_id: str, person: dict) -> None: if not isinstance(age, int) or isinstance(age, bool) or age < 0: raise ValueError(f"person {person_id!r} needs an integer age >= 0") dob = person.get("date_of_birth") - if dob is not None and not ( - isinstance(dob, str) and len(dob) == 10 and dob[4] == "-" and dob[7] == "-" - ): - raise ValueError(f"person {person_id!r} date_of_birth must be YYYY-MM-DD") + if dob is not None: + # a real calendar date, not just the YYYY-MM-DD shape + # ("2026-13-40" must fail) + try: + if not (isinstance(dob, str) and len(dob) == 10): + raise ValueError + date.fromisoformat(dob) + except ValueError: + raise ValueError( + f"person {person_id!r} date_of_birth must be a real " + f"YYYY-MM-DD date, got {dob!r}" + ) from None for key in PERSON_NUMERIC_KEYS & set(person): v = person[key] if isinstance(v, bool) or not isinstance(v, (int, float)) or v < 0: @@ -236,11 +250,23 @@ class CaseResult: classification: DiffClassification abs_diff: Optional[float] = None tolerance: Optional[float] = None + # Which tolerance rule applied — persisted so a stored row can be + # re-classified/audited without inferring the class from the + # free-text variable name. + variable_class: Optional[VariableClass] = None annotations: list = field(default_factory=list) + schema_version: int = SCHEMA_VERSION def __post_init__(self): self.oracle = Oracle(self.oracle) self.classification = DiffClassification(self.classification) + if self.variable_class is not None: + self.variable_class = VariableClass(self.variable_class) + if self.schema_version != SCHEMA_VERSION: + raise ValueError( + f"schema_version {self.schema_version!r} is not the current " + f"contract version {SCHEMA_VERSION}; migrate explicitly" + ) if not self.case_id or not self.variable: raise ValueError("case_id and variable are required") if not isinstance(self.annotations, list) or not all( @@ -296,6 +322,54 @@ def __post_init__(self): "requires a writeup in annotations (SCHEMA.md)" ) + @classmethod + def from_classification( + cls, + *, + case_id: str, + variable: str, + pe_value: Optional[float], + oracle_value: Optional[float], + variable_class: VariableClass, + oracle: Oracle, + engine_version: str, + oracle_version: str, + computed_at: str, + tolerance_table: dict = DEFAULT_TOLERANCES, + ) -> "CaseResult": + """The one wiring path from ``classify`` to a stored row. + + Derives abs_diff, threads the exact tolerance ``classify`` judged + against (only onto match_within_tolerance rows, per the field + contract), and persists the variable class — so no caller ever + re-derives the tolerance by hand and passes the wrong one. + """ + variable_class = VariableClass(variable_class) + classification = classify( + pe_value, oracle_value, variable_class, tolerance_table + ) + both = pe_value is not None and oracle_value is not None + abs_diff = abs(float(pe_value) - float(oracle_value)) if both else None + tolerance = ( + float(tolerance_table[variable_class]) + if classification == DiffClassification.MATCH_WITHIN_TOLERANCE + else None + ) + return cls( + case_id=case_id, + variable=variable, + pe_value=pe_value, + oracle_value=oracle_value, + oracle=oracle, + engine_version=engine_version, + oracle_version=oracle_version, + computed_at=computed_at, + classification=classification, + abs_diff=abs_diff, + tolerance=tolerance, + variable_class=variable_class, + ) + def classify( pe_value: Optional[float], @@ -325,7 +399,7 @@ def classify( return DiffClassification.UNCLASSIFIED -BATTERY_KEYS = frozenset({"schema", "description", "cases"}) +BATTERY_KEYS = frozenset({"schema", "schema_version", "description", "cases"}) CASE_KEYS = frozenset( { "case_id", @@ -347,6 +421,11 @@ def load_battery(path) -> list[CaseSpec]: unknown = set(raw) - BATTERY_KEYS if unknown: raise ValueError(f"battery has unknown keys: {sorted(unknown)}") + if raw.get("schema_version") != SCHEMA_VERSION: + raise ValueError( + f"battery schema_version must be {SCHEMA_VERSION}, " + f"got {raw.get('schema_version')!r}" + ) if not isinstance(raw.get("cases"), list) or not raw["cases"]: raise ValueError("battery.cases must be a non-empty list") cases = [] diff --git a/sources/ukmod-cases/SCHEMA.md b/sources/ukmod-cases/SCHEMA.md index 6426539..8ab6088 100644 --- a/sources/ukmod-cases/SCHEMA.md +++ b/sources/ukmod-cases/SCHEMA.md @@ -86,7 +86,7 @@ Person keys (closed set): | key | type | meaning | |---|---|---| | `age` | int, required | age at the start of the policy year | -| `date_of_birth` | `YYYY-MM-DD` | only when the exact date is load-bearing (two-child-limit protection, state pension age) | +| `date_of_birth` | `YYYY-MM-DD`, a real calendar date (`2026-13-40` fails) | only when the exact date is load-bearing (two-child-limit protection, state pension age) | | `employment_income` | number ≥ 0 | gross annual employee earnings | | `self_employment_income` | number ≥ 0 | annual trading profit | | `pension_income` | number ≥ 0 | private/occupational pension in payment | @@ -126,7 +126,9 @@ as `pe_results`). "computed_at": "2026-08-14T12:00:00Z", "abs_diff": 0.04, "tolerance": 0.52, - "classification": "match_within_tolerance" + "variable_class": "currency", + "classification": "match_within_tolerance", + "schema_version": 1 } ``` @@ -148,6 +150,17 @@ as `pe_results`). the numeric tolerance the row was judged against, satisfying `0 < abs_diff ≤ tolerance`. Stored so a tolerance-table change can never silently re-bless old rows. +- `variable_class` — which tolerance rule applied (`currency` / + `boolean`), persisted so a stored row can be re-classified and + audited without inferring the class from the free-text `variable`. +- `schema_version` — this contract's version (currently `1`), on the + battery file and every result row; a mismatched version raises, so + a future breaking change migrates stored artifacts explicitly. +- Connectors build rows via `CaseResult.from_classification(...)` — + the one wiring path from `classify` to a stored row. It derives + `abs_diff`, threads the exact tolerance `classify` judged against, + and persists `variable_class`, so no caller re-derives the + tolerance by hand. - `annotations` — a list of non-empty strings. Required (non-empty) on the adjudicated-only classifications `oracle_difference` and `rounding`: the traceable writeup naming the oracle defect / documented rounding rule. diff --git a/sources/ukmod-cases/battery/cases.json b/sources/ukmod-cases/battery/cases.json index 1afbadc..cd71c88 100644 --- a/sources/ukmod-cases/battery/cases.json +++ b/sources/ukmod-cases/battery/cases.json @@ -1,5 +1,6 @@ { "schema": "sources/ukmod-cases/SCHEMA.md", + "schema_version": 1, "description": "Initial UK mode-3 case battery for the UKMOD/EUROMOD lane (#41): standard EUROMOD-style hypothetical households plus the edge cases PE UK carries regression coverage for. Inputs + expected focus only — expected output values come from engine runs (PE and the oracle), never from this file. All monetary amounts are annual GBP; weekly-quoted amounts enter as weekly x 52.", "cases": [ { diff --git a/tests/test_case_schema.py b/tests/test_case_schema.py index 6f5e695..13475d6 100644 --- a/tests/test_case_schema.py +++ b/tests/test_case_schema.py @@ -8,6 +8,7 @@ import pytest from scorecard_db.case_diffs import ( + SCHEMA_VERSION, DEFAULT_TOLERANCES, CaseResult, CaseSpec, @@ -321,3 +322,100 @@ def test_annotations_must_be_nonempty_strings(self): CaseResult(**self.result_kwargs(annotations="a writeup")) with pytest.raises(ValueError, match="annotations"): CaseResult(**self.result_kwargs(annotations=[""])) + + +class TestDateOfBirthRealDate: + def test_impossible_date_rejected(self): + for bad in ("2026-13-40", "0000-00-00", "2025-02-30"): + hh = { + "people": {"adult_1": {"age": 30, "date_of_birth": bad}}, + "benefit_units": [{"adults": ["adult_1"], "children": []}], + } + with pytest.raises(ValueError, match="real"): + CaseSpec(**case_kwargs(household=hh)) + + def test_real_date_accepted(self): + hh = { + "people": {"adult_1": {"age": 30, "date_of_birth": "1996-02-29"}}, + "benefit_units": [{"adults": ["adult_1"], "children": []}], + } + CaseSpec(**case_kwargs(household=hh)) + + +class TestSchemaVersion: + def test_battery_carries_current_version(self): + raw = json.loads(BATTERY.read_text()) + assert raw["schema_version"] == SCHEMA_VERSION + + def test_battery_with_wrong_version_rejected(self, tmp_path): + raw = json.loads(BATTERY.read_text()) + raw["schema_version"] = SCHEMA_VERSION + 1 + p = tmp_path / "cases.json" + p.write_text(json.dumps(raw)) + with pytest.raises(ValueError, match="schema_version"): + load_battery(p) + + def test_battery_without_version_rejected(self, tmp_path): + raw = json.loads(BATTERY.read_text()) + del raw["schema_version"] + p = tmp_path / "cases.json" + p.write_text(json.dumps(raw)) + with pytest.raises(ValueError, match="schema_version"): + load_battery(p) + + def test_result_row_rejects_foreign_version(self): + kw = TestCaseResult().result_kwargs(schema_version=SCHEMA_VERSION + 1) + with pytest.raises(ValueError, match="schema_version"): + CaseResult(**kw) + + +class TestClassifyToResultSeam: + """The classify -> CaseResult wiring path (review: previously untested; + callers had to re-derive the tolerance by hand).""" + + def wired(self, pe, oracle_value, vclass=VariableClass.CURRENCY, **kw): + base = dict( + case_id="uk-uc-single-unemployed", + variable="universal_credit", + pe_value=pe, + oracle_value=oracle_value, + variable_class=vclass, + oracle=Oracle.UKMOD, + engine_version="policyengine-uk 2.0.0", + oracle_version="UKMOD B2026.08", + computed_at="2026-08-14T12:00:00Z", + ) + base.update(kw) + return CaseResult.from_classification(**base) + + def test_tolerance_match_threads_the_applied_tolerance(self): + r = self.wired(100.0, 100.30) + assert r.classification is DiffClassification.MATCH_WITHIN_TOLERANCE + assert r.tolerance == DEFAULT_TOLERANCES[VariableClass.CURRENCY] + assert r.variable_class is VariableClass.CURRENCY + assert abs(r.abs_diff - 0.30) < 1e-9 + + def test_exact_match_carries_no_tolerance(self): + r = self.wired(100.0, 100.0) + assert r.classification is DiffClassification.MATCH_EXACT + assert r.tolerance is None and r.abs_diff == 0.0 + + def test_null_pe_side_wires_to_pe_gap(self): + r = self.wired(None, 5.0) + assert r.classification is DiffClassification.PE_GAP + assert r.abs_diff is None and r.tolerance is None + + def test_custom_table_tolerance_is_the_one_stored(self): + table = {VariableClass.CURRENCY: 1.0} + r = self.wired(100.0, 100.9, tolerance_table=table) + assert r.classification is DiffClassification.MATCH_WITHIN_TOLERANCE + assert r.tolerance == 1.0 + + def test_above_tolerance_lands_unclassified(self): + r = self.wired(100.0, 200.0) + assert r.classification is DiffClassification.UNCLASSIFIED + assert r.tolerance is None + + def test_variable_class_is_a_closed_vocabulary(self): + with pytest.raises(ValueError): + self.wired(100.0, 100.0, vclass="percentage") From 9eff5901b842661bfefe31f981b000b5279e3f83 Mon Sep 17 00:00:00 2001 From: r Date: Thu, 20 Aug 2026 13:53:17 +0100 Subject: [PATCH 4/5] Restore the lane feed's derived updated date MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `updated` is derived, not authored: sync_lane_feed() sets it from the max of the lane rows' own updated_at (2026-08-19) and rewrites the file. The rebase conflict resolution hand-set it to 2026-08-20, so every ingest reverts it — invisible while the DB was committed, but #74 builds the DB during collection, so the drift now fails the no-drift gate and test_app_data_copies_match_committed_data (the build rewrites data/ but not the app/public/ copy). Co-Authored-By: Claude Fable 5 --- app/public/data/lanes.json | 2 +- data/lanes.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/public/data/lanes.json b/app/public/data/lanes.json index 68dab29..6f7e653 100644 --- a/app/public/data/lanes.json +++ b/app/public/data/lanes.json @@ -4,7 +4,7 @@ "country": "US | UK \u2014 the model instance the lane feeds (issue #42)", "note": "Repo-committed status feed (issue #7); graduate to Supabase when lane count makes commits noisy. Every lane writes its transition here." }, - "updated": "2026-08-20", + "updated": "2026-08-19", "lanes": [ { "id": "urban-sotsn", diff --git a/data/lanes.json b/data/lanes.json index 68dab29..6f7e653 100644 --- a/data/lanes.json +++ b/data/lanes.json @@ -4,7 +4,7 @@ "country": "US | UK \u2014 the model instance the lane feeds (issue #42)", "note": "Repo-committed status feed (issue #7); graduate to Supabase when lane count makes commits noisy. Every lane writes its transition here." }, - "updated": "2026-08-20", + "updated": "2026-08-19", "lanes": [ { "id": "urban-sotsn", From 7684f6421359abfe40a82c3361b3c5a19aa1deb4 Mon Sep 17 00:00:00 2001 From: r Date: Fri, 21 Aug 2026 12:55:32 +0100 Subject: [PATCH 5/5] Close the mode-3 seam structurally; five of six re-gate findings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1. The classify -> CaseResult seam is enforced, not conventional. __post_init__ now RECOMPUTES classify() and compares. A stored classification is valid only if it equals the classifier's, or if it adjudicates a row the classifier left `unclassified` — drawn from ADJUDICATABLE and carrying a writeup. Every round-2 probe now raises: 100 vs 100 as unclassified/pe_gap, a numeric mismatch as policy_scope_mismatch with nothing said, and a boolean 1 vs 0 as match_within_tolerance with a caller-supplied tolerance of 100. pe_gap and policy_scope_mismatch are two-sided: mechanical on a null side, adjudicated (and explained) on a numeric-vs-numeric row — previously only oracle_difference and rounding demanded a writeup. Tolerances now come from DEFAULT_TOLERANCES alone; from_classification no longer takes a table, and a row carrying a different tolerance is rejected. It DOES now take annotations, which is also what made every #64 calculator call raise. 2. variable_class is required (the validator cannot check a row without it, and the shipped valid-row test omitted it) and schema_version is required rather than defaulting to the current contract. Blank engine or oracle versions are rejected. 4. The two-child-limit family isolates one mechanism per pair. Three 2026 cases: `binding` (three SEPARATE births, so no exception can stand in for the limit) and `multiple-birth` (identical household, one date of birth apart) both in the registered `pre_ab2025` world, and `abolished` — byte-identical household, same year, same rates — under current law. binding/abolished attributes the abolition; binding/multiple-birth attributes the exception. The old pair used twins on both sides, so an engine that wrongly kept the limit but rightly applied the exception still paid three elements and passed, and its year-on-year delta moved ages, rates and policy year at once. Cases can now reference a registered baseline world, which is the instrument that made the same-year pair possible. 5. Identity closure reaches the edges. Regions are a closed per-country registry (region="MARS" raises), expected_focus is closed per country and may not repeat, NaN and infinity are rejected everywhere an amount is read, a boolean-class comparison may only hold 0 or 1, the battery's `schema` path must name a contract this repo defines, and a case baseline must already be registered in baselines.py. UK private-rent cases must now pin a BRMA — LHA is set per Broad Rental Market Area, so "the rent clears the cap in most Yorkshire BRMAs" was not a pinned world — and a BRMA outside its own region raises. 6. sync_lane_feed's `updated` is documented as the caller-supplied LITERAL it is, with the constraint that every caller in a build must pass the same constant, so the next caller does not reintroduce the drift. Deliberately NOT in this commit: finding 3, the case/result table, writer, exporter and build_db step, plus the epistemic columns and the status/diagnosis split that ride on them. That is the piece with repo-wide surface that Max offered to pair on, and guessing at it unilaterally is how it ends up re-litigated. SCHEMA.md and the module docstring now state the open shape explicitly rather than leaving it implied. Suite 306 passed / 4 skipped, ruff format clean. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_016HuXJFVme8HRbnke2Ey3Me --- scorecard_db/case_diffs.py | 337 +++++++++++++++-- scorecard_db/ingest_harvest.py | 13 +- sources/ukmod-cases/SCHEMA.md | 88 ++++- sources/ukmod-cases/battery/cases.json | 480 ++++++++++++++++++++----- tests/test_case_schema.py | 248 +++++++++++-- 5 files changed, 1012 insertions(+), 154 deletions(-) diff --git a/scorecard_db/case_diffs.py b/scorecard_db/case_diffs.py index dd1e2f4..09db78e 100644 --- a/scorecard_db/case_diffs.py +++ b/scorecard_db/case_diffs.py @@ -15,9 +15,29 @@ values; both sides of every comparison come from engine runs, so the battery cannot smuggle in hand-computed truth. 3. **The classifier never flatters.** ``classify`` emits match buckets, the - two null-side scope buckets, or ``unclassified``; ``oracle_difference`` - and ``rounding`` exist only as adjudicated outcomes with a traceable - writeup — misses stay visible until someone explains them. + two null-side scope buckets, or ``unclassified``; every other outcome + exists only as an ADJUDICATION of an ``unclassified`` row, with a + traceable writeup — misses stay visible until someone explains them. +4. **The seam is enforced, not conventional.** ``CaseResult`` RECOMPUTES + ``classify()`` and compares: a stored classification either equals what + the classifier says, or it is an adjudication of a row the classifier + left ``unclassified`` and carries a writeup. Nothing can persist + ``100`` vs ``100`` as ``pe_gap``, a boolean ``1`` vs ``0`` as a + tolerance match, or a numeric mismatch as ``policy_scope_mismatch`` + without an explanation — the failure modes the round-2 probes found. + Tolerances come from :data:`DEFAULT_TOLERANCES` alone; a caller cannot + hand in a wider one. +5. **Identities are closed at the edges too.** Regions, focus variables, + battery schema paths and baseline worlds route through registries; NaN + and infinity are not numbers a case may carry, and a boolean-class + comparison may only hold 0 or 1. + +Not yet here: the case/result TABLE, writer, exporter and build_db step. +Mode-3 rows therefore do not persist to the DB in this PR, and the +epistemic wiring that rides on those columns (calibration_relationship, +run/bundle pins, connector revision, citable adjudication links) lands +with them. That design has repo-wide surface and is being paired on +rather than decided unilaterally here; SCHEMA.md records the open shape. """ from __future__ import annotations @@ -55,9 +75,7 @@ class DiffClassification(str, Enum): UNCLASSIFIED = "unclassified" -# Classifications the automatic classifier may emit; the rest -# (oracle_difference, rounding, and pe_gap/policy_scope_mismatch on -# numeric-vs-numeric rows) require adjudication with a writeup. +# Classifications the automatic classifier may emit. CLASSIFIER_EMITTED = frozenset( { DiffClassification.MATCH_EXACT, @@ -68,9 +86,24 @@ class DiffClassification(str, Enum): } ) -# Adjudicated-only outcomes: a row may carry one of these only with a -# traceable writeup in ``annotations`` (SCHEMA.md — "misses stay visible -# until someone explains them"). +# Outcomes a human may adjudicate an ``unclassified`` row INTO, each +# requiring a traceable writeup in ``annotations``. Note that pe_gap and +# policy_scope_mismatch appear both here and in CLASSIFIER_EMITTED: the +# classifier emits them for a NULL side, where they are mechanical, and a +# human may also assign them to a numeric-vs-numeric difference, where +# they are a judgement and need explaining. The round-2 finding was that +# only oracle_difference and rounding demanded a writeup, so a +# numeric-vs-numeric row could be labelled pe_gap silently. +ADJUDICATABLE = frozenset( + { + DiffClassification.ORACLE_DIFFERENCE, + DiffClassification.ROUNDING, + DiffClassification.PE_GAP, + DiffClassification.POLICY_SCOPE_MISMATCH, + } +) + +# Back-compatible alias for the adjudicated-only pair. ADJUDICATED_ONLY = frozenset( { DiffClassification.ORACLE_DIFFERENCE, @@ -117,9 +150,99 @@ class DiffClassification(str, Enum): ) HOUSEHOLD_KEYS = frozenset( - {"people", "benefit_units", "region", "tenure", "rent", "council_tax"} + {"people", "benefit_units", "region", "tenure", "rent", "council_tax", "brma"} ) +# Regions are a CLOSED registry per country, not free text: region routes +# devolved policy and location-dependent amounts, so `region="MARS"` must +# not reach a connector that will silently fall back to a default rate. +VALID_REGIONS = { + # UK ITL-1 + "UK": frozenset( + { + "NORTH_EAST", + "NORTH_WEST", + "YORKSHIRE", + "EAST_MIDLANDS", + "WEST_MIDLANDS", + "EAST_OF_ENGLAND", + "LONDON", + "SOUTH_EAST", + "SOUTH_WEST", + "WALES", + "SCOTLAND", + "NORTHERN_IRELAND", + } + ), + # US state codes (+ DC), for the taxsim-cases lane + "US": frozenset( + "AL AK AZ AR CA CO CT DE DC FL GA HI ID IL IN IA KS KY LA ME MD MA " + "MI MN MS MO MT NE NV NH NJ NM NY NC ND OH OK OR PA RI SC SD TN TX " + "UT VT VA WA WV WI WY".split() + ), +} + +# Broad Rental Market Areas the private-rent cases may pin. A UK +# private-rent case MUST name one: LHA rates are set per BRMA, not per +# ITL-1 region, so "Yorkshire" does not fix a world — the round-2 finding +# that the rent clears the cap only in "most" Yorkshire BRMAs means the +# two engines were not guaranteed the same LHA rate. Extending the list +# is deliberate, and each entry names the region it sits in so a case +# cannot pin a BRMA outside its own region. +BRMA_REGION = { + "Leeds": "YORKSHIRE", + "Sheffield": "YORKSHIRE", + "Inner London": "LONDON", + "Outer London": "LONDON", + "Manchester": "NORTH_WEST", + "Nottingham": "EAST_MIDLANDS", + "Birmingham": "WEST_MIDLANDS", + "Southampton": "SOUTH_EAST", +} + +# expected_focus is claim-shaped: it names the output variables a case is +# curated to exercise, and a connector maps each to both engines. An +# invented name would silently produce no comparison at all, so the +# vocabulary is closed per country. +FOCUS_VARIABLES = { + "UK": frozenset( + { + "universal_credit", + "child_benefit", + "pension_credit", + "carers_allowance", + "benefit_cap", + "income_tax", + "national_insurance", + "scottish_child_payment", + "council_tax_reduction", + "housing_benefit", + } + ), + "US": frozenset( + { + "federal_income_tax", + "state_income_tax", + "payroll_tax", + "eitc", + "ctc", + "snap", + } + ), +} + +# Battery `schema` values: the contract document a battery declares +# itself against. An arbitrary path let a battery claim a schema nobody +# wrote. +VALID_BATTERY_SCHEMAS = frozenset({"sources/ukmod-cases/SCHEMA.md"}) + + +def _finite(value) -> bool: + """NaN and infinity are not amounts a case or a comparison may carry: + NaN silently defeats every comparison operator below, and an infinite + rent produces an infinite entitlement rather than an error.""" + return value == value and value not in (float("inf"), float("-inf")) + def _validate_person(person_id: str, person: dict) -> None: if not isinstance(person, dict): @@ -145,14 +268,16 @@ def _validate_person(person_id: str, person: dict) -> None: ) from None for key in PERSON_NUMERIC_KEYS & set(person): v = person[key] - if isinstance(v, bool) or not isinstance(v, (int, float)) or v < 0: + if isinstance(v, bool) or not isinstance(v, (int, float)) or not _finite(v): + raise ValueError(f"person {person_id!r} {key} must be a finite number >= 0") + if v < 0: raise ValueError(f"person {person_id!r} {key} must be a number >= 0") for key in PERSON_BOOL_KEYS & set(person): if not isinstance(person[key], bool): raise ValueError(f"person {person_id!r} {key} must be a boolean") -def _validate_household(household: dict) -> None: +def _validate_household(household: dict, country: str) -> None: if not isinstance(household, dict): raise ValueError("household must be a mapping") unknown = set(household) - HOUSEHOLD_KEYS @@ -193,11 +318,60 @@ def _validate_household(household: dict) -> None: for key in ("rent", "council_tax"): v = household.get(key) if v is not None and ( - isinstance(v, bool) or not isinstance(v, (int, float)) or v < 0 + isinstance(v, bool) + or not isinstance(v, (int, float)) + or not _finite(v) + or v < 0 ): - raise ValueError(f"household.{key} must be a number >= 0") + raise ValueError(f"household.{key} must be a finite number >= 0") if household.get("rent", 0) and tenure in ("owned_outright", "owned_mortgage"): raise ValueError("owner-occupier households carry no rent") + region = household.get("region") + if region is not None and region not in VALID_REGIONS[country]: + raise ValueError( + f"unregistered {country} region {region!r} — region routes " + "devolved policy and location-dependent amounts, so it is a " + "closed registry (VALID_REGIONS), never free text" + ) + brma = household.get("brma") + if country == "UK" and tenure == "rented_private": + if brma is None: + raise ValueError( + "a UK private-rent case must pin a BRMA: LHA rates are set " + "per Broad Rental Market Area, not per ITL-1 region, so a " + "region alone does not fix the world both engines run" + ) + if brma not in BRMA_REGION: + raise ValueError(f"unregistered BRMA: {brma!r}") + if region is not None and BRMA_REGION[brma] != region: + raise ValueError(f"BRMA {brma!r} sits in {BRMA_REGION[brma]}, not {region}") + elif brma is not None: + raise ValueError("brma is recorded only on UK private-rent cases") + + +def _validate_baseline(descriptor) -> None: + """A case may only name a baseline world the registry describes. + + Registration is what makes a counterfactual auditable: an unnamed + world is exactly the "which law did this run?" ambiguity mode 3 + exists to remove. Imported lazily — baselines.py reaches the DB + layer, and this module must stay importable on its own. + """ + if descriptor is None: + return + if not isinstance(descriptor, dict) or not descriptor.get("policy"): + raise ValueError( + f"case baseline must be a descriptor with a policy key: {descriptor!r}" + ) + from .baselines import BASELINES + from .models import baseline_key + + known = {baseline_key(d) for d, *_ in BASELINES} + if baseline_key(descriptor) not in known: + raise ValueError( + f"unregistered baseline world {descriptor!r} — register it in " + "scorecard_db/baselines.py before a case may be evaluated in it" + ) @dataclass @@ -211,6 +385,13 @@ class CaseSpec: household: dict expected_focus: list rationale: str = "" + # The policy world the case is evaluated in. None = current law. + # A descriptor of the same shape ReformRef.baseline carries, and it + # must already be REGISTERED in baselines.py — this is how a case + # pins a same-year counterfactual (e.g. {"policy": "pre_ab2025"}, + # the two-child limit reinstated) instead of leaning on a + # year-on-year delta that moves ages, rates and policy year at once. + baseline: Optional[dict] = None def __post_init__(self): if not self.case_id or not isinstance(self.case_id, str): @@ -232,7 +413,16 @@ def __post_init__(self): or not all(isinstance(v, str) and v for v in self.expected_focus) ): raise ValueError("expected_focus must be a non-empty list of variables") - _validate_household(self.household) + unknown_focus = sorted(set(self.expected_focus) - FOCUS_VARIABLES[self.country]) + if unknown_focus: + raise ValueError( + f"unregistered {self.country} focus variables: {unknown_focus} — " + "a name no connector maps produces no comparison at all" + ) + if len(set(self.expected_focus)) != len(self.expected_focus): + raise ValueError("expected_focus must not repeat a variable") + _validate_baseline(self.baseline) + _validate_household(self.household, self.country) @dataclass @@ -248,20 +438,28 @@ class CaseResult: oracle_version: str computed_at: str # ISO timestamp, caller-supplied classification: DiffClassification + # REQUIRED, not optional: the validator re-runs classify() against it, + # so a row without a variable class cannot be checked at all. (It was + # Optional, and the shipped valid-row test omitted it.) + variable_class: VariableClass abs_diff: Optional[float] = None tolerance: Optional[float] = None - # Which tolerance rule applied — persisted so a stored row can be - # re-classified/audited without inferring the class from the - # free-text variable name. - variable_class: Optional[VariableClass] = None annotations: list = field(default_factory=list) - schema_version: int = SCHEMA_VERSION + # REQUIRED: an omitted version used to default to the current one, + # which is precisely the silent reinterpretation the field exists to + # prevent. `None` raises rather than assuming 1. + schema_version: Optional[int] = None def __post_init__(self): self.oracle = Oracle(self.oracle) self.classification = DiffClassification(self.classification) - if self.variable_class is not None: - self.variable_class = VariableClass(self.variable_class) + self.variable_class = VariableClass(self.variable_class) + if self.schema_version is None: + raise ValueError( + "schema_version is required — an omitted version silently " + f"reinterprets a stored row; write {SCHEMA_VERSION} " + "explicitly" + ) if self.schema_version != SCHEMA_VERSION: raise ValueError( f"schema_version {self.schema_version!r} is not the current " @@ -269,6 +467,22 @@ def __post_init__(self): ) if not self.case_id or not self.variable: raise ValueError("case_id and variable are required") + if not str(self.engine_version).strip() or not str(self.oracle_version).strip(): + raise ValueError( + "engine_version and oracle_version are required — a blank " + "version makes a stored comparison unreproducible" + ) + for name in ("pe_value", "oracle_value"): + v = getattr(self, name) + if v is None: + continue + if isinstance(v, bool) or not isinstance(v, (int, float)) or not _finite(v): + raise ValueError(f"{name} must be a finite number or None, got {v!r}") + if self.variable_class is VariableClass.BOOLEAN and v not in (0, 1): + raise ValueError( + f"boolean-class {name} must be 0 or 1, got {v!r} — a " + "boolean comparison carrying 7 is not a boolean" + ) if not isinstance(self.annotations, list) or not all( isinstance(a, str) and a.strip() for a in self.annotations ): @@ -316,11 +530,45 @@ def __post_init__(self): raise ValueError( "tolerance is recorded only on match_within_tolerance rows" ) - if self.classification in ADJUDICATED_ONLY and not self.annotations: - raise ValueError( - f"{self.classification.value} is an adjudicated outcome and " - "requires a writeup in annotations (SCHEMA.md)" - ) + # THE SEAM. Recompute rather than trust the caller: the factory + # was a convention, and round-2 probes persisted 100-vs-100 as + # pe_gap, a numeric mismatch as policy_scope_mismatch, and a + # boolean 1-vs-0 as a tolerance match with a caller-supplied + # tolerance of 100. A stored classification is valid only if it + # is what the classifier says, or an explained adjudication of a + # row the classifier left unclassified. + computed = classify(self.pe_value, self.oracle_value, self.variable_class) + if self.classification != computed: + if computed != DiffClassification.UNCLASSIFIED: + raise ValueError( + f"classification {self.classification.value!r} contradicts " + f"the classifier, which says {computed.value!r} for " + f"pe={self.pe_value!r} oracle={self.oracle_value!r} " + f"({self.variable_class.value}). Only an `unclassified` " + "row may be adjudicated." + ) + if self.classification not in ADJUDICATABLE: + raise ValueError( + f"{self.classification.value!r} is not an outcome an " + f"unclassified row may be adjudicated into " + f"({sorted(c.value for c in ADJUDICATABLE)})" + ) + if not self.annotations: + raise ValueError( + f"{self.classification.value} is an adjudicated outcome " + "and requires a writeup in annotations (SCHEMA.md)" + ) + # Tolerance comes from the registry, never from the caller: a + # hand-passed tolerance is how a boolean mismatch became a match. + if self.classification == DiffClassification.MATCH_WITHIN_TOLERANCE: + registered = float(DEFAULT_TOLERANCES[self.variable_class]) + if abs(float(self.tolerance) - registered) > 1e-12: + raise ValueError( + f"tolerance {self.tolerance!r} is not the registered " + f"tolerance for {self.variable_class.value} " + f"({registered}) — tolerances are a registry decision, " + "never a per-row argument" + ) @classmethod def from_classification( @@ -335,24 +583,29 @@ def from_classification( engine_version: str, oracle_version: str, computed_at: str, - tolerance_table: dict = DEFAULT_TOLERANCES, + annotations: Optional[list] = None, + classification: Optional[DiffClassification] = None, ) -> "CaseResult": """The one wiring path from ``classify`` to a stored row. - Derives abs_diff, threads the exact tolerance ``classify`` judged - against (only onto match_within_tolerance rows, per the field - contract), and persists the variable class — so no caller ever - re-derives the tolerance by hand and passes the wrong one. + Derives abs_diff, threads the registered tolerance (only onto + match_within_tolerance rows, per the field contract), and + persists the variable class. + + ``annotations`` is accepted here because some lanes REQUIRE one — + the calculator oracles (#64) must carry an archive annotation, so + a factory that could not take annotations made every calculator + call raise. ``classification`` is accepted only to record an + adjudication of a row the classifier leaves unclassified; the + constructor re-checks it either way. """ variable_class = VariableClass(variable_class) - classification = classify( - pe_value, oracle_value, variable_class, tolerance_table - ) + computed = classify(pe_value, oracle_value, variable_class) both = pe_value is not None and oracle_value is not None abs_diff = abs(float(pe_value) - float(oracle_value)) if both else None tolerance = ( - float(tolerance_table[variable_class]) - if classification == DiffClassification.MATCH_WITHIN_TOLERANCE + float(DEFAULT_TOLERANCES[variable_class]) + if computed == DiffClassification.MATCH_WITHIN_TOLERANCE else None ) return cls( @@ -364,10 +617,12 @@ def from_classification( engine_version=engine_version, oracle_version=oracle_version, computed_at=computed_at, - classification=classification, + classification=computed if classification is None else classification, abs_diff=abs_diff, tolerance=tolerance, variable_class=variable_class, + annotations=list(annotations or []), + schema_version=SCHEMA_VERSION, ) @@ -409,6 +664,7 @@ def classify( "household", "expected_focus", "rationale", + "baseline", } ) @@ -421,6 +677,11 @@ def load_battery(path) -> list[CaseSpec]: unknown = set(raw) - BATTERY_KEYS if unknown: raise ValueError(f"battery has unknown keys: {sorted(unknown)}") + if raw.get("schema") not in VALID_BATTERY_SCHEMAS: + raise ValueError( + f"battery schema {raw.get('schema')!r} is not a contract this " + f"repo defines ({sorted(VALID_BATTERY_SCHEMAS)})" + ) if raw.get("schema_version") != SCHEMA_VERSION: raise ValueError( f"battery schema_version must be {SCHEMA_VERSION}, " diff --git a/scorecard_db/ingest_harvest.py b/scorecard_db/ingest_harvest.py index 5d6df6b..b55957d 100644 --- a/scorecard_db/ingest_harvest.py +++ b/scorecard_db/ingest_harvest.py @@ -99,7 +99,18 @@ def sync_lane_feed( ) -> int: """Merge DB lane rows into the committed feed. `lanes` maps lane id -> display meta; defaults to this module's harvest registry (other - exporters pass their own — the merge only touches listed ids).""" + exporters pass their own — the merge only touches listed ids). + + ``updated`` is a LITERAL the caller supplies, not a derived value. + That is the contract, stated here because it is easy to misread: the + feed's top-level `updated` stamp is whatever the LAST caller in a + build passes, so every ingest that syncs this feed must pass the SAME + constant. If two callers disagree, the committed data/lanes.json + drifts depending on ingest order and the no-drift gate fails — the + exact breakage that had to be repaired once already. A lane's own + `updated_at` comes from its DB row and is per-lane; only this + top-level stamp is the shared literal. + """ feed = json.loads(feed_path.read_text()) by_id = {lane["id"]: lane for lane in feed["lanes"]} n = 0 diff --git a/sources/ukmod-cases/SCHEMA.md b/sources/ukmod-cases/SCHEMA.md index 8ab6088..ba157ee 100644 --- a/sources/ukmod-cases/SCHEMA.md +++ b/sources/ukmod-cases/SCHEMA.md @@ -102,11 +102,48 @@ Person keys (closed set): Household keys (closed set): `people`, `benefit_units`, `region`, `tenure` (`owned_outright` | `owned_mortgage` | `rented_social` | `rented_private`), -`rent` (annual), `council_tax` (annual, optional). `region` uses the target -country's own geography vocabulary (UK: ITL-1 slugs like `LONDON`, -`SCOTLAND`; US: state codes) — it is what routes devolved policy (Scottish -income tax, Scottish Child Payment; US state taxes) and location-dependent -amounts (LHA rates, benefit-cap tier). +`rent` (annual), `council_tax` (annual, optional), `brma`. `region` uses +the target country's own geography vocabulary (UK: ITL-1 slugs like +`LONDON`, `SCOTLAND`; US: state codes) — it is what routes devolved policy +(Scottish income tax, Scottish Child Payment; US state taxes) and the +benefit-cap tier. It is a CLOSED registry (`VALID_REGIONS`), never free +text: an unrecognised region would otherwise reach a connector that +silently falls back to a default. + +`brma` is REQUIRED on a UK `rented_private` case and forbidden elsewhere. +Local Housing Allowance rates are set per Broad Rental Market Area, not +per ITL-1 region, so a region alone does not fix the world the two engines +run — "the rent clears the cap in most Yorkshire BRMAs" is not a pinned +comparison. `BRMA_REGION` is the registry, and each entry names the region +it sits in, so a case cannot pin a BRMA outside its own region. + +`expected_focus` is likewise closed per country (`FOCUS_VARIABLES`): it +names the output variables the case is curated to exercise, and a name no +connector maps produces no comparison at all rather than an error. + +Amounts must be FINITE. NaN silently defeats every comparison operator, +and an infinite rent produces an infinite entitlement rather than a +failure. + +### `baseline` — the policy world a case is evaluated in + +Optional; absent means current law. A descriptor of the same shape +`ReformRef.baseline` carries, and it must already be REGISTERED in +`scorecard_db/baselines.py` — an unnamed counterfactual is exactly the +"which law did this run?" ambiguity mode 3 exists to remove. + +This is how a case pins a SAME-YEAR counterfactual instead of leaning on a +year-on-year delta. The two-child-limit family uses it: three 2026 cases, +`uk-uc-two-child-limit-binding` and `uk-uc-two-child-limit-multiple-birth` +in the registered `pre_ab2025` world (limit reinstated at two children) +and `uk-uc-two-child-limit-abolished` under current law. The binding / +abolished pair differs only in the world, so it attributes the abolition; +the binding / multiple-birth pair differs only in `child_3`'s date of +birth, so it attributes the exception. An earlier design paired a 2025 +case against a 2026 one, which moved ages, uprated rates and the policy +year together, and used twins in the "limit in force" case — where an +engine that wrongly kept the limit but rightly applied the multiple-birth +exception still paid three elements and passed. ## Result row — `CaseResult` @@ -114,6 +151,47 @@ One row per case × output variable × run. History is preserved: re-running on a new engine or oracle version appends, never overwrites (same doctrine as `pe_results`). +`variable_class` and `schema_version` are REQUIRED. Both used to be +omissible, and an omitted `schema_version` defaulted to the current one — +precisely the silent reinterpretation the field exists to prevent. +`engine_version` and `oracle_version` may not be blank. A boolean-class +comparison may only carry 0 or 1 on either side. + +### The classify → CaseResult seam is enforced, not conventional + +`CaseResult.__post_init__` RECOMPUTES `classify()` and compares. A stored +classification is valid only if it is what the classifier says, or if it +is an ADJUDICATION of a row the classifier left `unclassified`, drawn from +`ADJUDICATABLE` (`oracle_difference`, `rounding`, `pe_gap`, +`policy_scope_mismatch`) and carrying a writeup in `annotations`. A row +the classifier has already decided is not up for reinterpretation, and +nothing may be adjudicated INTO a match. + +Note the two-sidedness of `pe_gap` and `policy_scope_mismatch`: the +classifier emits them mechanically for a null side, and a human may also +assign them to a numeric-vs-numeric difference — where they are a +judgement and need explaining. Previously only `oracle_difference` and +`rounding` demanded a writeup, so `100` vs `200` could be filed as a +`pe_gap` with nothing said. + +Tolerances come from `DEFAULT_TOLERANCES` alone. `from_classification` +no longer accepts a tolerance table and a hand-written row carrying a +tolerance other than the registered one is rejected — a caller-supplied +tolerance is how a boolean `1` vs `0` became a `match_within_tolerance`. +`from_classification` does accept `annotations`, because some lanes +require one (the #64 calculator rows must carry an archive annotation). + +### Not yet built + +There is no case/result TABLE, writer, exporter or `build_db` step, so +mode-3 rows do not persist to the DB yet. The epistemic wiring that rides +on those columns — `calibration_relationship`, the executed baseline key, +case/battery digests, engine bundle and run pins, the JRC connector +revision, and a citable adjudication link — lands with that table, as does +splitting the descriptive comparison status from the normative diagnosis +the way `external_scores` and `diagnoses` already do. That design has +repo-wide surface and is being paired on rather than decided here. + ```json { "case_id": "uk-uc-single-unemployed", diff --git a/sources/ukmod-cases/battery/cases.json b/sources/ukmod-cases/battery/cases.json index cd71c88..7667b30 100644 --- a/sources/ukmod-cases/battery/cases.json +++ b/sources/ukmod-cases/battery/cases.json @@ -1,7 +1,7 @@ { "schema": "sources/ukmod-cases/SCHEMA.md", "schema_version": 1, - "description": "Initial UK mode-3 case battery for the UKMOD/EUROMOD lane (#41): standard EUROMOD-style hypothetical households plus the edge cases PE UK carries regression coverage for. Inputs + expected focus only — expected output values come from engine runs (PE and the oracle), never from this file. All monetary amounts are annual GBP; weekly-quoted amounts enter as weekly x 52.", + "description": "Initial UK mode-3 case battery for the UKMOD/EUROMOD lane (#41): standard EUROMOD-style hypothetical households plus the edge cases PE UK carries regression coverage for. Inputs + expected focus only \u2014 expected output values come from engine runs (PE and the oracle), never from this file. All monetary amounts are annual GBP; weekly-quoted amounts enter as weekly x 52.", "cases": [ { "case_id": "uk-uc-single-unemployed", @@ -10,16 +10,27 @@ "country": "UK", "household": { "people": { - "adult_1": {"age": 30} + "adult_1": { + "age": 30 + } }, "benefit_units": [ - {"adults": ["adult_1"], "children": []} + { + "adults": [ + "adult_1" + ], + "children": [] + } ], "region": "NORTH_EAST", "tenure": "rented_social", "rent": 6240 }, - "expected_focus": ["universal_credit", "income_tax", "national_insurance"], + "expected_focus": [ + "universal_credit", + "income_tax", + "national_insurance" + ], "rationale": "The simplest UC case: standard allowance (single 25+) plus social-rent housing element with no deductions. Rent GBP 120/week = 6,240/year. Pins the standard allowance and housing element before any taper, cap, or element interacts." }, { @@ -29,65 +40,190 @@ "country": "UK", "household": { "people": { - "adult_1": {"age": 32}, - "child_1": {"age": 8}, - "child_2": {"age": 4} + "adult_1": { + "age": 32 + }, + "child_1": { + "age": 8 + }, + "child_2": { + "age": 4 + } }, "benefit_units": [ - {"adults": ["adult_1"], "children": ["child_1", "child_2"]} + { + "adults": [ + "adult_1" + ], + "children": [ + "child_1", + "child_2" + ] + } ], "region": "YORKSHIRE", "tenure": "rented_private", - "rent": 9360 + "rent": 9360, + "brma": "Leeds" }, - "expected_focus": ["universal_credit", "child_benefit"], + "expected_focus": [ + "universal_credit", + "child_benefit" + ], "rationale": "UC child elements (two children, both post-April-2017 so no protected rate) plus the private-rent housing element via LHA (2-bed rate, Yorkshire BRMA) and Child Benefit for two. Rent GBP 180/week = 9,360/year, below the LHA cap in most Yorkshire BRMAs so the LHA-vs-actual-rent minimum is exercised without binding." }, { - "case_id": "uk-uc-two-child-limit-multiple-birth", - "description": "Workless couple, three children incl. twins, limit in force (2025)", - "policy_year": 2025, + "case_id": "uk-uc-two-child-limit-binding", + "description": "Workless couple, three singleton-birth children, limit in force", + "policy_year": 2026, "country": "UK", "household": { "people": { - "adult_1": {"age": 36}, - "adult_2": {"age": 34}, - "child_1": {"age": 6, "date_of_birth": "2018-06-01"}, - "child_2": {"age": 2, "date_of_birth": "2022-09-01"}, - "child_3": {"age": 2, "date_of_birth": "2022-09-01"} + "adult_1": { + "age": 37 + }, + "adult_2": { + "age": 35 + }, + "child_1": { + "age": 7, + "date_of_birth": "2018-06-01" + }, + "child_2": { + "age": 3, + "date_of_birth": "2022-09-01" + }, + "child_3": { + "age": 3, + "date_of_birth": "2022-11-15" + } }, "benefit_units": [ - {"adults": ["adult_1", "adult_2"], "children": ["child_1", "child_2", "child_3"]} + { + "adults": [ + "adult_1", + "adult_2" + ], + "children": [ + "child_1", + "child_2", + "child_3" + ] + } ], "region": "WEST_MIDLANDS", "tenure": "rented_social", "rent": 8320 }, - "expected_focus": ["universal_credit", "child_benefit"], - "rationale": "Two-child limit with the multiple-birth exception, pinned to policy year 2025 because the limit was abolished from 6 April 2026 (AB2025) and only a pre-abolition year can exercise it. All three children are post-April-2017, so the limit binds on ordinal position alone: child_1 takes the first element, child_2 (first of the twins) the second, and child_3 - the second child of the multiple birth - is exempt and takes a third element the limit would otherwise deny. The exemption sits on a third-or-later child, so it strictly changes the entitlement (3 elements vs 2). Also pins that Child Benefit ignores the limit (paid for all three). Shared DOBs are explicit because the exception keys off the birth event." + "expected_focus": [ + "universal_credit", + "child_benefit" + ], + "baseline": { + "policy": "pre_ab2025" + }, + "rationale": "The limit itself, with nothing else to hide behind. Three post-April-2017 children, all SEPARATE births (child_2 and child_3 have different dates of birth), so no exception of any kind applies: an engine applying the two-child limit pays two child elements and an engine that has dropped it pays three. The earlier version of this case used twins, which meant an engine that wrongly retained the limit but rightly applied the multiple-birth exception still paid three elements \u2014 it could not catch the failure its own rationale named. Evaluated in the registered pre_ab2025 world (limit reinstated at two children), so the case does not depend on a pre-abolition policy year." }, { "case_id": "uk-uc-two-child-limit-abolished", - "description": "Same three-child family evaluated post-abolition (2026)", + "description": "The same 2026 household under current law (limit abolished)", + "policy_year": 2026, + "country": "UK", + "household": { + "people": { + "adult_1": { + "age": 37 + }, + "adult_2": { + "age": 35 + }, + "child_1": { + "age": 7, + "date_of_birth": "2018-06-01" + }, + "child_2": { + "age": 3, + "date_of_birth": "2022-09-01" + }, + "child_3": { + "age": 3, + "date_of_birth": "2022-11-15" + } + }, + "benefit_units": [ + { + "adults": [ + "adult_1", + "adult_2" + ], + "children": [ + "child_1", + "child_2", + "child_3" + ] + } + ], + "region": "WEST_MIDLANDS", + "tenure": "rented_social", + "rent": 8320 + }, + "expected_focus": [ + "universal_credit", + "child_benefit" + ], + "rationale": "The abolition counterpart to uk-uc-two-child-limit-binding: byte-identical household, SAME policy year, same rates, same ages \u2014 the only difference is the policy world (current law, where AB2025 removed the limit from 6 April 2026, versus the registered pre_ab2025 counterfactual). The pair therefore isolates the abolition itself. The earlier year-on-year pair moved ages, uprated rates and the policy year together, so its delta could not attribute anything." + }, + { + "case_id": "uk-uc-two-child-limit-multiple-birth", + "description": "Same household with a multiple birth, limit in force", "policy_year": 2026, "country": "UK", "household": { "people": { - "adult_1": {"age": 37}, - "adult_2": {"age": 35}, - "child_1": {"age": 7, "date_of_birth": "2018-06-01"}, - "child_2": {"age": 3, "date_of_birth": "2022-09-01"}, - "child_3": {"age": 3, "date_of_birth": "2022-09-01"} + "adult_1": { + "age": 37 + }, + "adult_2": { + "age": 35 + }, + "child_1": { + "age": 7, + "date_of_birth": "2018-06-01" + }, + "child_2": { + "age": 3, + "date_of_birth": "2022-09-01" + }, + "child_3": { + "age": 3, + "date_of_birth": "2022-09-01" + } }, "benefit_units": [ - {"adults": ["adult_1", "adult_2"], "children": ["child_1", "child_2", "child_3"]} + { + "adults": [ + "adult_1", + "adult_2" + ], + "children": [ + "child_1", + "child_2", + "child_3" + ] + } ], "region": "WEST_MIDLANDS", "tenure": "rented_social", "rent": 8320 }, - "expected_focus": ["universal_credit", "child_benefit"], - "rationale": "The abolition counterpart to uk-uc-two-child-limit-multiple-birth: the same family one year on, in the first year the two-child limit no longer exists (removed from 6 April 2026, AB2025). All three post-April-2017 children take child elements with no exception needed; an engine still applying the limit under-pays exactly one element. Paired with the 2025 case, the year-on-year delta isolates the abolition itself." + "expected_focus": [ + "universal_credit", + "child_benefit" + ], + "baseline": { + "policy": "pre_ab2025" + }, + "rationale": "The multiple-birth exception, isolated against uk-uc-two-child-limit-binding: identical household, identical ages, identical pre_ab2025 world, differing in exactly one input \u2014 child_3's date of birth, which makes children 2 and 3 a multiple birth. The exception sits on a third-or-later child, so it strictly changes entitlement (three elements vs the binding case's two), and the one-field difference is what attributes the change to the exception rather than to the limit. Also pins that Child Benefit ignores the limit in both." }, { "case_id": "uk-benefit-cap-london", @@ -96,19 +232,41 @@ "country": "UK", "household": { "people": { - "adult_1": {"age": 33}, - "adult_2": {"age": 31}, - "child_1": {"age": 6}, - "child_2": {"age": 3} + "adult_1": { + "age": 33 + }, + "adult_2": { + "age": 31 + }, + "child_1": { + "age": 6 + }, + "child_2": { + "age": 3 + } }, "benefit_units": [ - {"adults": ["adult_1", "adult_2"], "children": ["child_1", "child_2"]} + { + "adults": [ + "adult_1", + "adult_2" + ], + "children": [ + "child_1", + "child_2" + ] + } ], "region": "LONDON", "tenure": "rented_private", - "rent": 20800 + "rent": 20800, + "brma": "Inner London" }, - "expected_focus": ["benefit_cap", "universal_credit", "child_benefit"], + "expected_focus": [ + "benefit_cap", + "universal_credit", + "child_benefit" + ], "rationale": "Benefit cap binding: rent GBP 400/week = 20,800/year drives pre-cap UC (standard + 2 child elements + housing) above the Greater London couple cap, so the cap reduction is nonzero. Exercises the London cap tier and the cap-applies-to-UC-not-Child-Benefit accounting." }, { @@ -118,18 +276,37 @@ "country": "UK", "household": { "people": { - "adult_1": {"age": 29, "employment_income": 12000, "hours_worked_per_week": 20}, - "child_1": {"age": 5} + "adult_1": { + "age": 29, + "employment_income": 12000, + "hours_worked_per_week": 20 + }, + "child_1": { + "age": 5 + } }, "benefit_units": [ - {"adults": ["adult_1"], "children": ["child_1"]} + { + "adults": [ + "adult_1" + ], + "children": [ + "child_1" + ] + } ], "region": "NORTH_WEST", "tenure": "rented_private", - "rent": 7800 + "rent": 7800, + "brma": "Manchester" }, - "expected_focus": ["universal_credit", "income_tax", "national_insurance", "child_benefit"], - "rationale": "The 55% UC earnings taper above the lower (with-housing) work allowance. Earnings GBP 12,000 sit below the personal allowance (income tax GBP 0) but above the NI primary threshold pro-rata, so net-earnings-for-UC differs from gross — the taper base (net of tax and NI) is the thing under test. Rent GBP 150/week = 7,800/year." + "expected_focus": [ + "universal_credit", + "income_tax", + "national_insurance", + "child_benefit" + ], + "rationale": "The 55% UC earnings taper above the lower (with-housing) work allowance. Earnings GBP 12,000 sit below the personal allowance (income tax GBP 0) but above the NI primary threshold pro-rata, so net-earnings-for-UC differs from gross \u2014 the taper base (net of tax and NI) is the thing under test. Rent GBP 150/week = 7,800/year." }, { "case_id": "uk-uc-minimum-income-floor", @@ -138,16 +315,28 @@ "country": "UK", "household": { "people": { - "adult_1": {"age": 40, "self_employment_income": 6000, "gainfully_self_employed": true} + "adult_1": { + "age": 40, + "self_employment_income": 6000, + "gainfully_self_employed": true + } }, "benefit_units": [ - {"adults": ["adult_1"], "children": []} + { + "adults": [ + "adult_1" + ], + "children": [] + } ], "region": "EAST_MIDLANDS", "tenure": "rented_private", - "rent": 7280 + "rent": 7280, + "brma": "Nottingham" }, - "expected_focus": ["universal_credit"], + "expected_focus": [ + "universal_credit" + ], "rationale": "Minimum income floor: actual trading profit GBP 6,000/year is far below the assumed floor (35 hours x NLW net of notional tax/NI), so UC is tapered on the floor, not on actual income. Past the 12-month start-up period by assumption (gainfully_self_employed = true). Pins the floor construction, a known cross-model divergence spot." }, { @@ -157,16 +346,27 @@ "country": "UK", "household": { "people": { - "adult_1": {"age": 70, "state_pension": 9000} + "adult_1": { + "age": 70, + "state_pension": 9000 + } }, "benefit_units": [ - {"adults": ["adult_1"], "children": []} + { + "adults": [ + "adult_1" + ], + "children": [] + } ], "region": "SOUTH_WEST", "tenure": "owned_outright", "rent": 0 }, - "expected_focus": ["pension_credit", "income_tax"], + "expected_focus": [ + "pension_credit", + "income_tax" + ], "rationale": "Guarantee credit tops a GBP 9,000 state pension up to the single-person standard minimum guarantee; income below the personal allowance so income tax is zero. Deliberately below the full new state pension so the top-up is strictly positive." }, { @@ -176,17 +376,33 @@ "country": "UK", "household": { "people": { - "adult_1": {"age": 72, "state_pension": 9500, "pension_income": 1000}, - "adult_2": {"age": 70, "state_pension": 4500} + "adult_1": { + "age": 72, + "state_pension": 9500, + "pension_income": 1000 + }, + "adult_2": { + "age": 70, + "state_pension": 4500 + } }, "benefit_units": [ - {"adults": ["adult_1", "adult_2"], "children": []} + { + "adults": [ + "adult_1", + "adult_2" + ], + "children": [] + } ], "region": "WALES", "tenure": "owned_outright", "rent": 0 }, - "expected_focus": ["pension_credit", "income_tax"], + "expected_focus": [ + "pension_credit", + "income_tax" + ], "rationale": "Couple guarantee credit with joint income assessment: GBP 15,000 combined (two state pensions of 9,500 and 4,500 plus a 1,000 private pension) against the couple standard minimum guarantee. Pins joint aggregation and that private pension income counts in full." }, { @@ -196,17 +412,31 @@ "country": "UK", "household": { "people": { - "adult_1": {"age": 68, "state_pension": 9500}, - "adult_2": {"age": 60} + "adult_1": { + "age": 68, + "state_pension": 9500 + }, + "adult_2": { + "age": 60 + } }, "benefit_units": [ - {"adults": ["adult_1", "adult_2"], "children": []} + { + "adults": [ + "adult_1", + "adult_2" + ], + "children": [] + } ], "region": "NORTH_EAST", "tenure": "rented_social", "rent": 6760 }, - "expected_focus": ["universal_credit", "pension_credit"], + "expected_focus": [ + "universal_credit", + "pension_credit" + ], "rationale": "Since May 2019 a mixed-age couple (one over, one under state pension age) claims UC, not Pension Credit. Expect pension_credit = 0 and UC assessed with the state pension counted as unearned income (GBP-for-GBP deduction). A known routing rule models get wrong in both directions. Rent GBP 130/week = 6,760/year." }, { @@ -216,20 +446,44 @@ "country": "UK", "household": { "people": { - "adult_1": {"age": 35, "employment_income": 22000, "hours_worked_per_week": 35}, - "adult_2": {"age": 33}, - "child_1": {"age": 7}, - "child_2": {"age": 2} + "adult_1": { + "age": 35, + "employment_income": 22000, + "hours_worked_per_week": 35 + }, + "adult_2": { + "age": 33 + }, + "child_1": { + "age": 7 + }, + "child_2": { + "age": 2 + } }, "benefit_units": [ - {"adults": ["adult_1", "adult_2"], "children": ["child_1", "child_2"]} + { + "adults": [ + "adult_1", + "adult_2" + ], + "children": [ + "child_1", + "child_2" + ] + } ], "region": "SCOTLAND", "tenure": "rented_social", "rent": 8840 }, - "expected_focus": ["income_tax", "universal_credit", "scottish_child_payment", "child_benefit"], - "rationale": "Devolution double-check: GBP 22,000 earnings span the Scottish starter, basic, and intermediate bands (different liability than rUK at the same gross), while UC entitlement plus two under-16s triggers Scottish Child Payment for both children. Exercises whether the oracle models devolved Scottish payments at all — a candidate policy_scope_mismatch, which is exactly why it is in the battery." + "expected_focus": [ + "income_tax", + "universal_credit", + "scottish_child_payment", + "child_benefit" + ], + "rationale": "Devolution double-check: GBP 22,000 earnings span the Scottish starter, basic, and intermediate bands (different liability than rUK at the same gross), while UC entitlement plus two under-16s triggers Scottish Child Payment for both children. Exercises whether the oracle models devolved Scottish payments at all \u2014 a candidate policy_scope_mismatch, which is exactly why it is in the battery." }, { "case_id": "uk-hicbc-partial-taper", @@ -238,19 +492,40 @@ "country": "UK", "household": { "people": { - "adult_1": {"age": 41, "employment_income": 70000}, - "adult_2": {"age": 39}, - "child_1": {"age": 10}, - "child_2": {"age": 7} + "adult_1": { + "age": 41, + "employment_income": 70000 + }, + "adult_2": { + "age": 39 + }, + "child_1": { + "age": 10 + }, + "child_2": { + "age": 7 + } }, "benefit_units": [ - {"adults": ["adult_1", "adult_2"], "children": ["child_1", "child_2"]} + { + "adults": [ + "adult_1", + "adult_2" + ], + "children": [ + "child_1", + "child_2" + ] + } ], "region": "SOUTH_EAST", "tenure": "owned_mortgage", "rent": 0 }, - "expected_focus": ["income_tax", "child_benefit"], + "expected_focus": [ + "income_tax", + "child_benefit" + ], "rationale": "High Income Child Benefit Charge at the midpoint of the post-2024 GBP 60,000-80,000 taper: adjusted net income 70,000 claws back exactly half the two-child Child Benefit through the earner's income tax. Round midpoint chosen so the expected clawback fraction (1/2) is auditable by inspection." }, { @@ -260,18 +535,35 @@ "country": "UK", "household": { "people": { - "adult_1": {"age": 45, "employment_income": 12570}, - "adult_2": {"age": 44, "employment_income": 50270, "employee_pension_contributions": 2000, "salary_sacrifice": true} + "adult_1": { + "age": 45, + "employment_income": 12570 + }, + "adult_2": { + "age": 44, + "employment_income": 50270, + "employee_pension_contributions": 2000, + "salary_sacrifice": true + } }, "benefit_units": [ - {"adults": ["adult_1", "adult_2"], "children": []} + { + "adults": [ + "adult_1", + "adult_2" + ], + "children": [] + } ], "region": "EAST_OF_ENGLAND", "tenure": "owned_mortgage", "rent": 0 }, - "expected_focus": ["national_insurance", "income_tax"], - "rationale": "Threshold edges: adult_1 sits exactly at GBP 12,570 (personal allowance = NI primary threshold; both liabilities should be exactly zero). Adult_2 sits at the GBP 50,270 UEL with a GBP 2,000 salary sacrifice, so NI-able and taxable pay drop to 48,270 — pinning that sacrifice reduces both tax and Class 1 NI, and that the main-vs-additional NI rate boundary is applied to post-sacrifice pay." + "expected_focus": [ + "national_insurance", + "income_tax" + ], + "rationale": "Threshold edges: adult_1 sits exactly at GBP 12,570 (personal allowance = NI primary threshold; both liabilities should be exactly zero). Adult_2 sits at the GBP 50,270 UEL with a GBP 2,000 salary sacrifice, so NI-able and taxable pay drop to 48,270 \u2014 pinning that sacrifice reduces both tax and Class 1 NI, and that the main-vs-additional NI rate boundary is applied to post-sacrifice pay." }, { "case_id": "uk-carer-ca-uc", @@ -280,17 +572,28 @@ "country": "UK", "household": { "people": { - "adult_1": {"age": 45, "is_carer": true} + "adult_1": { + "age": 45, + "is_carer": true + } }, "benefit_units": [ - {"adults": ["adult_1"], "children": []} + { + "adults": [ + "adult_1" + ], + "children": [] + } ], "region": "NORTH_WEST", "tenure": "rented_social", "rent": 6240 }, - "expected_focus": ["carers_allowance", "universal_credit"], - "rationale": "Carer's Allowance plus UC carer element interaction: CA is unearned income deducted from UC GBP-for-GBP, while the carer element is added — the net gain should equal the carer element alone. The cared-for person (receiving a qualifying disability benefit) lives outside the household, per the is_carer flag's definition. A classic double-counting trap between models." + "expected_focus": [ + "carers_allowance", + "universal_credit" + ], + "rationale": "Carer's Allowance plus UC carer element interaction: CA is unearned income deducted from UC GBP-for-GBP, while the carer element is added \u2014 the net gain should equal the carer element alone. The cared-for person (receiving a qualifying disability benefit) lives outside the household, per the is_carer flag's definition. A classic double-counting trap between models." }, { "case_id": "uk-pa-taper-110k", @@ -299,16 +602,27 @@ "country": "UK", "household": { "people": { - "adult_1": {"age": 48, "employment_income": 110000} + "adult_1": { + "age": 48, + "employment_income": 110000 + } }, "benefit_units": [ - {"adults": ["adult_1"], "children": []} + { + "adults": [ + "adult_1" + ], + "children": [] + } ], "region": "LONDON", "tenure": "owned_mortgage", "rent": 0 }, - "expected_focus": ["income_tax", "national_insurance"], + "expected_focus": [ + "income_tax", + "national_insurance" + ], "rationale": "The GBP 100,000 cliff: at 110,000 the personal allowance is tapered by GBP 1 per 2 over 100,000 (5,000 of allowance lost), producing the 60% effective marginal band. Round 10,000 excess makes the expected allowance (12,570 - 5,000 = 7,570) auditable by inspection." } ] diff --git a/tests/test_case_schema.py b/tests/test_case_schema.py index 13475d6..6a668e2 100644 --- a/tests/test_case_schema.py +++ b/tests/test_case_schema.py @@ -8,6 +8,7 @@ import pytest from scorecard_db.case_diffs import ( + BRMA_REGION, SCHEMA_VERSION, DEFAULT_TOLERANCES, CaseResult, @@ -43,7 +44,7 @@ def case_kwargs(**kw): class TestBattery: def test_loads_and_validates(self): cases = load_battery(BATTERY) - assert 12 <= len(cases) <= 16 + assert 12 <= len(cases) <= 20 assert all(isinstance(c, CaseSpec) for c in cases) def test_unique_ids(self): @@ -55,32 +56,60 @@ def test_uk_battery_shape(self): for c in load_battery(BATTERY): assert c.country == "UK" assert c.case_id.startswith("uk-") - # 2025 is deliberate: the two-child limit ended 6 April 2026, - # so only a pre-abolition year can exercise its exemptions. - assert c.policy_year in (2025, 2026) + # Every case is a 2026 case now: the two-child family reaches + # the pre-abolition world through the registered pre_ab2025 + # baseline instead of through an earlier policy year, so no + # comparison has to straddle two uprating rounds. + assert c.policy_year == 2026 assert c.rationale # every case explains itself - def test_two_child_limit_pair(self): - # The exemption case must sit in a year the limit exists, and its - # abolition counterpart in the first year it does not. + def test_two_child_limit_cases_isolate_one_mechanism_each(self): + """Three cases, two same-year pairs, each differing in exactly + one thing — the round-2 finding was that the old pair changed the + policy world, the ages and the uprating round all at once, and + that its 'limit in force' case used twins, so an engine that + wrongly kept the limit but rightly applied the multiple-birth + exception still paid three elements and passed.""" by_id = {c.case_id: c for c in load_battery(BATTERY)} - binding = by_id["uk-uc-two-child-limit-multiple-birth"] + binding = by_id["uk-uc-two-child-limit-binding"] abolished = by_id["uk-uc-two-child-limit-abolished"] - assert binding.policy_year == 2025 - assert abolished.policy_year == 2026 - for case in (binding, abolished): + twins = by_id["uk-uc-two-child-limit-multiple-birth"] + + for case in (binding, abolished, twins): children = case.household["benefit_units"][0]["children"] assert len(children) == 3 # Every child post-April-2017, so no pre-2017 protection can - # stand in for the exemption / abolition under test. + # stand in for the mechanism under test. for child in children: - dob = case.household["people"][child]["date_of_birth"] - assert dob > "2017-04-06" - # The multiple birth spans a third-or-later child: shared DOBs on - # the 2nd and 3rd children, so the exception changes entitlement. - people = binding.household["people"] + assert case.household["people"][child]["date_of_birth"] > "2017-04-06" + assert case.policy_year == 2026 # one uprating round for all three + + # Pair 1 — abolition: same household, same year, different world. + assert binding.household == abolished.household + assert binding.baseline == {"policy": "pre_ab2025"} + assert abolished.baseline is None + + # Pair 2 — the exception: same world, same year, ONE input apart. + assert twins.baseline == binding.baseline + differing = [ + pid + for pid in binding.household["people"] + if binding.household["people"][pid] != twins.household["people"][pid] + ] + assert differing == ["child_3"] + assert ( + binding.household["people"]["child_3"]["age"] + == twins.household["people"]["child_3"]["age"] + ) + + # The binding case has no exception to hide behind: three + # separate births, so the limit itself decides the entitlement. + b_people = binding.household["people"] + assert len({b_people[c]["date_of_birth"] for c in ("child_2", "child_3")}) == 2 + # ...and the multiple birth spans a third-or-later child. + t_people = twins.household["people"] assert ( - people["child_2"]["date_of_birth"] == (people["child_3"]["date_of_birth"]) + t_people["child_2"]["date_of_birth"] == t_people["child_3"]["date_of_birth"] ) def test_battery_is_inputs_only(self): @@ -95,6 +124,7 @@ def test_battery_is_inputs_only(self): "household", "expected_focus", "rationale", + "baseline", } def test_focus_coverage(self): @@ -238,8 +268,10 @@ def result_kwargs(self, **kw): oracle_version="UKMOD B2026.08", computed_at="2026-08-14T12:00:00Z", classification="match_within_tolerance", + variable_class="currency", abs_diff=0.04, tolerance=0.52, + schema_version=SCHEMA_VERSION, ) base.update(kw) return base @@ -302,21 +334,52 @@ def test_tolerance_only_on_within_tolerance_rows(self): ) def test_adjudicated_classes_require_writeup(self): - for cls in ("oracle_difference", "rounding"): + """An adjudication applies to a row the classifier left + unclassified, and it needs a writeup.""" + # 4796.48 vs 4900.00 is well above the currency tolerance, so the + # classifier says `unclassified` — the adjudication queue. + unclassified = dict(oracle_value=4900.0, abs_diff=103.52, tolerance=None) + for cls in ("oracle_difference", "rounding", "pe_gap"): with pytest.raises(ValueError, match="adjudicated"): - CaseResult(**self.result_kwargs(classification=cls, tolerance=None)) + CaseResult(**self.result_kwargs(classification=cls, **unclassified)) r = CaseResult( **self.result_kwargs( classification=cls, - tolerance=None, annotations=[ "UKMOD rounds monthly amounts to the penny; " - "x12 explains the 0.04 (writeup ref)" + "x12 explains the difference (writeup ref)" ], + **unclassified, ) ) assert r.classification is DiffClassification(cls) + def test_adjudication_cannot_overrule_the_classifier(self): + """Only an `unclassified` row may be adjudicated: a row the + classifier has already decided is not up for reinterpretation.""" + with pytest.raises(ValueError, match="contradicts the classifier"): + # a within-tolerance row relabelled as an oracle difference + CaseResult( + **self.result_kwargs( + classification="oracle_difference", + tolerance=None, + annotations=["because I say so"], + ) + ) + + def test_a_match_is_never_an_adjudicated_outcome(self): + """`unclassified` may be explained, never flattered into a + match.""" + with pytest.raises(ValueError, match="abs_diff <= tolerance"): + CaseResult( + **self.result_kwargs( + oracle_value=4900.0, + abs_diff=103.52, + classification="match_within_tolerance", + annotations=["close enough"], + ) + ) + def test_annotations_must_be_nonempty_strings(self): with pytest.raises(ValueError, match="annotations"): CaseResult(**self.result_kwargs(annotations="a writeup")) @@ -405,11 +468,55 @@ def test_null_pe_side_wires_to_pe_gap(self): assert r.classification is DiffClassification.PE_GAP assert r.abs_diff is None and r.tolerance is None - def test_custom_table_tolerance_is_the_one_stored(self): - table = {VariableClass.CURRENCY: 1.0} - r = self.wired(100.0, 100.9, tolerance_table=table) - assert r.classification is DiffClassification.MATCH_WITHIN_TOLERANCE - assert r.tolerance == 1.0 + def test_a_caller_cannot_widen_the_tolerance(self): + """Tolerances are a registry decision. The factory no longer + takes a table, and a hand-written row carrying a wider tolerance + is rejected — the probe that turned a boolean 1-vs-0 into a + `match_within_tolerance` with tolerance 100.""" + with pytest.raises(TypeError): + self.wired(100.0, 100.9, tolerance_table={VariableClass.CURRENCY: 1.0}) + with pytest.raises(ValueError, match="registered tolerance"): + CaseResult( + case_id="uk-uc-single-unemployed", + variable="universal_credit", + pe_value=100.0, + oracle_value=100.3, + oracle=Oracle.UKMOD, + engine_version="policyengine-uk 2.0.0", + oracle_version="UKMOD B2026.08", + computed_at="2026-08-14T12:00:00Z", + classification="match_within_tolerance", + variable_class=VariableClass.CURRENCY, + abs_diff=0.3, + tolerance=1.0, + schema_version=SCHEMA_VERSION, + ) + + def test_a_boolean_mismatch_can_never_be_a_match(self): + r = self.wired(1.0, 0.0, vclass=VariableClass.BOOLEAN) + assert r.classification is DiffClassification.UNCLASSIFIED + with pytest.raises(ValueError, match="not an outcome"): + CaseResult( + case_id="uk-uc-single-unemployed", + variable="is_uc_eligible", + pe_value=1.0, + oracle_value=0.0, + oracle=Oracle.UKMOD, + engine_version="policyengine-uk 2.0.0", + oracle_version="UKMOD B2026.08", + computed_at="2026-08-14T12:00:00Z", + classification="match_within_tolerance", + variable_class=VariableClass.BOOLEAN, + abs_diff=1.0, + tolerance=100.0, + schema_version=SCHEMA_VERSION, + ) + + def test_the_factory_carries_annotations(self): + """#64's calculator rows REQUIRE an archive annotation; a factory + that could not take one made every calculator call raise.""" + r = self.wired(100.0, 100.0, annotations=["archive:https://example/x"]) + assert r.annotations == ["archive:https://example/x"] def test_above_tolerance_lands_unclassified(self): r = self.wired(100.0, 200.0) @@ -419,3 +526,90 @@ def test_above_tolerance_lands_unclassified(self): def test_variable_class_is_a_closed_vocabulary(self): with pytest.raises(ValueError): self.wired(100.0, 100.0, vclass="percentage") + + +class TestIdentityClosureAtTheEdges: + """Round-2 probes accepted region="MARS", invented focus variables, + NaN incomes, infinite rent, boolean values outside {0,1} and an + arbitrary battery schema. Each is now a closed decision.""" + + def test_region_is_a_closed_registry(self): + hh = dict(case_kwargs()["household"], region="MARS") + with pytest.raises(ValueError, match="unregistered UK region"): + CaseSpec(**case_kwargs(household=hh)) + CaseSpec(**case_kwargs(household=dict(hh, region="SCOTLAND"))) + + def test_focus_variables_are_closed_per_country(self): + with pytest.raises(ValueError, match="unregistered UK focus"): + CaseSpec(**case_kwargs(expected_focus=["vibes_credit"])) + # a US variable is not a UK variable + with pytest.raises(ValueError, match="unregistered UK focus"): + CaseSpec(**case_kwargs(expected_focus=["snap"])) + + def test_focus_variables_do_not_repeat(self): + with pytest.raises(ValueError, match="must not repeat"): + CaseSpec( + **case_kwargs(expected_focus=["universal_credit", "universal_credit"]) + ) + + def test_nan_and_infinity_are_not_amounts(self): + for bad in (float("nan"), float("inf")): + hh = case_kwargs()["household"] + people = dict(hh["people"], adult_1={"age": 30, "employment_income": bad}) + with pytest.raises(ValueError, match="finite"): + CaseSpec(**case_kwargs(household=dict(hh, people=people))) + with pytest.raises(ValueError, match="finite"): + CaseSpec(**case_kwargs(household=dict(hh, rent=bad))) + + def test_boolean_class_values_must_be_zero_or_one(self): + kw = TestCaseResult().result_kwargs( + variable_class="boolean", + pe_value=7.0, + oracle_value=7.0, + abs_diff=0.0, + tolerance=None, + classification="match_exact", + ) + with pytest.raises(ValueError, match="must be 0 or 1"): + CaseResult(**kw) + + def test_blank_engine_versions_rejected(self): + for field in ("engine_version", "oracle_version"): + with pytest.raises(ValueError, match="required"): + CaseResult(**TestCaseResult().result_kwargs(**{field: " "})) + + def test_battery_schema_path_is_closed(self, tmp_path): + raw = json.loads(BATTERY.read_text()) + raw["schema"] = "sources/made-up/SCHEMA.md" + p = tmp_path / "cases.json" + p.write_text(json.dumps(raw)) + with pytest.raises(ValueError, match="not a contract this repo defines"): + load_battery(p) + + def test_private_rent_cases_pin_a_brma(self): + """LHA rates are per BRMA, not per ITL-1 region: without one, the + two engines are not guaranteed the same world.""" + hh = dict( + case_kwargs()["household"], + tenure="rented_private", + rent=9360, + region="YORKSHIRE", + ) + with pytest.raises(ValueError, match="must pin a BRMA"): + CaseSpec(**case_kwargs(household=hh)) + with pytest.raises(ValueError, match="unregistered BRMA"): + CaseSpec(**case_kwargs(household=dict(hh, brma="Atlantis"))) + with pytest.raises(ValueError, match="sits in"): + CaseSpec(**case_kwargs(household=dict(hh, brma="Inner London"))) + CaseSpec(**case_kwargs(household=dict(hh, brma="Leeds"))) + + def test_every_private_rent_case_in_the_battery_pins_one(self): + for c in load_battery(BATTERY): + if c.household.get("tenure") == "rented_private": + assert c.household["brma"] in BRMA_REGION + + def test_case_baselines_must_be_registered(self): + with pytest.raises(ValueError, match="unregistered baseline world"): + CaseSpec(**case_kwargs(baseline={"policy": "a_world_nobody_named"})) + # the registered reinstated-limit world is accepted + CaseSpec(**case_kwargs(baseline={"policy": "pre_ab2025"}))