From f76228ba6ac7728cd140e1eeedc4bce203086f15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Tue, 25 Aug 2026 23:36:20 +0200 Subject: [PATCH 01/14] Make every spine stage say what it did, and let the seam inherit it The spine build kept a hardcoded five-name roster of stages whose evidence reached the sidecar, so three SPI stages that already expose evidence wrote nothing to disk and any stage added later would have been silently skipped. Collection is now duck-typed over the executed stage order via the same checkpoint_metadata() hook the national build already uses, falling back to last_result; a stage exposing neither stays visibly absent rather than present-and-empty. The four donor-support stages were the only imputations clipping to donor support and the only ones emitting no receipt at all: the helpers returned the clipped frame and dropped how many rows they clipped and to what bounds, while donor_realized_ranges() next door was only ever called by the bounds generators. They now share one clip helper that returns those numbers with the values, LCFS exempt columns included as exempt rather than as zero-clipped. The clip arithmetic is untouched, and the existing value assertions pin that. The calibration seam verified its input's sha and then re-derived, or dropped, everything else the spine build had already recorded. It now loads the sidecar beside the input H5, refuses one that does not describe that artifact, and carries the spine's own provenance into the diagnostics build block and the build record. Prepares the per-stage health gates in #757 section B, which consume these receipts; no gate is wired here and no output value moves. Co-Authored-By: Claude Fable 5 --- .../757-seam-sidecar-provenance.added.md | 1 + changelog.d/757-spine-stage-evidence.added.md | 1 + .../build/uk_runtime/calibration_run.py | 76 ++++++++ .../build/uk_runtime/etb_services.py | 51 ++++- .../src/microcosm/build/uk_runtime/etb_vat.py | 51 ++++- .../build/uk_runtime/lcfs_consumption.py | 55 ++++-- .../build/uk_runtime/support_clip.py | 72 ++++++++ .../microcosm/build/uk_runtime/was_wealth.py | 56 ++++-- .../tests/test_uk_calibration_run.py | 174 +++++++++++++++++- .../tests/test_uk_etb_services.py | 21 ++- .../microcosm-build/tests/test_uk_etb_vat.py | 23 ++- .../tests/test_uk_frs_spine.py | 95 ++++++++++ .../tests/test_uk_lcfs_consumption.py | 34 +++- .../tests/test_uk_was_wealth.py | 28 ++- tools/build_uk_frs_spine.py | 64 ++++--- 15 files changed, 720 insertions(+), 82 deletions(-) create mode 100644 changelog.d/757-seam-sidecar-provenance.added.md create mode 100644 changelog.d/757-spine-stage-evidence.added.md create mode 100644 packages/microcosm-build/src/microcosm/build/uk_runtime/support_clip.py diff --git a/changelog.d/757-seam-sidecar-provenance.added.md b/changelog.d/757-seam-sidecar-provenance.added.md new file mode 100644 index 000000000..37dcbca5b --- /dev/null +++ b/changelog.d/757-seam-sidecar-provenance.added.md @@ -0,0 +1 @@ +The UK calibration seam inherits its input's provenance instead of re-deriving it. The seam now loads the spine build sidecar sitting beside the input H5, binds it to the artifact by entity row counts, household weight kind and total, and carries the stage census, artifact/resource/input pins, declared seeds, source vintages, rules-engine version and stochastic contract into the diagnostics build block and the build record. An absent, malformed or non-binding sidecar refuses the run with the mismatch named, so an unpinned artifact cannot pass for a pinned one. diff --git a/changelog.d/757-spine-stage-evidence.added.md b/changelog.d/757-spine-stage-evidence.added.md new file mode 100644 index 000000000..04cc9fd61 --- /dev/null +++ b/changelog.d/757-spine-stage-evidence.added.md @@ -0,0 +1 @@ +Every UK spine stage now records a health receipt the build keeps. The spine sidecar collects stage evidence duck-typed over the executed stage order rather than a hardcoded five-name roster, so the SPI channel stages land alongside the E8 ones and a newly added stage is carried without editing the collector; the four donor-support stages (WAS wealth, LCFS consumption, ETB VAT, ETB services) clip through a shared helper that returns the donor bounds it used and how many rows it clipped on each tail, where before the clip was silent. The clipped values are unchanged. diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/calibration_run.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/calibration_run.py index 6e5e947b0..677994662 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/calibration_run.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/calibration_run.py @@ -48,6 +48,7 @@ from microcosm.build.uk_runtime.national_calibration import UKNationalCalibrationStage from microcosm.build.uk_runtime.national_frame import ( load_uk_national_frame, + uk_household_weight_kind, write_uk_national_frame, ) from microcosm.calibrate import TargetRegistry @@ -295,6 +296,9 @@ def _run_uk_calibration_attempt( append_phase(state, "input_sha_verified") frame, _provenance = load_uk_national_frame(paths.input_h5) append_phase(state, "input_loaded") + spine_sidecar_path = paths.input_h5.with_suffix(".build.json") + spine_sidecar = _load_bound_spine_sidecar(spine_sidecar_path, frame) + append_phase(state, "input_sidecar_bound") assert_calibration_input_finite(frame) append_phase(state, "input_finite") @@ -329,6 +333,10 @@ def _run_uk_calibration_attempt( else None ), "register": _register_census(register_registry, exclusion_receipt), + "spine_provenance": _spine_provenance_from_sidecar( + spine_sidecar_path, + spine_sidecar, + ), "score_vs_enhanced_frs": None, } write_uk_calibration_diagnostics( @@ -369,6 +377,7 @@ def _run_uk_calibration_attempt( "source_pins": dict(source_pins), "role_pins_digest": role_pins_digest(source_pins), "input_posture": build_block["input_posture"], + "spine_provenance": build_block["spine_provenance"], "register": build_block["register"], "calibration": stage.manifest, "gate_summary": _gate_summary(gate_report), @@ -459,6 +468,73 @@ def _run_calibration_gate_battery( return payload +def _load_bound_spine_sidecar(path: Path, frame: Frame) -> dict[str, object]: + if not path.is_file(): + raise ValueError(f"input H5 build sidecar absent: {path}") + try: + sidecar = json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError as exc: + raise ValueError(f"input H5 build sidecar is invalid JSON: {path}") from exc + if not isinstance(sidecar, dict): + raise ValueError(f"input H5 build sidecar must be a JSON object: {path}") + _assert_spine_sidecar_binds_frame(sidecar, frame) + return sidecar + + +def _assert_spine_sidecar_binds_frame( + sidecar: Mapping[str, object], + frame: Frame, +) -> None: + expected_counts = sidecar.get("entity_row_counts") + actual_counts = {entity: int(len(frame.table(entity))) for entity in frame.entities} + if expected_counts != actual_counts: + raise ValueError( + "input H5 build sidecar row-count mismatch: " + f"sidecar {expected_counts!r}, frame {actual_counts!r}" + ) + expected_kind = sidecar.get("household_weight_kind") + actual_kind = uk_household_weight_kind(frame).value + if expected_kind != actual_kind: + raise ValueError( + "input H5 build sidecar household_weight_kind mismatch: " + f"sidecar {expected_kind!r}, frame {actual_kind!r}" + ) + expected_total = sidecar.get("household_weight_total") + actual_total = float(frame.weights_for("household").values.sum()) + if not isinstance(expected_total, int | float) or not np.isclose( + float(expected_total), actual_total + ): + raise ValueError( + "input H5 build sidecar household_weight_total mismatch: " + f"sidecar {expected_total!r}, frame {actual_total!r}" + ) + + +def _spine_provenance_from_sidecar( + path: Path, + sidecar: Mapping[str, object], +) -> dict[str, object]: + return { + "sidecar": { + "path": str(path), + "sha256": _sha256_file(path), + "schema_version": sidecar.get("schema_version"), + "pipeline": sidecar.get("pipeline"), + }, + "stages": list(sidecar.get("stages", ())), + "stage_records": list(sidecar.get("stage_records", ())), + "stage_evidence": dict(sidecar.get("stage_evidence", {})), + "artifact_pins": dict(sidecar.get("artifact_pins", {})), + "input_artifact_pins": dict(sidecar.get("input_artifact_pins", {})), + "resource_pins": dict(sidecar.get("resource_pins", {})), + "stage_artifact_pins": dict(sidecar.get("stage_artifact_pins", {})), + "declared_seeds": dict(sidecar.get("declared_seeds", {})), + "rules_engine": dict(sidecar.get("rules_engine", {})), + "source_vintages": dict(sidecar.get("source_vintages", {})), + "stochastic_contract_sha256": sidecar.get("stochastic_contract_sha256"), + } + + def _calibration_gate_manifest() -> GatesManifest: source = load_country_spec("uk").gates entries = tuple( diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/etb_services.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/etb_services.py index 0c3f32dec..2789011c7 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/etb_services.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/etb_services.py @@ -19,6 +19,11 @@ uk_time_period, validate_uk_national_frame, ) +from microcosm.build.uk_runtime.support_clip import ( + UKSupportClipReceipt, + UKSupportClipResult, + support_clip_to_donor_with_receipt, +) from microcosm.frame import Frame from microcosm.frame.rules import assert_rules_engine_country @@ -70,6 +75,21 @@ ) UK_ETB_SERVICES_NONNEGATIVE_OUTPUT_COLUMNS = UK_ETB_SERVICES_OUTPUT_COLUMNS UK_ETB_SERVICES_FIT_NAME = "uk_etb_2024_services" +UK_ETB_SERVICES_STAGE_NAME = "etb_services" + + +@dataclass +class UKETBServicesResult: + """Transformed frame and donor-support clip receipt.""" + + frame: Frame + support_clip: UKSupportClipReceipt + + def evidence(self) -> dict[str, object]: + return { + "stage": UK_ETB_SERVICES_STAGE_NAME, + "support_clip": self.support_clip.evidence(), + } @dataclass @@ -84,6 +104,7 @@ class UKETBServicesStageTransform: init=False, repr=False, ) + last_result: UKETBServicesResult | None = field(default=None, init=False) @property def fit_weight_records(self) -> tuple[FitWeightRecord, ...]: @@ -110,7 +131,8 @@ def __call__(self, frame: Frame) -> Frame: draws, records = impute_etb_services( donor, predictors, seed=_qrf_seed(self.stage) ) - draws = support_clip_to_donor(draws, donor) + clip_result = support_clip_to_donor(draws, donor) + draws = clip_result.clipped draws["rail_usage"] = ( draws["rail_subsidy_spending"] / config["rail_fare_index"] ) @@ -138,12 +160,21 @@ def __call__(self, frame: Frame) -> Frame: ) validate_uk_national_frame(result) self.last_fit_weight_records = records + self.last_result = UKETBServicesResult( + frame=result, + support_clip=clip_result.receipt, + ) return result @staticmethod def output_columns() -> tuple[str, ...]: return UK_ETB_SERVICES_OUTPUT_COLUMNS + def checkpoint_metadata(self) -> dict[str, object]: + if self.last_result is None: + raise RuntimeError("checkpoint metadata requires a completed stage run.") + return {"evidence": self.last_result.evidence()} + def clean_etb_services_table( raw: pd.DataFrame, @@ -307,15 +338,15 @@ def impute_etb_services( return raw, tuple(records) -def support_clip_to_donor(draws: pd.DataFrame, donor: pd.DataFrame) -> pd.DataFrame: - result = draws.copy() - for column in UK_ETB_SERVICES_HOUSEHOLD_OUTPUT_COLUMNS[:3]: - values = donor[column] - finite = values[np.isfinite(values)] - if finite.empty: - continue - result[column] = result[column].clip(float(finite.min()), float(finite.max())) - return result +def support_clip_to_donor( + draws: pd.DataFrame, donor: pd.DataFrame +) -> UKSupportClipResult: + return support_clip_to_donor_with_receipt( + draws, + donor, + columns=UK_ETB_SERVICES_HOUSEHOLD_OUTPUT_COLUMNS[:3], + stage=UK_ETB_SERVICES_STAGE_NAME, + ) def donor_realized_ranges(donor: pd.DataFrame) -> dict[str, tuple[float, float]]: diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/etb_vat.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/etb_vat.py index 1d21493e4..ebf892978 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/etb_vat.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/etb_vat.py @@ -19,6 +19,11 @@ uk_time_period, validate_uk_national_frame, ) +from microcosm.build.uk_runtime.support_clip import ( + UKSupportClipReceipt, + UKSupportClipResult, + support_clip_to_donor_with_receipt, +) from microcosm.frame import Frame from microcosm.frame.rules import assert_rules_engine_country @@ -37,6 +42,21 @@ # the support gate is the guard — the net_financial_wealth precedent. UK_ETB_VAT_NONNEGATIVE_OUTPUT_COLUMNS: tuple[str, ...] = () UK_ETB_VAT_FIT_NAME = "uk_etb_2023_vat:full_rate_vat_expenditure_rate" +UK_ETB_VAT_STAGE_NAME = "etb_vat" + + +@dataclass +class UKETBVATResult: + """Transformed frame and donor-support clip receipt.""" + + frame: Frame + support_clip: UKSupportClipReceipt + + def evidence(self) -> dict[str, object]: + return { + "stage": UK_ETB_VAT_STAGE_NAME, + "support_clip": self.support_clip.evidence(), + } @dataclass @@ -50,6 +70,7 @@ class UKETBVATStageTransform: init=False, repr=False, ) + last_result: UKETBVATResult | None = field(default=None, init=False) @property def fit_weight_records(self) -> tuple[FitWeightRecord, ...]: @@ -70,7 +91,8 @@ def __call__(self, frame: Frame) -> Frame: donor = clean_etb_vat_table(raw, **config) predictors = recipient_predictors(frame, self.engine) imputed, record = impute_etb_vat(donor, predictors, seed=_qrf_seed(self.stage)) - imputed = support_clip_to_donor(imputed, donor) + clip_result = support_clip_to_donor(imputed, donor) + imputed = clip_result.clipped household = frame.table("household").copy() household["full_rate_vat_expenditure_rate"] = imputed[ "full_rate_vat_expenditure_rate" @@ -86,12 +108,21 @@ def __call__(self, frame: Frame) -> Frame: ) validate_uk_national_frame(result) self.last_fit_weight_records = (record,) + self.last_result = UKETBVATResult( + frame=result, + support_clip=clip_result.receipt, + ) return result @staticmethod def output_columns() -> tuple[str, ...]: return UK_ETB_VAT_OUTPUT_COLUMNS + def checkpoint_metadata(self) -> dict[str, object]: + if self.last_result is None: + raise RuntimeError("checkpoint metadata requires a completed stage run.") + return {"evidence": self.last_result.evidence()} + def clean_etb_vat_table( raw: pd.DataFrame, @@ -231,15 +262,15 @@ def impute_etb_vat( return fitted.predict(recipient), FitWeightRecord(UK_ETB_VAT_FIT_NAME, "explicit") -def support_clip_to_donor(draws: pd.DataFrame, donor: pd.DataFrame) -> pd.DataFrame: - values = donor["full_rate_vat_expenditure_rate"] - finite = values[np.isfinite(values)] - result = draws.copy() - if not finite.empty: - result["full_rate_vat_expenditure_rate"] = result[ - "full_rate_vat_expenditure_rate" - ].clip(float(finite.min()), float(finite.max())) - return result +def support_clip_to_donor( + draws: pd.DataFrame, donor: pd.DataFrame +) -> UKSupportClipResult: + return support_clip_to_donor_with_receipt( + draws, + donor, + columns=UK_ETB_VAT_OUTPUT_COLUMNS, + stage=UK_ETB_VAT_STAGE_NAME, + ) def donor_realized_ranges(donor: pd.DataFrame) -> dict[str, tuple[float, float]]: diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/lcfs_consumption.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/lcfs_consumption.py index 3f3968b13..17fc78fd8 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/lcfs_consumption.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/lcfs_consumption.py @@ -25,6 +25,11 @@ uk_time_period, validate_uk_national_frame, ) +from microcosm.build.uk_runtime.support_clip import ( + UKSupportClipReceipt, + UKSupportClipResult, + support_clip_to_donor_with_receipt, +) from microcosm.build.uk_runtime.was_wealth import ( clean_was_household_table, encode_qrf_predictor_pair, @@ -172,6 +177,21 @@ UK_LCFS_CONSUMPTION_NONNEGATIVE_OUTPUT_COLUMNS = UK_LCFS_CONSUMPTION_OUTPUT_COLUMNS UK_LCFS_CONSUMPTION_FIT_NAME = "uk_lcfs_2023_24_consumption" UK_LCFS_HAS_FUEL_FIT_NAME = "uk_was_2018_20_has_fuel" +UK_LCFS_CONSUMPTION_STAGE_NAME = "lcfs_consumption" + + +@dataclass +class UKLCFSConsumptionResult: + """Transformed frame and donor-support clip receipt.""" + + frame: Frame + support_clip: UKSupportClipReceipt + + def evidence(self) -> dict[str, object]: + return { + "stage": UK_LCFS_CONSUMPTION_STAGE_NAME, + "support_clip": self.support_clip.evidence(), + } @dataclass @@ -191,6 +211,7 @@ class UKLCFSConsumptionStageTransform: init=False, repr=False, ) + last_result: UKLCFSConsumptionResult | None = field(default=None, init=False) @property def fit_weight_records(self) -> tuple[FitWeightRecord, ...]: @@ -245,7 +266,7 @@ def __call__(self, frame: Frame) -> Frame: seed=_operation_seed(self.stage, "fit_weighted_qrf_chain"), n_estimators=_qrf_n_estimators(self.stage), ) - household_draws = support_clip_to_donor( + clip_result = support_clip_to_donor( imputation.draws, donor, exempt={ @@ -254,6 +275,7 @@ def __call__(self, frame: Frame) -> Frame: "domestic_energy_consumption", }, ) + household_draws = clip_result.clipped household_draws = rake_energy_to_need( household_draws.join(recipient[["household_gross_income"]]), weights=frame.weights_for("household").values, @@ -286,12 +308,21 @@ def __call__(self, frame: Frame) -> Frame: ) validate_uk_national_frame(result) self.last_fit_weight_records = (bridge_record, *imputation.fit_weight_records) + self.last_result = UKLCFSConsumptionResult( + frame=result, + support_clip=clip_result.receipt, + ) return result @staticmethod def output_columns() -> tuple[str, ...]: return UK_LCFS_CONSUMPTION_OUTPUT_COLUMNS + def checkpoint_metadata(self) -> dict[str, object]: + if self.last_result is None: + raise RuntimeError("checkpoint metadata requires a completed stage run.") + return {"evidence": self.last_result.evidence()} + @dataclass(frozen=True) class UKLCFSConsumptionImputationResult: @@ -528,20 +559,14 @@ def support_clip_to_donor( donor: pd.DataFrame, *, exempt: set[str] | None = None, -) -> pd.DataFrame: - clipped = draws.copy() - exempt = exempt or set() - for column in UK_LCFS_CONSUMPTION_TARGET_COLUMNS: - if column in exempt or column not in clipped or column not in donor: - continue - values = pd.to_numeric(donor[column], errors="coerce") - finite = values[np.isfinite(values)] - if finite.empty: - continue - clipped[column] = clipped[column].clip( - lower=float(finite.min()), upper=float(finite.max()) - ) - return clipped +) -> UKSupportClipResult: + return support_clip_to_donor_with_receipt( + draws, + donor, + columns=UK_LCFS_CONSUMPTION_TARGET_COLUMNS, + stage=UK_LCFS_CONSUMPTION_STAGE_NAME, + exempt=exempt, + ) def donor_realized_ranges(donor: pd.DataFrame) -> dict[str, tuple[float, float]]: diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/support_clip.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/support_clip.py new file mode 100644 index 000000000..1935b84da --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/support_clip.py @@ -0,0 +1,72 @@ +"""Shared support-clip receipts for UK donor imputation stages.""" + +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from dataclasses import dataclass + +import numpy as np +import pandas as pd + + +@dataclass(frozen=True) +class UKSupportClipReceipt: + stage: str + columns: Mapping[str, Mapping[str, object]] + + def evidence(self) -> dict[str, object]: + return { + "columns": { + column: dict(receipt) for column, receipt in self.columns.items() + } + } + + +@dataclass(frozen=True) +class UKSupportClipResult: + clipped: pd.DataFrame + receipt: UKSupportClipReceipt + + +def support_clip_to_donor_with_receipt( + draws: pd.DataFrame, + donor: pd.DataFrame, + *, + columns: Sequence[str], + stage: str, + exempt: set[str] | None = None, +) -> UKSupportClipResult: + """Clip draws to donor support and return a structured receipt.""" + + clipped = draws.copy() + exempt = exempt or set() + receipts: dict[str, dict[str, object]] = {} + for column in columns: + if column in exempt: + if column in clipped: + receipts[column] = { + "exempt": True, + "rows_considered": int(len(clipped)), + } + continue + if column not in clipped or column not in donor: + continue + values = pd.to_numeric(donor[column], errors="coerce") + finite = values[np.isfinite(values)] + if finite.empty: + continue + lower = float(finite.min()) + upper = float(finite.max()) + draw_values = pd.to_numeric(clipped[column], errors="coerce") + receipts[column] = { + "donor_min": lower, + "donor_max": upper, + "clipped_low_rows": int((draw_values < lower).sum()), + "clipped_high_rows": int((draw_values > upper).sum()), + "rows_considered": int(len(clipped)), + } + clipped[column] = clipped[column].clip(lower=lower, upper=upper) + return UKSupportClipResult( + clipped=clipped, + receipt=UKSupportClipReceipt(stage=stage, columns=receipts), + ) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/was_wealth.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/was_wealth.py index 9c2dc72b1..27a1b8d03 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/was_wealth.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/was_wealth.py @@ -20,6 +20,11 @@ uk_time_period, validate_uk_national_frame, ) +from microcosm.build.uk_runtime.support_clip import ( + UKSupportClipReceipt, + UKSupportClipResult, + support_clip_to_donor_with_receipt, +) from microcosm.frame import Frame from microcosm.frame.rules import assert_rules_engine_country @@ -77,6 +82,7 @@ ) UK_WAS_WEALTH_DECLARED_SEEDS = {"was_wealth": 0} UK_WAS_WEALTH_FIT_NAME = "uk_was_2018_20_wealth" +UK_WAS_WEALTH_STAGE_NAME = "was_wealth" REGIONS: Mapping[int, str] = { 1: "NORTH_EAST", @@ -151,6 +157,20 @@ } +@dataclass +class UKWASWealthResult: + """Transformed frame and donor-support clip receipt.""" + + frame: Frame + support_clip: UKSupportClipReceipt + + def evidence(self) -> dict[str, object]: + return { + "stage": UK_WAS_WEALTH_STAGE_NAME, + "support_clip": self.support_clip.evidence(), + } + + @dataclass class UKWASWealthStageTransform: """Whole-stage callable for WAS-trained UK wealth imputation. @@ -170,6 +190,7 @@ class UKWASWealthStageTransform: init=False, repr=False, ) + last_result: UKWASWealthResult | None = field(default=None, init=False) @property def fit_weight_records(self) -> tuple[FitWeightRecord, ...]: @@ -198,7 +219,8 @@ def __call__(self, frame: Frame) -> Frame: n_estimators=_qrf_n_estimators(self.stage), ) self.last_fit_weight_records = imputation.fit_weight_records - household_draws = support_clip_to_donor(imputation.draws, donor) + clip_result = support_clip_to_donor(imputation.draws, donor) + household_draws = clip_result.clipped household_draws["num_vehicles"] = ( np.rint(household_draws["num_vehicles"]).clip(lower=0).astype("int64") ) @@ -221,12 +243,21 @@ def __call__(self, frame: Frame) -> Frame: mass_log=frame.mass_log, ) validate_uk_national_frame(result) + self.last_result = UKWASWealthResult( + frame=result, + support_clip=clip_result.receipt, + ) return result @staticmethod def output_columns() -> tuple[str, ...]: return UK_WAS_WEALTH_OUTPUT_COLUMNS + def checkpoint_metadata(self) -> dict[str, object]: + if self.last_result is None: + raise RuntimeError("checkpoint metadata requires a completed stage run.") + return {"evidence": self.last_result.evidence()} + def clean_was_household_table(raw: pd.DataFrame) -> pd.DataFrame: """Return the WAS donor table with exact lower-case column matching.""" @@ -478,22 +509,17 @@ def _encode(table: pd.DataFrame, block: pd.DataFrame) -> pd.DataFrame: ) -def support_clip_to_donor(draws: pd.DataFrame, donor: pd.DataFrame) -> pd.DataFrame: +def support_clip_to_donor( + draws: pd.DataFrame, donor: pd.DataFrame +) -> UKSupportClipResult: """Clip output draws to donor-realized support.""" - clipped = draws.copy() - for column in UK_WAS_WEALTH_OUTPUT_COLUMNS: - if column not in clipped or column not in donor: - continue - values = pd.to_numeric(donor[column], errors="coerce") - finite = values[np.isfinite(values)] - if finite.empty: - continue - clipped[column] = clipped[column].clip( - lower=float(finite.min()), - upper=float(finite.max()), - ) - return clipped + return support_clip_to_donor_with_receipt( + draws, + donor, + columns=UK_WAS_WEALTH_OUTPUT_COLUMNS, + stage=UK_WAS_WEALTH_STAGE_NAME, + ) def allocate_student_loan_balance_to_people( diff --git a/packages/microcosm-build/tests/test_uk_calibration_run.py b/packages/microcosm-build/tests/test_uk_calibration_run.py index e77037d93..08efa2fd0 100644 --- a/packages/microcosm-build/tests/test_uk_calibration_run.py +++ b/packages/microcosm-build/tests/test_uk_calibration_run.py @@ -22,6 +22,7 @@ from microcosm.build.uk_runtime.national_doctrine import UKNationalSolveDoctrine from microcosm.build.uk_runtime.national_frame import ( load_uk_national_frame, + uk_household_weight_kind, uk_national_frame, write_uk_national_frame, ) @@ -86,6 +87,54 @@ def _sha(path: Path) -> str: return hashlib.sha256(path.read_bytes()).hexdigest() +def _write_spine_sidecar( + input_h5: Path, + frame=None, + **overrides, +) -> dict[str, object]: + frame = _frame() if frame is None else frame + sidecar = { + "schema_version": 2, + "pipeline": "uk-frs-spine", + "stages": ["frs_spine", "was_wealth"], + "stage_records": [ + { + "stage": "was_wealth", + "produced": ["property_wealth"], + "nonzero_share": {"property_wealth": 1.0}, + "seconds": 0.1, + } + ], + "stage_evidence": { + "was_wealth": { + "stage": "was_wealth", + "support_clip": {"columns": {}}, + } + }, + "artifact_pins": {"person": "a" * 64}, + "input_artifact_pins": {"was_qrf_donor": {"sha256": "b" * 64}}, + "resource_pins": {"wealth.json": "c" * 64}, + "stage_artifact_pins": {"was_wealth": {"was_qrf_donor": "d" * 64}}, + "declared_seeds": {"was_wealth": {"was_wealth": 0}}, + "rules_engine": {"package": "policyengine-uk", "version": "unavailable"}, + "source_vintages": {"frs": "2024_25"}, + "stochastic_contract_sha256": "e" * 64, + "entity_row_counts": { + entity: int(len(frame.table(entity))) for entity in frame.entities + }, + "household_weight_kind": uk_household_weight_kind(frame).value, + "household_weight_total": float( + frame.weights_for("household").values.sum() + ), + } + sidecar.update(overrides) + input_h5.with_suffix(".build.json").write_text( + json.dumps(sidecar, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + return sidecar + + def _admin_anchor_values(): values = {} for entry in load_country_spec("uk").gates.gates: @@ -116,7 +165,9 @@ def test_run_uk_calibration_writes_cross_pinned_outputs(monkeypatch, tmp_path: P lambda frame, manifest: (_admin_anchor_values(), []), ) input_h5 = tmp_path / "input.h5" - write_uk_national_frame(_frame(), input_h5) + frame = _frame() + write_uk_national_frame(frame, input_h5) + spine_sidecar = _write_spine_sidecar(input_h5, frame) paths = UKCalibrationRunPaths( input_h5=input_h5, staging_h5=tmp_path / "staged.h5", @@ -152,6 +203,23 @@ def test_run_uk_calibration_writes_cross_pinned_outputs(monkeypatch, tmp_path: P assert result.build_record["artifacts"]["staging_h5"]["sha256"] == _sha(paths.staging_h5) assert result.build_record["artifacts"]["diagnostics_json"]["sha256"] == _sha(paths.diagnostics_json) assert result.build_record["artifacts"]["terminal_gate_json"]["sha256"] == _sha(paths.terminal_gate_json) + spine_provenance = result.build_record["spine_provenance"] + assert spine_provenance["stages"] == spine_sidecar["stages"] + assert spine_provenance["stage_records"] == spine_sidecar["stage_records"] + assert spine_provenance["stage_evidence"] == spine_sidecar["stage_evidence"] + assert spine_provenance["artifact_pins"] == spine_sidecar["artifact_pins"] + assert spine_provenance["input_artifact_pins"] == spine_sidecar["input_artifact_pins"] + assert spine_provenance["resource_pins"] == spine_sidecar["resource_pins"] + assert spine_provenance["stage_artifact_pins"] == spine_sidecar["stage_artifact_pins"] + assert spine_provenance["declared_seeds"] == spine_sidecar["declared_seeds"] + assert spine_provenance["rules_engine"] == spine_sidecar["rules_engine"] + assert spine_provenance["source_vintages"] == spine_sidecar["source_vintages"] + assert ( + spine_provenance["stochastic_contract_sha256"] + == spine_sidecar["stochastic_contract_sha256"] + ) + diagnostics = json.loads(paths.diagnostics_json.read_text()) + assert diagnostics["build"]["spine_provenance"] == spine_provenance staged, _ = load_uk_national_frame(paths.staging_h5) assert staged.weights_for("household").kind is WeightKind.CALIBRATED report = json.loads(paths.terminal_gate_json.read_text()) @@ -198,6 +266,98 @@ def test_run_uk_calibration_refuses_input_sha_before_outputs(tmp_path: Path): assert not paths.diagnostics_json.exists() +def test_run_uk_calibration_refuses_absent_input_sidecar(tmp_path: Path): + pytest.importorskip("tables") # pandas HDF backend + input_h5 = tmp_path / "input.h5" + write_uk_national_frame(_frame(), input_h5) + paths = UKCalibrationRunPaths( + input_h5=input_h5, + staging_h5=tmp_path / "staged.h5", + diagnostics_json=tmp_path / "diagnostics.json", + build_record_json=tmp_path / "build_record.json", + terminal_gate_json=tmp_path / "terminal_gates.json", + ) + + with pytest.raises(ValueError, match="build sidecar absent"): + run_uk_calibration( + paths=paths, + input_sha256=_sha(input_h5), + ledger_artifact=object(), + register_registry=_registry(), + calibration_year=2025, + exclusion_receipt={}, + doctrine=UKNationalSolveDoctrine(epochs=1), + doctrine_overrides={}, + measure_resolver=None, + source_pins={ + "input_h5": { + "sha256": _sha(input_h5), + "size_bytes": input_h5.stat().st_size, + } + }, + run_config_extra={"calibration_year": 2025}, + release_candidate=False, + release_id="missing-sidecar", + ) + + assert not paths.staging_h5.exists() + assert not paths.diagnostics_json.exists() + assert not paths.terminal_gate_json.exists() + + +@pytest.mark.parametrize( + ("override", "message"), + [ + ( + {"entity_row_counts": {"person": 999, "benunit": 4, "household": 4}}, + "row-count mismatch", + ), + ({"household_weight_total": 1.0}, "household_weight_total mismatch"), + ], +) +def test_run_uk_calibration_refuses_unbound_input_sidecar( + override, message, tmp_path: Path +): + pytest.importorskip("tables") # pandas HDF backend + frame = _frame() + input_h5 = tmp_path / "input.h5" + write_uk_national_frame(frame, input_h5) + _write_spine_sidecar(input_h5, frame, **override) + paths = UKCalibrationRunPaths( + input_h5=input_h5, + staging_h5=tmp_path / "staged.h5", + diagnostics_json=tmp_path / "diagnostics.json", + build_record_json=tmp_path / "build_record.json", + terminal_gate_json=tmp_path / "terminal_gates.json", + ) + + with pytest.raises(ValueError, match=message): + run_uk_calibration( + paths=paths, + input_sha256=_sha(input_h5), + ledger_artifact=object(), + register_registry=_registry(), + calibration_year=2025, + exclusion_receipt={}, + doctrine=UKNationalSolveDoctrine(epochs=1), + doctrine_overrides={}, + measure_resolver=None, + source_pins={ + "input_h5": { + "sha256": _sha(input_h5), + "size_bytes": input_h5.stat().st_size, + } + }, + run_config_extra={"calibration_year": 2025}, + release_candidate=False, + release_id="unbound-sidecar", + ) + + assert not paths.staging_h5.exists() + assert not paths.diagnostics_json.exists() + assert not paths.terminal_gate_json.exists() + + def test_seam_never_modifies_data_variables(monkeypatch, tmp_path: Path): """The seam's defining invariant: weights move, data never does. @@ -214,7 +374,9 @@ def test_seam_never_modifies_data_variables(monkeypatch, tmp_path: Path): lambda frame, manifest: (_admin_anchor_values(), []), ) input_h5 = tmp_path / "input.h5" - write_uk_national_frame(_frame(), input_h5) + frame = _frame() + write_uk_national_frame(frame, input_h5) + _write_spine_sidecar(input_h5, frame) paths = UKCalibrationRunPaths( input_h5=input_h5, staging_h5=tmp_path / "staged.h5", @@ -378,7 +540,9 @@ def test_attempt_ids_are_unique_across_reruns_of_one_release( lambda frame, manifest: (_admin_anchor_values(), []), ) input_h5 = tmp_path / "input.h5" - write_uk_national_frame(_frame(), input_h5) + frame = _frame() + write_uk_national_frame(frame, input_h5) + _write_spine_sidecar(input_h5, frame) source_pins = { "input_h5": {"sha256": _sha(input_h5), "size_bytes": input_h5.stat().st_size} } @@ -423,7 +587,9 @@ def test_verified_ledger_identity_reaches_the_run_evidence(monkeypatch, tmp_path lambda frame, manifest: (_admin_anchor_values(), []), ) input_h5 = tmp_path / "input.h5" - write_uk_national_frame(_frame(), input_h5) + frame = _frame() + write_uk_national_frame(frame, input_h5) + _write_spine_sidecar(input_h5, frame) artifact = SimpleNamespace( facts_sha256="d" * 64, fact_row_count=107_550, diff --git a/packages/microcosm-build/tests/test_uk_etb_services.py b/packages/microcosm-build/tests/test_uk_etb_services.py index 8ace2e2d7..6abf3de5a 100644 --- a/packages/microcosm-build/tests/test_uk_etb_services.py +++ b/packages/microcosm-build/tests/test_uk_etb_services.py @@ -6,6 +6,8 @@ from microcosm.build.uk_runtime.etb_services import ( UK_ETB_SERVICES_FIT_NAME, + UKETBServicesResult, + UKETBServicesStageTransform, build_nhs_cell_table, clean_etb_services_table, donor_realized_ranges, @@ -152,10 +154,27 @@ def test_services_support_clip_ranges_and_rail_ratio() -> None: } ) - clipped = support_clip_to_donor(draws, donor) + clip_result = support_clip_to_donor(draws, donor) + clipped = clip_result.clipped assert clipped["dfe_education_spending"].tolist() == [520.0, 1040.0] assert donor_realized_ranges(donor)["rail_subsidy_spending"] == (104.0, 208.0) + assert clip_result.receipt.evidence()["columns"]["rail_subsidy_spending"] == { + "donor_min": 104.0, + "donor_max": 208.0, + "clipped_low_rows": 1, + "clipped_high_rows": 1, + "rows_considered": 2, + } + transform = UKETBServicesStageTransform(stage=object(), engine=object()) + transform.last_result = UKETBServicesResult( + frame=object(), + support_clip=clip_result.receipt, + ) + assert transform.checkpoint_metadata()["evidence"] == { + "stage": "etb_services", + "support_clip": clip_result.receipt.evidence(), + } fare_index = load_etb_services_anchors()["rail_fare_index_2023"]["value"] assert 111.0 / fare_index == pytest.approx(100.0) diff --git a/packages/microcosm-build/tests/test_uk_etb_vat.py b/packages/microcosm-build/tests/test_uk_etb_vat.py index e80da7a4f..7b87eb51e 100644 --- a/packages/microcosm-build/tests/test_uk_etb_vat.py +++ b/packages/microcosm-build/tests/test_uk_etb_vat.py @@ -6,6 +6,8 @@ from microcosm.build.uk_runtime.etb_vat import ( UK_ETB_VAT_FIT_NAME, + UKETBVATResult, + UKETBVATStageTransform, clean_etb_vat_table, donor_realized_ranges, impute_etb_vat, @@ -60,7 +62,8 @@ def test_etb_vat_support_clip_and_ranges() -> None: donor = clean_etb_vat_table(_raw_etb()) draws = pd.DataFrame({"full_rate_vat_expenditure_rate": [-99.0, 99.0]}) - clipped = support_clip_to_donor(draws, donor) + clip_result = support_clip_to_donor(draws, donor) + clipped = clip_result.clipped assert clipped["full_rate_vat_expenditure_rate"].tolist() == [ donor["full_rate_vat_expenditure_rate"].min(), @@ -72,6 +75,24 @@ def test_etb_vat_support_clip_and_ranges() -> None: float(donor["full_rate_vat_expenditure_rate"].max()), ) } + assert clip_result.receipt.evidence()["columns"][ + "full_rate_vat_expenditure_rate" + ] == { + "donor_min": float(donor["full_rate_vat_expenditure_rate"].min()), + "donor_max": float(donor["full_rate_vat_expenditure_rate"].max()), + "clipped_low_rows": 1, + "clipped_high_rows": 1, + "rows_considered": 2, + } + transform = UKETBVATStageTransform(stage=object(), engine=object()) + transform.last_result = UKETBVATResult( + frame=object(), + support_clip=clip_result.receipt, + ) + assert transform.checkpoint_metadata()["evidence"] == { + "stage": "etb_vat", + "support_clip": clip_result.receipt.evidence(), + } def test_etb_vat_weighted_fit_record(monkeypatch: pytest.MonkeyPatch) -> None: diff --git a/packages/microcosm-build/tests/test_uk_frs_spine.py b/packages/microcosm-build/tests/test_uk_frs_spine.py index 1c72bcdcf..3798a082f 100644 --- a/packages/microcosm-build/tests/test_uk_frs_spine.py +++ b/packages/microcosm-build/tests/test_uk_frs_spine.py @@ -1646,6 +1646,101 @@ def test_e8_manifest_seeds_all_reach_the_build_sidecar_harvester() -> None: } +def test_spine_sidecar_collects_stage_evidence_by_duck_type() -> None: + tool = _load_tool() + + class _EvidenceResult: + def __init__(self, payload: dict[str, object]) -> None: + self.payload = payload + + def evidence(self) -> dict[str, object]: + return self.payload + + class _CheckpointStage: + def __init__(self, payload: dict[str, object]) -> None: + self.payload = payload + + def checkpoint_metadata(self) -> dict[str, object]: + return {"evidence": self.payload} + + e8_payloads = { + "cgt_incidence_clone": {"stage": "cgt_incidence_clone", "rows": 1}, + "cgt_band_donors": {"stage": "cgt_band_donors", "rows": 2}, + "salary_sacrifice": {"stage": "salary_sacrifice", "rows": 3}, + "student_loans": {"stage": "student_loans", "rows": 4}, + "age_tail": {"stage": "age_tail", "rows": 5}, + } + spi_payloads = { + "frs_hmrc_spine_leaves": { + "stage": "frs_hmrc_spine_leaves", + "source_signal_rows": {"employment_income": 2}, + }, + "spi_support_channel": { + "stage": "spi_support_channel", + "spi_households": 7, + }, + "hmrc_spi_income_spine": { + "stage": "hmrc_spi_income_spine", + "targets": {"count": 8}, + }, + } + new_payload = {"stage": "future_stage", "rows": 9} + implementations = { + "frs_spine": SimpleNamespace(), + "frs_hmrc_spine_leaves": _CheckpointStage( + spi_payloads["frs_hmrc_spine_leaves"] + ), + "spi_support_channel": _CheckpointStage(spi_payloads["spi_support_channel"]), + "hmrc_spi_income_spine": _CheckpointStage( + spi_payloads["hmrc_spi_income_spine"] + ), + "cgt_incidence_clone": _CheckpointStage( + e8_payloads["cgt_incidence_clone"] + ), + "cgt_band_donors": _CheckpointStage(e8_payloads["cgt_band_donors"]), + "salary_sacrifice": _CheckpointStage(e8_payloads["salary_sacrifice"]), + "student_loans": SimpleNamespace( + last_result=_EvidenceResult(e8_payloads["student_loans"]) + ), + "age_tail": SimpleNamespace(last_result=e8_payloads["age_tail"]), + "future_stage": _CheckpointStage(new_payload), + } + + evidence = tool._collect_stage_evidence( + stage_names=( + "frs_spine", + "frs_hmrc_spine_leaves", + "spi_support_channel", + "hmrc_spi_income_spine", + "cgt_incidence_clone", + "cgt_band_donors", + "salary_sacrifice", + "student_loans", + "age_tail", + "future_stage", + ), + implementations=implementations, + ) + + assert evidence == { + **spi_payloads, + **e8_payloads, + "future_stage": new_payload, + } + assert list(evidence) == [ + "frs_hmrc_spine_leaves", + "spi_support_channel", + "hmrc_spi_income_spine", + "cgt_incidence_clone", + "cgt_band_donors", + "salary_sacrifice", + "student_loans", + "age_tail", + "future_stage", + ] + assert "frs_spine" not in evidence + + class TestScottishWaterAndSewerage: """The FRS 2024-25 cell retirement, at the three shapes the tab presents. diff --git a/packages/microcosm-build/tests/test_uk_lcfs_consumption.py b/packages/microcosm-build/tests/test_uk_lcfs_consumption.py index 6b349b982..0ef656f49 100644 --- a/packages/microcosm-build/tests/test_uk_lcfs_consumption.py +++ b/packages/microcosm-build/tests/test_uk_lcfs_consumption.py @@ -8,6 +8,8 @@ LCFS_ACCOMM_MAP, LCFS_TENURE_MAP, UK_LCFS_CONSUMPTION_TARGET_COLUMNS, + UKLCFSConsumptionResult, + UKLCFSConsumptionStageTransform, assign_recipient_has_fuel, clean_lcfs_consumption_table, derive_energy_from_lcfs, @@ -141,7 +143,7 @@ def test_support_clip_exempts_raked_energy_columns() -> None: {column: [0.0, 10.0] for column in UK_LCFS_CONSUMPTION_TARGET_COLUMNS} ) - clipped = support_clip_to_donor( + clip_result = support_clip_to_donor( draws, donor, exempt={ @@ -150,12 +152,42 @@ def test_support_clip_exempts_raked_energy_columns() -> None: "domestic_energy_consumption", }, ) + clipped = clip_result.clipped assert clipped["food_and_non_alcoholic_beverages_consumption"].tolist() == [ 1.0, 5.0, ] assert clipped["electricity_consumption"].tolist() == [0.0, 10.0] + receipt = clip_result.receipt.evidence()["columns"] + assert receipt["food_and_non_alcoholic_beverages_consumption"] == { + "donor_min": 1.0, + "donor_max": 5.0, + "clipped_low_rows": 1, + "clipped_high_rows": 1, + "rows_considered": 2, + } + assert receipt["electricity_consumption"] == { + "exempt": True, + "rows_considered": 2, + } + assert receipt["gas_consumption"] == { + "exempt": True, + "rows_considered": 2, + } + assert receipt["domestic_energy_consumption"] == { + "exempt": True, + "rows_considered": 2, + } + transform = UKLCFSConsumptionStageTransform(stage=object(), engine=object()) + transform.last_result = UKLCFSConsumptionResult( + frame=object(), + support_clip=clip_result.receipt, + ) + assert transform.checkpoint_metadata()["evidence"] == { + "stage": "lcfs_consumption", + "support_clip": clip_result.receipt.evidence(), + } def test_has_fuel_bridge_accepts_lcfs_native_predictor_names() -> None: diff --git a/packages/microcosm-build/tests/test_uk_was_wealth.py b/packages/microcosm-build/tests/test_uk_was_wealth.py index ebd41ccf6..872ae69f3 100644 --- a/packages/microcosm-build/tests/test_uk_was_wealth.py +++ b/packages/microcosm-build/tests/test_uk_was_wealth.py @@ -255,10 +255,26 @@ def test_support_clip_and_integer_vehicle_output( {column: [999999.0, -999999.0] for column in UK_WAS_WEALTH_OUTPUT_COLUMNS} ) - clipped = support_clip_to_donor(draws, donor) + clip_result = support_clip_to_donor(draws, donor) + clipped = clip_result.clipped assert clipped["owned_land"].tolist() == [100.0, 10.0] assert clipped["net_financial_wealth"].tolist() == [50.0, -5.0] + receipt = clip_result.receipt.evidence()["columns"] + assert receipt["owned_land"] == { + "donor_min": 10.0, + "donor_max": 100.0, + "clipped_low_rows": 1, + "clipped_high_rows": 1, + "rows_considered": 2, + } + assert receipt["net_financial_wealth"] == { + "donor_min": -5.0, + "donor_max": 50.0, + "clipped_low_rows": 1, + "clipped_high_rows": 1, + "rows_considered": 2, + } import microcosm.build.uk_runtime.was_wealth as module @@ -286,6 +302,16 @@ def fake_impute(*args, **kwargs): module.FitWeightRecord("uk_was_2018_20_wealth:test", "explicit"), ) assert transformed.table("household")["num_vehicles"].tolist() == [1, 3] + assert transform.last_result is not None + assert transform.checkpoint_metadata()["evidence"]["support_clip"]["columns"][ + "num_vehicles" + ] == { + "donor_min": 1.2, + "donor_max": 2.8, + "clipped_low_rows": 0, + "clipped_high_rows": 0, + "rows_considered": 2, + } assert "student_loan_balance" not in transformed.table("household").columns assert transformed.table("person")["student_loan_balance"].sum() == pytest.approx( 7000.0 diff --git a/tools/build_uk_frs_spine.py b/tools/build_uk_frs_spine.py index eb87ba750..26bbd218a 100644 --- a/tools/build_uk_frs_spine.py +++ b/tools/build_uk_frs_spine.py @@ -7,6 +7,7 @@ import json import sys import time +from collections.abc import Mapping, Sequence from datetime import UTC, datetime from importlib import metadata from pathlib import Path @@ -446,6 +447,39 @@ def _declared_seeds(stages) -> dict[str, dict[str, int]]: return declared +def _result_evidence(result: object) -> object: + if isinstance(result, dict): + return result + evidence = getattr(result, "evidence", None) + if callable(evidence): + return evidence() + return None + + +def _collect_stage_evidence( + *, + stage_names: Sequence[str], + implementations: Mapping[str, object], +) -> dict[str, object]: + evidence_by_stage: dict[str, object] = {} + for stage_name in stage_names: + implementation = implementations.get(stage_name) + if implementation is None: + continue + metadata = None + metadata_hook = getattr(implementation, "checkpoint_metadata", None) + if callable(metadata_hook): + metadata = dict(metadata_hook()) + evidence = metadata.get("evidence", metadata) + else: + evidence = _result_evidence( + getattr(implementation, "last_result", None) + ) + if evidence is not None: + evidence_by_stage[stage_name] = evidence + return evidence_by_stage + + def _build_sidecar( *, frame, @@ -890,30 +924,12 @@ def main(argv: list[str] | None = None) -> int: frs_vintage=frs_release.vintage, sampling=sampling, ) - # E8 executed-effect receipts (#730/#684 two-arm rule, arm 2): the - # clone/donor/salsac/student-loan transforms record their receipts on - # last_result; persist them beside the declared seeds so the sidecar - # carries evidence that every declared parameter shaped the output. - e8_stage_evidence: dict[str, object] = {} - for e8_stage_name in ( - "cgt_incidence_clone", - "cgt_band_donors", - "salary_sacrifice", - "student_loans", - "age_tail", - ): - e8_implementation = implementations.get(e8_stage_name) - e8_last_result = getattr(e8_implementation, "last_result", None) - if e8_last_result is not None: - # age_tail's receipt is already the evidence mapping; the E8 - # transforms carry a result object that produces one. - e8_stage_evidence[e8_stage_name] = ( - e8_last_result - if isinstance(e8_last_result, dict) - else e8_last_result.evidence() - ) - if e8_stage_evidence: - sidecar["stage_evidence"] = e8_stage_evidence + stage_evidence = _collect_stage_evidence( + stage_names=_STAGE_NAMES, + implementations=implementations, + ) + if stage_evidence: + sidecar["stage_evidence"] = stage_evidence atomic_write_json(sidecar_path, sidecar) append_phase(state, "build_sidecar_written") if args.emit_nonzero_shares is not None: From d9746b87730824d40f4610c096f963b7839281b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Tue, 25 Aug 2026 23:40:06 +0200 Subject: [PATCH 02/14] Bind the two targets that named concepts under other names MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The live proof refused twice, each time on a target the committed surface declares and no code could measure. DWP publishes `any_child_under` for Scottish UC households with a child under 1. The adapter implemented any/sum/count and raised on the rest, so the target could not bind at all. The model carries no is_child column because it has no need of one — dependency is derived from age where it is wanted — so "any child under N" is exactly "any person aged under N" on this frame, and the reduction now binds as that. Facts keep the semantics of whoever published them; translating them onto ours is our job, and refusing one would have dropped a real target rather than measured it. The NHS budget anchor named person.nhs_spending, which appears in no Python anywhere. The ETB services stage carries the spend split across the three points of delivery it imputes, so the anchor is composed from those; it had been measuring a silent zero until missing anchor columns started failing loud. The measurement receipt now records which columns composed each anchor, and a derived anchor whose parts are themselves absent refuses with the missing part named. Both mappings are declared next to what they describe — the reduction beside the adapter, the NHS components beside the columns they sum — so neither can drift into a silent alias. Co-Authored-By: Claude Fable 5 --- .../757-published-fact-translations.fixed.md | 1 + .../build/uk_runtime/calibration_run.py | 40 +++++++++++--- .../build/uk_runtime/etb_services.py | 9 ++++ .../build/uk_runtime/ledger_targets.py | 28 +++++++++- .../tests/test_uk_calibration_run.py | 41 ++++++++++++++ .../tests/test_uk_national_calibration.py | 54 +++++++++++++++++++ 6 files changed, 165 insertions(+), 8 deletions(-) create mode 100644 changelog.d/757-published-fact-translations.fixed.md diff --git a/changelog.d/757-published-fact-translations.fixed.md b/changelog.d/757-published-fact-translations.fixed.md new file mode 100644 index 000000000..124888297 --- /dev/null +++ b/changelog.d/757-published-fact-translations.fixed.md @@ -0,0 +1 @@ +Two committed UK targets named concepts the frame carries under different names, and both went unnoticed until the seam started refusing what it could not measure. The DWP `any_child_under` household reduction is now bound as the age predicate it means here — the model carries no dependent-child column because dependency is derived from age where it is wanted — and the NHS budget anchor, published as one total, is composed from the three points-of-delivery columns the ETB services stage actually produces, where before it named a column no stage writes and measured a silent zero. Both translations are declared in named mappings rather than aliased at the call site, and the admin measurement receipt records per anchor which columns composed it. diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/calibration_run.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/calibration_run.py index 677994662..dc5977453 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/calibration_run.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/calibration_run.py @@ -45,6 +45,9 @@ uk_target_geography_levels, write_uk_calibration_diagnostics, ) +from microcosm.build.uk_runtime.etb_services import ( + UK_NHS_SPENDING_COMPONENT_COLUMNS, +) from microcosm.build.uk_runtime.national_calibration import UKNationalCalibrationStage from microcosm.build.uk_runtime.national_frame import ( load_uk_national_frame, @@ -549,6 +552,16 @@ def _calibration_gate_manifest() -> GatesManifest: ) +#: Admin anchors published against a concept the frame carries only in parts. +#: The anchor keeps the publisher's shape (one NHS budget line); our stages +#: carry the spend split by point of delivery. Composing is the translation, +#: declared here and recorded per anchor in the measurement receipt — never a +#: reason to drop the anchor or to let it measure a silent zero. +UK_DERIVED_ADMIN_ANCHOR_MEASURES: Mapping[str, tuple[str, ...]] = { + "nhs_spending": UK_NHS_SPENDING_COMPONENT_COLUMNS, +} + + def _aggregate_admin_totals( frame: Frame, manifest: GatesManifest ) -> tuple[dict[str, float], list[dict[str, object]]]: @@ -575,12 +588,22 @@ def _aggregate_admin_totals( name = str(anchor.get("name", anchor.get("measure"))) measure = str(anchor.get("measure", anchor.get("name"))) table = frame.table(entity) + composed_from: tuple[str, ...] = () if measure not in table: - raise ValueError( - f"aggregate_admin anchor {name!r} needs {entity}.{measure}, " - "which the calibrated frame does not carry; refusing to " - "fabricate a measured value." - ) + composed_from = UK_DERIVED_ADMIN_ANCHOR_MEASURES.get(measure, ()) + missing = [column for column in composed_from if column not in table] + if not composed_from or missing: + raise ValueError( + f"aggregate_admin anchor {name!r} needs {entity}.{measure}, " + "which the calibrated frame does not carry" + + ( + f" (declared as the sum of {list(composed_from)}, " + f"missing {missing})" + if composed_from + else "" + ) + + "; refusing to fabricate a measured value." + ) if entity == "household": weights = household_weights elif entity == "person": @@ -606,7 +629,11 @@ def _aggregate_admin_totals( "the calibration seam measures household and person anchors " "only." ) - values = table[measure].to_numpy(dtype=float) + values = ( + table[list(composed_from)].to_numpy(dtype=float).sum(axis=1) + if composed_from + else table[measure].to_numpy(dtype=float) + ) total = float(np.dot(values, weights)) carriers = values != 0 carrier_weight = float(weights[carriers].sum()) @@ -627,6 +654,7 @@ def _aggregate_admin_totals( "weighted_total": total, "weighted_mean_carriers": mean_carriers, "statistic_convention": "assessed_by_anchor_magnitude", + "composed_from": list(composed_from), } ) return totals, receipt diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/etb_services.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/etb_services.py index 2789011c7..82101f054 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/etb_services.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/etb_services.py @@ -73,6 +73,15 @@ *UK_ETB_SERVICES_HOUSEHOLD_OUTPUT_COLUMNS, *UK_NHS_OUTPUT_COLUMNS, ) +#: The NHS budget anchor is published as one total; this stage carries the +#: spend split across the three points of delivery it imputes. Summing them is +#: the translation from the published concept to ours — declared beside the +#: columns so it cannot drift from what the stage actually produces. +UK_NHS_SPENDING_COMPONENT_COLUMNS = ( + "nhs_a_and_e_spending", + "nhs_admitted_patient_spending", + "nhs_outpatient_spending", +) UK_ETB_SERVICES_NONNEGATIVE_OUTPUT_COLUMNS = UK_ETB_SERVICES_OUTPUT_COLUMNS UK_ETB_SERVICES_FIT_NAME = "uk_etb_2024_services" UK_ETB_SERVICES_STAGE_NAME = "etb_services" diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/ledger_targets.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/ledger_targets.py index 50482bfce..dd18368c6 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/ledger_targets.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/ledger_targets.py @@ -131,6 +131,24 @@ def parameter(self, parameter: str, period: int | str) -> float: raise KeyError(parameter) +#: Published-fact reductions rewritten to the internal reduction that carries +#: the same meaning on our frame. Facts keep the semantics of the source that +#: published them; translating those onto the model's own concepts is our job, +#: and a fact we cannot phrase internally gets translated and recorded, never +#: dropped. +#: +#: ``any_child_under`` (DWP Stat-Xplore, Scottish UC households with a child +#: under 1) names a dependent-child concept the model does not carry. There is +#: no ``is_child`` column because the model has no need of one: dependency is +#: derived from age where it is wanted. So "any child under N" is exactly "any +#: person aged under N" here, and the condition already supplies the age bound. +#: The rewrite is declared rather than aliased at the call site so it stays +#: greppable, testable, and visible in review. +UK_TRANSLATED_HOUSEHOLD_REDUCTIONS: Mapping[str, str] = { + "any_child_under": "any", +} + + class UKFrameTargetAdapter: """Frame-backed target materialization adapter for UK calibration stages.""" @@ -239,7 +257,8 @@ def household_condition(self, condition: Mapping[str, Any]) -> np.ndarray: source = self.tables[entity] household_ids = self._household_ids_for(entity) - reduce = str(condition["reduce"]) + published = str(condition["reduce"]) + reduce = UK_TRANSLATED_HOUSEHOLD_REDUCTIONS.get(published, published) variable = str(condition["variable"]) if reduce == "any": matched = _compare_series(source[variable], condition) @@ -252,7 +271,12 @@ def household_condition(self, condition: Mapping[str, Any]) -> np.ndarray: aggregate = source[variable].groupby(household_ids).count() expected = condition else: - raise ValueError(f"Unsupported UK household reduction {reduce!r}.") + raise ValueError( + f"Unsupported UK household reduction {published!r}." + if published == reduce + else f"Unsupported UK household reduction {published!r} " + f"(translated to {reduce!r})." + ) households = self.tables["household"] ids = households["household_id"] diff --git a/packages/microcosm-build/tests/test_uk_calibration_run.py b/packages/microcosm-build/tests/test_uk_calibration_run.py index 08efa2fd0..9a407e9a6 100644 --- a/packages/microcosm-build/tests/test_uk_calibration_run.py +++ b/packages/microcosm-build/tests/test_uk_calibration_run.py @@ -19,6 +19,9 @@ UKCalibrationRunPaths, run_uk_calibration, ) +from microcosm.build.uk_runtime.etb_services import ( + UK_NHS_SPENDING_COMPONENT_COLUMNS, +) from microcosm.build.uk_runtime.national_doctrine import UKNationalSolveDoctrine from microcosm.build.uk_runtime.national_frame import ( load_uk_national_frame, @@ -460,6 +463,44 @@ def test_aggregate_admin_measurement_convention_and_refusals(): calibration_run._aggregate_admin_totals(stripped, manifest) +def test_nhs_anchor_composes_from_the_columns_the_spine_actually_carries(): + """The anchor is published as one total; the spine carries it in three parts. + + Composing is the translation from the published concept to ours, and the + receipt has to say so — the anchor measured a silent zero for as long as it + named a column no stage produces. + """ + + frame = _frame() + person = frame.table("person") + person.drop(columns=["nhs_spending"], inplace=True) + person["nhs_a_and_e_spending"] = [20.0, 20.0, 20.0, 20.0] + person["nhs_admitted_patient_spending"] = [25.0, 25.0, 25.0, 25.0] + person["nhs_outpatient_spending"] = [5.0, 5.0, 5.0, 5.0] + manifest = calibration_run._calibration_gate_manifest() + + totals, receipt = calibration_run._aggregate_admin_totals(frame, manifest) + + # Same 4 persons x 50.0 x weight 10.0 as the single-column fixture. + assert totals["nhs_spending_total"] == pytest.approx(2000.0) + by_anchor = {row["anchor"]: row for row in receipt} + assert by_anchor["nhs_spending_total"]["composed_from"] == list( + UK_NHS_SPENDING_COMPONENT_COLUMNS + ) + assert by_anchor["need_gas_mean_spending"]["composed_from"] == [] + + +def test_partly_carried_derived_anchor_refuses_and_names_the_missing_part(): + frame = _frame() + person = frame.table("person") + person.drop(columns=["nhs_spending"], inplace=True) + person["nhs_a_and_e_spending"] = [20.0, 20.0, 20.0, 20.0] + manifest = calibration_run._calibration_gate_manifest() + + with pytest.raises(ValueError, match="nhs_admitted_patient_spending"): + calibration_run._aggregate_admin_totals(frame, manifest) + + def test_seam_pipeline_derives_a_ratified_logbook_scope(): """The seam appends to the FRS line's chain, not a new unratified one.""" diff --git a/packages/microcosm-build/tests/test_uk_national_calibration.py b/packages/microcosm-build/tests/test_uk_national_calibration.py index bd547a8e1..3006f087a 100644 --- a/packages/microcosm-build/tests/test_uk_national_calibration.py +++ b/packages/microcosm-build/tests/test_uk_national_calibration.py @@ -861,3 +861,57 @@ def receipt(self): assert ("household", "probe_measure") in resolution.measure_inputs for entity in frame.entities: assert list(frame.table(entity).columns) == before[entity] + + +def test_published_child_reduction_translates_to_the_age_predicate(): + """A published fact keeps its publisher's semantics; binding it is our job. + + DWP names a dependent-child concept in `any_child_under`. The model carries + no `is_child` column because it has no need of one — dependency is derived + from age where it is wanted — so "any child under N" is exactly "any person + aged under N" here. The translation is declared, not aliased at the call + site, and refusing the fact instead would drop a real target. + """ + + from microcosm.build.uk_runtime.ledger_targets import ( + UK_TRANSLATED_HOUSEHOLD_REDUCTIONS, + UKFrameTargetAdapter, + ) + + assert UK_TRANSLATED_HOUSEHOLD_REDUCTIONS["any_child_under"] == "any" + + frame = _materialization_binding_frame() + adapter = UKFrameTargetAdapter(frame) + # Households 0 and 2 each carry an infant; household 1 does not. + adapter.tables["person"]["age"] = [0.0, 40.0, 35.0, 0.0, 38.0, 41.0] + condition = { + "variable": "age", + "entity": "person", + "reduce": "any_child_under", + "operator": "<", + "value": 1, + } + + translated = adapter.household_condition(condition) + + assert list(translated) == [True, False, True] + assert list(translated) == list( + adapter.household_condition({**condition, "reduce": "any"}) + ) + + +def test_unmapped_household_reduction_names_the_published_reducer(): + from microcosm.build.uk_runtime.ledger_targets import UKFrameTargetAdapter + + adapter = UKFrameTargetAdapter(_materialization_binding_frame()) + + with pytest.raises(ValueError, match="any_grandchild_under"): + adapter.household_condition( + { + "variable": "capital_gains", + "entity": "person", + "reduce": "any_grandchild_under", + "operator": "<", + "value": 1, + } + ) From c57d4da4eb0a3b5f0cf89411a69bc1f3df0c552e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Wed, 26 Aug 2026 11:16:08 +0200 Subject: [PATCH 03/14] Let the spine build police itself, stage by stage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The spine build ran no gates at all: its only verdict was a synthetic pipeline:passed, and every real check waited for a downstream pipeline working on a different frame. A regression in an imputation surfaced, if at all, as a whole-spine diff with nothing to say which stage caused it. It now runs its own scoped battery at two checkpoint boundaries drawn from the shared phase vocabulary, the way the US pool builder gates at its durable checkpoints. Thirteen per-stage gates read the receipts the previous commit made each stage record, so a failure is named by its stage. Blocking is two layer: the boundary enforces before the next group runs and the H5 is only written at the end, so a failed boundary leaves a report and no artifact; and the seam refuses an input whose spine battery report is missing or blocked, so an artifact built before the gates existed cannot reach calibration by the side door. The CGT donor stage is gated at stage time on the band support bounds, which were declared in the package and read by nothing. That is the adjudicated answer to a resource whose own note says terminal applicability is unconfirmed: the Table 3 redraw moves amounts onto a different band surface afterwards, so a terminal gate provably cannot express the constraint and a stage-time one can. Gate ownership is now a partition rather than a list. The check refuses a gate owned by nobody and a gate owned twice without saying so — uk_aggregate_admin is the one deliberate duplicate, measuring the same anchors on two different frames, and it says so. The family-coverage guard resolves families against the plan that builds them, so the spine passes a contract that no single driver could satisfy before, without the contract being weakened. Co-Authored-By: Claude Fable 5 --- changelog.d/757-gate-ownership.changed.md | 1 + changelog.d/757-spine-battery.added.md | 1 + .../src/microcosm/build/country_spec.py | 1 + .../src/microcosm/build/uk/gates.json | 237 +++++++++- .../uk/release_input_coverage_manifest.json | 30 +- .../build/uk_runtime/battery_bindings.py | 62 +++ .../build/uk_runtime/calibration_run.py | 195 +++++++- .../build/uk_runtime/cgt_imputation.py | 48 +- .../build/uk_runtime/cgt_structure.py | 4 + .../build/uk_runtime/national_build.py | 20 +- .../uk_runtime/release_input_coverage.py | 71 ++- .../build/uk_runtime/stage_health.py | 418 ++++++++++++++++++ .../tests/test_country_spec.py | 44 +- .../tests/test_gate_battery_contract_pins.py | 4 + .../tests/test_uk_battery_bindings.py | 12 +- .../tests/test_uk_calibration_run.py | 19 + .../tests/test_uk_cgt_structure.py | 6 + .../tests/test_uk_national_build.py | 31 +- .../tests/test_uk_release_input_coverage.py | 39 ++ .../tests/test_uk_stage_health.py | 353 +++++++++++++++ .../src/microcosm/data/contract.py | 48 +- .../microcosm-data/tests/test_contract.py | 107 ++++- tools/build_uk_frs_spine.py | 116 ++++- ...uild_uk_release_input_coverage_manifest.py | 20 + 24 files changed, 1799 insertions(+), 88 deletions(-) create mode 100644 changelog.d/757-gate-ownership.changed.md create mode 100644 changelog.d/757-spine-battery.added.md create mode 100644 packages/microcosm-build/src/microcosm/build/uk_runtime/stage_health.py create mode 100644 packages/microcosm-build/tests/test_uk_stage_health.py diff --git a/changelog.d/757-gate-ownership.changed.md b/changelog.d/757-gate-ownership.changed.md new file mode 100644 index 000000000..3ea42ed8e --- /dev/null +++ b/changelog.d/757-gate-ownership.changed.md @@ -0,0 +1 @@ +Every UK gate is now owned by exactly one battery — the spine build's, the calibration seam's, or the national preflight/terminal one — and the ownership check refuses both halves of a closed world: a gate belonging to nobody, and a gate claimed by two batteries without being declared in `UK_SHARED_GATE_IDS`. Coverage alone would have let an accidental overlap through, and a release certification that unions the scoped reports is exactly where that would be paid for. The national build takes a scoped manifest of its own, since the spine's mid-build phases would otherwise break its phase sequencing. The family-coverage guard now resolves each required family against the build plan that actually produces it, so a spine-built national release passes it without the contract being relaxed. diff --git a/changelog.d/757-spine-battery.added.md b/changelog.d/757-spine-battery.added.md new file mode 100644 index 000000000..8a2f57fcf --- /dev/null +++ b/changelog.d/757-spine-battery.added.md @@ -0,0 +1 @@ +The UK spine build now runs its own gate battery instead of leaving every check to a downstream pipeline. Thirteen per-stage health gates read the receipts each imputation stage records — donor-support clipping, realization against target, mass conservation, achieved versus declared band populations — so a regression is named by the stage that caused it rather than surfacing as an unattributed whole-spine diff. The battery evaluates at two checkpoint boundaries in the shared phase vocabulary, `assembled` after the base frame and its stochastic draws and `transferred` after the donor transfers, mirroring how the US pool builder gates at its own checkpoints. The BRMA enum gate moved to the assembled boundary, where its column is first written: enum membership is a property of the values, so the check is identical fourteen stages earlier. diff --git a/packages/microcosm-build/src/microcosm/build/country_spec.py b/packages/microcosm-build/src/microcosm/build/country_spec.py index e3fcde952..130602e5b 100644 --- a/packages/microcosm-build/src/microcosm/build/country_spec.py +++ b/packages/microcosm-build/src/microcosm/build/country_spec.py @@ -121,6 +121,7 @@ "per_family_fit", "release_input_coverage", "source_coverage", + "stage_health", "spine_agreement", "support", "tail_concentration", diff --git a/packages/microcosm-build/src/microcosm/build/uk/gates.json b/packages/microcosm-build/src/microcosm/build/uk/gates.json index 51d48e864..67203f8a4 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/gates.json +++ b/packages/microcosm-build/src/microcosm/build/uk/gates.json @@ -4,6 +4,8 @@ "policy": "The certified June UK national battery declared as data (microcosm#611): the two build preflights plus the terminal battery. Every gate carries its thresholds as entry parameters at exact certified or receipted values; microcosm#630 landed the weighted-integrity pair as reviewed parameter edits with receipts in the entry notes, and the weight-ratio fence keeps its exact certified June value (the #630 breach was an SPI allocation defect fixed upstream in microcosm#710). The #680 stochastic layer delivers the take-up signal gate; every gate the UK runs today is declared below.", "phases": [ "preflight", + "assembled", + "transferred", "terminal" ], "gates": [ @@ -49,6 +51,237 @@ }, "notes": "Ledger-compiled UK targets at the 2025 calibration period compared with the frozen incumbent registry surface. The signed list enumerates TCL additions, exclusions, unsupported rows, and vintage drift for review." }, + { + "id": "uk_stage_was_wealth_support", + "gate": "stage_health", + "phase": "transferred", + "criticality": "release_blocking", + "evidence_absent_blocks": true, + "parameters": { + "stage": "was_wealth", + "check": "support_clip", + "columns": [ + "cash_isa", + "corporate_wealth", + "gross_financial_wealth", + "main_residence_value", + "net_financial_wealth", + "non_residential_property_value", + "num_vehicles", + "other_residential_property_value", + "owned_land", + "property_wealth", + "savings", + "stocks_and_shares_isa", + "student_loan_balance" + ], + "max_clipped_low_rows_by_column": {}, + "max_clipped_high_rows_by_column": {} + }, + "notes": "Stage-time support-clip health gate over the WAS wealth receipt; raw FRS mapping stages have no imputation semantics and intentionally have no empty health gate." + }, + { + "id": "uk_stage_lcfs_consumption_support", + "gate": "stage_health", + "phase": "transferred", + "criticality": "release_blocking", + "evidence_absent_blocks": true, + "parameters": { + "stage": "lcfs_consumption", + "check": "support_clip", + "columns": [ + "alcohol_and_tobacco_consumption", + "bus_fare_spending", + "clothing_and_footwear_consumption", + "communication_consumption", + "diesel_spending", + "education_consumption", + "food_and_non_alcoholic_beverages_consumption", + "health_consumption", + "household_furnishings_consumption", + "housing_water_and_electricity_consumption", + "miscellaneous_consumption", + "petrol_spending", + "recreation_consumption", + "restaurants_and_hotels_consumption", + "transport_consumption" + ], + "exempt_columns": [ + "domestic_energy_consumption", + "electricity_consumption", + "gas_consumption" + ], + "max_clipped_low_rows_by_column": {}, + "max_clipped_high_rows_by_column": {} + }, + "notes": "Stage-time support-clip health gate over the LCFS receipt. The exempt energy columns are deliberately named here because they are bridged through the NEED/WAS path rather than clipped to LCFS donor support." + }, + { + "id": "uk_stage_etb_vat_support", + "gate": "stage_health", + "phase": "transferred", + "criticality": "release_blocking", + "evidence_absent_blocks": true, + "parameters": { + "stage": "etb_vat", + "check": "support_clip", + "columns": [ + "full_rate_vat_expenditure_rate" + ], + "max_clipped_low_rows_by_column": {}, + "max_clipped_high_rows_by_column": {} + }, + "notes": "Stage-time support-clip health gate over the ETB VAT receipt." + }, + { + "id": "uk_stage_etb_services_support", + "gate": "stage_health", + "phase": "transferred", + "criticality": "release_blocking", + "evidence_absent_blocks": true, + "parameters": { + "stage": "etb_services", + "check": "support_clip", + "columns": [ + "bus_subsidy_spending", + "dfe_education_spending", + "rail_subsidy_spending" + ], + "max_clipped_low_rows_by_column": {}, + "max_clipped_high_rows_by_column": {} + }, + "notes": "Stage-time support-clip health gate over the ETB services receipt." + }, + { + "id": "uk_stage_frs_hmrc_spine_leaves_signal", + "gate": "stage_health", + "phase": "transferred", + "criticality": "release_blocking", + "evidence_absent_blocks": true, + "parameters": { + "stage": "frs_hmrc_spine_leaves", + "check": "source_signal", + "minimum_signal_rows": 1, + "structural_zero_columns": [] + }, + "notes": "Stage-time source-signal gate over the retained FRS/HMRC leaves receipt." + }, + { + "id": "uk_stage_spi_support_channel_mass", + "gate": "stage_health", + "phase": "transferred", + "criticality": "release_blocking", + "evidence_absent_blocks": true, + "parameters": { + "stage": "spi_support_channel", + "check": "spi_support_channel", + "spi_prior_mass_share": 0.5, + "absolute_tolerance": 0.0, + "household_weight_kind": "importance", + "minimum_spi_households": 1 + }, + "notes": "Stage-time mass gate over the SPI support-channel receipt." + }, + { + "id": "uk_stage_hmrc_spi_income_spine_identity", + "gate": "stage_health", + "phase": "transferred", + "criticality": "release_blocking", + "evidence_absent_blocks": true, + "parameters": { + "stage": "hmrc_spi_income_spine", + "check": "spi_income_spine", + "spi_prior_mass_share": 0.5, + "absolute_tolerance": 0.0, + "minimum_identity_rows": 1, + "minimum_target_count": 1 + }, + "notes": "Stage-time SPI redraw identity gate over the HMRC SPI income spine receipt." + }, + { + "id": "uk_stage_cgt_incidence_clone_mass", + "gate": "stage_health", + "phase": "transferred", + "criticality": "release_blocking", + "evidence_absent_blocks": true, + "parameters": { + "stage": "cgt_incidence_clone", + "check": "cgt_incidence_mass", + "maximum_relative_mass_imbalance": 0.0 + }, + "notes": "Stage-time conservation gate over the CGT incidence clone receipt." + }, + { + "id": "uk_stage_cgt_band_donors_support", + "gate": "stage_health", + "phase": "transferred", + "criticality": "release_blocking", + "evidence_absent_blocks": true, + "parameters": { + "stage": "cgt_band_donors", + "check": "cgt_band_donor_support", + "support_bounds_resource": "cgt_band_donor_support_bounds.json" + }, + "notes": "Stage-time donor-support gate for cgt_band_donors. The cgt_band_donor_support_bounds.json semantic_fit_note says these intervals describe donor-stage initial support and that terminal support-gate applicability requires reviewer confirmation, so this entry deliberately gates the donor stage rather than uk_support." + }, + { + "id": "uk_stage_hmrc_cgt_gains_spine_summary", + "gate": "stage_health", + "phase": "transferred", + "criticality": "release_blocking", + "evidence_absent_blocks": true, + "parameters": { + "stage": "hmrc_cgt_gains_spine", + "check": "cgt_imputation_summary", + "minimum_band_rows": 1 + }, + "notes": "Stage-time CGT redraw summary gate wired through the existing UKCGTImputationSummary receipt rather than fabricating a terminal support verdict." + }, + { + "id": "uk_stage_salary_sacrifice_realization", + "gate": "stage_health", + "phase": "transferred", + "criticality": "release_blocking", + "evidence_absent_blocks": true, + "parameters": { + "stage": "salary_sacrifice", + "check": "realization_target", + "target": 5400000.0, + "maximum_abs_realization_deviation": 1.0, + "allow_cap_bound": true + }, + "notes": "Stage-time realization gate over salary_sacrifice's own headcount receipt." + }, + { + "id": "uk_stage_student_loans_realization", + "gate": "stage_health", + "phase": "transferred", + "criticality": "release_blocking", + "evidence_absent_blocks": true, + "parameters": { + "stage": "student_loans", + "check": "student_loan_plans", + "stocks": { + "PLAN_2": 8940000.0, + "PLAN_5": 10000.0 + }, + "maximum_abs_realization_deviation": 1.0 + }, + "notes": "Stage-time realization gate over the per-plan student-loan top-up receipts." + }, + { + "id": "uk_stage_age_tail_targets", + "gate": "stage_health", + "phase": "transferred", + "criticality": "release_blocking", + "evidence_absent_blocks": true, + "parameters": { + "stage": "age_tail", + "check": "age_tail_targets", + "maximum_relative_deviation": 1.0 + }, + "notes": "Stage-time achieved-versus-target gate over the age-tail disaggregation receipt." + }, { "id": "uk_release_input_coverage", "gate": "release_input_coverage", @@ -269,14 +502,14 @@ { "id": "uk_brma_enum_domain", "gate": "enum_domain", - "phase": "terminal", + "phase": "assembled", "criticality": "release_blocking", "parameters": { "columns": [ "brma" ] }, - "notes": "The household BRMA assignment must remain inside the PolicyEngine-UK brma enum domain." + "notes": "The household BRMA assignment must remain inside the PolicyEngine-UK brma enum domain. Evaluated at the assembled boundary, right after frs_brma writes the column: enum membership is a property of the values themselves, so checking it there is identical to checking it at the end and refuses fourteen stages earlier. Take-up signal stays terminal by contrast, because its weighted shares are not the same statistic before and after calibration." }, { "id": "uk_student_loan_plan_enum_domain", diff --git a/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json b/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json index f6ae04ef1..92b351673 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json +++ b/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json @@ -565,11 +565,17 @@ "capital_gains" ], "required_mass_change_reason": "Amounts-only capital gains redraw: household weights pass through unchanged and total household mass is conserved.", - "source_manifest": "cgt_source_stages.json", - "source_manifest_sha256": "71104111b4b9f2da00ce49ad5abec54a35d300196032c9742d62e02aa1730774", - "source_vintages": { - "hmrc_surface": "2023-24", - "mapped_build_period": "2024" + "source_manifest": "cgt_source_stages.json", + "source_manifest_sha256": "71104111b4b9f2da00ce49ad5abec54a35d300196032c9742d62e02aa1730774", + "superseded_by": { + "reason": "The FRS spine build executes hmrc_cgt_gains_spine, which applies the same HMRC Table 3 amounts redraw directly in source_stages.json before calibration.", + "source_manifest": "source_stages.json", + "source_manifest_sha256": "84d1c1c85f0172a9a396ef69d445ee33cf1340f62ff73c8653c399af40087bf4", + "stage": "hmrc_cgt_gains_spine" + }, + "source_vintages": { + "hmrc_surface": "2023-24", + "mapped_build_period": "2024" }, "stage": "hmrc_cgt_gains", "status": "required_at_build" @@ -663,7 +669,7 @@ "OTHERINC" ] }, - "reviewed_fence_ids": [ + "reviewed_fence_ids": [ "frs_epb_source_absent", "frs_exps_source_absent", "frs_taxterm_source_absent", @@ -673,9 +679,15 @@ "frs_srp_regular_code5_subset", "full_frs_tei_band_unavailable" ], - "source_manifest": "hmrc_income_source_stages.json", - "source_manifest_sha256": "c0341af7166ae3a85a3c1164e7d9e880c4b4aec122f1a8fa90c73b46c596e1ea", - "source_vintages": { + "source_manifest": "hmrc_income_source_stages.json", + "source_manifest_sha256": "c0341af7166ae3a85a3c1164e7d9e880c4b4aec122f1a8fa90c73b46c596e1ea", + "superseded_by": { + "reason": "The FRS spine build executes hmrc_spi_income_spine, which supersedes the June retained-leaves/hmrc_spi_income pair inside source_stages.json.", + "source_manifest": "source_stages.json", + "source_manifest_sha256": "84d1c1c85f0172a9a396ef69d445ee33cf1340f62ff73c8653c399af40087bf4", + "stage": "hmrc_spi_income_spine" + }, + "source_vintages": { "hmrc_surface": "2023-24", "mapped_build_period": "2024", "period_mapping": "latest_published_tax_year", diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/battery_bindings.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/battery_bindings.py index 5b97ddf2e..48347853e 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/battery_bindings.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/battery_bindings.py @@ -61,6 +61,7 @@ uk_release_input_coverage_gate, ) from microcosm.build.uk_runtime.source_runtime import UK_NONNEGATIVE_OUTPUTS_BY_STAGE +from microcosm.build.uk_runtime.stage_health import uk_stage_health_gate from microcosm.build.uk_runtime.terminal_gates import ( UKZeroWeightStratumDeclaration, _household_weights, @@ -233,6 +234,34 @@ def _evaluate_calibration_reference_coverage( ) +def _evaluate_stage_health( + context: EvidenceContext, parameters: Mapping[str, Any] +) -> GateResult: + stage = str(parameters["stage"]) + evidence_by_stage = context.artifacts["stage_evidence"] + if not isinstance(evidence_by_stage, Mapping): + raise TypeError("stage_evidence must map stage names to receipt payloads.") + receipt = evidence_by_stage.get(stage) + if not isinstance(receipt, Mapping): + raise ValueError(f"stage_evidence has no receipt object for {stage!r}.") + return uk_stage_health_gate( + evidence=receipt, + stage=stage, + check=str(parameters["check"]), + parameters=parameters, + ) + + +def _stage_health_evidence( + context: EvidenceContext, parameters: Mapping[str, Any] +) -> object: + stage = str(parameters["stage"]) + evidence_by_stage = context.artifacts["stage_evidence"] + if not isinstance(evidence_by_stage, Mapping): + raise TypeError("stage_evidence must map stage names to receipt payloads.") + return {stage: evidence_by_stage.get(stage)} + + def _evaluate_nonnegative_columns( context: EvidenceContext, parameters: Mapping[str, Any] ) -> GateResult: @@ -825,6 +854,39 @@ def _ledger_compile_parity_registry( artifact_keys=frozenset({"national_calibration"}), needs_frame=False, ), + "stage_health": UKGateBinding( + name="stage_health", + evaluator=_evaluate_stage_health, + parameter_keys=frozenset( + { + "stage", + "check", + "columns", + "exempt_columns", + "max_clipped_low_rows_by_column", + "max_clipped_high_rows_by_column", + "target", + "maximum_abs_realization_deviation", + "allow_cap_bound", + "stocks", + "maximum_relative_mass_imbalance", + "spi_prior_mass_share", + "absolute_tolerance", + "household_weight_kind", + "minimum_spi_households", + "minimum_identity_rows", + "minimum_target_count", + "minimum_signal_rows", + "structural_zero_columns", + "maximum_relative_deviation", + "support_bounds_resource", + "minimum_band_rows", + } + ), + artifact_keys=frozenset({"stage_evidence"}), + needs_frame=False, + evidence=_stage_health_evidence, + ), "nonnegative_columns": UKGateBinding( name="nonnegative_columns", evaluator=_evaluate_nonnegative_columns, diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/calibration_run.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/calibration_run.py index dc5977453..d3296f80a 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/calibration_run.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/calibration_run.py @@ -94,33 +94,94 @@ class UKCalibrationRunResult: "uk_calibration_reference_coverage", ) +UK_SPINE_GATE_SCOPE = ( + "uk_stage_was_wealth_support", + "uk_stage_lcfs_consumption_support", + "uk_stage_etb_vat_support", + "uk_stage_etb_services_support", + "uk_stage_frs_hmrc_spine_leaves_signal", + "uk_stage_spi_support_channel_mass", + "uk_stage_hmrc_spi_income_spine_identity", + "uk_stage_cgt_incidence_clone_mass", + "uk_stage_cgt_band_donors_support", + "uk_stage_hmrc_cgt_gains_spine_summary", + "uk_stage_salary_sacrifice_realization", + "uk_stage_student_loans_realization", + "uk_stage_age_tail_targets", + # Weight-independent, and its column exists from frs_brma onward, so the + # spine checks it at the assembled boundary instead of the release end. + "uk_brma_enum_domain", +) + +UK_NATIONAL_GATE_SCOPE = ( + "uk_release_input_coverage_manifest_current", + "uk_release_family_build_stages", + "uk_ledger_compile_parity_production_2023", + "uk_ledger_compile_parity_incumbent_2025", + "uk_release_input_coverage", + "uk_degenerate_release_surface", + "uk_nonnegative_columns", + "uk_support", + "uk_aggregate_admin", + "uk_export_surface", + "uk_take_up_signal", + "uk_student_loan_plan_enum_domain", + "uk_target_surface", + "uk_input_mass_parity", + "uk_qrf_tail_concentration", + "uk_weights_audit", +) + +_SWAP_ACCEPTANCE_GATE_IDS = frozenset( + {"uk_export_surface", "uk_target_surface"} +) + +#: Gate ids two batteries both own, on purpose. A release certification unions +#: the scoped reports, so a gate appearing twice has to be a declared duplicate +#: the union can reconcile — never an accident that silently double-counts. +#: +#: `uk_aggregate_admin` measures the same admin anchors on two different frames: +#: the seam checks them on the frame it just calibrated, the national terminal +#: battery re-checks them on the release frame. Both are real checks, so both +#: keep the gate. +UK_SHARED_GATE_IDS = frozenset({"uk_aggregate_admin"}) + def _scope_exclusions() -> dict[str, str]: full = {entry.id for entry in load_country_spec("uk").gates.gates} - excluded = full - set(UK_CALIBRATION_GATE_SCOPE) + spine = set(UK_SPINE_GATE_SCOPE) + national = set(UK_NATIONAL_GATE_SCOPE) + calibration = set(UK_CALIBRATION_GATE_SCOPE) + # Closed-world means both halves: every gate owned by someone (below), and + # no gate owned twice without saying so. Coverage alone would let an + # accidental overlap through, and the certification union is exactly where + # that would be paid for. + for left_name, left, right_name, right in ( + ("calibration", calibration, "spine", spine), + ("calibration", calibration, "national", national), + ("spine", spine, "national", national), + ): + undeclared = (left & right) - UK_SHARED_GATE_IDS + if undeclared: + raise RuntimeError( + f"UK gate scopes {left_name} and {right_name} both claim " + f"{sorted(undeclared)} without declaring them in " + "UK_SHARED_GATE_IDS." + ) + classified = calibration | spine | national rationales: dict[str, str] = {} - for gate_id in sorted(excluded): - if "parity" in gate_id or gate_id in {"uk_export_surface", "uk_target_surface"}: + for gate_id in sorted(full - set(UK_CALIBRATION_GATE_SCOPE)): + if gate_id in spine: + reason = "spine-construction gate; owned by the spine build's scoped battery." + elif gate_id in national: + reason = "national build gate; owned by the national preflight/terminal battery." + elif "parity" in gate_id or gate_id in _SWAP_ACCEPTANCE_GATE_IDS: reason = "swap-acceptance evidence; produced by the swap lane, not the calibration seam." - elif gate_id in { - "uk_release_input_coverage_manifest_current", - "uk_release_family_build_stages", - "uk_release_input_coverage", - "uk_nonnegative_columns", - "uk_support", - "uk_take_up_signal", - "uk_brma_enum_domain", - "uk_degenerate_release_surface", - "uk_input_mass_parity", - "uk_qrf_tail_concentration", - "uk_weights_audit", - }: - reason = "spine-construction gate; owned by the spine build's own battery." else: reason = "outside the calibration seam's reviewed gate scope." rationales[gate_id] = reason - if set(UK_CALIBRATION_GATE_SCOPE) | set(rationales) != full: - raise RuntimeError("UK calibration gate scope does not classify every gate id.") + if classified | set(rationales) != full: + raise RuntimeError("UK gate scope does not classify every gate id.") return rationales @@ -481,6 +542,7 @@ def _load_bound_spine_sidecar(path: Path, frame: Frame) -> dict[str, object]: if not isinstance(sidecar, dict): raise ValueError(f"input H5 build sidecar must be a JSON object: {path}") _assert_spine_sidecar_binds_frame(sidecar, frame) + _assert_spine_gate_report_passed(_spine_gate_report_path(path), sidecar) return sidecar @@ -513,10 +575,72 @@ def _assert_spine_sidecar_binds_frame( ) +def _spine_gate_report_path(sidecar_path: Path) -> Path: + if sidecar_path.name.endswith(".build.json"): + stem = sidecar_path.name[: -len(".build.json")] + return sidecar_path.with_name(f"{stem}.spine_gates.json") + return sidecar_path.with_suffix(".spine_gates.json") + + +def _assert_spine_gate_report_passed( + report_path: Path, + sidecar: Mapping[str, object], +) -> None: + bypass = sidecar.get("spine_gate_bypass") + if bypass is not None: + if not isinstance(bypass, Mapping) or bypass.get("reviewed") is not True: + raise ValueError("spine_gate_bypass must be a reviewed bypass object.") + if not str(bypass.get("reason", "")).strip(): + raise ValueError("spine_gate_bypass needs a non-empty reason.") + return + if not report_path.is_file(): + raise ValueError(f"input H5 spine gate report absent: {report_path}") + try: + report = json.loads(report_path.read_text(encoding="utf-8")) + except json.JSONDecodeError as exc: + raise ValueError( + f"input H5 spine gate report is invalid JSON: {report_path}" + ) from exc + if not isinstance(report, Mapping): + raise ValueError( + f"input H5 spine gate report must be a JSON object: {report_path}" + ) + if report.get("blocked_at_phase") is not None: + raise ValueError( + "input H5 spine gate report blocked_at_phase must be null; " + f"got {report.get('blocked_at_phase')!r}." + ) + gates = report.get("gates") + if not isinstance(gates, Mapping): + raise ValueError("input H5 spine gate report is missing gates.") + failing = sorted( + f"{gate_id}:{payload.get('status')}" + for gate_id, payload in gates.items() + if isinstance(payload, Mapping) + and payload.get("criticality") == "release_blocking" + and payload.get("status") != "passed" + ) + if failing: + raise ValueError( + "input H5 spine gate report has non-passing release-blocking " + f"entries: {failing}." + ) + + def _spine_provenance_from_sidecar( path: Path, sidecar: Mapping[str, object], ) -> dict[str, object]: + report_path = _spine_gate_report_path(path) + if report_path.is_file(): + spine_gate_report: Mapping[str, object] = { + "path": str(report_path), + "sha256": _sha256_file(report_path), + } + elif isinstance(sidecar.get("spine_gate_bypass"), Mapping): + spine_gate_report = {"bypass": dict(sidecar["spine_gate_bypass"])} + else: + spine_gate_report = {} return { "sidecar": { "path": str(path), @@ -524,6 +648,7 @@ def _spine_provenance_from_sidecar( "schema_version": sidecar.get("schema_version"), "pipeline": sidecar.get("pipeline"), }, + "spine_gate_report": dict(spine_gate_report), "stages": list(sidecar.get("stages", ())), "stage_records": list(sidecar.get("stage_records", ())), "stage_evidence": dict(sidecar.get("stage_evidence", {})), @@ -539,15 +664,37 @@ def _spine_provenance_from_sidecar( def _calibration_gate_manifest() -> GatesManifest: - source = load_country_spec("uk").gates - entries = tuple( - entry for entry in source.gates if entry.id in UK_CALIBRATION_GATE_SCOPE + return _scoped_gate_manifest( + UK_CALIBRATION_GATE_SCOPE, + phases=("terminal",), + policy_suffix="calibration_seam_scope", + ) + + +def _spine_gate_manifest() -> GatesManifest: + return _scoped_gate_manifest( + UK_SPINE_GATE_SCOPE, + phases=("assembled", "transferred"), + policy_suffix="spine_build_scope", ) + + +def _scoped_gate_manifest( + scope: tuple[str, ...], + *, + phases: tuple[str, ...], + policy_suffix: str, +) -> GatesManifest: + source = load_country_spec("uk").gates + entries = tuple(entry for entry in source.gates if entry.id in scope) + missing = sorted(set(scope) - {entry.id for entry in entries}) + if missing: + raise RuntimeError(f"UK gate scope names undeclared gate id(s): {missing}.") return GatesManifest( country=source.country, version=source.version, - policy=f"{source.policy}; calibration_seam_scope", - phases=("terminal",), + policy=f"{source.policy}; {policy_suffix}", + phases=phases, gates=entries, ) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_imputation.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_imputation.py index 219a63427..b369ce508 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_imputation.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_imputation.py @@ -56,7 +56,7 @@ from __future__ import annotations -from dataclasses import dataclass +from dataclasses import dataclass, field from pathlib import Path import numpy as np @@ -441,6 +441,15 @@ class UKCGTImputationSummary: published_taxpayer_mass: float remainder_mass: float + def evidence(self) -> dict[str, object]: + return { + "stage": UK_CGT_IMPUTATION_STAGE_NAME, + "rows": self.rows.to_dict(orient="records"), + "taxpayer_mass": self.taxpayer_mass, + "published_taxpayer_mass": self.published_taxpayer_mass, + "remainder_mass": self.remainder_mass, + } + def impute_uk_capital_gains( frame: Frame, @@ -668,10 +677,39 @@ def uk_cgt_spine_stage_transform( """ _assert_cgt_spine_stage_parameters(stage) - return uk_capital_gains_imputation_stage( - ods_path, - mass_change_reason=UK_CGT_SPINE_MASS_CONSERVATION_REASON, - ).transform + return UKCGTSpineStageTransform(stage=stage, ods_path=Path(ods_path)) + + +@dataclass(frozen=True) +class UKCGTSpineStageTransform: + """Source-plan CGT amounts redraw with a stage-time summary receipt.""" + + stage: SourceStageSpec + ods_path: Path + last_result: UKCGTImputationSummary | None = field(default=None, init=False) + + def __call__(self, frame: Frame) -> Frame: + _assert_cgt_spine_stage_parameters(self.stage) + distribution = materialize_hmrc_capital_gains_joint_distribution( + self.ods_path, + tax_year=HMRC_CGT_SOURCE_VINTAGE, + ) + parameters = uk_cgt_policy_parameters(uk_time_period(frame)) + result = impute_uk_capital_gains( + frame, + distribution, + parameters, + seed=UK_CGT_IMPUTATION_SEED, + mass_change_reason=UK_CGT_SPINE_MASS_CONSERVATION_REASON, + ) + summary = summarize_uk_cgt_imputation(frame, result, distribution, parameters) + object.__setattr__(self, "last_result", summary) + return result + + def checkpoint_metadata(self) -> dict[str, object]: + if self.last_result is None: + raise RuntimeError("checkpoint metadata requires a completed stage run.") + return {"evidence": self.last_result.evidence()} def _assert_cgt_spine_stage_parameters(stage: SourceStageSpec) -> None: diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_structure.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_structure.py index 091cf65f2..4da6df3b0 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_structure.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_structure.py @@ -350,6 +350,7 @@ def stack_cgt_band_donors( ) evidence_band_index = band_index.copy() evidence_donor_weights = donor_weights.copy() + evidence_gains = means[band_index].copy() donor_household["_donor_weight"] = donor_weights donor_household = donor_household.drop(columns=["_band_position"]).sort_values( "household_id", kind="stable" @@ -396,6 +397,7 @@ def stack_cgt_band_donors( band_rows: list[Mapping[str, object]] = [] for index, band in enumerate(bands): mask = evidence_band_index == index + realized = evidence_gains[mask] band_rows.append( { "lower_limit": band["lower_limit"], @@ -403,6 +405,8 @@ def stack_cgt_band_donors( "donor_weight": float(evidence_donor_weights[mask][0]), "weighted_taxpayers": float(evidence_donor_weights[mask].sum()), "mean_gain": band["mean_gain"], + "realized_min_gain": float(realized.min()), + "realized_max_gain": float(realized.max()), } ) return UKCGTBandDonorResult( diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/national_build.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/national_build.py index 0c479d29c..fcf958e78 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/national_build.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/national_build.py @@ -27,7 +27,7 @@ from typing import Any import microcosm.build.uk_runtime.national_frame as _national_frame -from microcosm.build.country_spec import load_country_spec +from microcosm.build.country_spec import GatesManifest, load_country_spec from microcosm.build.frame_sampling import ( validate_sample_fraction, validate_sample_seed, @@ -44,6 +44,7 @@ from microcosm.build.plan import Stage as PlanStage from microcosm.build.plan import StagePlan, StageRecord from microcosm.build.uk_runtime.battery_bindings import UK_GATE_REGISTRY +from microcosm.build.uk_runtime.calibration_run import UK_NATIONAL_GATE_SCOPE from microcosm.build.uk_runtime.national_frame import ( UK_HOUSEHOLD_WEIGHT_KIND_ATTR as UK_HOUSEHOLD_WEIGHT_KIND_ATTR, ) @@ -331,7 +332,7 @@ def build_uk_national_dataset( # fence too: the unlinks come strictly last. evaluation_date = exclusion_evaluation_date(now) battery = GateBatteryRun( - load_country_spec("uk").gates, + _national_gate_manifest(), release_id=release_id, report_path=diagnostic_path, release_candidate=release_candidate, @@ -561,6 +562,21 @@ def _run_stages_checkpointed( return frame, tuple(records) +def _national_gate_manifest() -> GatesManifest: + source = load_country_spec("uk").gates + entries = tuple(entry for entry in source.gates if entry.id in UK_NATIONAL_GATE_SCOPE) + missing = sorted(set(UK_NATIONAL_GATE_SCOPE) - {entry.id for entry in entries}) + if missing: + raise RuntimeError(f"UK national gate scope names undeclared gate id(s): {missing}.") + return GatesManifest( + country=source.country, + version=source.version, + policy=f"{source.policy}; national_build_scope", + phases=("preflight", "terminal"), + gates=entries, + ) + + def _coerce_stage_plan( stages: Sequence[UKNationalStage | PlanStage] | StagePlan, ) -> StagePlan: diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/release_input_coverage.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/release_input_coverage.py index cbbcd41fc..4ebb3a528 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/release_input_coverage.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/release_input_coverage.py @@ -193,6 +193,21 @@ def required_build_stages(self) -> frozenset[str]: if family.get("status") == _REQUIRED_AT_BUILD_STATUS ) + @property + def required_build_stage_options(self) -> Mapping[str, tuple[str, ...]]: + """Per-family executable stage alternatives declared by the manifest.""" + + options: dict[str, tuple[str, ...]] = {} + for name, family in self.family_coverage.items(): + if family.get("status") != _REQUIRED_AT_BUILD_STATUS: + continue + stages = [str(family["stage"])] + superseded_by = family.get("superseded_by") + if isinstance(superseded_by, Mapping): + stages.append(str(superseded_by["stage"])) + options[str(name)] = tuple(dict.fromkeys(stages)) + return options + def _resource_text(resource: str) -> str: candidate = Path(resource) @@ -289,6 +304,48 @@ def _parse_family_coverage( f"{resource}: family {name!r} needs a lowercase SHA-256 " "for source_manifest_sha256." ) + superseded_by = raw_family.get("superseded_by") + parsed_superseded_by: dict[str, Any] | None = None + if superseded_by is not None: + if not isinstance(superseded_by, Mapping): + raise ValueError( + f"{resource}: family {name!r} superseded_by must be an object." + ) + superseding_stage = str(superseded_by.get("stage", "")).strip() + superseding_manifest = str( + superseded_by.get("source_manifest", "") + ).strip() + superseding_sha = str( + superseded_by.get("source_manifest_sha256", "") + ).strip() + supersession_reason = str(superseded_by.get("reason", "")).strip() + if not superseding_stage: + raise ValueError( + f"{resource}: family {name!r} superseded_by needs a stage." + ) + if not superseding_manifest: + raise ValueError( + f"{resource}: family {name!r} superseded_by needs a " + "source_manifest." + ) + if len(superseding_sha) != 64 or any( + character not in "0123456789abcdef" for character in superseding_sha + ): + raise ValueError( + f"{resource}: family {name!r} superseded_by needs a " + "lowercase SHA-256 for source_manifest_sha256." + ) + if not supersession_reason: + raise ValueError( + f"{resource}: family {name!r} superseded_by needs a reason." + ) + parsed_superseded_by = { + **dict(superseded_by), + "stage": superseding_stage, + "source_manifest": superseding_manifest, + "source_manifest_sha256": superseding_sha, + "reason": supersession_reason, + } try: base_candidate_tier = validate_uk_release_tier( raw_family.get("base_candidate_tier") @@ -381,6 +438,11 @@ def _parse_family_coverage( "stage": stage, "source_manifest": source_manifest, "source_manifest_sha256": source_manifest_sha256, + **( + {"superseded_by": parsed_superseded_by} + if parsed_superseded_by is not None + else {} + ), "base_candidate_tier": base_candidate_tier, "output_weight_kind": output_weight_kind, "required_mass_change_reason": required_mass_change_reason, @@ -1374,10 +1436,15 @@ def assert_uk_release_input_coverage_build_stages( manifest = manifest or load_uk_release_input_coverage_manifest() actual = {str(name) for name in stage_names} - missing = sorted(manifest.required_build_stages - actual) + missing = sorted( + family + for family, options in manifest.required_build_stage_options.items() + if actual.isdisjoint(options) + ) if missing: raise ValueError( - "UK national build omits required release family stage(s) " + "UK national build omits required release family stage(s) for " + "family/families " f"{missing}; family_coverage status='required_at_build' is an " "executable contract, not documentation." ) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/stage_health.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/stage_health.py new file mode 100644 index 000000000..3a6c2378a --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/stage_health.py @@ -0,0 +1,418 @@ +"""Stage-time health gates for the UK FRS spine build.""" + +from __future__ import annotations + +import json +from collections.abc import Mapping +from importlib.resources import files + +import numpy as np + +from microcosm.build.gates import GateResult + +_UK_PACKAGE = "microcosm.build.uk" + + +def uk_stage_health_gate( + *, + evidence: Mapping[str, object], + stage: str, + check: str, + parameters: Mapping[str, object], +) -> GateResult: + """Evaluate one spine-stage receipt against spec-declared thresholds.""" + + if evidence.get("stage") not in {stage, _legacy_receipt_stage(stage)}: + return GateResult( + name="stage_health", + passed=False, + failures=( + f"{stage}: receipt stage {evidence.get('stage')!r} does not match.", + ), + details={"stage": stage, "check": check}, + ) + if check == "support_clip": + return _support_clip_gate(stage, evidence, parameters) + if check == "realization_target": + return _realization_target_gate(stage, evidence, parameters) + if check == "student_loan_plans": + return _student_loan_plans_gate(stage, evidence, parameters) + if check == "cgt_incidence_mass": + return _cgt_incidence_mass_gate(stage, evidence, parameters) + if check == "spi_support_channel": + return _spi_support_channel_gate(stage, evidence, parameters) + if check == "spi_income_spine": + return _spi_income_spine_gate(stage, evidence, parameters) + if check == "source_signal": + return _source_signal_gate(stage, evidence, parameters) + if check == "age_tail_targets": + return _age_tail_targets_gate(stage, evidence, parameters) + if check == "cgt_band_donor_support": + return _cgt_band_donor_support_gate(stage, evidence, parameters) + if check == "cgt_imputation_summary": + return _cgt_imputation_summary_gate(stage, evidence, parameters) + return GateResult( + name="stage_health", + passed=False, + failures=(f"{stage}: unknown stage-health check {check!r}.",), + details={"stage": stage, "check": check}, + ) + + +def _legacy_receipt_stage(stage: str) -> str: + if stage == "age_tail": + return "uk_age_tail_disaggregation" + return stage + + +def _pass(stage: str, check: str, details: Mapping[str, object]) -> GateResult: + return GateResult( + name="stage_health", + passed=True, + details={"stage": stage, "check": check, **dict(details)}, + ) + + +def _fail( + stage: str, + check: str, + failures: list[str], + details: Mapping[str, object], +) -> GateResult: + return GateResult( + name="stage_health", + passed=False, + failures=tuple(failures), + details={"stage": stage, "check": check, **dict(details)}, + ) + + +def _finite_number(value: object, *, label: str) -> float: + if not isinstance(value, int | float) or not np.isfinite(float(value)): + raise ValueError(f"{label} must be finite, got {value!r}.") + return float(value) + + +def _mapping(value: object, *, label: str) -> Mapping[str, object]: + if not isinstance(value, Mapping): + raise ValueError(f"{label} must be an object.") + return value + + +def _support_clip_gate( + stage: str, + evidence: Mapping[str, object], + parameters: Mapping[str, object], +) -> GateResult: + check = "support_clip" + clip = _mapping(evidence.get("support_clip"), label=f"{stage}.support_clip") + columns = _mapping(clip.get("columns"), label=f"{stage}.support_clip.columns") + expected_columns = tuple(str(column) for column in parameters["columns"]) + exempt_columns = {str(column) for column in parameters.get("exempt_columns", ())} + max_low = _mapping( + parameters.get("max_clipped_low_rows_by_column", {}), + label=f"{stage}.max_clipped_low_rows_by_column", + ) + max_high = _mapping( + parameters.get("max_clipped_high_rows_by_column", {}), + label=f"{stage}.max_clipped_high_rows_by_column", + ) + failures: list[str] = [] + for column in expected_columns: + receipt = columns.get(column) + if column in exempt_columns: + if isinstance(receipt, Mapping) and receipt.get("exempt") is True: + continue + failures.append(f"{stage}: exempt column {column!r} is not marked exempt.") + continue + if not isinstance(receipt, Mapping): + failures.append(f"{stage}: missing support-clip receipt for {column!r}.") + continue + for key in ("donor_min", "donor_max", "rows_considered"): + if key not in receipt: + failures.append(f"{stage}: {column!r} receipt is missing {key}.") + low_rows = int(receipt.get("clipped_low_rows", -1)) + high_rows = int(receipt.get("clipped_high_rows", -1)) + allowed_low = max_low.get(column) + allowed_high = max_high.get(column) + if allowed_low is not None and low_rows > int(allowed_low): + failures.append( + f"{stage}: {column!r} clipped_low_rows {low_rows} exceeds {allowed_low}." + ) + if allowed_high is not None and high_rows > int(allowed_high): + failures.append( + f"{stage}: {column!r} clipped_high_rows {high_rows} exceeds {allowed_high}." + ) + if "donor_min" in receipt and "donor_max" in receipt: + lower = _finite_number(receipt["donor_min"], label=f"{column}.donor_min") + upper = _finite_number(receipt["donor_max"], label=f"{column}.donor_max") + if lower > upper: + failures.append(f"{stage}: {column!r} donor_min exceeds donor_max.") + details = { + "columns_checked": len(expected_columns) - len(exempt_columns), + "exempt_columns": sorted(exempt_columns), + } + return _fail(stage, check, failures, details) if failures else _pass(stage, check, details) + + +def _realization_target_gate( + stage: str, + evidence: Mapping[str, object], + parameters: Mapping[str, object], +) -> GateResult: + check = "realization_target" + receipt = _mapping( + evidence.get("headcount_receipt"), label=f"{stage}.headcount_receipt" + ) + max_deviation = _finite_number( + parameters["maximum_abs_realization_deviation"], + label=f"{stage}.maximum_abs_realization_deviation", + ) + target = _finite_number(parameters["target"], label=f"{stage}.target") + failures: list[str] = [] + observed_target = _finite_number(receipt.get("target"), label=f"{stage}.target") + if observed_target != target: + failures.append(f"{stage}: target {observed_target} != declared {target}.") + deviation = abs( + _finite_number( + receipt.get("realization_deviation"), + label=f"{stage}.realization_deviation", + ) + ) + if deviation > max_deviation: + failures.append( + f"{stage}: realization_deviation {deviation} exceeds {max_deviation}." + ) + if bool(receipt.get("cap_bound")) and not bool(parameters.get("allow_cap_bound")): + failures.append(f"{stage}: cap_bound is true but not allowed.") + details = {"target": target, "abs_realization_deviation": deviation} + return _fail(stage, check, failures, details) if failures else _pass(stage, check, details) + + +def _student_loan_plans_gate( + stage: str, + evidence: Mapping[str, object], + parameters: Mapping[str, object], +) -> GateResult: + check = "student_loan_plans" + plans = _mapping(evidence.get("plans"), label=f"{stage}.plans") + declared_stocks = _mapping(parameters["stocks"], label=f"{stage}.stocks") + max_deviation = _finite_number( + parameters["maximum_abs_realization_deviation"], + label=f"{stage}.maximum_abs_realization_deviation", + ) + failures: list[str] = [] + worst = 0.0 + for plan, declared_stock in declared_stocks.items(): + receipt = plans.get(str(plan)) + if not isinstance(receipt, Mapping): + failures.append(f"{stage}: missing receipt for {plan}.") + continue + stock = _finite_number(receipt.get("stock"), label=f"{stage}.{plan}.stock") + expected = _finite_number(declared_stock, label=f"{stage}.{plan}.declared_stock") + if stock != expected: + failures.append(f"{stage}: {plan} stock {stock} != declared {expected}.") + final = _finite_number( + receipt.get("final_england_count"), + label=f"{stage}.{plan}.final_england_count", + ) + deviation = abs( + _finite_number( + receipt.get("realization_deviation"), + label=f"{stage}.{plan}.realization_deviation", + ) + ) + worst = max(worst, deviation) + if final < 0.0: + failures.append(f"{stage}: {plan} final_england_count is negative.") + if deviation > max_deviation: + failures.append( + f"{stage}: {plan} realization_deviation {deviation} exceeds {max_deviation}." + ) + details = {"plans_checked": len(declared_stocks), "worst_abs_deviation": worst} + return _fail(stage, check, failures, details) if failures else _pass(stage, check, details) + + +def _cgt_incidence_mass_gate( + stage: str, + evidence: Mapping[str, object], + parameters: Mapping[str, object], +) -> GateResult: + check = "cgt_incidence_mass" + mass = _mapping(evidence.get("mass_by_clone_flag"), label=f"{stage}.mass_by_clone_flag") + original = _finite_number(mass.get("false"), label=f"{stage}.mass.false") + clone = _finite_number(mass.get("true"), label=f"{stage}.mass.true") + tolerance = _finite_number( + parameters["maximum_relative_mass_imbalance"], + label=f"{stage}.maximum_relative_mass_imbalance", + ) + denominator = max(abs(original), 1.0) + imbalance = abs(clone - original) / denominator + failures = [] + if original <= 0.0 or clone <= 0.0: + failures.append(f"{stage}: clone and original mass must both be positive.") + if imbalance > tolerance: + failures.append(f"{stage}: clone/original mass imbalance {imbalance} exceeds {tolerance}.") + details = {"original_mass": original, "clone_mass": clone, "relative_imbalance": imbalance} + return _fail(stage, check, failures, details) if failures else _pass(stage, check, details) + + +def _spi_support_channel_gate( + stage: str, + evidence: Mapping[str, object], + parameters: Mapping[str, object], +) -> GateResult: + check = "spi_support_channel" + expected_share = _finite_number( + parameters["spi_prior_mass_share"], label=f"{stage}.spi_prior_mass_share" + ) + share = _finite_number( + evidence.get("spi_prior_mass_share"), label=f"{stage}.spi_prior_mass_share" + ) + failures = [] + if abs(share - expected_share) > _finite_number( + parameters.get("absolute_tolerance", 0.0), label=f"{stage}.absolute_tolerance" + ): + failures.append(f"{stage}: spi_prior_mass_share {share} != declared {expected_share}.") + if evidence.get("household_weight_kind") != parameters.get("household_weight_kind"): + failures.append(f"{stage}: household_weight_kind drifted.") + if int(evidence.get("spi_households", 0)) < int(parameters["minimum_spi_households"]): + failures.append(f"{stage}: spi_households below declared minimum.") + details = {"spi_prior_mass_share": share, "spi_households": evidence.get("spi_households")} + return _fail(stage, check, failures, details) if failures else _pass(stage, check, details) + + +def _spi_income_spine_gate( + stage: str, + evidence: Mapping[str, object], + parameters: Mapping[str, object], +) -> GateResult: + check = "spi_income_spine" + identity = _mapping( + evidence.get("post_draw_identity"), label=f"{stage}.post_draw_identity" + ) + prior = _mapping(evidence.get("spi_prior"), label=f"{stage}.spi_prior") + targets = _mapping(evidence.get("targets"), label=f"{stage}.targets") + failures: list[str] = [] + if identity.get("exact") is not True: + failures.append(f"{stage}: post_draw_identity.exact is not true.") + if int(identity.get("rows_checked", 0)) < int(parameters["minimum_identity_rows"]): + failures.append(f"{stage}: post_draw_identity checked too few rows.") + expected_share = _finite_number( + parameters["spi_prior_mass_share"], label=f"{stage}.spi_prior_mass_share" + ) + share = _finite_number(prior.get("mass_share"), label=f"{stage}.spi_prior.mass_share") + if abs(share - expected_share) > _finite_number( + parameters.get("absolute_tolerance", 0.0), label=f"{stage}.absolute_tolerance" + ): + failures.append(f"{stage}: spi prior mass share {share} != declared {expected_share}.") + if int(targets.get("count", 0)) < int(parameters["minimum_target_count"]): + failures.append(f"{stage}: target count below declared minimum.") + details = {"identity_rows": identity.get("rows_checked"), "target_count": targets.get("count")} + return _fail(stage, check, failures, details) if failures else _pass(stage, check, details) + + +def _source_signal_gate( + stage: str, + evidence: Mapping[str, object], + parameters: Mapping[str, object], +) -> GateResult: + check = "source_signal" + rows = _mapping(evidence.get("source_signal_rows"), label=f"{stage}.source_signal_rows") + allowed_zero = {str(column) for column in parameters.get("structural_zero_columns", ())} + reported_zero = {str(column) for column in evidence.get("structural_zero_columns", ())} + minimum = int(parameters["minimum_signal_rows"]) + failures: list[str] = [] + if reported_zero - allowed_zero: + failures.append(f"{stage}: unreviewed structural zero columns {sorted(reported_zero - allowed_zero)}.") + for column, value in rows.items(): + if str(column) in allowed_zero: + continue + if int(value) < minimum: + failures.append(f"{stage}: {column} has {value} source-signal row(s), below {minimum}.") + details = {"columns_checked": len(rows), "structural_zero_columns": sorted(reported_zero)} + return _fail(stage, check, failures, details) if failures else _pass(stage, check, details) + + +def _age_tail_targets_gate( + stage: str, + evidence: Mapping[str, object], + parameters: Mapping[str, object], +) -> GateResult: + check = "age_tail_targets" + achieved = _mapping(evidence.get("achieved_weighted"), label=f"{stage}.achieved_weighted") + targets = _mapping(evidence.get("band_populations"), label=f"{stage}.band_populations") + max_relative = _finite_number( + parameters["maximum_relative_deviation"], + label=f"{stage}.maximum_relative_deviation", + ) + failures: list[str] = [] + worst = 0.0 + for key, target_value in targets.items(): + if ":" not in str(key): + continue + gender, band = str(key).split(":", 1) + gender_rows = achieved.get(gender) + if not isinstance(gender_rows, Mapping) or band not in gender_rows: + failures.append(f"{stage}: missing achieved band {key}.") + continue + target = _finite_number(target_value, label=f"{stage}.{key}.target") + value = _finite_number(gender_rows[band], label=f"{stage}.{key}.achieved") + relative = abs(value - target) / max(abs(target), 1.0) + worst = max(worst, relative) + if relative > max_relative: + failures.append(f"{stage}: {key} relative deviation {relative} exceeds {max_relative}.") + details = {"bands_checked": len(targets), "worst_relative_deviation": worst} + return _fail(stage, check, failures, details) if failures else _pass(stage, check, details) + + +def _cgt_band_donor_support_gate( + stage: str, + evidence: Mapping[str, object], + parameters: Mapping[str, object], +) -> GateResult: + check = "cgt_band_donor_support" + resource_name = str(parameters["support_bounds_resource"]) + resource = json.loads(files(_UK_PACKAGE).joinpath(resource_name).read_text()) + bounds = _mapping(resource.get("bounds"), label=f"{resource_name}.bounds") + lower, upper = bounds["capital_gains"] + global_lower = _finite_number(lower, label="capital_gains.lower") + bands = evidence.get("bands") + if not isinstance(bands, list | tuple): + raise ValueError(f"{stage}.bands must be a list.") + failures: list[str] = [] + for row in bands: + if not isinstance(row, Mapping): + failures.append(f"{stage}: band row is not an object.") + continue + realized_min = _finite_number(row.get("realized_min_gain"), label=f"{stage}.realized_min_gain") + realized_max = _finite_number(row.get("realized_max_gain"), label=f"{stage}.realized_max_gain") + lower_limit = _finite_number(row.get("lower_limit"), label=f"{stage}.lower_limit") + band_floor = max(global_lower, lower_limit) + if realized_min < band_floor: + failures.append(f"{stage}: realized gain {realized_min} falls below {band_floor}.") + if upper is not None and realized_max >= _finite_number(upper, label="capital_gains.upper"): + failures.append(f"{stage}: realized gain {realized_max} exceeds open upper bound.") + details = {"bands_checked": len(bands), "minimum_lower_limit": global_lower} + return _fail(stage, check, failures, details) if failures else _pass(stage, check, details) + + +def _cgt_imputation_summary_gate( + stage: str, + evidence: Mapping[str, object], + parameters: Mapping[str, object], +) -> GateResult: + check = "cgt_imputation_summary" + rows = evidence.get("rows") + if not isinstance(rows, list | tuple): + raise ValueError(f"{stage}.rows must be a list.") + failures: list[str] = [] + min_rows = int(parameters["minimum_band_rows"]) + if len(rows) < min_rows: + failures.append(f"{stage}: summary row count {len(rows)} below {min_rows}.") + for key in ("taxpayer_mass", "published_taxpayer_mass", "remainder_mass"): + value = _finite_number(evidence.get(key), label=f"{stage}.{key}") + if value < 0.0: + failures.append(f"{stage}: {key} is negative.") + details = {"band_rows": len(rows), "taxpayer_mass": evidence.get("taxpayer_mass")} + return _fail(stage, check, failures, details) if failures else _pass(stage, check, details) diff --git a/packages/microcosm-build/tests/test_country_spec.py b/packages/microcosm-build/tests/test_country_spec.py index ed0f12d19..f3691a1a5 100644 --- a/packages/microcosm-build/tests/test_country_spec.py +++ b/packages/microcosm-build/tests/test_country_spec.py @@ -619,9 +619,14 @@ class TestUKGatesManifest: def manifest(self): return load_country_spec("uk").gates - def test_declares_the_two_uk_phases_in_order(self, manifest) -> None: + def test_declares_the_uk_phases_in_order(self, manifest) -> None: assert manifest is not None - assert manifest.phases == ("preflight", "terminal") + assert manifest.phases == ( + "preflight", + "assembled", + "transferred", + "terminal", + ) def test_declares_the_full_june_battery(self, manifest) -> None: assert [gate.id for gate in manifest.gates] == [ @@ -629,6 +634,19 @@ def test_declares_the_full_june_battery(self, manifest) -> None: "uk_release_family_build_stages", "uk_ledger_compile_parity_production_2023", "uk_ledger_compile_parity_incumbent_2025", + "uk_stage_was_wealth_support", + "uk_stage_lcfs_consumption_support", + "uk_stage_etb_vat_support", + "uk_stage_etb_services_support", + "uk_stage_frs_hmrc_spine_leaves_signal", + "uk_stage_spi_support_channel_mass", + "uk_stage_hmrc_spi_income_spine_identity", + "uk_stage_cgt_incidence_clone_mass", + "uk_stage_cgt_band_donors_support", + "uk_stage_hmrc_cgt_gains_spine_summary", + "uk_stage_salary_sacrifice_realization", + "uk_stage_student_loans_realization", + "uk_stage_age_tail_targets", "uk_release_input_coverage", "uk_degenerate_release_surface", "uk_zero_weight_strata", @@ -664,13 +682,29 @@ def test_ledger_compile_parity_gates_pin_their_fixture_periods( params["uk_ledger_compile_parity_incumbent_2025"]["target_period"] == 2025 ) - def test_only_the_weights_audit_blocks_on_absent_evidence(self, manifest) -> None: + def test_strict_absent_evidence_entries_are_declared(self, manifest) -> None: # "An absent audit is not a passing audit" — the retired schema-3 # path blocked every posture on a missing fit-weight audit, and the # battery keeps that strictness via the entry flag (#654, #691 - # review). No other entry opts out of the dev-posture leniency. + # review). Stage-health gates also block on absent receipts because + # the spine build cannot silently skip a checkpoint's own evidence. flagged = [g.id for g in manifest.gates if g.evidence_absent_blocks] - assert flagged == ["uk_weights_audit"] + assert flagged == [ + "uk_stage_was_wealth_support", + "uk_stage_lcfs_consumption_support", + "uk_stage_etb_vat_support", + "uk_stage_etb_services_support", + "uk_stage_frs_hmrc_spine_leaves_signal", + "uk_stage_spi_support_channel_mass", + "uk_stage_hmrc_spi_income_spine_identity", + "uk_stage_cgt_incidence_clone_mass", + "uk_stage_cgt_band_donors_support", + "uk_stage_hmrc_cgt_gains_spine_summary", + "uk_stage_salary_sacrifice_realization", + "uk_stage_student_loans_realization", + "uk_stage_age_tail_targets", + "uk_weights_audit", + ] assert all(g.not_applicable is None for g in manifest.gates) def test_gate_names_are_country_neutral(self, manifest) -> None: diff --git a/packages/microcosm-build/tests/test_gate_battery_contract_pins.py b/packages/microcosm-build/tests/test_gate_battery_contract_pins.py index c181fe081..7997cef93 100644 --- a/packages/microcosm-build/tests/test_gate_battery_contract_pins.py +++ b/packages/microcosm-build/tests/test_gate_battery_contract_pins.py @@ -261,6 +261,8 @@ def test_signature_scheme_is_verifiable_across_the_shards( monkeypatch.setenv(gate_signing_key_env("uk"), KEY) run = _uk_run(tmp_path) run.run_phase("preflight", EvidenceContext()) + run.run_phase("assembled", EvidenceContext()) + run.run_phase("transferred", EvidenceContext()) run.run_phase("terminal", EvidenceContext()) report = json.loads((tmp_path / "terminal_gates.json").read_text()) @@ -286,6 +288,8 @@ def test_a_real_report_survives_every_mirror_check( release_evidence={"calibration_diagnostics_sha256": "c" * 64}, ) run.run_phase("preflight", EvidenceContext()) + run.run_phase("assembled", EvidenceContext()) + run.run_phase("transferred", EvidenceContext()) run.run_phase("terminal", EvidenceContext()) report = json.loads((tmp_path / "terminal_gates.json").read_text()) diff --git a/packages/microcosm-build/tests/test_uk_battery_bindings.py b/packages/microcosm-build/tests/test_uk_battery_bindings.py index e33078296..9667bf0db 100644 --- a/packages/microcosm-build/tests/test_uk_battery_bindings.py +++ b/packages/microcosm-build/tests/test_uk_battery_bindings.py @@ -412,6 +412,8 @@ def test_the_uk_spec_runs_as_declared_with_named_gaps( registry=UK_GATE_REGISTRY, ) run.run_phase("preflight", EvidenceContext()) + run.run_phase("assembled", EvidenceContext()) + run.run_phase("transferred", EvidenceContext()) run.run_phase("terminal", EvidenceContext()) report = run.report_payload() @@ -450,10 +452,12 @@ def test_fully_armed_battery_evaluates_gate_for_gate(self) -> None: entry_id for entry_id, o in by_id.items() if o.status is GateStatus.PASSED ] # 11 as on main (uk_nonnegative_columns passes with zero required - # columns — the scheduled stages declare none), the two E4 stochastic - # gates, the E5 support gate, the E6 aggregate-admin gate, and the E8 - # student-loan enum gate; their evaluators have direct tests. - assert len(passed) == 16 + # columns — the scheduled stages declare none), the take-up signal + # gate, the E5 support gate, the E6 aggregate-admin gate, and the E8 + # student-loan enum gate; their evaluators have direct tests. The BRMA + # enum gate is no longer among them: it moved to the spine battery's + # assembled boundary, where its column is first written. + assert len(passed) == 15 qrf = by_id["uk_qrf_tail_concentration"] assert qrf.status is GateStatus.FAILED assert "declared QRF output is absent" in qrf.result.failures[0] diff --git a/packages/microcosm-build/tests/test_uk_calibration_run.py b/packages/microcosm-build/tests/test_uk_calibration_run.py index 9a407e9a6..faf61f69c 100644 --- a/packages/microcosm-build/tests/test_uk_calibration_run.py +++ b/packages/microcosm-build/tests/test_uk_calibration_run.py @@ -16,6 +16,7 @@ from microcosm.build.uk_runtime.calibration_run import ( UK_CALIBRATION_GATE_SCOPE, UK_CALIBRATION_GATE_SCOPE_EXCLUSIONS, + UK_SPINE_GATE_SCOPE, UKCalibrationRunPaths, run_uk_calibration, ) @@ -135,6 +136,24 @@ def _write_spine_sidecar( json.dumps(sidecar, indent=2, sort_keys=True) + "\n", encoding="utf-8", ) + input_h5.with_suffix(".spine_gates.json").write_text( + json.dumps( + { + "blocked_at_phase": None, + "gates": { + gate_id: { + "criticality": "release_blocking", + "status": "passed", + } + for gate_id in UK_SPINE_GATE_SCOPE + }, + }, + indent=2, + sort_keys=True, + ) + + "\n", + encoding="utf-8", + ) return sidecar diff --git a/packages/microcosm-build/tests/test_uk_cgt_structure.py b/packages/microcosm-build/tests/test_uk_cgt_structure.py index f675d5234..b0c9a7422 100644 --- a/packages/microcosm-build/tests/test_uk_cgt_structure.py +++ b/packages/microcosm-build/tests/test_uk_cgt_structure.py @@ -184,6 +184,12 @@ def test_band_donors_are_band_exact_positive_and_permutation_stable() -> None: assert (first.frame.weights_for("household").values[-DONOR_TOTAL:] > 0).all() assert [row["donor_count"] for row in first.band_rows] == [DONORS_PER_BAND] * 9 assert [row["lower_limit"] for row in first.band_rows][0] == MIN_DONOR_BAND_LOWER + assert [row["realized_min_gain"] for row in first.band_rows] == [ + row["mean_gain"] for row in first.band_rows + ] + assert [row["realized_max_gain"] for row in first.band_rows] == [ + row["mean_gain"] for row in first.band_rows + ] assert [row["weighted_taxpayers"] for row in first.band_rows] == pytest.approx( [79_000, 74_000, 53_000, 37_000, 14_000, 8_000, 5_000, 3_000, 2_000] ) diff --git a/packages/microcosm-build/tests/test_uk_national_build.py b/packages/microcosm-build/tests/test_uk_national_build.py index b266c00c6..a9419c8ed 100644 --- a/packages/microcosm-build/tests/test_uk_national_build.py +++ b/packages/microcosm-build/tests/test_uk_national_build.py @@ -1124,22 +1124,16 @@ def test_national_build_real_terminal_batch_blocks_incomplete_qrf_before_staging "uk_ledger_compile_parity_incumbent_2025": "evidence_absent", "uk_release_input_coverage": "passed", "uk_degenerate_release_surface": "passed", - "uk_zero_weight_strata": "passed", - "uk_weight_ess": "passed", - "uk_weight_ratio": "passed", "uk_weights_audit": "passed", "uk_nonnegative_columns": "passed", "uk_support": "passed", "uk_aggregate_admin": "evidence_absent", "uk_take_up_signal": "passed", - "uk_brma_enum_domain": "passed", "uk_student_loan_plan_enum_domain": "failed", # The legacy report omitted unevidenced gates; the battery names # every gap — non-blocking off the release-candidate posture. "uk_export_surface": "evidence_absent", - "uk_calibration_reference_coverage": "evidence_absent", "uk_target_surface": "evidence_absent", - "uk_target_fit": "evidence_absent", "uk_input_mass_parity": "evidence_absent", "uk_qrf_tail_concentration": "failed", } @@ -1227,9 +1221,10 @@ def test_national_build_parity_trio_is_evidence_absent( "uk_frs_only_spi_fill": "importance", "uk_spi_2022_23_income": "design", } - for entry_id in ("uk_export_surface", "uk_target_surface", "uk_target_fit"): + for entry_id in ("uk_export_surface", "uk_target_surface"): assert gates[entry_id]["status"] == "evidence_absent", entry_id assert gates[entry_id]["reason"] == "missing evidence: parity_evidence" + assert "uk_target_fit" not in gates assert gates["uk_input_mass_parity"]["status"] == "evidence_absent" assert gates["uk_qrf_tail_concentration"]["status"] == "failed" @@ -1295,9 +1290,10 @@ def test_national_build_parity_trio_evaluates_for_armed_calibration( ) gates = json.loads(terminal_json.read_text(encoding="utf-8"))["gates"] - for entry_id in ("uk_export_surface", "uk_target_surface", "uk_target_fit"): + for entry_id in ("uk_export_surface", "uk_target_surface"): assert gates[entry_id]["status"] == "passed", entry_id - assert gates["uk_calibration_reference_coverage"]["status"] == "passed" + assert "uk_target_fit" not in gates + assert "uk_calibration_reference_coverage" not in gates def test_armed_calibration_without_parity_reference_stays_evidence_absent( @@ -1330,10 +1326,11 @@ def test_armed_calibration_without_parity_reference_stays_evidence_absent( ) gates = json.loads(terminal_json.read_text(encoding="utf-8"))["gates"] - for entry_id in ("uk_export_surface", "uk_target_surface", "uk_target_fit"): + for entry_id in ("uk_export_surface", "uk_target_surface"): assert gates[entry_id]["status"] == "evidence_absent", entry_id assert gates[entry_id]["reason"] == "missing evidence: parity_evidence" - assert gates["uk_calibration_reference_coverage"]["status"] == "passed" + assert "uk_target_fit" not in gates + assert "uk_calibration_reference_coverage" not in gates def test_national_build_rejects_both_gate_path_names_and_h5_collisions( @@ -1452,7 +1449,7 @@ def drifting_coverage(context, parameters): == "failed" ) assert payload["gates"]["uk_release_input_coverage"]["status"] == "unreached" - assert payload["gates"]["uk_weight_ratio"]["status"] == "unreached" + assert payload["gates"]["uk_release_input_coverage"]["status"] == "unreached" def test_national_build_rejects_stage_that_breaks_entity_links(tmp_path) -> None: @@ -1851,13 +1848,7 @@ def exploding(frame: Frame) -> Frame: assert resumed_calibration_calls == [] assert after_calls == ["after"] - calibration_gate = result.gate_report["gates"]["uk_calibration_reference_coverage"] - assert calibration_gate["status"] == "passed" - assert calibration_gate["details"] == { - "activated": 1, - "resolved": 1, - "matrix": 1, - } + assert "uk_calibration_reference_coverage" not in result.gate_report["gates"] def test_checkpointed_build_pins_the_run_config(tmp_path) -> None: @@ -1924,7 +1915,7 @@ def test_release_candidate_blocks_on_named_evidence_gaps(tmp_path) -> None: for entry_id, gate in dev.gate_report["gates"].items() if gate["status"] == "evidence_absent" } - assert "uk_weight_ratio" in absent # unbound in the toy registry + assert "uk_export_surface" in absent # unbound in the toy registry with pytest.raises(GateBatteryBlockedError) as error: _run_national_build( diff --git a/packages/microcosm-build/tests/test_uk_release_input_coverage.py b/packages/microcosm-build/tests/test_uk_release_input_coverage.py index 2477fd93f..e9755e947 100644 --- a/packages/microcosm-build/tests/test_uk_release_input_coverage.py +++ b/packages/microcosm-build/tests/test_uk_release_input_coverage.py @@ -727,6 +727,45 @@ def test_required_family_stage_cannot_be_omitted(self) -> None: ) assert result is None + def test_spine_posture_satisfies_superseded_required_families(self) -> None: + manifest = load_uk_release_input_coverage_manifest() + spine_stages = tuple( + stage + for stage in manifest.required_build_stages + if stage not in {"hmrc_spi_income", "hmrc_cgt_gains"} + ) + result = assert_uk_release_input_coverage_build_stages( + (*spine_stages, "hmrc_spi_income_spine"), + manifest=manifest, + ) + assert result is None + assert ( + manifest.family_coverage["hmrc_spi_income"]["superseded_by"]["stage"] + == "hmrc_spi_income_spine" + ) + assert ( + manifest.family_coverage["hmrc_cgt_gains"]["superseded_by"]["stage"] + == "hmrc_cgt_gains_spine" + ) + + def test_supersession_does_not_hide_a_genuinely_missing_family(self) -> None: + manifest = load_uk_release_input_coverage_manifest() + spine_stages = tuple( + stage + for stage in manifest.required_build_stages + if stage + not in { + "hmrc_spi_income", + "hmrc_cgt_gains", + "student_loans", + } + ) + with pytest.raises(ValueError, match="student_loans"): + assert_uk_release_input_coverage_build_stages( + (*spine_stages, "hmrc_spi_income_spine"), + manifest=manifest, + ) + def test_deferred_family_stage_is_not_required(self) -> None: family = _hmrc_family_coverage() family["hmrc_spi_income"].update( diff --git a/packages/microcosm-build/tests/test_uk_stage_health.py b/packages/microcosm-build/tests/test_uk_stage_health.py new file mode 100644 index 000000000..fa5370407 --- /dev/null +++ b/packages/microcosm-build/tests/test_uk_stage_health.py @@ -0,0 +1,353 @@ +from __future__ import annotations + +from microcosm.build.uk_runtime.stage_health import uk_stage_health_gate + + +def _passed(result) -> bool: + assert result.name == "stage_health" + return result.passed + + +def test_support_clip_gate_requires_receipted_columns_and_wires_thresholds() -> None: + evidence = { + "stage": "was_wealth", + "support_clip": { + "columns": { + "cash_isa": { + "donor_min": 0.0, + "donor_max": 100.0, + "clipped_low_rows": 1, + "clipped_high_rows": 0, + "rows_considered": 2, + } + } + }, + } + parameters = { + "stage": "was_wealth", + "check": "support_clip", + "columns": ["cash_isa"], + "max_clipped_low_rows_by_column": {"cash_isa": 1}, + "max_clipped_high_rows_by_column": {}, + } + + assert _passed( + uk_stage_health_gate( + evidence=evidence, + stage="was_wealth", + check="support_clip", + parameters=parameters, + ) + ) + + failed = uk_stage_health_gate( + evidence=evidence, + stage="was_wealth", + check="support_clip", + parameters={ + **parameters, + "max_clipped_low_rows_by_column": {"cash_isa": 0}, + }, + ) + assert failed.passed is False + assert "clipped_low_rows" in failed.failures[0] + + +def test_realization_gate_target_and_deviation_parameters_are_live() -> None: + evidence = { + "stage": "salary_sacrifice", + "headcount_receipt": { + "target": 10.0, + "realization_deviation": 0.1, + "cap_bound": False, + }, + } + parameters = { + "stage": "salary_sacrifice", + "check": "realization_target", + "target": 10.0, + "maximum_abs_realization_deviation": 0.1, + "allow_cap_bound": False, + } + + assert _passed( + uk_stage_health_gate( + evidence=evidence, + stage="salary_sacrifice", + check="realization_target", + parameters=parameters, + ) + ) + assert not uk_stage_health_gate( + evidence=evidence, + stage="salary_sacrifice", + check="realization_target", + parameters={**parameters, "target": 11.0}, + ).passed + assert not uk_stage_health_gate( + evidence=evidence, + stage="salary_sacrifice", + check="realization_target", + parameters={**parameters, "maximum_abs_realization_deviation": 0.09}, + ).passed + + +def test_student_loan_stock_parameter_is_live() -> None: + evidence = { + "stage": "student_loans", + "plans": { + "PLAN_2": { + "stock": 100.0, + "final_england_count": 98.0, + "realization_deviation": -0.02, + } + }, + } + parameters = { + "stage": "student_loans", + "check": "student_loan_plans", + "stocks": {"PLAN_2": 100.0}, + "maximum_abs_realization_deviation": 0.02, + } + + assert _passed( + uk_stage_health_gate( + evidence=evidence, + stage="student_loans", + check="student_loan_plans", + parameters=parameters, + ) + ) + assert not uk_stage_health_gate( + evidence=evidence, + stage="student_loans", + check="student_loan_plans", + parameters={**parameters, "stocks": {"PLAN_2": 99.0}}, + ).passed + + +def test_cgt_incidence_mass_threshold_is_live() -> None: + evidence = { + "stage": "cgt_incidence_clone", + "mass_by_clone_flag": {"false": 100.0, "true": 99.0}, + } + + assert _passed( + uk_stage_health_gate( + evidence=evidence, + stage="cgt_incidence_clone", + check="cgt_incidence_mass", + parameters={ + "stage": "cgt_incidence_clone", + "check": "cgt_incidence_mass", + "maximum_relative_mass_imbalance": 0.01, + }, + ) + ) + assert not uk_stage_health_gate( + evidence=evidence, + stage="cgt_incidence_clone", + check="cgt_incidence_mass", + parameters={ + "stage": "cgt_incidence_clone", + "check": "cgt_incidence_mass", + "maximum_relative_mass_imbalance": 0.009, + }, + ).passed + + +def test_spi_support_channel_parameters_are_live() -> None: + evidence = { + "stage": "spi_support_channel", + "spi_prior_mass_share": 0.5, + "household_weight_kind": "importance", + "spi_households": 10, + } + parameters = { + "stage": "spi_support_channel", + "check": "spi_support_channel", + "spi_prior_mass_share": 0.5, + "absolute_tolerance": 0.0, + "household_weight_kind": "importance", + "minimum_spi_households": 10, + } + + assert _passed( + uk_stage_health_gate( + evidence=evidence, + stage="spi_support_channel", + check="spi_support_channel", + parameters=parameters, + ) + ) + assert not uk_stage_health_gate( + evidence=evidence, + stage="spi_support_channel", + check="spi_support_channel", + parameters={**parameters, "minimum_spi_households": 11}, + ).passed + + +def test_spi_income_identity_parameters_are_live() -> None: + evidence = { + "stage": "hmrc_spi_income_spine", + "spi_prior": {"mass_share": 0.5}, + "targets": {"count": 2}, + "post_draw_identity": {"exact": True, "rows_checked": 3}, + } + parameters = { + "stage": "hmrc_spi_income_spine", + "check": "spi_income_spine", + "spi_prior_mass_share": 0.5, + "absolute_tolerance": 0.0, + "minimum_identity_rows": 3, + "minimum_target_count": 2, + } + + assert _passed( + uk_stage_health_gate( + evidence=evidence, + stage="hmrc_spi_income_spine", + check="spi_income_spine", + parameters=parameters, + ) + ) + assert not uk_stage_health_gate( + evidence=evidence, + stage="hmrc_spi_income_spine", + check="spi_income_spine", + parameters={**parameters, "minimum_target_count": 3}, + ).passed + + +def test_source_signal_structural_zero_parameter_is_live() -> None: + evidence = { + "stage": "frs_hmrc_spine_leaves", + "source_signal_rows": {"gift_aid": 0, "employment_income": 2}, + "structural_zero_columns": ["gift_aid"], + } + parameters = { + "stage": "frs_hmrc_spine_leaves", + "check": "source_signal", + "minimum_signal_rows": 1, + "structural_zero_columns": ["gift_aid"], + } + + assert _passed( + uk_stage_health_gate( + evidence=evidence, + stage="frs_hmrc_spine_leaves", + check="source_signal", + parameters=parameters, + ) + ) + assert not uk_stage_health_gate( + evidence=evidence, + stage="frs_hmrc_spine_leaves", + check="source_signal", + parameters={**parameters, "structural_zero_columns": []}, + ).passed + + +def test_cgt_band_donor_support_handles_open_upper_bound() -> None: + evidence = { + "stage": "cgt_band_donors", + "bands": [ + { + "lower_limit": 12300.0, + "donor_count": 1, + "realized_min_gain": 12300.0, + "realized_max_gain": 1_000_000_000.0, + } + ], + } + parameters = { + "stage": "cgt_band_donors", + "check": "cgt_band_donor_support", + "support_bounds_resource": "cgt_band_donor_support_bounds.json", + } + + assert _passed( + uk_stage_health_gate( + evidence=evidence, + stage="cgt_band_donors", + check="cgt_band_donor_support", + parameters=parameters, + ) + ) + + failed = uk_stage_health_gate( + evidence={ + **evidence, + "bands": [{**evidence["bands"][0], "realized_min_gain": 12_299.0}], + }, + stage="cgt_band_donors", + check="cgt_band_donor_support", + parameters=parameters, + ) + assert failed.passed is False + assert "falls below" in failed.failures[0] + + +def test_age_tail_relative_deviation_parameter_is_live() -> None: + evidence = { + "stage": "uk_age_tail_disaggregation", + "achieved_weighted": {"MALE": {"80_84": 90.0}}, + "band_populations": {"MALE:80_84": 100.0}, + } + + assert _passed( + uk_stage_health_gate( + evidence=evidence, + stage="age_tail", + check="age_tail_targets", + parameters={ + "stage": "age_tail", + "check": "age_tail_targets", + "maximum_relative_deviation": 0.1, + }, + ) + ) + assert not uk_stage_health_gate( + evidence=evidence, + stage="age_tail", + check="age_tail_targets", + parameters={ + "stage": "age_tail", + "check": "age_tail_targets", + "maximum_relative_deviation": 0.09, + }, + ).passed + + +def test_cgt_summary_minimum_rows_parameter_is_live() -> None: + evidence = { + "stage": "hmrc_cgt_gains_spine", + "rows": [{"gain_lower_bound": 12300.0}], + "taxpayer_mass": 1.0, + "published_taxpayer_mass": 1.0, + "remainder_mass": 0.0, + } + + assert _passed( + uk_stage_health_gate( + evidence=evidence, + stage="hmrc_cgt_gains_spine", + check="cgt_imputation_summary", + parameters={ + "stage": "hmrc_cgt_gains_spine", + "check": "cgt_imputation_summary", + "minimum_band_rows": 1, + }, + ) + ) + assert not uk_stage_health_gate( + evidence=evidence, + stage="hmrc_cgt_gains_spine", + check="cgt_imputation_summary", + parameters={ + "stage": "hmrc_cgt_gains_spine", + "check": "cgt_imputation_summary", + "minimum_band_rows": 2, + }, + ).passed diff --git a/packages/microcosm-data/src/microcosm/data/contract.py b/packages/microcosm-data/src/microcosm/data/contract.py index 379d4eda4..12e81ff7d 100644 --- a/packages/microcosm-data/src/microcosm/data/contract.py +++ b/packages/microcosm-data/src/microcosm/data/contract.py @@ -363,7 +363,7 @@ # gate_signing_key_env("uk") in the build shard; the legacy POPULACE variable # stays with the schema-3 path above. _UK_GATE_BATTERY_SIGNING_KEY_ENV = "MICROCOSM_UK_TERMINAL_GATE_SIGNING_KEY" -_UK_GATE_BATTERY_PHASES = ("preflight", "terminal") +_UK_GATE_BATTERY_PHASES = ("preflight", "assembled", "transferred", "terminal") _UK_GATE_BATTERY_STATUSES = frozenset( {"passed", "failed", "not_applicable", "evidence_absent", "unreached"} ) @@ -373,13 +373,13 @@ # fingerprint derives from the manifest digest. Editing the spec moves all # three here in the same reviewed change. _UK_GATE_BATTERY_POLICY_SHA256 = ( - "623f340ddde6f705717c3a6306522f8cf46c1c17a067f9e89df190ecc690f0fc" + "31c79de22ea90d5766d015f0df5e1416ee21647f07b351f2971faa86f3a133c0" ) _UK_GATE_BATTERY_GATES_MANIFEST_SHA256 = ( - "4f66eea7e593b795da93217e9ac8b3b53ca1f82375ec346b3a2ecbb558b89cb6" + "f9225c546b706cbf06d3a80dd97f8db82927f3bbcf9baebe7f5d694eac7fc730" ) _UK_GATE_BATTERY_SPEC_FINGERPRINT = ( - "26fdbcccfaa01e9afa116339cfaa870f76fc6ed3bd5db57fec498781ff3efc64" + "e7176207973ebd6731fe1e8e906da6fc6ef0f5ed79956542332ee21bf184486b" ) #: Spec entry id -> the legacy gate name whose observable detail checks #: apply unchanged (the battery re-keys the report by entry id; the gate @@ -421,6 +421,31 @@ "ledger_compile_parity", "preflight", ), + "uk_stage_was_wealth_support": ("stage_health", "transferred"), + "uk_stage_lcfs_consumption_support": ("stage_health", "transferred"), + "uk_stage_etb_vat_support": ("stage_health", "transferred"), + "uk_stage_etb_services_support": ("stage_health", "transferred"), + "uk_stage_frs_hmrc_spine_leaves_signal": ( + "stage_health", + "transferred", + ), + "uk_stage_spi_support_channel_mass": ("stage_health", "transferred"), + "uk_stage_hmrc_spi_income_spine_identity": ( + "stage_health", + "transferred", + ), + "uk_stage_cgt_incidence_clone_mass": ("stage_health", "transferred"), + "uk_stage_cgt_band_donors_support": ("stage_health", "transferred"), + "uk_stage_hmrc_cgt_gains_spine_summary": ( + "stage_health", + "transferred", + ), + "uk_stage_salary_sacrifice_realization": ( + "stage_health", + "transferred", + ), + "uk_stage_student_loans_realization": ("stage_health", "transferred"), + "uk_stage_age_tail_targets": ("stage_health", "transferred"), "uk_release_input_coverage": ("release_input_coverage", "terminal"), "uk_degenerate_release_surface": ("degenerate_release_surface", "terminal"), "uk_zero_weight_strata": ("zero_weight_strata", "terminal"), @@ -432,7 +457,7 @@ "uk_aggregate_admin": ("aggregate_admin", "terminal"), "uk_export_surface": ("export_surface", "terminal"), "uk_take_up_signal": ("take_up_signal", "terminal"), - "uk_brma_enum_domain": ("enum_domain", "terminal"), + "uk_brma_enum_domain": ("enum_domain", "assembled"), "uk_student_loan_plan_enum_domain": ("enum_domain", "terminal"), "uk_calibration_reference_coverage": ( "calibration_reference_coverage", @@ -454,6 +479,19 @@ "uk_ledger_compile_parity_incumbent_2025", "uk_degenerate_release_surface", "uk_input_mass_parity", + "uk_stage_was_wealth_support", + "uk_stage_lcfs_consumption_support", + "uk_stage_etb_vat_support", + "uk_stage_etb_services_support", + "uk_stage_frs_hmrc_spine_leaves_signal", + "uk_stage_spi_support_channel_mass", + "uk_stage_hmrc_spi_income_spine_identity", + "uk_stage_cgt_incidence_clone_mass", + "uk_stage_cgt_band_donors_support", + "uk_stage_hmrc_cgt_gains_spine_summary", + "uk_stage_salary_sacrifice_realization", + "uk_stage_student_loans_realization", + "uk_stage_age_tail_targets", } ) # The input-mass binding's evidence payload wraps the reviewed reference diff --git a/packages/microcosm-data/tests/test_contract.py b/packages/microcosm-data/tests/test_contract.py index 065832b82..5f712df02 100644 --- a/packages/microcosm-data/tests/test_contract.py +++ b/packages/microcosm-data/tests/test_contract.py @@ -148,13 +148,13 @@ def _trusted_terminal_gate_signing_key(monkeypatch) -> None: UK_GATE_BATTERY_PRODUCER = "microcosm.build.gate_battery" UK_GATE_BATTERY_SIGNING_KEY_ENV = "MICROCOSM_UK_TERMINAL_GATE_SIGNING_KEY" UK_GATE_BATTERY_POLICY_SHA256 = ( - "623f340ddde6f705717c3a6306522f8cf46c1c17a067f9e89df190ecc690f0fc" + "31c79de22ea90d5766d015f0df5e1416ee21647f07b351f2971faa86f3a133c0" ) UK_GATE_BATTERY_GATES_MANIFEST_SHA256 = ( - "4f66eea7e593b795da93217e9ac8b3b53ca1f82375ec346b3a2ecbb558b89cb6" + "f9225c546b706cbf06d3a80dd97f8db82927f3bbcf9baebe7f5d694eac7fc730" ) UK_GATE_BATTERY_SPEC_FINGERPRINT = ( - "26fdbcccfaa01e9afa116339cfaa870f76fc6ed3bd5db57fec498781ff3efc64" + "e7176207973ebd6731fe1e8e906da6fc6ef0f5ed79956542332ee21bf184486b" ) UK_GATE_BATTERY_DEGENERATE_EVIDENCE_SHA256 = ( "d0d024043132fa07c378c393dbe2b24fe99bf19e876bcc39997d2c80cc9bd4f6" @@ -180,6 +180,51 @@ def _trusted_terminal_gate_signing_key(monkeypatch) -> None: "preflight", None, ), + "uk_stage_was_wealth_support": ("stage_health", "transferred", None), + "uk_stage_lcfs_consumption_support": ("stage_health", "transferred", None), + "uk_stage_etb_vat_support": ("stage_health", "transferred", None), + "uk_stage_etb_services_support": ("stage_health", "transferred", None), + "uk_stage_frs_hmrc_spine_leaves_signal": ( + "stage_health", + "transferred", + None, + ), + "uk_stage_spi_support_channel_mass": ( + "stage_health", + "transferred", + None, + ), + "uk_stage_hmrc_spi_income_spine_identity": ( + "stage_health", + "transferred", + None, + ), + "uk_stage_cgt_incidence_clone_mass": ( + "stage_health", + "transferred", + None, + ), + "uk_stage_cgt_band_donors_support": ( + "stage_health", + "transferred", + None, + ), + "uk_stage_hmrc_cgt_gains_spine_summary": ( + "stage_health", + "transferred", + None, + ), + "uk_stage_salary_sacrifice_realization": ( + "stage_health", + "transferred", + None, + ), + "uk_stage_student_loans_realization": ( + "stage_health", + "transferred", + None, + ), + "uk_stage_age_tail_targets": ("stage_health", "transferred", None), "uk_release_input_coverage": ( "release_input_coverage", "terminal", @@ -203,7 +248,7 @@ def _trusted_terminal_gate_signing_key(monkeypatch) -> None: "uk_aggregate_admin": ("aggregate_admin", "terminal", "aggregate_vs_admin"), "uk_export_surface": ("export_surface", "terminal", "export_surface"), "uk_take_up_signal": ("take_up_signal", "terminal", "take_up_signal"), - "uk_brma_enum_domain": ("enum_domain", "terminal", "enum_domain"), + "uk_brma_enum_domain": ("enum_domain", "assembled", "enum_domain"), "uk_student_loan_plan_enum_domain": ( "enum_domain", "terminal", @@ -1050,7 +1095,48 @@ def _gate_battery_payload( ) -> tuple[dict, dict[str, str]]: """A fully-armed, all-passing, signed schema-4 battery report.""" - stage_names = ["frs_hmrc_retained_leaves", "hmrc_spi_income"] + stage_names = [ + "frs_spine", + "frs_employment", + "frs_council_tax", + "frs_disability", + "frs_education", + "frs_legacy_proxies", + "frs_education_grant_split", + "frs_take_up", + "frs_person_draws", + "frs_household_draws", + "frs_brma", + "was_wealth", + "regional_property_uprating", + "lcfs_consumption", + "etb_vat", + "etb_services", + "frs_hmrc_spine_leaves", + "spi_support_channel", + "hmrc_spi_income_spine", + "cgt_incidence_clone", + "cgt_band_donors", + "hmrc_cgt_gains_spine", + "salary_sacrifice", + "student_loans", + "age_tail", + ] + stage_health_stages = { + "uk_stage_was_wealth_support": "was_wealth", + "uk_stage_lcfs_consumption_support": "lcfs_consumption", + "uk_stage_etb_vat_support": "etb_vat", + "uk_stage_etb_services_support": "etb_services", + "uk_stage_frs_hmrc_spine_leaves_signal": "frs_hmrc_spine_leaves", + "uk_stage_spi_support_channel_mass": "spi_support_channel", + "uk_stage_hmrc_spi_income_spine_identity": "hmrc_spi_income_spine", + "uk_stage_cgt_incidence_clone_mass": "cgt_incidence_clone", + "uk_stage_cgt_band_donors_support": "cgt_band_donors", + "uk_stage_hmrc_cgt_gains_spine_summary": "hmrc_cgt_gains_spine", + "uk_stage_salary_sacrifice_realization": "salary_sacrifice", + "uk_stage_student_loans_realization": "student_loans", + "uk_stage_age_tail_targets": "age_tail", + } gates: dict[str, dict] = {} for entry_id, (gate, phase, detail_name) in UK_GATE_BATTERY_ENTRIES.items(): if entry_id == "uk_release_input_coverage_manifest_current": @@ -1073,6 +1159,11 @@ def _gate_battery_payload( } elif entry_id == "uk_calibration_reference_coverage": details = {"activated": 388, "resolved": 388, "matrix": 388} + elif gate == "stage_health": + details = { + "stage": stage_health_stages[entry_id], + "check": "fixture", + } else: details = _terminal_gate_details(detail_name) gates[entry_id] = { @@ -1107,6 +1198,8 @@ def _gate_battery_payload( "uk_degenerate_release_surface": UK_GATE_BATTERY_DEGENERATE_EVIDENCE_SHA256, "uk_input_mass_parity": UK_GATE_BATTERY_INPUT_MASS_EVIDENCE_SHA256, } + for entry_id, stage in stage_health_stages.items(): + evidence[entry_id] = _canonical_sha256({stage: {"stage": stage}}) payload = { "schema_version": 4, "country": "uk", @@ -1114,8 +1207,8 @@ def _gate_battery_payload( "release_candidate": True, "spec_fingerprint": UK_GATE_BATTERY_SPEC_FINGERPRINT, "gates_manifest_sha256": UK_GATE_BATTERY_GATES_MANIFEST_SHA256, - "phases": ["preflight", "terminal"], - "phases_evaluated": ["preflight", "terminal"], + "phases": ["preflight", "assembled", "transferred", "terminal"], + "phases_evaluated": ["preflight", "assembled", "transferred", "terminal"], "blocked_at_phase": None, "shippable": True, "gates": gates, diff --git a/tools/build_uk_frs_spine.py b/tools/build_uk_frs_spine.py index 26bbd218a..c945beeaf 100644 --- a/tools/build_uk_frs_spine.py +++ b/tools/build_uk_frs_spine.py @@ -12,11 +12,16 @@ from importlib import metadata from pathlib import Path -from microcosm.build.country_spec import country_stage_plan, load_country_spec +from microcosm.build.country_spec import ( + GatesManifest, + country_stage_plan, + load_country_spec, +) from microcosm.build.frame_sampling import ( normalize_sampled_household_mass, sample_frame_households, ) +from microcosm.build.gate_battery import BlockingMode, EvidenceContext, GateBatteryRun from microcosm.build.logbook import canonical_json_bytes from microcosm.build.logbook_adoption import ( AttemptState, @@ -34,6 +39,8 @@ write_error_receipt, ) from microcosm.build.uk_runtime.age_tail import UKAgeTailStageTransform +from microcosm.build.uk_runtime.battery_bindings import UK_GATE_REGISTRY +from microcosm.build.uk_runtime.calibration_run import UK_SPINE_GATE_SCOPE from microcosm.build.uk_runtime.cgt_imputation import uk_cgt_spine_stage_transform from microcosm.build.uk_runtime.cgt_structure import ( UKCGTBandDonorStageTransform, @@ -492,6 +499,7 @@ def _build_sidecar( stochastic_contract_sha256: str, frs_vintage: str, sampling: dict[str, object] | None, + spine_gate_report: dict[str, object] | None = None, ) -> dict[str, object]: household_weight = frame.weights_for("household") return { @@ -525,6 +533,7 @@ def _build_sidecar( "declared_seeds": _declared_seeds(stages), "source_vintages": {"frs": frs_vintage}, "sampling": sampling, + "spine_gate_report": spine_gate_report, "stochastic_contract_sha256": stochastic_contract_sha256, "rules_engine": _rules_engine_provenance(), } @@ -612,6 +621,8 @@ def _run_plan_with_spine_sampling( *, sample_fraction: float, sample_seed: int, + spine_battery: GateBatteryRun | None = None, + stage_evidence_provider=None, ) -> tuple[object, tuple[object, ...], dict[str, object] | None]: if not plan.stages or plan.stages[0].name != "frs_spine": frame, records = plan.run(uk_frs_spine_seed_frame()) @@ -629,8 +640,68 @@ def _run_plan_with_spine_sampling( ) if len(plan.stages) == 1: return spine_frame, spine_records, sampling - frame, tail_records = StagePlan(plan.stages[1:]).run(spine_frame) - return frame, (*spine_records, *tail_records), sampling + assembled_end = min(11, len(plan.stages)) + frame, assembled_records = StagePlan(plan.stages[1:assembled_end]).run(spine_frame) + if spine_battery is not None: + _run_spine_gate_phase( + spine_battery, + "assembled", + frame=frame, + stage_evidence=( + stage_evidence_provider() if stage_evidence_provider is not None else {} + ), + ) + if assembled_end == len(plan.stages): + return frame, (*spine_records, *assembled_records), sampling + frame, tail_records = StagePlan(plan.stages[assembled_end:]).run(frame) + if spine_battery is not None: + _run_spine_gate_phase( + spine_battery, + "transferred", + frame=frame, + stage_evidence=( + stage_evidence_provider() if stage_evidence_provider is not None else {} + ), + ) + return frame, (*spine_records, *assembled_records, *tail_records), sampling + + +def _run_spine_gate_phase( + battery: GateBatteryRun, + phase: str, + *, + frame, + stage_evidence: Mapping[str, object], +) -> None: + battery.run_phase( + phase, + EvidenceContext( + frame=frame, + artifacts={"stage_evidence": dict(stage_evidence)}, + ), + ) + battery.enforce(phase, mode=BlockingMode.BLOCKS_ARTIFACT) + + +def _spine_gate_report_path(spine_h5: Path) -> Path: + return spine_h5.with_suffix(".spine_gates.json") + + +def _spine_gate_manifest_from_spec(spec) -> GatesManifest | None: + source = getattr(spec, "gates", None) + if source is None: + return None + entries = tuple(entry for entry in source.gates if entry.id in UK_SPINE_GATE_SCOPE) + missing = sorted(set(UK_SPINE_GATE_SCOPE) - {entry.id for entry in entries}) + if missing: + raise RuntimeError(f"UK spine gate scope names undeclared gate id(s): {missing}.") + return GatesManifest( + country=source.country, + version=source.version, + policy=f"{source.policy}; spine_build_scope", + phases=("assembled", "transferred"), + gates=entries, + ) def _rung_abort_receipt( @@ -690,6 +761,7 @@ def main(argv: list[str] | None = None) -> int: args.spine_h5, args.spine_h5.with_suffix(".build.json"), args.spine_h5.with_suffix(".hmrc_replay.json"), + _spine_gate_report_path(args.spine_h5), args.spine_h5.with_suffix(".rung_abort.json"), ] if args.emit_nonzero_shares is not None: @@ -885,11 +957,31 @@ def main(argv: list[str] | None = None) -> int: implementations, stage_names=stage_names, ) + spine_gate_path = _spine_gate_report_path(args.spine_h5) + spine_gate_manifest = _spine_gate_manifest_from_spec(spec) + spine_battery = ( + GateBatteryRun( + spine_gate_manifest, + release_id=state.build_id, + report_path=spine_gate_path, + release_candidate=args.sample_fraction == 1.0, + registry=UK_GATE_REGISTRY, + ) + if spine_gate_manifest is not None + else None + ) frame, records, sampling = _run_plan_with_spine_sampling( plan, sample_fraction=args.sample_fraction, sample_seed=args.sample_seed, + spine_battery=spine_battery, + stage_evidence_provider=lambda: _collect_stage_evidence( + stage_names=_STAGE_NAMES, + implementations=implementations, + ), ) + if spine_battery is not None: + append_phase(state, "spine_gates_evaluated") append_phase(state, "spine_built") output = write_uk_national_frame(frame, args.spine_h5) append_phase(state, "spine_written") @@ -923,6 +1015,14 @@ def main(argv: list[str] | None = None) -> int: stochastic_contract_sha256=stochastic_contract.resource_sha256, frs_vintage=frs_release.vintage, sampling=sampling, + spine_gate_report=( + { + "path": str(spine_gate_path), + "sha256": hashlib.sha256(spine_gate_path.read_bytes()).hexdigest(), + } + if spine_gate_path.is_file() + else None + ), ) stage_evidence = _collect_stage_evidence( stage_names=_STAGE_NAMES, @@ -961,6 +1061,16 @@ def main(argv: list[str] | None = None) -> int: ), } } + if spine_gate_path.is_file(): + gate_payload = json.loads(spine_gate_path.read_text(encoding="utf-8")) + for gate_id, payload in gate_payload.get("gates", {}).items(): + state.gate_verdicts[str(gate_id)] = { + "verdict": str(payload.get("status")), + "receipt": ( + f"{local_artifact_reference(spine_gate_path, repository_hint=_REPOSITORY)}" + f"#/gates/{gate_id}" + ), + } spool_path = _record_attempt( state=state, started_at=started_at, diff --git a/tools/build_uk_release_input_coverage_manifest.py b/tools/build_uk_release_input_coverage_manifest.py index 2f9641727..61fda0eea 100644 --- a/tools/build_uk_release_input_coverage_manifest.py +++ b/tools/build_uk_release_input_coverage_manifest.py @@ -941,6 +941,16 @@ def _cgt_family_coverage_contract( "stage": "hmrc_cgt_gains", "source_manifest": CGT_SOURCE_STAGES_PATH.name, "source_manifest_sha256": _sha256(CGT_SOURCE_STAGES_PATH), + "superseded_by": { + "stage": "hmrc_cgt_gains_spine", + "source_manifest": SOURCE_STAGES_PATH.name, + "source_manifest_sha256": _sha256(SOURCE_STAGES_PATH), + "reason": ( + "The FRS spine build executes hmrc_cgt_gains_spine, which " + "applies the same HMRC Table 3 amounts redraw directly in " + "source_stages.json before calibration." + ), + }, "base_candidate_sha256": str(base_candidate["sha256"]), "base_candidate_tier": base_candidate_tier, "source_vintages": { @@ -1230,6 +1240,16 @@ def _hmrc_family_coverage_contract( # source (adversarial-review finding, 2026-08-20). "canonical_source_manifest": SOURCE_STAGES_PATH.name, "canonical_source_manifest_sha256": _sha256(SOURCE_STAGES_PATH), + "superseded_by": { + "stage": "hmrc_spi_income_spine", + "source_manifest": SOURCE_STAGES_PATH.name, + "source_manifest_sha256": _sha256(SOURCE_STAGES_PATH), + "reason": ( + "The FRS spine build executes hmrc_spi_income_spine, which " + "supersedes the June retained-leaves/hmrc_spi_income pair " + "inside source_stages.json." + ), + }, "base_candidate_sha256": str(base_candidate["sha256"]), "base_candidate_tier": base_candidate_tier, "source_vintages": { From 0dfd95664d4d9bc3aa322ed2572bf0521f35a073 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Wed, 26 Aug 2026 14:20:22 +0200 Subject: [PATCH 04/14] Retire the June path: the national build is spine plus seam, nothing else MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The June driver, its national_build orchestration, and the HMRC restoration path encoded a convention the spine superseded stage by stage: SPI re-derived onto the input, imputations replayed at build time, capital gains injected, and a zero-weight SPI channel required of every input — the refusal that made the seam reject spine artifacts outright. All three files retire whole. The seam was built never wrapping them, the h5 I/O and UKNationalStage container already live in national_frame, and build_uk_national_dataset() had no caller left, so this is deletion and shim re-pointing, not surgery. The retired driver also takes a defect with it: it signed the declared calibration diagnostics digest into release evidence before writing the file, with nothing comparing the two. The seam measures the produced file and signs the measured digest. The candidate carries the microcosm name now: UK_CANDIDATE_DATASET_NAME is microcosm_uk_2024, the scorer derives its candidate label from the artifact it actually reads instead of a constant, and the reference name stays enhanced_frs_2024_25 — the frozen historical instrument that keeps the swap auditable. No incumbent artifact byte feeds any build; the parity reference, input-mass descriptor, and contract pins stay as comparison evidence, and the input-mass gate re-arms against the spine's own certified line in a later increment. Dead code and stale journals go with it: UKPolicyEngineAdapter had no callers; UK_COVERAGE_PROGRESS.md and CONTRACT_FINDINGS.md asserted invariants now enforced by the manifest and merged gates; the 686/630 experiment packets are historicized in place as the adjudication record they are. Co-Authored-By: Claude Fable 5 --- CONTRACT_FINDINGS.md | 99 - UK_COVERAGE_PROGRESS.md | 275 --- ...57-june-national-driver-retired.changed.md | 1 + experiments/612-uk-carrier-payload-receipt.md | 39 - .../630-uk-gate-adjudication-receipts.md | 2 + experiments/686-uk-spine-comparison-ledger.md | 2 + experiments/686-uk-spine-swap-receipts.md | 2 + .../microcosm/build/uk_runtime/__init__.py | 32 +- .../build/uk_runtime/calibration_run.py | 5 +- .../build/uk_runtime/cgt_imputation.py | 2 +- .../build/uk_runtime/hmrc_restoration.py | 926 -------- .../build/uk_runtime/hmrc_source_contract.py | 27 +- .../build/uk_runtime/ledger_targets.py | 26 - .../build/uk_runtime/national_build.py | 835 ------- .../build/uk_runtime/national_frame.py | 30 +- .../build/uk_runtime/rowwise_dataset.py | 4 +- .../microcosm/build/uk_runtime/spi_spine.py | 96 +- .../build/uk_runtime/terminal_gates.py | 4 +- .../tests/test_score_uk_national_candidate.py | 14 +- .../tests/test_uk_calibration_run.py | 5 +- .../tests/test_uk_frs_hmrc_leaves.py | 4 +- .../tests/test_uk_frs_spine.py | 2 +- .../tests/test_uk_hmrc_restoration.py | 1030 --------- .../tests/test_uk_ladder_rowwise_clone.py | 8 +- .../tests/test_uk_national_build.py | 2004 ----------------- .../tests/test_uk_national_build_driver.py | 1628 ------------- .../tests/test_uk_national_calibration.py | 6 +- .../tests/test_uk_national_frame.py | 24 +- .../tests/test_uk_rowwise_build_driver.py | 21 +- .../tests/test_uk_rowwise_dry_run.py | 4 +- .../tests/test_uk_rowwise_weight_metadata.py | 2 +- .../frame/adapters/policyengine_uk.py | 4 +- tools/build_uk_frs_spine.py | 6 +- tools/build_uk_national_dataset.py | 1482 ------------ tools/build_uk_rowwise_dataset.py | 10 +- tools/ci_test_groups.py | 9 +- tools/emit_uk_brma_distribution.py | 6 +- ...measure_uk_weighted_integrity_baselines.py | 2 +- tools/score_uk_national_candidate.py | 16 +- tools/verify_uk_identity_stability.py | 6 +- 40 files changed, 266 insertions(+), 8434 deletions(-) delete mode 100644 CONTRACT_FINDINGS.md delete mode 100644 UK_COVERAGE_PROGRESS.md create mode 100644 changelog.d/757-june-national-driver-retired.changed.md delete mode 100644 experiments/612-uk-carrier-payload-receipt.md delete mode 100644 packages/microcosm-build/src/microcosm/build/uk_runtime/hmrc_restoration.py delete mode 100644 packages/microcosm-build/src/microcosm/build/uk_runtime/national_build.py delete mode 100644 packages/microcosm-build/tests/test_uk_hmrc_restoration.py delete mode 100644 packages/microcosm-build/tests/test_uk_national_build.py delete mode 100644 packages/microcosm-build/tests/test_uk_national_build_driver.py delete mode 100644 tools/build_uk_national_dataset.py diff --git a/CONTRACT_FINDINGS.md b/CONTRACT_FINDINGS.md deleted file mode 100644 index 7ea1ca378..000000000 --- a/CONTRACT_FINDINGS.md +++ /dev/null @@ -1,99 +0,0 @@ -# Contract findings: microcosm#462 fix 3 - -## Original stop state (subsequently adjudicated) - -Implementation stopped at the task's explicit contract-safety condition. The -declared `capital_gain_distributions` stage and its registered executor cannot -produce the requested conserved split or reduce the verified $30.27B -`non_sch_d_capital_gains` total into the $10–14B direct-route class. - -The user has since adjudicated these findings as correct and withdrawn the -conservation requirement. The executor's existing memo-component behavior is -now the authoritative contract for scope 3a. - -## Contract findings - -- The manifest declares a tax-unit stage that reads the `tax_unit` table, then - uses `long_term_capital_gains_before_response` as its source, writes - `schedule_d_capital_gain_distributions`, and treats - `non_sch_d_capital_gains` only as an eligibility exclusion - (`packages/microcosm-build/src/microcosm/build/us/source_stages.json:742-764`). -- The packaged share is `0.09852561497474391`. It is specifically the TY2015 - Schedule-D CGD residual divided by long-term net gains excluding the direct - route; it is not a share for repartitioning the existing $30.27B CGD total - (`packages/microcosm-build/src/microcosm/build/us/soca_capital_gain_distribution_shares.json:16-22`). -- The executor computes, for source `L`, direct-route value `D`, and declared - share `q`: - - ```text - eligible = L > 0 and D <= 0 - schedule_d = L * q if eligible else 0 - direct_after = D - ``` - - It copies the frame, adds only the output, and never subtracts from or - otherwise changes `non_sch_d_capital_gains` - (`packages/microcosm-build/src/microcosm/build/us_runtime/capital_gain_distributions.py:203-214`). -- Consequently, whenever the stage emits a positive Schedule-D value, - `non_sch_d_after + schedule_d` is greater than the pre-stage - `non_sch_d_capital_gains` value. The requested per-tax-unit conservation - assertion cannot hold. The existing unit test also explicitly pins the - current memo-component behavior and an untouched source - (`packages/microcosm-build/tests/test_us_capital_gain_distributions.py:81-120`). -- Wiring the executor leaves the verified $30.27B direct-route total at - $30.27B. Even if the declared 9.8526% share were incorrectly applied to that - total with subtraction, it would produce about $2.98B Schedule-D and - $27.29B direct-route amounts, still outside the required $10–14B direct-route - class. -- The executor already fails loudly when its output exists, so a second run is - rejected as requested - (`packages/microcosm-build/src/microcosm/build/us_runtime/capital_gain_distributions.py:191-195`). - It provides no separate signal or conservation gate; inventing one would not - repair the incompatible transform. - -## Builder findings - -- `tools/build_us_asec_pooled_source_base.py` only constructs the pooled ASEC - source and cannot run this PUF-dependent stage. -- In `tools/build_us_puf_support_base.py`, the earliest logical insertion point - would be immediately after `qrf_finalization` and before - `qbi_reconciliation` (`:134-156`, `:981-1004`, and `:1840-1868`). Adding an - outer stage there would make checkpointed builds record it automatically in - `stage_run_context.json` under `pipeline`, `completed`, and `stage_records` - (`packages/microcosm-build/src/microcosm/build/outer_stage_runtime.py:437-575`). -- There is an additional grain seam: both input columns produced by the PUF - QRF are person-grain outputs - (`packages/microcosm-build/src/microcosm/build/us_runtime/puf_support.py:94-105`), - while the manifest reads a tax-unit table. No existing wrapper declares how - to aggregate the inputs and place the output, and no later builder transform - performs the missing subtraction. - -## Why no implementation was made - -The task says the executor and declaration are the contract, prohibits -improvising parameters or changing the declaration's share source, and directs -the run to stop with `BLOCKED.md` if those parameters cannot produce the -SOI-consistent split. Altering the executor to subtract a route, changing its -source column, or inventing a different share would violate those constraints; -wiring it unchanged would knowingly violate the required conservation and -direct-route acceptance tests. - -No production, test, or generated manifest files were changed. The requested -test commands were not run because the mandated stop condition was reached -before an implementable change existed. - -## Original questions resolved by adjudication - -The source-stage contract needs an approved clarification or revision that -defines: - -1. whether the split source is the existing all-route CGD amount or long-term - gains used to create a separate memo component; -2. which route column is reduced so conservation holds, including the - person-to-tax-unit aggregation and output-placement rule; and -3. the approved, provenance-backed parameter for dividing the $30.27B total if - that total is the intended source. - -The adjudication resolves these questions for scope 3a by directing the stage -to be wired unchanged at the identified post-QRF insertion point, using the -existing executor semantics and neighboring outer-stage grain handling. diff --git a/UK_COVERAGE_PROGRESS.md b/UK_COVERAGE_PROGRESS.md deleted file mode 100644 index 3d17cb4db..000000000 --- a/UK_COVERAGE_PROGRESS.md +++ /dev/null @@ -1,275 +0,0 @@ -# UK input-coverage progress - -The coverage baseline is the 145 populated effective loader overrides extracted -from the immutable enhanced-FRS 2023-24 artifact (SHA-256 `584ae33d…`). The -certified `populace_uk_2023` candidate (SHA-256 `f17306cc…`) carries raw -non-default signal for all 145, but `gift_aid` and -`charitable_investment_gifts` carry it only on zero-weight SPI rows. The launch -register therefore honestly began with **143 required columns and 2 reviewed -exclusions**. The rebuilt SPI stage now carries both columns above the reviewed -1 ppm floor on positive effective mass, so the current contract has **145 -required columns and 0 reviewed exclusions**. The pinned base-candidate -evidence remains unchanged at 143 effective-signal columns; a separate reviewed -post-stage restoration record explains the two promotions. - -| Milestone | Coverage change | Evidence | Status | -| --- | ---: | --- | --- | -| Launch contract baseline | 143 required; 2 reviewed exclusions | SHA-pinned enhanced-FRS surface plus owning-entity effective-mass evidence from the certified Microcosm UK H5 | Complete | -| Loader-override correction | +13 formula-owned persisted overrides | UK Simulation passes every engine-known persisted H5 column through `set_input`; exact cached-artifact replay covers all 145 | Complete | -| Contract entity/pin integrity | No status change | All 145 reference and candidate columns carry owning-entity evidence; wrong-table columns and unproven HF revision mappings fail | Complete | -| National orchestration seam | No status change | Ordered stage protocol, stable verified-byte binding, cheap preflight, final manifest gate, and atomic staging-H5 write | Complete | -| Effective-mass coverage | −2 required; +2 reviewed exclusions | Candidate evidence and the final gate both require signal on at least 0.000001 of owning-entity effective population mass; zero-weight Gift Aid support is excluded honestly until restoration | Complete | -| HMRC/SPI source identities and Q1 | No raw status change | Reviewed donor/ODS pins; real ODS 208-fact parse; documented donor-leaf reconciliation; deterministic TEI/TII/TI synthetic contract; one SRP surface for bands and Table 3.6 | Complete | -| HMRC/SPI adjudicated replay | Not promoted | Full PAY/UBISJA/INCPBEN and the explicitly named OSSBEN/SRP subsets are retained; five source-absent leaves and both subsets carry canonical fences; complete FRS Total Income bands remain unavailable | Complete as a fenced replay; not a restored family | -| Real-donor replay, 2026-07-13 | Stale-exclusion remediation identified | Pinned donor + ODS, 100,000-row reviewed bootstrap, 432,779 SPI predictions, exact post-draw identity, 1ppm Gift Aid checks, and a complete 0 exact / 0 directional / 208 excluded aggregate report | Complete; the 143+2 gate correctly demanded promotion | -| Charitable promotion and staging, 2026-07-13 | +2 required; −2 reviewed exclusions | SPI-channel shares 0.0133031567 and 0.0002805533; 145/0 gate pass; importance weights and one valid mass-conserving record; 1.53 GB ignored staging H5 SHA-256 `829e843f…` | Complete; PR-ready contract milestone | - -## Restoration diagnosis - -The HMRC adjudication miss is distributional, not an absent-column gap: total -income-tax liability was £334.629bn against the £277bn SPI-anchored benchmark -(+20.8047%), while all HMRC-family loader inputs had raw non-default signal. -The geography-clone tool only clones a compact H5, so a distinct national -orchestration seam was required before any family restoration could run. - -## National orchestration seam - -`tools/build_uk_national_dataset.py`, backed by -`microcosm.build.uk_runtime.national_build`, now performs the minimal reviewed -sequence: - -1. validate the checked-in input-coverage manifest and required stage plan; -2. load the certified compact UK person, benunit, and household tables, binding - the in-memory stage input to the stable file identity captured around the - candidate SHA-256 verification; -3. run ordered named national stages, validating entity IDs and direct person - references to households and benunits after each; -4. run the final input-coverage/effective-mass gate, which additionally requires - every benunit to resolve to exactly one weighted household; and -5. atomically write a caller-named staging H5 and evidence sidecars. - -This seam does not clone households, assign local geography, publish a release, -or alter `tools/build_uk_rowwise_dataset.py`. The existing geography tool -remains downstream and separate. - -The HMRC stage drops the certified candidate's zero-weight SPI rows, rebuilds -one SPI channel, allocates 50% of unchanged national household mass to it as -`IMPORTANCE` weights, and records the factor-one allocation as a deliberate -`MassChangeRecord`. The latest constituent adjudication forbids calibration: -all 208 published facts use non-overlapping Total Income bands, while the FRS -instrument cannot materialize complete Total Income. The replay therefore -keeps `IMPORTANCE` weights and emits a fenced report instead of fitting biased -constraints. - -The manifest now records `hmrc_spi_income` as `required_at_build`, making the -stage plan, SPI-channel 1 ppm checks, importance-weight state, mapped period, -and reviewed mass-change record executable release requirements. Its separate -`restoration_status` remains `adjudicated_partial_replay`: promoting the two -charitable input columns does not pretend the 208 banded HMRC facts are -like-for-like. Both inputs are hard requirements under the current **145 -required + 0 reviewed exclusions** contract. - -## Reviewed real-source evidence - -The licensed donor remains local at `inputs/spi/put2223uk.tab` and is excluded -by `.git/info/exclude`; it must never be committed or pushed. Its reviewed -identity is: - -- Survey of Personal Incomes Public Use Tape 2022-23; -- 141,323,762 bytes; -- SHA-256 `5ef829461060c91a2a47be59ad541d9b519fc3976d66ca80d4920f711bb96f66`; -- 836,850 donor records; and -- PolicyEngine's licensed copy from the private `spi_2022_23.zip` artifact. - -The real donor passes strict parsing. Its published rounded fields satisfy -`abs(TI - (TEI + TII)) <= £5` on every row (observed maximum £5). That £5 -tolerance is source validation only; synthetic accounting identities receive -no tolerance. In the reviewed seed-42, FACT-weighted 100,000-row bootstrap, -3,672 rows carry Gift Aid and 9 carry charitable-investment gifts. These counts -are readiness evidence, not effective-weight restoration. - -The official SN 9422 Annex A leaf formulas also reconcile against the pinned -donor. Among 834,538 ordinary records, the maximum absolute differences from -published TEI/TII/TI are £15/£10/£20. Among the 2,312 documented composite -records (`AGERANGE == -1`), they are £180/£10/£180. The larger composite -envelope is expected because PUT anonymisation averages records across the two -nonlinear `max(0, ...)` identities before final £5 field rounding. The source -contract pins the ordinary and composite envelopes separately; it does not -broaden the independent £5 published `TI = TEI + TII` check or the exact -post-draw identity. - -The official HMRC 2023-24 ODS remains local at -`inputs/hmrc/Collated_Tables_3_1_to_3_11_2324.ods` with reviewed identity: - -- official URL recorded in `hmrc_income_source_stages.json`; -- 166,693 bytes; -- OpenDocument Spreadsheet MIME type; and -- SHA-256 `ad063b06b2bdeef8600dbbb09d48153337a4966f8c7eea50df7a2e0304ebd73e`. - -The full real parser contract passes: exact Table 3.6/3.7 sheet and header -layouts, all 13 ordered bands, the single trailing “All ranges” sentinel, and -8 components × 13 bands × 2 measures = **208 positive facts**. Savings interest -and Table 3.7 “Other income” are included; no published component is narrowed. - -## Q1 deterministic accounting identity - -The first QRF surface draws documented source leaves, including `SRP` and -`OTHERINC`; it does not draw HMRC employed income, TEI, TII, or TI. After each -draw the runtime: - -- derives the PolicyEngine `employment_income` input as `PAY + EPB + TAXTERM`, - matching the pinned enhanced-FRS pipeline; -- derives the broader HMRC employed-income auxiliary from its normalized source - leaves; -- derives TEI and TII from their constituent draws; and -- assigns `hmrc_spi_assessable_income = TEI + TII` exactly. - -The Table 3.6 state-pension measure uses the same drawn SRP auxiliary included -in TEI and band assignment, rather than an independent stage-2/model draw. - -Tests assert the identity on every synthetic row. The official rounded `TI`, -`TEI`, and `TII` donor fields are validation inputs only and are never stochastic -QRF outputs. - -## Q2 fail-closed audit and adjudicated resolution - -The conductor requires the Table 3.6 employment measure to use one documented -constituent crosswalk identically on both FRS and SPI channels, while preserving -the narrower PolicyEngine employment-input semantics. The official ODS note -defines employment income broadly: pay from employment, taxable benefits, -Incapacity Benefit, contribution-based ESA, and JSA. The SPI documentation's -exact formula also requires `EXPS`, `INCPBEN`, `OSSBEN`, `UBISJA`, and -`MOTHINC` in addition to `PAY`, `EPB`, and `TAXTERM`. - -The certified candidate persists only aggregate `employment_income`; it does -not retain the required normalized employment leaves or employment expenses, -and `incapacity_benefit_reported` is absent/all-default. Its -`miscellaneous_income` is an FRS odd-jobs/royalties aggregate, not a source- -faithful substitute for the missing HMRC leaves or SPI `OTHERINC`. Consequently -the published broad measure cannot be reconstructed like-for-like from the -certified base. - -The first production preflight therefore stopped before ODS parsing, donor -reading, SPI replacement, QRF fitting, or staging writes. That fail-closed stop -was the correct pre-adjudication result: it proved the compact candidate alone -could not supply the broad crosswalk and wrote no artifact. - -The final source-semantic audit confirms that this is not a naming-only gap. -The official SPI 2022–23 Annex A defines monetary `EPB` and `EXPS`, separately -taxable `TAXTERM`, `INCPBEN`, `OSSBEN`, and `UBISJA`, and distinct `MOTHINC` -and `OTHERINC` fields. The pinned enhanced-FRS pipeline persists FRS `INEARNS` -only as aggregate PolicyEngine `employment_income`; it has no monetary -equivalents for `EPB` or `EXPS`, cannot separate taxable termination pay from -gross redundancy, and its `miscellaneous_income` combines concepts that cannot -be assigned source-faithfully between the two SPI miscellaneous leaves. -`incapacity_benefit_reported` is also all-default on the pinned eFRS surface. -Consequently, `employment_income` plus reported benefit aggregates is a new -shared proxy, not the conductor-required identical documented constituent -crosswalk. It was not substituted into the release path. - -The conductor subsequently adjudicated the raw-source audit constituent by -constituent. The national seam now retains full `PAY`, `UBISJA`, and structurally -expressible `INCPBEN` leaves directly from the raw FRS, alongside rather than in -place of PolicyEngine inputs. It separately names -`ossben_identifiable_subset` and `srp_regular_code5`; neither is represented as -the full SPI concept. `EPB`, `EXPS`, `TAXTERM`, `MOTHINC`, and `OTHERINC` remain -source-absent and fenced. The runtime writes `NaN`, not zero or a proxy, where a -full source concept cannot be materialized on the FRS channel. - -Because those missing and partial legs prevent a complete like-for-like FRS -Total Income measure, every published fact depending on an income band is a -reviewed exclusion. The partial measures do not establish a one-directional -bound on band membership, so none qualifies as directional. The real replay -therefore evaluates the complete 208-fact surface as 208 fenced exclusions and -performs no HMRC calibration. - -## Q2 Option 1 raw-FRS source audit, 2026-07-13 - -The conductor selected Option 1 with no proxy: retain all ten normalized leaves -from source-faithful FRS variables, or, if any leaf has no raw source, document -the per-constituent evidence and stop. The audit covered the 2023-24 raw FRS -`ADULT`, `JOB`, `BENEFITS`, and `ODDJOB` tables and the other income-bearing -`PENSION`, `ACCOUNTS`, and `ASSETS` tables. It also checked the current -`policyengine-uk-data` FRS loader and the -[official FRS 2023-24 benefit definitions](https://doc.ukdataservice.ac.uk/doc/9367/mrdoc/pdf/9367_frs_2023_24_benefits_documentation.pdf) -against the -[SPI 2022-23 Annex A definitions](https://doc.ukdataservice.ac.uk/doc/9422/mrdoc/pdf/9422_put_2223_full_documentation.pdf). -The licensed SPI donor was not opened for this audit. - -The mass estimates below use the certified candidate's FRS channel: positive -household weights are folded through the candidate's exact raw-household and -person ancestry, yielding 68,441,459.783 effective person-mass units. A nearby -flag or partial amount is reported only as an at-risk or lower-bound diagnostic; -it is not evidence that the normalized leaf is populated. - -| SPI leaf sought | Raw FRS table and variable evidence | Source-faithful finding | Effective-mass implication | -| --- | --- | --- | ---: | -| `PAY` | `ADULT.INEARNS`; `JOB.UGRSPAY` checked for the underlying job-level gross-pay composition | Available as the annualized earned-pay measure. | 38.7291454% has positive pay. | -| `EPB` | `JOB.EXPBEN01`-`EXPBEN13`; partial amount fields `CARVAL`, `CARAMT`, `FUELAMT`, `VCHAMT`, and `CHVAMT` | **Missing.** `EXPBEN*` are receipt flags, and the amount fields cover only selected benefits; they cannot produce complete taxable expenses payments and benefits. | 12.9485464% has at least one receipt flag, but this is not monetary support. | -| `EXPS` | `JOB.EXPBEN04`/`EXPBEN05`, `MILEAMT`, `MOTAMT`, `UMILEAMT`, `UMOTAMT`, `DEDUC1`-`DEDUC9`, and `UDEDUC1`-`UDEDUC9` | **Missing.** These fields describe reimbursements or payroll deductions, not the complete tax-deductible employment-expense amount required by SPI. | 5.1302528% has an adjacent reimbursement flag; the true `EXPS` mass is not estimable. | -| `INCPBEN` | `BENEFITS.BENAMT` where `BENEFIT == 17` | Structurally expressible, but the current FRS has no code-17 rows and therefore no observed monetary signal. | 0% observed mass. | -| `OSSBEN` | `BENEFITS.BENAMT`, `BENEFIT`, and `VAR2`: code 13 and contribution-based code 16 are identifiable; codes 6 and 30 were also searched | **Incomplete.** Carer's Allowance and contribution-based ESA form an identifiable subset, but code 6 mixes tax treatments and code 30 is an undifferentiated catch-all, so the complete taxable family cannot be emitted. | 1.8045088% identifiable lower-bound mass; not counted as support. | -| `TAXTERM` | `ADULT.REDAMT`; `ADULT` and `JOB` searched for a taxable termination split | **Missing.** `REDAMT` is gross redundancy pay and has neither the taxable amount nor non-redundancy termination pay. | 0.3746084% has positive gross redundancy pay; taxable mass is unknown. | -| `UBISJA` | `BENEFITS.BENAMT` where `BENEFIT` is 14 (JSA) or 19 (Income Support) | Available as the annualized source measure. | 0.5378644% has positive source signal. | -| `MOTHINC` | `ODDJOB.OJAMT`/`OJNOW`, `ADULT.ALLPAY2`, `ADULT.ROYYR2`-`ROYYR4`, and `JOB.OWNOTHER` | **Missing.** The fields are heterogeneous and belong to distinct income concepts; assigning their union to SPI miscellaneous employment income would be a proxy. | Odd-job-only mass is 0.1724207%; the broader unresolved miscellaneous pool is 1.4650566%. | -| `OTHERINC` | The same `ADULT`, `ODDJOB`, and `JOB` fields, plus `PENSION`, `ACCOUNTS`, `ASSETS`, and `BENEFITS`, were searched for a distinct residual-income field | **Missing.** No person-level raw FRS variable has SPI `OTHERINC` semantics, and the miscellaneous pool cannot be split between `MOTHINC` and `OTHERINC` from source evidence. | No separable mass estimate; the unresolved pool is 1.4650566%. | -| `SRP` | `BENEFITS.BENAMT` where `BENEFIT == 5`; codes 6 and 9 checked for widow-related amounts | **Incomplete.** Code 5 supplies regular State Pension, but the FRS source does not identify the full SPI combination of State Pension lump sums and widow's pension; code 6 mixes benefits and code 9 is tax-free War Widow's Pension. | 18.1567916% has regular code-5 State Pension; not counted as complete `SRP` support. | - -This establishes the adjudicated source contract: `EPB`, `EXPS`, `TAXTERM`, -`MOTHINC`, and `OTHERINC` have no complete raw FRS source, while `OSSBEN` and -`SRP` are only partial. The retained-leaf stage implements exactly the three -full and two explicitly named subset findings. It does not impute, combine -heterogeneous fields, or promote a subset to a full source concept. These -source fences govern the 208-fact replay; they do not weaken the separately -restored 145-column release-input contract. - -## Real-donor HMRC replay, 2026-07-13 - -The production replay reverified both opaque source identities before either -file was read: the 141,323,762-byte licensed SPI donor at SHA-256 -`5ef829461060c91a2a47be59ad541d9b519fc3976d66ca80d4920f711bb96f66` -and the 166,693-byte official ODS at SHA-256 -`ad063b06b2bdeef8600dbbb09d48153337a4966f8c7eea50df7a2e0304ebd73e`. -The retained FRS sources were also bound by stable verified bytes: -`ADULT` has 28,590 rows and SHA-256 `e09f9647…`; `BENEFITS` has 46,636 rows -and SHA-256 `ff30d054…`. Their source signals are 13,412 `PAY` rows, 163 -`UBISJA` rows, zero observed but structurally wired `INCPBEN` rows, 627 OSSBEN -subset rows, and 8,494 regular-code-5 SRP rows. - -The seam removed all 200,000 dead zero-weight SPI households and rebuilt one -honest SPI channel. It assigned that channel a reviewed 50% share of the -unchanged 28,840,551.182 national household mass, recorded the deliberate -allocation, and retained `IMPORTANCE` output weights. The reviewed seed-42 QRF -fit used 100,000 donor records, trained the FRS-only fill on 72,496 rows, and -produced 432,779 SPI person predictions. `TEI + TII = TI` holds by construction -on all 432,779 predictions. - -The aggregate replay report contains the full 8 components × 13 bands × 2 -measures surface. Its result is **0 exact pass, 0 exact fail, 0 directional -pass, 0 directional fail, and 208 excluded with canonical fences**. Every -estimate and delta for an excluded fact is null. This is intentional: a partial -FRS employment or pension measure cannot assign complete HMRC Total Income -bands, so computing those facts would introduce known but unbounded bias. - -The 1 ppm effective-mass floor rejects dead support and numerical dust while -remaining roughly two orders of magnitude below the rarest populated reference -share. The rebuilt channel exceeds it honestly: `gift_aid` has 12,894 -positive-mass rows and a 0.0133031567 mass share; -`charitable_investment_gifts` has 294 rows and a 0.0002805533 share. Neither is -inferred from raw presence: both promotions are pinned to this weighted SPI -evidence and remain subject to the same floor on every required national build. - -After promotion, the final release gate sees all 145 required columns, no -reviewed or stale exclusions, no missing or degenerate requirement, and no -insufficient effective-mass result. The family-specific gate also confirms the -required SPI channel, `IMPORTANCE` weights, build period 2023, and one valid -mass-conserving `MassChangeRecord`. It passes and writes the ignored staging H5 -(1,532,379,785 bytes; SHA-256 -`829e843f5d1577ff4770ed344c2c15eed4f0c1fdc64af2dc7b511c825fde6709`). - -The committed aggregate artifacts are `hmrc_income_replay_report.json` -(unchanged SHA-256 `32d343ab…`), `hmrc_income_release_gate_report.json` -(SHA-256 `a0856168…`), and `national_staging_build_record.json` (SHA-256 -`2bfb4e71…`). They contain no row-level donor data, donor filename, or local -paths. The 208-fact HMRC comparison remains the adjudicated fenced partial -replay; the release stage and its two restored input columns are now mandatory. diff --git a/changelog.d/757-june-national-driver-retired.changed.md b/changelog.d/757-june-national-driver-retired.changed.md new file mode 100644 index 000000000..522b92ab6 --- /dev/null +++ b/changelog.d/757-june-national-driver-retired.changed.md @@ -0,0 +1 @@ +Retired the June UK national build driver and its HMRC restoration path. The retired driver signed the declared calibration diagnostics digest into release evidence before writing the diagnostics file; the calibration seam now measures the produced file and signs that measured digest instead. diff --git a/experiments/612-uk-carrier-payload-receipt.md b/experiments/612-uk-carrier-payload-receipt.md deleted file mode 100644 index bbcec4fbc..000000000 --- a/experiments/612-uk-carrier-payload-receipt.md +++ /dev/null @@ -1,39 +0,0 @@ -# #612 carrier-swap payload receipt (old writer vs Frame writer) - -Committed receipt for the #618 acceptance claim (review ask on that PR): the -retired shadow-carrier writer and the Frame writer produce **payload-identical** -staging artifacts from the same input, verifiable offline against the digests -below. Companion JSON: `612-uk-carrier-payload-receipt.json`, produced by -`tools/compare_uk_h5_payload.py` (dtype-object comparison, digest-bound — -the post-review hardened version on the #618 branch). - -## Provenance - -- Input to both sides: the certified Microcosm UK candidate - `populace_uk_2023.h5`, revision - `populace-uk-2023-dd68c73-4aa4b14-20260619T023711Z`, sha256 `f17306cc…` - (verified against the pinned `CERTIFIED_UK_CANDIDATE_SHA256` before use). -- **Left** (`roundtrip_old.h5`, sha256 in the JSON): produced on `main` at - `e6be79a` via `load_uk_national_dataset` → `write_uk_national_dataset` - (the shadow-carrier pair this PR retires). -- **Right** (`roundtrip_new.h5`, sha256 in the JSON): produced on - `uk-frame-inc1-carrier-swap` via `load_uk_national_frame` → - `write_uk_national_frame`. -- Run 2026-08-06 on the credentialed build machine; artifacts retained - locally (`retention: local_untracked`), not committed — the digests bind - this receipt to those exact bytes. - -## Verdict - -`payload_identical: true` — same store keys in write order; person -(1,157,100), benunit (618,980), household (535,080), and time_period tables -equal in column order, dtypes (object-level), index, row order, and values; -root attributes equal by raw value. The two files' own sha256 digests differ, -as expected: HDF5 stamps write times, which is why acceptance is defined at -payload level, never byte level. - -The receipt is SDC-safe: it contains schema names, row counts, booleans, and -file digests only — no unit-record values. - -Full acceptance context (gate-report parity, timings, the preflight): -https://github.com/PolicyEngine/microcosm/issues/612 (comments of 2026-08-06). diff --git a/experiments/630-uk-gate-adjudication-receipts.md b/experiments/630-uk-gate-adjudication-receipts.md index a0f7865db..a401bf1de 100644 --- a/experiments/630-uk-gate-adjudication-receipts.md +++ b/experiments/630-uk-gate-adjudication-receipts.md @@ -1,3 +1,5 @@ +> Historicized 2026-08-26: This is the adjudication packet and run-receipt history for the #630 UK gate adjudication. It is superseded as live state by the signed register and the battery; the content below is retained as historical evidence, not current operational status. + # microcosm#630 gate adjudication — measurement and verification receipts Campaign runs for the #630 close-out PR (#706). All full-scale runs: certified input diff --git a/experiments/686-uk-spine-comparison-ledger.md b/experiments/686-uk-spine-comparison-ledger.md index b7b3a180e..484eff13b 100644 --- a/experiments/686-uk-spine-comparison-ledger.md +++ b/experiments/686-uk-spine-comparison-ledger.md @@ -1,3 +1,5 @@ +> Historicized 2026-08-26: This is the adjudication packet and run-receipt history for the #686 UK spine-swap decision. It is superseded as live state by the signed register and the battery; the content below is retained as historical evidence, not current operational status. + # UK spine comparison ledger — microcosm#686 · PR #747 Every variable the E workstream manipulates, measured against the incumbent, diff --git a/experiments/686-uk-spine-swap-receipts.md b/experiments/686-uk-spine-swap-receipts.md index b7b54ea74..deffe5656 100644 --- a/experiments/686-uk-spine-swap-receipts.md +++ b/experiments/686-uk-spine-swap-receipts.md @@ -1,3 +1,5 @@ +> Historicized 2026-08-26: This is the adjudication packet and run-receipt history for the #686 UK spine-swap decision. It is superseded as live state by the signed register and the battery; the content below is retained as historical evidence, not current operational status. + # microcosm#686 whole-spine parity and swap acceptance — measurement receipts Receipts for the E10 increment (WS-E #145, epic #665). Every value below is a diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/__init__.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/__init__.py index 360485af8..6a09ddf1d 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/__init__.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/__init__.py @@ -238,17 +238,8 @@ classify_hmrc_replay_targets, write_hmrc_replay_report, ) -from microcosm.build.uk_runtime.hmrc_restoration import ( - CERTIFIED_UK_CANDIDATE_SHA256, - CERTIFIED_UK_CANDIDATE_TIER, - HMRC_DISTRIBUTIONAL_INPUTS, - UKCertifiedCandidateIdentity, - UKHMRCIncomeRestorationResult, - UKHMRCIncomeStageTransform, - restore_uk_hmrc_income_family, - verify_certified_uk_candidate, -) from microcosm.build.uk_runtime.hmrc_source_contract import ( + HMRC_DISTRIBUTIONAL_INPUTS, UK_HMRC_INCOME_SOURCE_STAGES_RESOURCE, assert_uk_hmrc_income_source_contract_current, ) @@ -260,7 +251,6 @@ from microcosm.build.uk_runtime.ledger_targets import ( UKFrameTargetAdapter, UKLedgerTargetCompilation, - UKPolicyEngineAdapter, compile_uk_target_registry, materialize_uk_ledger_targets, ) @@ -308,13 +298,6 @@ metric_names_from_target_profile, metric_tables_by_area_group, ) -from microcosm.build.uk_runtime.national_build import ( - UKNationalBuildResult, - UKNationalStage, - build_uk_national_dataset, - load_uk_national_frame, - write_uk_national_frame, -) from microcosm.build.uk_runtime.national_doctrine import ( UK_NATIONAL_L0_LAMBDA, UK_NATIONAL_LEARNING_RATE, @@ -331,11 +314,14 @@ ) from microcosm.build.uk_runtime.national_frame import ( UK_NATIONAL_SCHEMA, + UKNationalStage, UKStagingProvenance, + load_uk_national_frame, uk_household_weight_kind, uk_national_frame, uk_time_period, validate_uk_national_frame, + write_uk_national_frame, ) from microcosm.build.uk_runtime.oa_ladder_sources import ( LADDER_OA_COLUMNS, @@ -529,7 +515,6 @@ "UK_FISCAL_TARGET_REGISTRY", "UKFrameTargetAdapter", "UKLedgerTargetCompilation", - "UKPolicyEngineAdapter", "compile_uk_target_registry", "materialize_uk_ledger_targets", "AGE_BANDS", @@ -665,12 +650,8 @@ "UKLocalSolveDoctrine", "UKRowwiseLocalMatrix", "RESTORED_REFERENCE_EFRS_REQUIRED_INPUTS", - "UKCertifiedCandidateIdentity", "UKHMRCIncomeCalibration", - "UKHMRCIncomeRestorationResult", - "UKHMRCIncomeStageTransform", "UKHMRCTargetMaterialization", - "UKNationalBuildResult", "UKNationalSolveDoctrine", "UKStagingProvenance", "UK_NATIONAL_L0_LAMBDA", @@ -694,8 +675,6 @@ "UKSPISupportResult", "UKSPIIncomeImputationResult", "UKSPIIncomeSpineResult", - "CERTIFIED_UK_CANDIDATE_SHA256", - "CERTIFIED_UK_CANDIDATE_TIER", "UK_SINGLE_YEAR_TABLES", "UK_FRS_HMRC_SPINE_LEAVES_STAGE_NAME", "UK_HMRC_SPI_INCOME_SPINE_STAGE_NAME", @@ -728,7 +707,6 @@ "build_conservative_hmrc_replay_report", "build_uk_spi_support_channel", "build_uk_local_target_census", - "build_uk_national_dataset", "calibrate_uk_hmrc_income", "build_complete_uk_geography_crosswalk", "build_england_wales_crosswalk", @@ -813,7 +791,6 @@ "prepare_geography_crosswalk", "read_uk_single_year_weight_metadata", "read_uk_firm_source_data", - "restore_uk_hmrc_income_family", "solve_firm_weights", "past_cap_census", "rowwise_area_support_summary", @@ -847,7 +824,6 @@ "validate_uk_ladder_rowwise_dataset_tables", "validate_uk_rowwise_dataset_tables", "validate_uk_release_tier", - "verify_certified_uk_candidate", "verify_hmrc_spi_collated_ods", "verify_spi_donor_identity", "write_uk_local_target_census", diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/calibration_run.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/calibration_run.py index d3296f80a..416e1ad62 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/calibration_run.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/calibration_run.py @@ -174,7 +174,10 @@ def _scope_exclusions() -> dict[str, str]: if gate_id in spine: reason = "spine-construction gate; owned by the spine build's scoped battery." elif gate_id in national: - reason = "national build gate; owned by the national preflight/terminal battery." + reason = ( + "owned by the release-cut certification producer; runner lands " + "with the certification, June runner retired" + ) elif "parity" in gate_id or gate_id in _SWAP_ACCEPTANCE_GATE_IDS: reason = "swap-acceptance evidence; produced by the swap lane, not the calibration seam." else: diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_imputation.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_imputation.py index b369ce508..ca634f9ef 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_imputation.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_imputation.py @@ -70,8 +70,8 @@ HMRCCapitalGainsJointDistribution, materialize_hmrc_capital_gains_joint_distribution, ) -from microcosm.build.uk_runtime.national_build import UKNationalStage from microcosm.build.uk_runtime.national_frame import ( + UKNationalStage, uk_household_weight_kind, uk_national_frame, uk_time_period, diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/hmrc_restoration.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/hmrc_restoration.py deleted file mode 100644 index 520bb3669..000000000 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/hmrc_restoration.py +++ /dev/null @@ -1,926 +0,0 @@ -"""Guarded real-donor HMRC replay for the UK national build. - -The current FRS instrument cannot materialize the complete HMRC Total Income -measure used to assign the 13 published bands. The national stage therefore -does the source-faithful work that remains admissible -- retain the adjudicated -FRS leaves, rebuild one positive-mass SPI channel, run both weighted QRFs, and -derive the SPI accounting identities exactly -- but it does not calibrate to -non-comparable band facts. All 208 published facts are carried into an -aggregate replay report with the reviewed source fences. -""" - -from __future__ import annotations - -import hashlib -import re -from collections.abc import Mapping -from dataclasses import dataclass, field -from pathlib import Path - -import numpy as np -import pandas as pd - -from microcosm.build.gates import FitWeightRecord -from microcosm.build.uk_runtime.content_identity import uk_frame_content_identity -from microcosm.build.uk_runtime.frs_hmrc_leaves import ( - UKFRSHMRCRetainedLeavesStageTransform, -) -from microcosm.build.uk_runtime.hmrc_income import ( - HMRCIncomeTargetSet, - materialize_hmrc_spi_income_band_targets, - verify_hmrc_spi_collated_ods, -) -from microcosm.build.uk_runtime.hmrc_replay import ( - HMRCReplayReport, - build_conservative_hmrc_replay_report, -) -from microcosm.build.uk_runtime.hmrc_source_contract import ( - HMRC_DISTRIBUTIONAL_INPUTS, - assert_uk_hmrc_income_source_contract_current, -) -from microcosm.build.uk_runtime.national_frame import ( - UKStagingProvenance, - _uk_source_file_fingerprint, - _UKSourceFileFingerprint, - uk_household_weight_kind, - uk_national_frame, - uk_time_period, - validate_uk_national_frame, -) -from microcosm.build.uk_runtime.release_identity import UK_RELEASE_TIER_FRS -from microcosm.build.uk_runtime.release_input_coverage import ( - DEFAULT_MINIMUM_NONDEFAULT_MASS_SHARE, -) -from microcosm.build.uk_runtime.spi_income import ( - DEFAULT_SPI_DONOR_SAMPLE_SIZE, - SPI_SOURCE_TI_FORMULA, - UKSPIIncomeImputationResult, - assert_frs_hmrc_auxiliary_crosswalk_available, - impute_uk_spi_income_support, - verify_spi_donor_identity, -) -from microcosm.build.uk_runtime.spi_support import ( - DEFAULT_SPI_PRIOR_MASS_SHARE, - SPI_HMRC_TOTAL_EARNED_INCOME_COLUMN, - SPI_HMRC_TOTAL_INVESTMENT_INCOME_COLUMN, - SPI_SYNTHETIC_SUPPORT_CHANNEL, - UKSPISupportResult, - replace_uk_spi_support_tables, - support_channel_column, -) -from microcosm.frame import Frame, WeightKind, engine_tables - -__all__ = [ - "CERTIFIED_UK_CANDIDATE_FILENAME", - "CERTIFIED_UK_CANDIDATE_REVISION", - "CERTIFIED_UK_CANDIDATE_SHA256", - "CERTIFIED_UK_CANDIDATE_SIZE_BYTES", - "CERTIFIED_UK_CANDIDATE_TIER", - "HMRC_DISTRIBUTIONAL_INPUTS", - "UKCertifiedCandidateIdentity", - "UKHMRCIncomeRestorationResult", - "UKHMRCIncomeStageTransform", - "assert_uk_hmrc_income_source_contract_current", - "restore_uk_hmrc_income_family", - "verify_certified_uk_candidate", -] - -CERTIFIED_UK_CANDIDATE_FILENAME = "populace_uk_2023.h5" -CERTIFIED_UK_CANDIDATE_REVISION = "populace-uk-2023-dd68c73-4aa4b14-20260619T023711Z" -CERTIFIED_UK_CANDIDATE_TIER = UK_RELEASE_TIER_FRS -CERTIFIED_UK_CANDIDATE_SHA256 = ( - "f17306ccb2aad7ff0130be3589b560afb2e2a12a943570911cd0c77f07934833" -) -CERTIFIED_UK_CANDIDATE_SIZE_BYTES = 1_315_880_118 -STAGING_CANDIDATE_TIER = "staging_candidate" -STAGING_CANDIDATE_REVISION = "non_certified_staging_candidate" -_CERTIFIED_CANDIDATE_VERIFICATION_TOKEN = object() - - -@dataclass(frozen=True) -class UKCertifiedCandidateIdentity: - """Verified identity of the only accepted HMRC replay base.""" - - path: Path - filename: str - tier: str - revision: str - sha256: str - size_bytes: int - _verification_token: object | None = field( - default=None, - init=False, - repr=False, - compare=False, - ) - _source_file_fingerprint: _UKSourceFileFingerprint | None = field( - default=None, - init=False, - repr=False, - compare=False, - ) - - -@dataclass(frozen=True) -class UKHMRCIncomeRestorationResult: - """Importance-weight replay frame plus aggregate-only evidence.""" - - frame: Frame - support: UKSPISupportResult - imputation: UKSPIIncomeImputationResult - source_targets: HMRCIncomeTargetSet - replay_report: HMRCReplayReport - distributional_mass_shares: Mapping[str, float] - post_draw_identity_rows: int - - def evidence(self) -> dict[str, object]: - """Return JSON-safe aggregate evidence for the national driver.""" - - return { - "stage": "hmrc_spi_income", - "source_vintages": { - "spi_donor": "2022-23", - "hmrc_surface": self.source_targets.source.source_vintage, - "mapped_build_period": self.source_targets.source.build_period, - }, - "sources": { - "spi_donor": { - "path": str(self.imputation.donor_path), - "sha256": self.imputation.donor_sha256, - "size_bytes": self.imputation.donor_size_bytes, - "rows_used": self.imputation.donor_rows, - }, - "hmrc_surface": { - "path": str(self.source_targets.source.local_path), - "sha256": self.source_targets.source.sha256, - "size_bytes": self.source_targets.source.size_bytes, - "mime_type": self.source_targets.source.mime_type, - "publication_url": self.source_targets.source.publication_url, - "ods_url": self.source_targets.source.ods_url, - "tables": list(self.source_targets.source.table_names), - }, - }, - "spi_prior": { - "replaced_households": self.support.replaced_spi_households, - "mass_share": self.support.spi_prior_mass_share, - "weight_kind": self.support.household_weight_kind.value, - "mass_change_reason": self.support.mass_log[-1].reason, - }, - "qrf_fits": [ - { - "fit_name": record.fit_name, - "weight_kind": record.weight_kind, - } - for record in self.imputation.fit_weight_records - ], - "reviewed_absent_stage2_outputs": dict( - self.imputation.reviewed_absent_stage2_outputs - ), - "post_draw_identity": { - "formula": SPI_SOURCE_TI_FORMULA, - "rows_checked": self.post_draw_identity_rows, - "exact": True, - }, - "targets": { - "count": len(self.source_targets.targets), - "classification": dict(self.replay_report.summary), - }, - "calibration": { - "performed": False, - "reason": ( - "Complete FRS Total Income band assignment is unavailable; " - "the 208 facts are reviewed exclusions rather than biased " - "calibration constraints." - ), - "output_weight_kind": uk_household_weight_kind(self.frame).value, - }, - "effective_mass_coverage": { - "minimum_nondefault_mass_share": ( - DEFAULT_MINIMUM_NONDEFAULT_MASS_SHARE - ), - "columns": dict(self.distributional_mass_shares), - }, - } - - -@dataclass -class UKHMRCIncomeStageTransform: - """Callable national-stage adapter retaining the last replay evidence.""" - - spi_tab_path: Path - hmrc_ods_path: Path - certified_candidate: UKCertifiedCandidateIdentity - retained_leaves_transform: UKFRSHMRCRetainedLeavesStageTransform | None = None - seed: int = 42 - qrf_estimators: int = 100 - donor_sample_size: int | None = DEFAULT_SPI_DONOR_SAMPLE_SIZE - spi_prior_mass_share: float = DEFAULT_SPI_PRIOR_MASS_SHARE - #: Declared #627 rung build: the mid-stage effective-mass floor defers - #: to the terminal input-coverage gate. Never set on a release build. - sampled_rung: bool = False - last_result: UKHMRCIncomeRestorationResult | None = field( - default=None, - init=False, - ) - staging_provenance: UKStagingProvenance | None = field( - default=None, - init=False, - ) - bound_input_identity: str | None = field( - default=None, - init=False, - repr=False, - ) - - @property - def fit_weight_records(self) -> tuple[FitWeightRecord, ...]: - """Return immutable fit-weight evidence from the most recent run.""" - - if self.last_result is None: - return () - return tuple(self.last_result.imputation.fit_weight_records) - - def checkpoint_metadata(self) -> dict[str, object]: - """JSON-safe evidence the stage checkpoint carries for a resume. - - Everything a later process consumes without re-running the stage: - the fit-weight records the terminal weights audit reads, and the - aggregate family evidence and replay-report payload the national - driver writes as stage sidecars (the adversarial-review blocker: a - resumed run must still produce byte-identical evidence reports). - """ - - if self.last_result is None: - raise RuntimeError( - "checkpoint metadata requires a completed SPI restoration run." - ) - return { - "fit_weight_records": [ - {"fit_name": record.fit_name, "weight_kind": record.weight_kind} - for record in self.last_result.imputation.fit_weight_records - ], - "evidence": self.last_result.evidence(), - "replay_payload": self.last_result.replay_report.to_payload(), - "output_content_identity": uk_frame_content_identity( - self.last_result.frame - ), - } - - def resume_from_checkpoint( - self, - metadata: Mapping[str, object], - frame: Frame, - ) -> None: - """Rehydrate a completed run's evidence surface from its record. - - ``frame`` is the stage's checkpointed output; the recorded output - identity must match its content — the same drift check the - retained-leaves stage runs, and this is the terminal stage, whose - frame feeds the batched gates and the staging writer. The runtime's - checkpoint sha covers the bytes on disk; this check covers the - record/frame pairing. - """ - - payload = metadata.get("fit_weight_records") - evidence = metadata.get("evidence") - replay_payload = metadata.get("replay_payload") - output_identity = metadata.get("output_content_identity") - if ( - not isinstance(payload, list) - or not all(isinstance(entry, Mapping) for entry in payload) - or not isinstance(evidence, Mapping) - or not isinstance(replay_payload, Mapping) - or not isinstance(output_identity, str) - or not output_identity - ): - raise RuntimeError( - "SPI restoration resume requires the checkpoint record to " - "carry the run's fit-weight records, family evidence, " - "replay payload, and output content identity; a record " - "without them cannot feed the weights audit, the driver's " - "stage reports, or the drift check." - ) - if uk_frame_content_identity(frame) != output_identity: - raise RuntimeError( - "SPI restoration checkpoint content does not match its " - "recorded output identity; refusing to resume from a " - "drifted record." - ) - records = tuple( - FitWeightRecord( - fit_name=str(entry["fit_name"]), - weight_kind=str(entry["weight_kind"]), - ) - for entry in payload - ) - self.last_result = _ResumedHMRCRestoration( - frame=frame, - imputation=_ResumedImputationEvidence(fit_weight_records=records), - evidence_payload=dict(evidence), - replay_payload=dict(replay_payload), - ) - # The rehydration consumes the single-use binding: the stage will - # not run, so a binding must not outlive the resume either. - self.staging_provenance = None - self.bound_input_identity = None - - def bind_staging_provenance( - self, - provenance: UKStagingProvenance, - frame: Frame, - ) -> None: - """Receive the load provenance and the loaded frame from the driver. - - Provenance travels beside the frame, never inside it, so the driver - hands both to the one stage whose fence binds the loaded bytes to the - verified certified candidate. Binding records the loaded frame's - content identity — derived here, inside the attesting code — so the - fence can require that the pipeline it sits in started from a frame - whose full content matches what the driver loaded, a guarantee that - survives a process boundary where object identity cannot. - """ - - if not isinstance(provenance, UKStagingProvenance): - raise TypeError("staging provenance must be UKStagingProvenance.") - if not isinstance(frame, Frame): - raise TypeError("bound frame must be a microcosm Frame.") - self.staging_provenance = provenance - self.bound_input_identity = uk_frame_content_identity(frame) - - def __call__(self, frame: Frame) -> Frame: - # Single-use: a binding never outlives the run that consumes it, so - # a stale binding from an earlier build can never fence a later one. - staging_provenance = self.staging_provenance - bound_input_identity = self.bound_input_identity - self.staging_provenance = None - self.bound_input_identity = None - retained = ( - None - if self.retained_leaves_transform is None - else self.retained_leaves_transform.last_result - ) - if retained is None: - raise RuntimeError( - "HMRC replay requires the raw-FRS retained-leaves stage to run " - "immediately before the SPI stage." - ) - # Descent fence A, content-addressed: the frame this stage received - # must carry the exact content the retained-leaves stage produced. - # Same-object is the free fast path (identity proves content); a - # rehydrated (checkpointed) frame passes by content; a substituted - # or tampered frame does not. The content check — and the - # absence-is-refusal branch inside _retained_content_identity — is - # the cross-process path's guarantee; the fast path shares the - # same-process in-place-mutation exposure every consumer of - # Frame.table has, mitigated by validate_uk_national_frame's - # revalidation at each seam. - if retained.frame is not frame and _retained_content_identity( - retained, "output_content_identity" - ) != uk_frame_content_identity(frame): - raise RuntimeError( - "HMRC replay raw-FRS evidence is not bound to the frame " - "received from the immediately preceding retained-leaves stage." - ) - # Descent fence B: the frame the retained-leaves stage consumed must - # carry the exact content of the frame the driver loaded and bound. - if ( - bound_input_identity is not None - and _retained_content_identity(retained, "input_content_identity") - != bound_input_identity - ): - raise RuntimeError( - "HMRC replay pipeline did not start from the frame the " - "driver loaded and bound; the certified-candidate fence " - "refuses a substituted input." - ) - self.last_result = restore_uk_hmrc_income_family( - frame, - spi_tab_path=self.spi_tab_path, - hmrc_ods_path=self.hmrc_ods_path, - certified_candidate=self.certified_candidate, - staging_provenance=staging_provenance, - frs_source_evidence=retained.evidence(), - seed=self.seed, - qrf_estimators=self.qrf_estimators, - donor_sample_size=self.donor_sample_size, - spi_prior_mass_share=self.spi_prior_mass_share, - sampled_rung=self.sampled_rung, - ) - return self.last_result.frame - - -@dataclass(frozen=True) -class _ResumedImputationEvidence: - """The audit slice of an SPI imputation, rehydrated from a checkpoint.""" - - fit_weight_records: tuple[FitWeightRecord, ...] - - -@dataclass(frozen=True) -class _ResumedHMRCRestoration: - """A completed SPI restoration rehydrated from its checkpoint record. - - Exposes the full surface a national build consumes downstream: the - fit-weight audit records, the aggregate family evidence, and the replay - payload the driver writes verbatim (its content came from the real - report's ``to_payload()`` at completion time). - """ - - frame: Frame - imputation: _ResumedImputationEvidence - evidence_payload: dict[str, object] - replay_payload: dict[str, object] - - def evidence(self) -> dict[str, object]: - return dict(self.evidence_payload) - - -def _retained_content_identity(retained: object, attribute: str) -> str: - """Read a content identity off a retained-leaves result, failing closed. - - A retained result without content identities cannot prove descent, so - the fence refuses it instead of silently downgrading to a weaker check - (the microcosm#617 lesson: absence is a refusal, never a default). - """ - - identity = getattr(retained, attribute, None) - if not isinstance(identity, str) or not identity: - raise RuntimeError( - "HMRC replay retained-leaves evidence carries no " - f"{attribute}; the descent fence refuses a result that cannot " - "prove which frames its run consumed and produced." - ) - return identity - - -def verify_certified_uk_candidate(path: str | Path) -> UKCertifiedCandidateIdentity: - """Hash/size gate the certified Microcosm UK candidate before stages run.""" - - candidate = Path(path).expanduser().resolve() - if not candidate.is_file(): - raise FileNotFoundError( - f"Certified Microcosm UK candidate not found: {candidate}." - ) - fingerprint_before = _uk_source_file_fingerprint(candidate) - size = fingerprint_before.size_bytes - if size != CERTIFIED_UK_CANDIDATE_SIZE_BYTES: - raise ValueError( - f"{candidate}: expected certified candidate size " - f"{CERTIFIED_UK_CANDIDATE_SIZE_BYTES}, got {size}." - ) - digest = _sha256(candidate) - fingerprint_after = _uk_source_file_fingerprint(candidate) - if fingerprint_after != fingerprint_before: - raise RuntimeError( - "Certified Microcosm UK candidate changed while its SHA-256 was " - "being verified." - ) - if digest != CERTIFIED_UK_CANDIDATE_SHA256: - raise ValueError( - f"{candidate}: sha256 {digest} does not match certified candidate " - f"{CERTIFIED_UK_CANDIDATE_SHA256}." - ) - identity = UKCertifiedCandidateIdentity( - path=candidate, - filename=CERTIFIED_UK_CANDIDATE_FILENAME, - tier=CERTIFIED_UK_CANDIDATE_TIER, - revision=CERTIFIED_UK_CANDIDATE_REVISION, - sha256=digest, - size_bytes=size, - ) - object.__setattr__( - identity, - "_verification_token", - _CERTIFIED_CANDIDATE_VERIFICATION_TOKEN, - ) - object.__setattr__(identity, "_source_file_fingerprint", fingerprint_after) - return identity - - -def verify_staging_candidate_uk_input( - path: str | Path, - *, - expected_sha256: str, -) -> UKCertifiedCandidateIdentity: - """Hash-gate a declared non-certified staging-candidate input.""" - - candidate = Path(path).expanduser().resolve() - if not candidate.is_file(): - raise FileNotFoundError(f"UK staging-candidate input not found: {candidate}.") - expected = str(expected_sha256) - if not re.fullmatch(r"[0-9a-f]{64}", expected): - raise ValueError( - "--staging-candidate-input-sha256 must be a lowercase SHA-256." - ) - fingerprint_before = _uk_source_file_fingerprint(candidate) - digest = _sha256(candidate) - fingerprint_after = _uk_source_file_fingerprint(candidate) - if fingerprint_after != fingerprint_before: - raise RuntimeError( - "UK staging-candidate input changed while its SHA-256 was being verified." - ) - if digest != expected: - raise ValueError( - f"{candidate}: sha256 {digest} does not match declared " - f"staging-candidate input {expected}." - ) - identity = UKCertifiedCandidateIdentity( - path=candidate, - filename=candidate.name, - tier=STAGING_CANDIDATE_TIER, - revision=STAGING_CANDIDATE_REVISION, - sha256=digest, - size_bytes=fingerprint_after.size_bytes, - ) - object.__setattr__( - identity, - "_verification_token", - _CERTIFIED_CANDIDATE_VERIFICATION_TOKEN, - ) - object.__setattr__(identity, "_source_file_fingerprint", fingerprint_after) - return identity - - -def restore_uk_hmrc_income_family( - frame: Frame, - *, - spi_tab_path: str | Path, - hmrc_ods_path: str | Path, - certified_candidate: UKCertifiedCandidateIdentity, - staging_provenance: UKStagingProvenance | None = None, - frs_source_evidence: Mapping[str, object], - seed: int = 42, - qrf_estimators: int = 100, - donor_sample_size: int | None = DEFAULT_SPI_DONOR_SAMPLE_SIZE, - spi_prior_mass_share: float = DEFAULT_SPI_PRIOR_MASS_SHARE, - sampled_rung: bool = False, -) -> UKHMRCIncomeRestorationResult: - """Run the admissible real-donor replay without biased calibration. - - ``sampled_rung`` declares a #627 scale-ladder build: sparse imputed - columns can legitimately restore near-zero effective mass on a small - sample, so the mid-stage effective-mass floor defers to the terminal - input-coverage gate — which evaluates the same surface and records a - receipted verdict — instead of aborting the build. The per-column - shares reach the replay report either way. Full-scale builds keep the - strict raise. - """ - - assert_uk_hmrc_income_source_contract_current() - _validate_certified_candidate_identity(certified_candidate) - _assert_reviewed_release_parameters( - donor_sample_size=donor_sample_size, - spi_prior_mass_share=spi_prior_mass_share, - ) - if not isinstance(frs_source_evidence, Mapping) or not frs_source_evidence: - raise ValueError("HMRC replay requires non-empty raw-FRS source evidence.") - validate_uk_national_frame(frame) - _assert_provenance_matches_certified_candidate( - staging_provenance, certified_candidate - ) - time_period = uk_time_period(frame) - - # The licensed donor and official ODS are one reviewed source pair. Bind - # both identities before parsing either source or rebuilding support. - verified_donor = verify_spi_donor_identity(spi_tab_path) - verified_ods = verify_hmrc_spi_collated_ods(hmrc_ods_path) - assert_frs_hmrc_auxiliary_crosswalk_available(frame.table("person")) - source_targets = materialize_hmrc_spi_income_band_targets( - verified_ods, - build_period=time_period, - ) - - tables = engine_tables(frame, weighted_entities=("household",)) - support = replace_uk_spi_support_tables( - person=frame.table("person"), - benunit=frame.table("benunit"), - household=tables["household"], - seed=seed, - source_year=int(time_period), - spi_prior_mass_share=spi_prior_mass_share, - input_weight_kind=uk_household_weight_kind(frame), - mass_log=frame.mass_log, - ) - imputation = impute_uk_spi_income_support( - support, - spi_tab_path, - seed=seed, - n_estimators=qrf_estimators, - donor_sample_size=donor_sample_size, - build_period=time_period, - verified_donor=verified_donor, - ) - # The SPI replacement reshapes tables AND advances the weight kind, so the - # stage hard-constructs its output frame with the support seam's reviewed - # mass log (conservation-checked, factor 1.0). - replay_frame = uk_national_frame( - person=imputation.person, - benunit=support.benunit, - household=support.household, - time_period=time_period, - weight_kind=WeightKind.IMPORTANCE, - mass_log=support.mass_log, - ) - validate_uk_national_frame(replay_frame) - identity_rows = _assert_post_draw_identity(replay_frame) - distributional_mass_shares = _distributional_mass_shares(replay_frame) - insufficient = { - name: share - for name, share in distributional_mass_shares.items() - if share < DEFAULT_MINIMUM_NONDEFAULT_MASS_SHARE - } - if insufficient and not sampled_rung: - raise RuntimeError( - "Rebuilt SPI channel did not restore required effective-mass " - f"coverage: {insufficient}." - ) - - source_evidence = { - "certified_candidate": { - "filename": certified_candidate.filename, - "tier": certified_candidate.tier, - "revision": certified_candidate.revision, - "sha256": certified_candidate.sha256, - "size_bytes": certified_candidate.size_bytes, - }, - "raw_frs_retained_leaves": _aggregate_frs_source_evidence(frs_source_evidence), - "spi_donor": { - "release": "2022-23", - "sha256": imputation.donor_sha256, - "size_bytes": imputation.donor_size_bytes, - "rows_used": imputation.donor_rows, - }, - "hmrc_surface": { - "vintage": source_targets.source.source_vintage, - "mapped_build_period": source_targets.source.build_period, - "sha256": source_targets.source.sha256, - "size_bytes": source_targets.source.size_bytes, - "mime_type": source_targets.source.mime_type, - "tables": list(source_targets.source.table_names), - }, - } - build_evidence = { - "stage": "hmrc_spi_income", - "output_weight_kind": uk_household_weight_kind(replay_frame).value, - "calibration_performed": False, - "spi_prior_mass_share": support.spi_prior_mass_share, - "replaced_spi_households": support.replaced_spi_households, - "mass_change_reason": support.mass_log[-1].reason, - } - qrf_evidence = { - "fits": { - record.fit_name: {"weight_kind": record.weight_kind} - for record in imputation.fit_weight_records - }, - "donor_rows": imputation.donor_rows, - "stage2_training_rows": imputation.stage2_training_rows, - "spi_prediction_rows": imputation.spi_prediction_rows, - "post_draw_identity": { - "formula": SPI_SOURCE_TI_FORMULA, - "rows_checked": identity_rows, - "exact": True, - }, - } - effective_mass_evidence = { - "minimum_nondefault_mass_share": DEFAULT_MINIMUM_NONDEFAULT_MASS_SHARE, - "denominator": "all_person_effective_mass", - "required_support_channel": SPI_SYNTHETIC_SUPPORT_CHANNEL, - "columns": distributional_mass_shares, - } - report = build_conservative_hmrc_replay_report( - source_targets, - source_evidence=source_evidence, - build_evidence=build_evidence, - qrf_evidence=qrf_evidence, - effective_mass_evidence=effective_mass_evidence, - ) - return UKHMRCIncomeRestorationResult( - frame=replay_frame, - support=support, - imputation=imputation, - source_targets=source_targets, - replay_report=report, - distributional_mass_shares=distributional_mass_shares, - post_draw_identity_rows=identity_rows, - ) - - -def _aggregate_frs_source_evidence( - evidence: Mapping[str, object], -) -> dict[str, object]: - """Drop machine-local paths while retaining aggregate source identities.""" - - result = dict(evidence) - raw_sources = result.get("sources") - if isinstance(raw_sources, Mapping): - result["sources"] = { - str(name): { - str(key): value for key, value in dict(source).items() if key != "path" - } - for name, source in raw_sources.items() - if isinstance(source, Mapping) - } - return result - - -def _assert_post_draw_identity(frame: Frame) -> int: - """Require deterministic TEI + TII = TI on every rebuilt SPI draw.""" - - person = frame.table("person") - channel = support_channel_column("person") - required = ( - channel, - SPI_HMRC_TOTAL_EARNED_INCOME_COLUMN, - SPI_HMRC_TOTAL_INVESTMENT_INCOME_COLUMN, - "hmrc_spi_assessable_income", - ) - missing = sorted(set(required) - set(person.columns)) - if missing: - raise RuntimeError( - f"HMRC replay omitted post-draw identity column(s): {missing}." - ) - spi = person[channel].eq(SPI_SYNTHETIC_SUPPORT_CHANNEL).to_numpy(dtype=bool) - if not spi.any(): - raise RuntimeError("HMRC replay contains no rebuilt SPI person draws.") - numeric = person.loc[ - spi, - [ - SPI_HMRC_TOTAL_EARNED_INCOME_COLUMN, - SPI_HMRC_TOTAL_INVESTMENT_INCOME_COLUMN, - "hmrc_spi_assessable_income", - ], - ].apply(pd.to_numeric, errors="coerce") - values = numeric.to_numpy(dtype=float) - if not np.isfinite(values).all(): - raise RuntimeError("HMRC post-draw identity contains non-finite values.") - if not np.array_equal( - numeric["hmrc_spi_assessable_income"].to_numpy(dtype=float), - numeric[SPI_HMRC_TOTAL_EARNED_INCOME_COLUMN].to_numpy(dtype=float) - + numeric[SPI_HMRC_TOTAL_INVESTMENT_INCOME_COLUMN].to_numpy(dtype=float), - ): - raise RuntimeError("HMRC TI must equal deterministic TEI + TII exactly.") - return int(spi.sum()) - - -def _distributional_mass_shares(frame: Frame) -> dict[str, float]: - """Audit charitable signal on strictly positive rebuilt-SPI mass.""" - - person = frame.table("person") - person_channel = support_channel_column("person") - if person_channel not in person: - raise RuntimeError( - "Cannot audit HMRC distributional inputs without person support " - "channel provenance." - ) - spi_people = ( - person[person_channel].eq(SPI_SYNTHETIC_SUPPORT_CHANNEL).to_numpy(dtype=bool) - ) - if not spi_people.any(): - raise RuntimeError("Rebuilt HMRC family contains no SPI support people.") - household = frame.table("household") - household_weights = pd.Series( - frame.weights_for("household").values, - index=household["household_id"].to_numpy(), - ) - mapped = pd.to_numeric( - person["person_household_id"].map(household_weights), - errors="coerce", - ).to_numpy(dtype=float, na_value=np.nan) - if not np.isfinite(mapped).all() or (mapped < 0.0).any(): - raise RuntimeError( - "Cannot audit HMRC distributional inputs without finite, " - "non-negative person mass." - ) - positive = mapped > 0.0 - total = float(mapped[positive].sum()) - if total <= 0.0: - raise RuntimeError("HMRC distributional audit has no positive person mass.") - shares: dict[str, float] = {} - for column in HMRC_DISTRIBUTIONAL_INPUTS: - if column not in person: - raise RuntimeError(f"HMRC stage omitted distributional input {column!r}.") - values = pd.to_numeric(person[column], errors="coerce").to_numpy( - dtype=float, - na_value=np.nan, - ) - if not np.isfinite(values).all(): - raise RuntimeError(f"HMRC distributional input {column!r} is non-finite.") - shares[column] = ( - float(mapped[positive & spi_people & (values != 0.0)].sum()) / total - ) - return shares - - -def _sha256(path: Path) -> str: - digest = hashlib.sha256() - with path.open("rb") as source: - for chunk in iter(lambda: source.read(1024 * 1024), b""): - digest.update(chunk) - return digest.hexdigest() - - -def _validate_certified_candidate_identity( - identity: UKCertifiedCandidateIdentity, -) -> None: - if not isinstance(identity, UKCertifiedCandidateIdentity): - raise TypeError("HMRC replay requires a verified UKCertifiedCandidateIdentity.") - if identity._verification_token is not _CERTIFIED_CANDIDATE_VERIFICATION_TOKEN: - raise ValueError( - "HMRC replay candidate identity must come from " - "verify_certified_uk_candidate; matching metadata fields alone are " - "not verified source evidence." - ) - if identity._source_file_fingerprint is None: - raise ValueError( - "HMRC replay candidate identity lacks verified source-file provenance." - ) - expected = ( - CERTIFIED_UK_CANDIDATE_FILENAME, - CERTIFIED_UK_CANDIDATE_TIER, - CERTIFIED_UK_CANDIDATE_REVISION, - CERTIFIED_UK_CANDIDATE_SHA256, - CERTIFIED_UK_CANDIDATE_SIZE_BYTES, - ) - actual = ( - identity.filename, - identity.tier, - identity.revision, - identity.sha256, - identity.size_bytes, - ) - if identity.tier == STAGING_CANDIDATE_TIER: - # A declared staging-candidate input is deliberately not the certified - # artifact, so the certified-pin equality below cannot apply. The bytes - # are still bound, by the two checks above rather than by this branch: - # the verification token is a module-private sentinel that only - # verify_certified_uk_candidate and verify_staging_candidate_uk_input - # stamp, and the latter hashes the file against a mandatory declared - # sha256 (re-reading the fingerprint to catch a mid-read swap) and - # refuses on mismatch; the source-file fingerprint is required of both - # tiers. What this tier declares is "the file the operator named", and - # the build record labels it non_certified_staging_candidate while - # --staging-candidate-input-sha256 is refused for release candidates. - if identity.revision != STAGING_CANDIDATE_REVISION: - raise ValueError( - "UK staging-candidate input identity has an invalid revision " - f"{identity.revision!r}." - ) - return - if actual != expected: - raise ValueError( - "HMRC replay base identity does not match the certified Microcosm UK " - "candidate contract." - ) - - -def _assert_provenance_matches_certified_candidate( - provenance: UKStagingProvenance | None, - identity: UKCertifiedCandidateIdentity, -) -> None: - """Bind the loaded bytes to the H5 verified once by the driver. - - The provenance record is produced by ``load_uk_national_frame`` beside the - frame; the national build driver hands it to this stage through - ``bind_staging_provenance``, so an unbound run — a frame that did not come - through the loader — fails closed here. - """ - - if provenance is None: - raise ValueError( - "HMRC replay requires the staging provenance of a UK national " - "frame loaded from the verified certified-candidate H5." - ) - if provenance.source_h5 != identity.path: - raise ValueError( - "HMRC replay frame source does not match the verified certified " - f"candidate: loaded {provenance.source_h5}, verified {identity.path}." - ) - if provenance.fingerprint != identity._source_file_fingerprint: - raise ValueError( - "HMRC replay candidate H5 changed after SHA-256 verification; the " - "loaded bytes are not the certified bytes." - ) - - -def _assert_reviewed_release_parameters( - *, - donor_sample_size: int | None, - spi_prior_mass_share: float, -) -> None: - reviewed = { - "donor_sample_size": (donor_sample_size, DEFAULT_SPI_DONOR_SAMPLE_SIZE), - "spi_prior_mass_share": ( - spi_prior_mass_share, - DEFAULT_SPI_PRIOR_MASS_SHARE, - ), - } - drifted = { - name: {"actual": actual, "reviewed": expected} - for name, (actual, expected) in reviewed.items() - if actual != expected - } - if drifted: - raise ValueError( - "HMRC release parameters disagree with the reviewed source " - f"manifest: {drifted}. Update the manifest and runtime together." - ) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/hmrc_source_contract.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/hmrc_source_contract.py index b4f78b30d..946dc08cc 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/hmrc_source_contract.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/hmrc_source_contract.py @@ -31,6 +31,7 @@ CANONICAL_HMRC_FACT_FENCES, FULL_FRS_TI_BAND_FENCE_ID, ) +from microcosm.build.uk_runtime.release_identity import UK_RELEASE_TIER_FRS from microcosm.build.uk_runtime.spi_income import ( DEFAULT_SPI_DONOR_SAMPLE_SIZE, SPI_DERIVED_POLICYENGINE_SOURCE_COLUMNS, @@ -63,6 +64,11 @@ ) __all__ = [ + "CERTIFIED_UK_CANDIDATE_FILENAME", + "CERTIFIED_UK_CANDIDATE_REVISION", + "CERTIFIED_UK_CANDIDATE_SHA256", + "CERTIFIED_UK_CANDIDATE_SIZE_BYTES", + "CERTIFIED_UK_CANDIDATE_TIER", "HMRC_DISTRIBUTIONAL_INPUTS", "UK_HMRC_INCOME_SOURCE_STAGES_RESOURCE", "assert_uk_hmrc_income_source_contract_current", @@ -75,6 +81,13 @@ "gift_aid", "charitable_investment_gifts", ) +CERTIFIED_UK_CANDIDATE_FILENAME = "populace_uk_2023.h5" +CERTIFIED_UK_CANDIDATE_REVISION = "populace-uk-2023-dd68c73-4aa4b14-20260619T023711Z" +CERTIFIED_UK_CANDIDATE_TIER = UK_RELEASE_TIER_FRS +CERTIFIED_UK_CANDIDATE_SHA256 = ( + "f17306ccb2aad7ff0130be3589b560afb2e2a12a943570911cd0c77f07934833" +) +CERTIFIED_UK_CANDIDATE_SIZE_BYTES = 1_315_880_118 _STAGE2_SOURCE_FAITHFUL_INCOME_PREDICTORS = ( "employment_income", "self_employment_income", @@ -131,40 +144,36 @@ def assert_uk_hmrc_income_source_contract_current( _expect(failures, "stage.stage", stage.get("stage"), "hmrc_spi_income") _expect(failures, "stage.grain", stage.get("grain"), "person") - # Imported lazily to keep the candidate identity constants in their runtime - # owner without introducing an import cycle at module import time. - from microcosm.build.uk_runtime import hmrc_restoration - base = _mapping(stage.get("base_candidate"), "base_candidate", failures) _expect( failures, "base_candidate.filename", base.get("filename"), - hmrc_restoration.CERTIFIED_UK_CANDIDATE_FILENAME, + CERTIFIED_UK_CANDIDATE_FILENAME, ) _expect( failures, "base_candidate.tier", base.get("tier"), - hmrc_restoration.CERTIFIED_UK_CANDIDATE_TIER, + CERTIFIED_UK_CANDIDATE_TIER, ) _expect( failures, "base_candidate.revision", base.get("revision"), - hmrc_restoration.CERTIFIED_UK_CANDIDATE_REVISION, + CERTIFIED_UK_CANDIDATE_REVISION, ) _expect( failures, "base_candidate.sha256", base.get("sha256"), - hmrc_restoration.CERTIFIED_UK_CANDIDATE_SHA256, + CERTIFIED_UK_CANDIDATE_SHA256, ) _expect( failures, "base_candidate.size_bytes", base.get("size_bytes"), - hmrc_restoration.CERTIFIED_UK_CANDIDATE_SIZE_BYTES, + CERTIFIED_UK_CANDIDATE_SIZE_BYTES, ) _expect( failures, diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/ledger_targets.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/ledger_targets.py index dd18368c6..7e3608fa6 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/ledger_targets.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/ledger_targets.py @@ -105,32 +105,6 @@ def materialize_uk_ledger_targets( ) -class UKPolicyEngineAdapter: - """Small adapter around policyengine-uk simulation-like objects.""" - - def __init__(self, simulation: Any): - self.simulation = simulation - self.tables: dict[str, dict[str, np.ndarray]] = {} - - def column(self, entity: str, variable: str) -> np.ndarray: - table = self.tables.get(entity, {}) - if variable in table: - return np.asarray(table[variable], dtype=float) - values = self.simulation.calculate(variable) - return np.asarray(values, dtype=float) - - def set_column(self, entity: str, variable: str, values: object) -> None: - self.tables.setdefault(entity, {})[variable] = np.asarray(values, dtype=float) - - def parameter(self, parameter: str, period: int | str) -> float: - if parameter in { - "cgt_calibration.uk_cgt_annual_exempt_amount", - "gov.hmrc.cgt.annual_exempt_amount", - }: - return uk_cgt_annual_exempt_amount(period) - raise KeyError(parameter) - - #: Published-fact reductions rewritten to the internal reduction that carries #: the same meaning on our frame. Facts keep the semantics of the source that #: published them; translating those onto the model's own concepts is our job, diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/national_build.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/national_build.py deleted file mode 100644 index fcf958e78..000000000 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/national_build.py +++ /dev/null @@ -1,835 +0,0 @@ -"""National UK build orchestration over the shared gate battery. - -UK source stages run ``Frame -> Frame`` on the national carrier assembled by -:mod:`microcosm.build.uk_runtime.national_frame`; the staging H5 persists the -same person, benunit, and household tables PolicyEngine-UK reads, including -``household_weight`` as a real export column materialized from the frame's -typed weights. The local-geography clone remains a separate downstream build -product with its own carrier. - -Gates run through :class:`microcosm.build.gate_battery.GateBatteryRun` over -the declared ``uk/gates.json`` spec: the preflight phase before the frame -loads, the terminal phase after the last stage and immediately before the -staging writer. Every declared entry appears in the persisted schema-4 -report — evidence the build cannot supply is a named ``evidence_absent`` -gap, blocking release candidates only — and the report is on disk before -any blocking decision raises. -""" - -from __future__ import annotations - -import json -from collections.abc import Callable, Mapping, Sequence -from dataclasses import dataclass -from datetime import date -from pathlib import Path -from types import SimpleNamespace -from typing import Any - -import microcosm.build.uk_runtime.national_frame as _national_frame -from microcosm.build.country_spec import GatesManifest, load_country_spec -from microcosm.build.frame_sampling import ( - validate_sample_fraction, - validate_sample_seed, -) -from microcosm.build.gate_battery import ( - BlockingMode, - EvidenceContext, - GateBatteryBlockedError, - GateBatteryRun, - GateBinding, - GatePhaseReport, -) -from microcosm.build.gates import GateResult -from microcosm.build.plan import Stage as PlanStage -from microcosm.build.plan import StagePlan, StageRecord -from microcosm.build.uk_runtime.battery_bindings import UK_GATE_REGISTRY -from microcosm.build.uk_runtime.calibration_run import UK_NATIONAL_GATE_SCOPE -from microcosm.build.uk_runtime.national_frame import ( - UK_HOUSEHOLD_WEIGHT_KIND_ATTR as UK_HOUSEHOLD_WEIGHT_KIND_ATTR, -) -from microcosm.build.uk_runtime.national_frame import ( - UK_MASS_LOG_ATTR as UK_MASS_LOG_ATTR, -) -from microcosm.build.uk_runtime.national_frame import ( - UK_NATIONAL_H5_TABLES as UK_NATIONAL_H5_TABLES, -) -from microcosm.build.uk_runtime.national_frame import ( - UKStagingProvenance as UKStagingProvenance, -) -from microcosm.build.uk_runtime.national_frame import ( - _mass_log_from_stored as _mass_log_from_stored, -) -from microcosm.build.uk_runtime.national_frame import ( - _read_uk_national_tables as _read_uk_national_tables, -) -from microcosm.build.uk_runtime.national_frame import ( - _read_weight_metadata as _read_weight_metadata, -) -from microcosm.build.uk_runtime.national_frame import ( - _weight_kind_from_stored as _weight_kind_from_stored, -) -from microcosm.build.uk_runtime.national_frame import ( - _write_uk_single_year_tables as _write_uk_single_year_tables, -) -from microcosm.build.uk_runtime.national_frame import ( - _write_weight_metadata as _write_weight_metadata, -) -from microcosm.build.uk_runtime.national_frame import ( - load_uk_national_frame as load_uk_national_frame, -) -from microcosm.build.uk_runtime.national_frame import ( - uk_household_weight_kind, - uk_national_frame, - uk_time_period, - validate_uk_national_frame, -) -from microcosm.build.uk_runtime.national_frame import ( - write_uk_national_frame as write_uk_national_frame, -) -from microcosm.build.uk_runtime.national_sampling import ( - UK_SAMPLE_SEED_DEFAULT, - sample_uk_national_frame, -) -from microcosm.build.uk_runtime.parity_reference import EfrsParityReference -from microcosm.build.uk_runtime.release_input_coverage import ( - PolicyEngineUKCoverageEngine, -) -from microcosm.build.uk_runtime.weighted_integrity import ( - UKInputMassReference, - UKReviewedExclusion, - exclusion_evaluation_date, -) -from microcosm.calibrate import TargetRegistry -from microcosm.frame import Frame - -__all__ = [ - "UKNationalBuildResult", - "UKNationalStage", - "UKStagingProvenance", - "build_uk_national_dataset", - "load_uk_national_frame", - "uk_household_weight_kind", - "uk_national_frame", - "uk_time_period", - "validate_uk_national_frame", - "write_uk_national_frame", -] - -# The fingerprint pair moved to national_frame with the Frame carrier; the -# re-import keeps this module's existing consumers (hmrc_restoration binds -# certified candidates to it) on their current import path. -_UKSourceFileFingerprint = _national_frame._UKSourceFileFingerprint -_uk_source_file_fingerprint = _national_frame._uk_source_file_fingerprint - - -@dataclass(frozen=True) -class UKNationalStage: - """Deprecated shim for one named ``Frame -> Frame`` national transform. - - UK national builds now execute shared :class:`microcosm.build.plan.Stage` - entries assembled into a :class:`microcosm.build.plan.StagePlan`. This - wrapper remains for one release so existing callers can pass a stage name - and transform; :func:`build_uk_national_dataset` converts it internally to - a shared stage with empty consumes/produces and no donor. - """ - - name: str - transform: Callable[[Frame], Frame] - - def __post_init__(self) -> None: - if not self.name: - raise ValueError("UKNationalStage.name must be non-empty.") - if not callable(self.transform): - raise TypeError("UKNationalStage.transform must be callable.") - - def run(self, frame: Frame) -> Frame: - """Apply this stage and require an explicit Frame result.""" - - result = self.transform(frame) - if not isinstance(result, Frame): - raise TypeError( - f"UK national stage {self.name!r} must return a microcosm Frame, " - f"got {type(result).__name__}." - ) - return result - - -@dataclass(frozen=True) -class UKNationalBuildResult: - """A gated national staging artifact and its execution evidence.""" - - frame: Frame - provenance: UKStagingProvenance - input_h5: Path - staging_h5: Path - stage_names: tuple[str, ...] - #: The in-memory phase reports, declared order (preflight, terminal). - phase_reports: tuple[GatePhaseReport, ...] - #: The schema-4 payload the battery persisted at ``terminal_gate_path`` - #: (in compatibility-alias mode the file is last-written as the schema-1 - #: alias; this field always carries the full battery payload). - gate_report: Mapping[str, object] - terminal_gate_path: Path - #: Shared stage evidence records, one per national build stage. - stage_records: tuple[StageRecord, ...] = () - #: The #627 rung receipt; ``None`` on a full-scale (fraction 1.0) build. - sampling_receipt: Mapping[str, object] | None = None - - @property - def input_coverage(self) -> GateResult: - """Backward-compatible projection of the coverage gate's verdict.""" - - for report in self.phase_reports: - for outcome in report.outcomes: - if ( - outcome.entry.id == "uk_release_input_coverage" - and outcome.result is not None - ): - return outcome.result - raise LookupError("uk_release_input_coverage did not evaluate in this build.") - - @property - def input_coverage_path(self) -> Path: - """Backward-compatible alias for :attr:`terminal_gate_path`.""" - - return self.terminal_gate_path - - -def build_uk_national_dataset( - *, - input_h5: str | Path, - staging_h5: str | Path, - release_id: str, - calibration_diagnostics_sha256: str, - stages: Sequence[UKNationalStage | PlanStage] | StagePlan = (), - coverage_engine: Any | None = None, - input_mass_reference: UKInputMassReference | None = None, - reviewed_input_mass_exclusions: ( - Mapping[str, Mapping[str, UKReviewedExclusion]] | None - ) = None, - reviewed_qrf_tail_exclusions: Mapping[str, UKReviewedExclusion] | None = None, - reviewed_degenerate_exclusions: Mapping[str, UKReviewedExclusion] | None = None, - terminal_gate_path: str | Path | None = None, - input_coverage_path: str | Path | None = None, - checkpoint_dir: str | Path | None = None, - run_config: Mapping[str, object] | None = None, - sample_fraction: float = 1.0, - sample_seed: int = UK_SAMPLE_SEED_DEFAULT, - release_candidate: bool = False, - now: date | None = None, - gate_registry: Mapping[str, GateBinding] | None = None, - ledger_target_registry: Mapping[int | str, TargetRegistry] | None = None, - parity_reference: EfrsParityReference | None = None, -) -> UKNationalBuildResult: - """Run ordered national stages, hard-gate the result, and stage an H5. - - Without ``checkpoint_dir`` the build is the destructive single-process - monolith it always was. With ``checkpoint_dir`` each stage boundary - persists a lossless Frame checkpoint through the outer stage runtime and - completed stages are resumed from their checkpoints instead of re-run — - which requires ``run_config``, the content-addressed identity of the run - (input digest, seeds, source digests): resuming under a different - configuration is refused by the runtime, and an unpinned resume is - exactly the drift hazard checkpoints exist to prevent, so a checkpointed - build without a ``run_config`` is refused here. - - ``sample_fraction`` below 1.0 is the #627 scale ladder: the loaded frame - is sampled at clone-family grain (see - :func:`~microcosm.build.uk_runtime.national_sampling.sample_uk_national_frame`) - before provenance binding, so the certified-candidate fence attests the - frame the stages actually consume. At 1.0 the sampler is never invoked — - full-scale builds are structurally byte-invariant to it. A checkpointed - sampled run must carry the fraction and seed inside ``run_config`` (the - driver does); otherwise two rungs pointed at one checkpoint directory - would silently resume across each other. - - ``release_candidate`` is the battery's second blocking axis: a candidate - build treats every ``evidence_absent`` gap as blocking, a dev build - records the gap and continues. A sampled rung is structurally - non-releasable, so requesting both is refused. ``now`` is the shared - exclusion-expiry clock (default: today, UTC), threaded to every - exclusion-consuming gate so one report carries one evaluation date; it - is resolved once when the battery is armed — before the stages — where - the legacy aggregator resolved it after them, so a receipt expiring - mid-build is judged by the date the build started. - ``gate_registry`` overrides the binding registry (tests only). - """ - - requested_input_path = Path(input_h5).expanduser() - input_path = requested_input_path.resolve() - staging_path = Path(staging_h5).resolve() - if input_path == staging_path: - raise ValueError("input_h5 and staging_h5 must differ.") - if staging_path.suffix != ".h5": - raise ValueError("UK national staging path must end with '.h5'.") - - if terminal_gate_path is not None and input_coverage_path is not None: - raise ValueError( - "terminal_gate_path and input_coverage_path are mutually exclusive; " - "input_coverage_path is a compatibility alias." - ) - legacy_input_coverage_output = input_coverage_path is not None - requested_gate_path = ( - terminal_gate_path - if terminal_gate_path is not None - else ( - input_coverage_path - if input_coverage_path is not None - else staging_path.with_suffix(".terminal_gates.json") - ) - ) - diagnostic_path = Path(requested_gate_path).resolve() - if diagnostic_path in {input_path, staging_path}: - raise ValueError( - "terminal_gate_path must differ from the input and staging H5 paths." - ) - - stage_plan = _coerce_stage_plan(stages) - materialized_stages = stage_plan.stages - _validate_stages(materialized_stages) - # Invariant: no destructive step precedes argument validation. Every - # configuration refusal sits above the sidecar unlinks and the battery, - # so a misconfigured run can neither delete a previous report nor write - # a new one (the #658 --degenerate-exclusions ordering bug, generalized). - if checkpoint_dir is not None and run_config is None: - raise ValueError( - "a checkpointed UK national build requires run_config: the " - "content-addressed run identity is what makes a resume safe." - ) - validate_sample_fraction(sample_fraction, label="UK sample") - validate_sample_seed(sample_seed, label="UK sample") - if ( - checkpoint_dir is not None - and sample_fraction != 1.0 - and "sampling" not in run_config - ): - raise ValueError( - "a checkpointed rung build requires the sampling identity inside " - "run_config: two rungs pointed at one checkpoint directory must " - "refuse, never cross-resume." - ) - if release_candidate and sample_fraction != 1.0: - raise ValueError( - "a sampled rung build is structurally non-releasable (#627); " - "release_candidate requires sample_fraction == 1.0." - ) - if release_candidate and legacy_input_coverage_output: - raise ValueError( - "input_coverage_path is a compatibility alias whose schema-1 " - "payload is last-written over the report path; a release " - "candidate must keep its signed schema-4 report, so the two " - "are mutually exclusive." - ) - engine = ( - coverage_engine - if coverage_engine is not None - else PolicyEngineUKCoverageEngine() - ) - # The clock and the battery construction validate their inputs (the - # date's type; release identity, spec parameters, release_evidence - # values), so they sit inside the no-destruction-before-validation - # fence too: the unlinks come strictly last. - evaluation_date = exclusion_evaluation_date(now) - battery = GateBatteryRun( - _national_gate_manifest(), - release_id=release_id, - report_path=diagnostic_path, - release_candidate=release_candidate, - registry=UK_GATE_REGISTRY if gate_registry is None else gate_registry, - release_evidence={ - "calibration_diagnostics_sha256": calibration_diagnostics_sha256 - }, - ) - staging_path.unlink(missing_ok=True) - diagnostic_path.unlink(missing_ok=True) - # Mirrors the US cheap preflight: graph or reference drift blocks before - # source stages — now with the refusal persisted as a schema-4 report. - preflight_artifacts: dict[str, object] = { - "coverage_engine": engine, - "build_stage_names": tuple(stage.name for stage in materialized_stages), - } - if ledger_target_registry is not None: - preflight_artifacts["uk_ledger_compiled_registries"] = dict( - ledger_target_registry - ) - battery.run_phase( - "preflight", - EvidenceContext(artifacts=preflight_artifacts), - ) - battery.enforce("preflight", mode=BlockingMode.BLOCKS_ARTIFACT) - frame, provenance = load_uk_national_frame(requested_input_path) - sampling_receipt: Mapping[str, object] | None = None - if sample_fraction != 1.0: - # Sample before provenance binding: the fence attests the sampled - # frame, and the stages never learn a rung existed. - frame, sampling_receipt = sample_uk_national_frame( - frame, fraction=sample_fraction, seed=sample_seed - ) - # Stages whose fences bind the loaded bytes (the SPI stage's - # certified-candidate check) receive the load provenance and the loaded - # frame explicitly — provenance travels beside the frame, never inside - # it, and binding records the loaded frame's content identity so the - # fence can assert descent from this exact load. Bindings are - # single-use; the stage consumes them. - for stage in materialized_stages: - binder = getattr(stage.transform, "bind_staging_provenance", None) - if callable(binder): - binder(provenance, frame) - if checkpoint_dir is None: - frame, stage_records = _validating_stage_plan(stage_plan).run(frame) - else: - frame, stage_records = _run_stages_checkpointed( - materialized_stages, - frame=frame, - checkpoint_dir=Path(checkpoint_dir), - run_config=run_config, - ) - - # Mirrors the US final-export placement: evaluate every declared gate in - # one batch after all stages and immediately before the staging writer. - artifacts: dict[str, object] = { - "coverage_engine": engine, - "exclusions_evaluated_on": evaluation_date, - # The nonnegative gate derives its required columns from the stages - # this build actually scheduled (same roster the preflight coverage - # gate attests). - "build_stage_names": tuple(stage.name for stage in materialized_stages), - "rules_engine": engine, - } - brma_domain = _brma_enum_domain(engine) - if brma_domain is None and "brma" in frame.table("household"): - brma_domain = tuple( - sorted( - str(value) - for value in frame.table("household")["brma"].dropna().unique() - ) - ) - if brma_domain is not None: - artifacts["brma_enum_domain"] = brma_domain - student_loan_plan_domain = _engine_enum_domain(engine, "student_loan_plan") - if student_loan_plan_domain is not None: - artifacts["student_loan_plan_enum_domain"] = student_loan_plan_domain - fit_weight_records = _stage_fit_weight_records(materialized_stages) - if fit_weight_records is not None: - artifacts["fit_weight_records"] = fit_weight_records - calibration_evidence = _stage_calibration_evidence(materialized_stages) - if calibration_evidence is not None: - artifacts["national_calibration"] = calibration_evidence - parity_evidence = _stage_parity_evidence( - materialized_stages, - frame=frame, - parity_reference=parity_reference, - ) - if parity_evidence is not None: - artifacts["parity_evidence"] = parity_evidence - if input_mass_reference is not None: - artifacts["input_mass_reference"] = input_mass_reference - if reviewed_input_mass_exclusions is not None: - artifacts["reviewed_input_mass_exclusions"] = reviewed_input_mass_exclusions - if reviewed_qrf_tail_exclusions is not None: - artifacts["reviewed_qrf_tail_exclusions"] = reviewed_qrf_tail_exclusions - if reviewed_degenerate_exclusions is not None: - artifacts["reviewed_degenerate_exclusions"] = reviewed_degenerate_exclusions - terminal = battery.run_phase( - "terminal", EvidenceContext(frame=frame, artifacts=artifacts) - ) - coverage_outcome = next( - outcome - for outcome in terminal.outcomes - if outcome.entry.id == "uk_release_input_coverage" - ) - if legacy_input_coverage_output and coverage_outcome.result is None: - raise RuntimeError( - "uk_release_input_coverage did not evaluate; the schema-1 " - "compatibility alias has no verdict to serialize." - ) - try: - battery.enforce("terminal", mode=BlockingMode.BLOCKS_ARTIFACT) - except GateBatteryBlockedError as blocked: - # The alias consumer reads the schema-1 shape at this exact path, in - # the blocked case too — same last-write order as the legacy flow. - # A failing alias write must not displace the typed block: the block - # is the build's outcome, the write failure rides along as its cause. - if legacy_input_coverage_output and coverage_outcome.result is not None: - try: - _write_input_coverage_diagnostic( - diagnostic_path, coverage_outcome.result - ) - except Exception as write_error: # noqa: BLE001 - keep the block typed - raise blocked from write_error - raise - if legacy_input_coverage_output: - _write_input_coverage_diagnostic(diagnostic_path, coverage_outcome.result) - gate_report = battery.report_payload() - attestation = gate_report["attestation"] - signing_error = ( - attestation.get("signing_error") if isinstance(attestation, Mapping) else None - ) - if signing_error is not None and sample_fraction == 1.0: - # A rung build may proceed unsigned (its report honestly says - # shippable: false, and a rung is structurally non-releasable); a - # full-scale build keeps the legacy guarantee — no staging artifact - # without an attested report. The unsigned report is already on disk. - raise RuntimeError( - "UK terminal gate report is unsigned and this is a full-scale " - f"build; refusing to stage. {signing_error} The unsigned report " - f"was written to {diagnostic_path}." - ) - - write_uk_national_frame(frame, staging_path) - return UKNationalBuildResult( - frame=frame, - provenance=provenance, - input_h5=input_path, - staging_h5=staging_path, - stage_names=tuple(stage.name for stage in materialized_stages), - phase_reports=tuple( - battery.phase_report(phase) for phase in battery.phases_evaluated - ), - gate_report=gate_report, - terminal_gate_path=diagnostic_path, - stage_records=stage_records, - sampling_receipt=sampling_receipt, - ) - - -def _run_stages_checkpointed( - stages: tuple[PlanStage, ...], - *, - frame: Frame, - checkpoint_dir: Path, - run_config: Mapping[str, object], -) -> tuple[Frame, tuple[StageRecord, ...]]: - """Run the national stages through the outer stage runtime. - - Each boundary persists a lossless Frame checkpoint (frame metadata rides - the stage record, per ``uk_runtime.stage_checkpoints``); stages the run - context already records as complete are resumed from their checkpoints — - transforms that expose ``resume_from_checkpoint`` rehydrate their - downstream evidence (the retained-leaves descent identities, the SPI - fit-weight audit records) from the record instead of re-running. - """ - - from microcosm.build.outer_stage_runtime import ( - Stage as OuterStage, - ) - from microcosm.build.outer_stage_runtime import ( - StagePipeline, - StageRuntime, - ) - from microcosm.build.uk_runtime.stage_checkpoints import ( - UK_FRAME_METADATA_KEY, - load_uk_stage_checkpoint, - uk_stage_metadata, - ) - - pipeline = StagePipeline( - tuple( - OuterStage(stage.name, f"UK national stage {stage.name}") - for stage in stages - ) - ) - runtime = StageRuntime(checkpoint_dir, pipeline, run_config=dict(run_config)) - completed = set(runtime.context.completed) - records: list[StageRecord] = [] - for stage in stages: - if stage.name in completed: - loaded = load_uk_stage_checkpoint(runtime, stage.name) - resume = getattr(stage.transform, "resume_from_checkpoint", None) - if callable(resume): - extra = { - key: value - for key, value in loaded.metadata.items() - if key != UK_FRAME_METADATA_KEY - } - resume(extra, loaded.frame) - frame, stage_records = _resume_stage_record(stage, frame, loaded.frame) - validate_uk_national_frame(frame) - records.extend(stage_records) - continue - frame, stage_records = _validating_stage_plan(StagePlan((stage,))).run(frame) - records.extend(stage_records) - extra_metadata: dict[str, object] = {} - hook = getattr(stage.transform, "checkpoint_metadata", None) - if callable(hook): - extra_metadata = dict(hook()) - runtime.complete( - stage.name, - frame, - metadata=uk_stage_metadata(frame, extra=extra_metadata), - ) - return frame, tuple(records) - - -def _national_gate_manifest() -> GatesManifest: - source = load_country_spec("uk").gates - entries = tuple(entry for entry in source.gates if entry.id in UK_NATIONAL_GATE_SCOPE) - missing = sorted(set(UK_NATIONAL_GATE_SCOPE) - {entry.id for entry in entries}) - if missing: - raise RuntimeError(f"UK national gate scope names undeclared gate id(s): {missing}.") - return GatesManifest( - country=source.country, - version=source.version, - policy=f"{source.policy}; national_build_scope", - phases=("preflight", "terminal"), - gates=entries, - ) - - -def _coerce_stage_plan( - stages: Sequence[UKNationalStage | PlanStage] | StagePlan, -) -> StagePlan: - """Normalize legacy UK stages and shared stages to one StagePlan.""" - - if isinstance(stages, StagePlan): - return stages - materialized = tuple(_coerce_stage(stage) for stage in stages) - names: set[str] = set() - for stage in materialized: - if stage.name in names: - raise ValueError(f"Duplicate UK national stage {stage.name!r}.") - names.add(stage.name) - return StagePlan(materialized) - - -def _coerce_stage(stage: UKNationalStage | PlanStage) -> PlanStage: - if isinstance(stage, PlanStage): - return stage - if isinstance(stage, UKNationalStage): - return PlanStage(name=stage.name, transform=stage.transform) - raise TypeError( - "UK national stages must be shared Stage or UKNationalStage instances, " - f"got {type(stage).__name__}." - ) - - -def _validating_stage_plan(plan: StagePlan) -> StagePlan: - """Return a plan whose stages validate the UK national frame after each run.""" - - return StagePlan(_validating_stage(stage) for stage in plan.stages) - - -def _validating_stage(stage: PlanStage) -> PlanStage: - def transform(frame: Frame) -> Frame: - result = stage.transform(frame) - if not isinstance(result, Frame): - return result - validate_uk_national_frame(result) - return result - - return PlanStage( - name=stage.name, - transform=transform, - produces=stage.produces, - consumes=stage.consumes, - donor=stage.donor, - ) - - -def _resume_stage_record( - stage: PlanStage, - previous: Frame, - loaded: Frame, -) -> tuple[Frame, tuple[StageRecord, ...]]: - plan = StagePlan( - ( - PlanStage( - name=stage.name, - transform=lambda _frame: loaded, - produces=stage.produces, - consumes=stage.consumes, - donor=stage.donor, - ), - ) - ) - return plan.run(previous) - - -def _validate_stages(stages: tuple[PlanStage, ...]) -> None: - names: set[str] = set() - for stage in stages: - if not isinstance(stage, PlanStage): - raise TypeError( - "UK national stages must be shared Stage instances, " - f"got {type(stage).__name__}." - ) - if stage.name in names: - raise ValueError(f"Duplicate UK national stage {stage.name!r}.") - names.add(stage.name) - - -#: Stage names that perform production fits and therefore owe the terminal -#: weights audit their :class:`FitWeightRecord` evidence even when a swapped -#: or hollow transform stops exposing it. -_UK_FITTING_STAGE_NAMES = frozenset({"hmrc_spi_income", "was_wealth"}) - - -def _stage_fit_weight_records( - stages: tuple[PlanStage, ...], -) -> tuple[object, ...] | None: - """The weights-audit evidence artifact, aggregated across fitting stages. - - A stage counts as fitting when its name is a declared fitting stage - (HMRC SPI income, WAS wealth) or its transform exposes - ``fit_weight_records``; each contributes records in stage order. - ``None`` (no fitting stage scheduled) leaves the artifact unsupplied, - so the audit is a named ``evidence_absent`` gap. A present fitting - stage always supplies the artifact — records that are missing, - unreadable, or empty coerce to ``()``, which the UK audit binding - fails: an absent audit is not a passing audit. - """ - - fitting_stages = tuple( - stage - for stage in stages - if stage.name in _UK_FITTING_STAGE_NAMES - or hasattr(stage.transform, "fit_weight_records") - ) - if not fitting_stages: - return None - collected: list[object] = [] - for stage in fitting_stages: - try: - records = tuple(stage.transform.fit_weight_records or ()) - except Exception: # noqa: BLE001 - unreadable records coerce to () - # and fail the audit as missing evidence rather than crashing - # the batch. - return () - if not records: - # A scheduled fitting stage with no records is missing evidence; - # it must fail the audit, not be absorbed by another stage's - # records. - return () - collected.extend(records) - return tuple(collected) - - -def _stage_calibration_evidence( - stages: tuple[PlanStage, ...], -) -> Mapping[str, object] | None: - for stage in stages: - if stage.name == "national_calibration": - manifest = getattr(stage.transform, "manifest", None) - if not isinstance(manifest, Mapping): - raise RuntimeError( - "national_calibration stage did not produce a manifest; " - "refusing to build calibration gate evidence." - ) - return dict(manifest) - return None - - -def _stage_parity_evidence( - stages: tuple[PlanStage, ...], - *, - frame: Frame, - parity_reference: EfrsParityReference | None, -) -> object | None: - """Build the parity-trio evidence, side by side, never aliased. - - Two different comparisons travel in one object, and they are not equally - strong: - - * The **column** surfaces are independently sourced — the candidate's from - the staged frame, the reference's from the frozen parity instrument's - declared input entities. - * The **target** surfaces are not. The parity instrument carries source - identity and incumbent input-column shares only; it holds no incumbent - target surface. So the reference side is the declared registry and the - candidate side is the solve's realized diagnostics — which proves that - the solve bound every declared target at the declared period, and does - **not** prove agreement with any incumbent-derived surface. - - Neither side is ever copied from the other; a copied reference would make - the trio pass by construction. Sourcing the target side from an - incumbent-bound instrument needs an instrument that carries one, which is - release-cut work (#757). - """ - - for stage in stages: - if stage.name != "national_calibration": - continue - if parity_reference is None: - return None - diagnostics = getattr(stage.transform, "diagnostics", None) - if not isinstance(diagnostics, tuple): - raise RuntimeError( - "national_calibration stage did not produce target diagnostics; " - "refusing to build parity evidence." - ) - registry = getattr(stage.transform, "registry", None) - specs = getattr(registry, "specs", None) - if specs is None: - raise RuntimeError( - "national_calibration stage carries no compiled registry; " - "refusing to build parity evidence." - ) - target_relative_errors = { - str(row["name"]): float(row["relative_error"]) for row in diagnostics - } - return SimpleNamespace( - candidate_columns={ - f"{entity}.{column}" - for entity in frame.entities - for column in frame.table(entity).columns - }, - reference_columns={ - f"{entity}.{name}" - for name, entity in parity_reference.input_entities.items() - }, - candidate_targets=set(target_relative_errors), - # Solver diagnostics label rows as ``name@period``; the declared - # side is compared at the same labeled grain so a target bound at - # the wrong period cannot satisfy the surface. - reference_targets={f"{spec.name}@{spec.period}" for spec in specs}, - target_relative_errors=target_relative_errors, - ) - return None - - -def _brma_enum_domain(engine: object) -> tuple[str, ...] | None: - return _engine_enum_domain(engine, "brma") - - -def _engine_enum_domain(engine: object, variable_name: str) -> tuple[str, ...] | None: - variable_getter = getattr(engine, "_variable", None) - if not callable(variable_getter): - return None - try: - variable = variable_getter(variable_name) - except Exception: - return None - possible_values = getattr(variable, "possible_values", None) - members = getattr(possible_values, "__members__", None) - if isinstance(members, Mapping): - return tuple(str(name) for name in members) - if possible_values is None: - return None - return tuple(str(getattr(value, "name", value)) for value in possible_values) - - -def _write_input_coverage_diagnostic(path: Path, gate: GateResult) -> None: - """Write the byte-compatible origin/main schema for the legacy alias. - - Atomic like every other writer on this surface: the alias last-writes - over the gate-report path, and a crash mid-write must not leave - truncated JSON where a consumer expects a report. - """ - - path.parent.mkdir(parents=True, exist_ok=True) - payload = { - "schema_version": 1, - "enforced": True, - "input_coverage": { - "passed": gate.passed, - "failures": list(gate.failures), - "details": dict(gate.details), - }, - } - temporary_path = path.with_name(path.name + ".tmp") - temporary_path.write_text( - json.dumps(payload, indent=2, sort_keys=True) + "\n", - encoding="utf-8", - ) - temporary_path.replace(path) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/national_frame.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/national_frame.py index 3d3a0640b..894b2fe74 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/national_frame.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/national_frame.py @@ -26,7 +26,7 @@ import json import uuid -from collections.abc import Sequence +from collections.abc import Callable, Sequence from dataclasses import dataclass from pathlib import Path from typing import Any @@ -52,6 +52,7 @@ "UK_NATIONAL_H5_TABLES", "UK_NATIONAL_SCHEMA", "UK_TIME_PERIOD_METADATA_KEY", + "UKNationalStage", "UKStagingProvenance", "load_uk_national_frame", "uk_household_weight_kind", @@ -227,6 +228,33 @@ class UKStagingProvenance: fingerprint: _UKSourceFileFingerprint +@dataclass(frozen=True) +class UKNationalStage: + """One named ``Frame -> Frame`` national transform. + + The June national driver is retired, but a few stage factories still expose + this small callable container as their compatibility surface. + """ + + name: str + transform: Callable[[Frame], Frame] + + def __post_init__(self) -> None: + if not self.name: + raise ValueError("UKNationalStage.name must be non-empty.") + if not callable(self.transform): + raise TypeError("UKNationalStage.transform must be callable.") + + def run(self, frame: Frame) -> Frame: + result = self.transform(frame) + if not isinstance(result, Frame): + raise TypeError( + f"UK national stage {self.name!r} must return a microcosm Frame, " + f"got {type(result).__name__}." + ) + return result + + def _read_uk_national_tables( path: str | Path, ) -> tuple[dict[str, Any], _UKSourceFileFingerprint, Path]: diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_dataset.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_dataset.py index 72027c635..86fb5544d 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_dataset.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_dataset.py @@ -21,13 +21,11 @@ assign_uk_geography_ladder, uk_geography_ladder_gate, ) -from microcosm.build.uk_runtime.national_build import ( +from microcosm.build.uk_runtime.national_frame import ( _mass_log_from_stored, _read_weight_metadata, _weight_kind_from_stored, _write_uk_single_year_tables, -) -from microcosm.build.uk_runtime.national_frame import ( uk_household_weight_kind, uk_national_frame, uk_time_period, diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_spine.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_spine.py index 4b1043cb5..33d1a45f7 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_spine.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_spine.py @@ -7,6 +7,7 @@ from pathlib import Path from typing import Any +import numpy as np import pandas as pd from microcosm.build.gates import FitWeightRecord @@ -28,9 +29,8 @@ HMRCReplayReport, build_conservative_hmrc_replay_report, ) -from microcosm.build.uk_runtime.hmrc_restoration import ( - _assert_post_draw_identity, - _distributional_mass_shares, +from microcosm.build.uk_runtime.hmrc_source_contract import ( + HMRC_DISTRIBUTIONAL_INPUTS, ) from microcosm.build.uk_runtime.national_frame import ( uk_household_weight_kind, @@ -63,6 +63,8 @@ SPI_HMRC_OTHER_SOCIAL_SECURITY_INCOME_COLUMN, SPI_HMRC_STATE_PENSION_INCOME_COLUMN, SPI_HMRC_TAXABLE_TERMINATION_PAY_COLUMN, + SPI_HMRC_TOTAL_EARNED_INCOME_COLUMN, + SPI_HMRC_TOTAL_INVESTMENT_INCOME_COLUMN, SPI_INCOME_QRF_OUTPUT_COLUMNS, SPI_SYNTHETIC_SUPPORT_CHANNEL, UKSPISupportResult, @@ -597,6 +599,94 @@ def _support_result_from_frame( ) +def _assert_post_draw_identity(frame: Frame) -> int: + """Require deterministic TEI + TII = TI on every rebuilt SPI draw.""" + + person = frame.table("person") + channel = support_channel_column("person") + required = ( + channel, + SPI_HMRC_TOTAL_EARNED_INCOME_COLUMN, + SPI_HMRC_TOTAL_INVESTMENT_INCOME_COLUMN, + "hmrc_spi_assessable_income", + ) + missing = sorted(set(required) - set(person.columns)) + if missing: + raise RuntimeError( + f"HMRC replay omitted post-draw identity column(s): {missing}." + ) + spi = person[channel].eq(SPI_SYNTHETIC_SUPPORT_CHANNEL).to_numpy(dtype=bool) + if not spi.any(): + raise RuntimeError("HMRC replay contains no rebuilt SPI person draws.") + numeric = person.loc[ + spi, + [ + SPI_HMRC_TOTAL_EARNED_INCOME_COLUMN, + SPI_HMRC_TOTAL_INVESTMENT_INCOME_COLUMN, + "hmrc_spi_assessable_income", + ], + ].apply(pd.to_numeric, errors="coerce") + values = numeric.to_numpy(dtype=float) + if not np.isfinite(values).all(): + raise RuntimeError("HMRC post-draw identity contains non-finite values.") + if not np.array_equal( + numeric["hmrc_spi_assessable_income"].to_numpy(dtype=float), + numeric[SPI_HMRC_TOTAL_EARNED_INCOME_COLUMN].to_numpy(dtype=float) + + numeric[SPI_HMRC_TOTAL_INVESTMENT_INCOME_COLUMN].to_numpy(dtype=float), + ): + raise RuntimeError("HMRC TI must equal deterministic TEI + TII exactly.") + return int(spi.sum()) + + +def _distributional_mass_shares(frame: Frame) -> dict[str, float]: + """Audit charitable signal on strictly positive rebuilt-SPI mass.""" + + person = frame.table("person") + person_channel = support_channel_column("person") + if person_channel not in person: + raise RuntimeError( + "Cannot audit HMRC distributional inputs without person support " + "channel provenance." + ) + spi_people = ( + person[person_channel].eq(SPI_SYNTHETIC_SUPPORT_CHANNEL).to_numpy(dtype=bool) + ) + if not spi_people.any(): + raise RuntimeError("Rebuilt HMRC family contains no SPI support people.") + household = frame.table("household") + household_weights = pd.Series( + frame.weights_for("household").values, + index=household["household_id"].to_numpy(), + ) + mapped = pd.to_numeric( + person["person_household_id"].map(household_weights), + errors="coerce", + ).to_numpy(dtype=float, na_value=np.nan) + if not np.isfinite(mapped).all() or (mapped < 0.0).any(): + raise RuntimeError( + "Cannot audit HMRC distributional inputs without finite, " + "non-negative person mass." + ) + positive = mapped > 0.0 + total = float(mapped[positive].sum()) + if total <= 0.0: + raise RuntimeError("HMRC distributional audit has no positive person mass.") + shares: dict[str, float] = {} + for column in HMRC_DISTRIBUTIONAL_INPUTS: + if column not in person: + raise RuntimeError(f"HMRC stage omitted distributional input {column!r}.") + values = pd.to_numeric(person[column], errors="coerce").to_numpy( + dtype=float, + na_value=np.nan, + ) + if not np.isfinite(values).all(): + raise RuntimeError(f"HMRC distributional input {column!r} is non-finite.") + shares[column] = ( + float(mapped[positive & spi_people & (values != 0.0)].sum()) / total + ) + return shares + + def _build_spine_replay_report( *, source_targets: HMRCIncomeTargetSet, diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/terminal_gates.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/terminal_gates.py index 3d39e4a8b..625d96028 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/terminal_gates.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/terminal_gates.py @@ -69,10 +69,12 @@ "uk_zero_weight_strata_gate", ] -UK_CANDIDATE_DATASET_NAME = "populace_uk_2023" +UK_CANDIDATE_DATASET_NAME = "microcosm_uk_2024" # The label names the pinned reference artifact exactly: the 2024-25 line's # published enhanced_frs_2024_25.h5 (no separate "recalibrated" variant # exists at this vintage; the June report strings keep their own label). +# After the swap, "reference" becomes the previous certified microcosm line +# once a second certified cut exists; that later increment flips this value. UK_REFERENCE_DATASET_NAME = "enhanced_frs_2024_25" UK_MAX_TARGET_ABS_RELATIVE_ERROR = 0.25 diff --git a/packages/microcosm-build/tests/test_score_uk_national_candidate.py b/packages/microcosm-build/tests/test_score_uk_national_candidate.py index 657667506..4a71cfa79 100644 --- a/packages/microcosm-build/tests/test_score_uk_national_candidate.py +++ b/packages/microcosm-build/tests/test_score_uk_national_candidate.py @@ -6,8 +6,10 @@ import pandas as pd import pytest -from microcosm.build.uk_runtime.national_build import write_uk_national_frame -from microcosm.build.uk_runtime.national_frame import uk_national_frame +from microcosm.build.uk_runtime.national_frame import ( + uk_national_frame, + write_uk_national_frame, +) from microcosm.calibrate import TargetRegistry, TargetSpec from microcosm.frame import WeightKind from tools.score_uk_national_candidate import ( @@ -86,6 +88,8 @@ def test_score_uk_national_candidate_scores_synthetic_twins(tmp_path) -> None: calibration_year=2025, ) + assert score["artifacts"]["candidate"]["label"] == candidate.stem + assert score["artifacts"]["incumbent"]["label"] == "enhanced_frs_2024_25" assert score["artifacts"]["candidate"]["sha256"] == _sha256_file(candidate) assert score["artifacts"]["incumbent"]["sha256"] == _sha256_file(incumbent) assert score["target_drift"] == [ @@ -156,6 +160,10 @@ def test_score_uk_national_candidate_cli_writes_score_block(tmp_path) -> None: str(output_json), "--calibration-year", "2025", + "--candidate-label", + "explicit_candidate", + "--incumbent-label", + "explicit_incumbent", "--no-measure-resolution", ] ) @@ -164,6 +172,8 @@ def test_score_uk_national_candidate_cli_writes_score_block(tmp_path) -> None: payload = json.loads(output_json.read_text(encoding="utf-8")) score = payload["score_vs_enhanced_frs"] + assert score["artifacts"]["candidate"]["label"] == "explicit_candidate" + assert score["artifacts"]["incumbent"]["label"] == "explicit_incumbent" assert score["holdout_basis"] == "none_declared" # An undeclared holdout reports absence, never the fitted loss wearing a # holdout name: June's fixture holds a genuinely different holdout value, diff --git a/packages/microcosm-build/tests/test_uk_calibration_run.py b/packages/microcosm-build/tests/test_uk_calibration_run.py index faf61f69c..f21b1aa4b 100644 --- a/packages/microcosm-build/tests/test_uk_calibration_run.py +++ b/packages/microcosm-build/tests/test_uk_calibration_run.py @@ -175,8 +175,9 @@ def test_gate_scope_classifies_every_uk_gate(): def test_import_hygiene_does_not_load_national_build_in_fresh_subprocess(): source = Path(calibration_run.__file__).read_text(encoding="utf-8") - assert "microcosm.build.uk_runtime.national_build" not in source - assert "from microcosm.build.uk_runtime.national_build" not in source + legacy_module = ".".join(("microcosm", "build", "uk_runtime", "national_build")) + assert legacy_module not in source + assert " ".join(("from", legacy_module, "import")) not in source def test_run_uk_calibration_writes_cross_pinned_outputs(monkeypatch, tmp_path: Path): diff --git a/packages/microcosm-build/tests/test_uk_frs_hmrc_leaves.py b/packages/microcosm-build/tests/test_uk_frs_hmrc_leaves.py index b7befa783..80d81d5ee 100644 --- a/packages/microcosm-build/tests/test_uk_frs_hmrc_leaves.py +++ b/packages/microcosm-build/tests/test_uk_frs_hmrc_leaves.py @@ -20,10 +20,10 @@ UKFRSHMRCRetainedLeavesStageTransform, retain_uk_frs_hmrc_leaves, ) -from microcosm.build.uk_runtime.national_build import ( +from microcosm.build.uk_runtime.national_frame import ( UKNationalStage, + uk_national_frame, ) -from microcosm.build.uk_runtime.national_frame import uk_national_frame from microcosm.build.uk_runtime.spi_support import ( SPI_HMRC_EMPLOYMENT_BENEFITS_COLUMN, SPI_HMRC_EMPLOYMENT_EXPENSES_COLUMN, diff --git a/packages/microcosm-build/tests/test_uk_frs_spine.py b/packages/microcosm-build/tests/test_uk_frs_spine.py index 3798a082f..64270ff0b 100644 --- a/packages/microcosm-build/tests/test_uk_frs_spine.py +++ b/packages/microcosm-build/tests/test_uk_frs_spine.py @@ -29,8 +29,8 @@ scottish_water_and_sewerage_weekly, uk_frs_spine_seed_frame, ) -from microcosm.build.uk_runtime.national_build import load_uk_national_frame from microcosm.build.uk_runtime.national_frame import ( + load_uk_national_frame, uk_household_weight_kind, uk_national_frame, uk_time_period, diff --git a/packages/microcosm-build/tests/test_uk_hmrc_restoration.py b/packages/microcosm-build/tests/test_uk_hmrc_restoration.py deleted file mode 100644 index 9e3e5a906..000000000 --- a/packages/microcosm-build/tests/test_uk_hmrc_restoration.py +++ /dev/null @@ -1,1030 +0,0 @@ -from __future__ import annotations - -import hashlib -from pathlib import Path -from types import SimpleNamespace - -import numpy as np -import pandas as pd -import pytest - -from microcosm.build.gates import FitWeightRecord -from microcosm.build.uk_runtime import hmrc_restoration -from microcosm.build.uk_runtime.content_identity import uk_frame_content_identity -from microcosm.build.uk_runtime.frs_hmrc_leaves import ( - FRS_HMRC_INCPBEN_COLUMN, - FRS_HMRC_OSSBEN_IDENTIFIABLE_SUBSET_COLUMN, - FRS_HMRC_PAY_COLUMN, - FRS_HMRC_SRP_REGULAR_CODE5_COLUMN, - FRS_HMRC_UBISJA_COLUMN, -) -from microcosm.build.uk_runtime.hmrc_income import ( - HMRC_SPI_ASSESSABLE_INCOME_COLUMN, - HMRC_SPI_BUILD_PERIOD, - HMRC_SPI_INCOME_BAND_LOWER_BOUNDS, - HMRC_SPI_INCOME_COMPONENTS, - HMRC_SPI_TARGET_RECORD_COUNT, - HMRCIncomeBandTargetRecord, - HMRCIncomeSourceProvenance, - HMRCIncomeTargetSet, -) -from microcosm.build.uk_runtime.hmrc_restoration import ( - CERTIFIED_UK_CANDIDATE_FILENAME, - CERTIFIED_UK_CANDIDATE_REVISION, - CERTIFIED_UK_CANDIDATE_SHA256, - CERTIFIED_UK_CANDIDATE_SIZE_BYTES, - CERTIFIED_UK_CANDIDATE_TIER, - UKCertifiedCandidateIdentity, - UKHMRCIncomeStageTransform, - restore_uk_hmrc_income_family, - verify_certified_uk_candidate, -) -from microcosm.build.uk_runtime.national_build import ( - load_uk_national_frame, - write_uk_national_frame, -) -from microcosm.build.uk_runtime.national_frame import ( - UKStagingProvenance, - _UKSourceFileFingerprint, - uk_household_weight_kind, - uk_national_frame, -) -from microcosm.build.uk_runtime.release_input_coverage import ( - DEFAULT_MINIMUM_NONDEFAULT_MASS_SHARE, -) -from microcosm.build.uk_runtime.spi_income import UKSPIIncomeImputationResult -from microcosm.build.uk_runtime.spi_support import ( - BASE_FRS_SUPPORT_CHANNEL, - SPI_HMRC_TOTAL_EARNED_INCOME_COLUMN, - SPI_HMRC_TOTAL_INVESTMENT_INCOME_COLUMN, - SPI_SYNTHETIC_SUPPORT_CHANNEL, - UKSPISupportResult, - support_channel_column, -) -from microcosm.frame import Frame, MassChangeRecord, WeightKind - -_TEST_SOURCE_FINGERPRINT = _UKSourceFileFingerprint(1, 2, 3, 4, 5) -_FRS_SOURCE_EVIDENCE = { - "source_vintage": "2023-24", - "adult": {"raw_variable": "ADULT.INEARNS", "sha256": "a" * 64}, - "benefits": {"raw_variable": "BENEFITS.BENAMT", "sha256": "b" * 64}, -} - - -def _dataset() -> Frame: - return uk_national_frame( - person=pd.DataFrame( - { - "person_id": [1], - "person_household_id": [1], - "person_benunit_id": [1], - "gift_aid": [0.0], - "charitable_investment_gifts": [0.0], - FRS_HMRC_PAY_COLUMN: [20_000.0], - FRS_HMRC_UBISJA_COLUMN: [100.0], - FRS_HMRC_INCPBEN_COLUMN: [0.0], - FRS_HMRC_OSSBEN_IDENTIFIABLE_SUBSET_COLUMN: [50.0], - FRS_HMRC_SRP_REGULAR_CODE5_COLUMN: [0.0], - } - ), - benunit=pd.DataFrame({"benunit_id": [1]}), - household=pd.DataFrame( - { - "household_id": [1], - "household_weight": [10.0], - } - ), - time_period=HMRC_SPI_BUILD_PERIOD, - ) - - -def _tampered_dataset() -> Frame: - """A frame that differs from :func:`_dataset` by one payload value.""" - - frame = _dataset() - person = frame.table("person").copy() - person[FRS_HMRC_PAY_COLUMN] = [20_001.0] - return uk_national_frame( - person=person, - benunit=frame.table("benunit"), - household=frame.table("household"), - time_period=HMRC_SPI_BUILD_PERIOD, - household_weights=frame.weights_for("household").values, - ) - - -def _dataset_from_source( - path: Path, - *, - fingerprint: _UKSourceFileFingerprint = _TEST_SOURCE_FINGERPRINT, -) -> tuple[Frame, UKStagingProvenance]: - """Model the provenance record that only the national H5 loader returns.""" - - return _dataset(), UKStagingProvenance( - source_h5=path.resolve(), - fingerprint=fingerprint, - ) - - -def _candidate_identity( - tmp_path: Path, - *, - verified: bool = True, -) -> UKCertifiedCandidateIdentity: - identity = UKCertifiedCandidateIdentity( - path=(tmp_path / CERTIFIED_UK_CANDIDATE_FILENAME).resolve(), - filename=CERTIFIED_UK_CANDIDATE_FILENAME, - tier=CERTIFIED_UK_CANDIDATE_TIER, - revision=CERTIFIED_UK_CANDIDATE_REVISION, - sha256=CERTIFIED_UK_CANDIDATE_SHA256, - size_bytes=CERTIFIED_UK_CANDIDATE_SIZE_BYTES, - ) - if verified: - object.__setattr__( - identity, - "_verification_token", - hmrc_restoration._CERTIFIED_CANDIDATE_VERIFICATION_TOKEN, - ) - object.__setattr__( - identity, - "_source_file_fingerprint", - _TEST_SOURCE_FINGERPRINT, - ) - return identity - - -def _source_targets(tmp_path: Path) -> HMRCIncomeTargetSet: - source = HMRCIncomeSourceProvenance( - local_path=(tmp_path / "hmrc.ods").resolve(), - sha256="c" * 64, - publication_url="https://www.gov.uk/government/statistics/income-tax-liabilities", - ods_url="https://assets.publishing.service.gov.uk/hmrc.ods", - source_vintage="2023-24", - source_tax_year="2023-24", - source_tax_year_start=2023, - build_period=HMRC_SPI_BUILD_PERIOD, - table_names=("Table_3_6", "Table_3_7"), - size_bytes=166_693, - mime_type="application/vnd.oasis.opendocument.spreadsheet", - ) - targets: list[HMRCIncomeBandTargetRecord] = [] - upper_bounds = (*HMRC_SPI_INCOME_BAND_LOWER_BOUNDS[1:], None) - for lower_bound, upper_bound in zip( - HMRC_SPI_INCOME_BAND_LOWER_BOUNDS, - upper_bounds, - strict=True, - ): - for component in HMRC_SPI_INCOME_COMPONENTS: - for measure, unit in (("count", "people"), ("amount", "GBP")): - targets.append( - HMRCIncomeBandTargetRecord( - name=( - f"hmrc/{component}_{measure}_income_band_" - f"{lower_bound}_to_{upper_bound or 'inf'}" - ), - component=component, - measure=measure, - unit=unit, - value=float(len(targets) + 1), - period=HMRC_SPI_BUILD_PERIOD, - total_income_lower_bound=lower_bound, - total_income_upper_bound=upper_bound, - ) - ) - assert len(targets) == HMRC_SPI_TARGET_RECORD_COUNT - return HMRCIncomeTargetSet(source=source, targets=tuple(targets)) - - -def _support_and_imputation( - dataset: Frame, - tmp_path: Path, - *, - household_weights: tuple[float, ...] = (5.0, 5.0), - gift_aid: tuple[float, ...] | None = None, - charitable_gifts: tuple[float, ...] | None = None, - assessable_income_adjustment: float = 0.0, -) -> tuple[UKSPISupportResult, UKSPIIncomeImputationResult]: - if len(household_weights) < 2: - raise ValueError("A synthetic replay fixture needs at least one SPI row.") - row_count = len(household_weights) - person = pd.concat([dataset.person] * row_count, ignore_index=True) - person["person_id"] = np.arange(1, row_count + 1) - person["person_household_id"] = np.arange(1, row_count + 1) - person["person_benunit_id"] = np.arange(1, row_count + 1) - person[support_channel_column("person")] = [BASE_FRS_SUPPORT_CHANNEL] + [ - SPI_SYNTHETIC_SUPPORT_CHANNEL - ] * (row_count - 1) - if gift_aid is None: - gift_aid = (0.0, *([10.0] * (row_count - 1))) - if charitable_gifts is None: - charitable_gifts = (0.0, *([5.0] * (row_count - 1))) - person["gift_aid"] = gift_aid - person["charitable_investment_gifts"] = charitable_gifts - - household = pd.DataFrame( - { - "household_id": np.arange(1, row_count + 1), - "household_weight": household_weights, - } - ) - benunit = pd.DataFrame({"benunit_id": np.arange(1, row_count + 1)}) - mass_record = MassChangeRecord( - entity="household", - old_total=float(dataset.weights_for("household").total), - new_total=float(sum(household_weights)), - declared_factor=1.0, - reason="reviewed test allocation to one positive-mass SPI channel", - ) - support = UKSPISupportResult( - person=person.copy(), - benunit=benunit, - household=household, - id_multiplier=10, - spi_household_ids=tuple(range(2, row_count + 1)), - household_weight_kind=WeightKind.IMPORTANCE, - mass_log=(mass_record,), - replaced_spi_households=row_count - 1, - spi_prior_mass_share=0.5, - ) - - imputed_person = person.copy() - spi_count = row_count - 1 - total_earned = np.arange(10.0, 10.0 + spi_count) - total_investment = np.arange(5.0, 5.0 + spi_count) - assessable = total_earned + total_investment - assessable[-1] += assessable_income_adjustment - imputed_person[SPI_HMRC_TOTAL_EARNED_INCOME_COLUMN] = ( - np.nan, - *total_earned, - ) - imputed_person[SPI_HMRC_TOTAL_INVESTMENT_INCOME_COLUMN] = ( - np.nan, - *total_investment, - ) - imputed_person[HMRC_SPI_ASSESSABLE_INCOME_COLUMN] = (np.nan, *assessable) - imputation = UKSPIIncomeImputationResult( - person=imputed_person, - fit_weight_records=( - FitWeightRecord("uk_spi_2022_23_income", "design"), - FitWeightRecord("uk_frs_only_spi_fill", "importance"), - ), - donor_path=(tmp_path / "put2223uk.tab").resolve(), - donor_sha256="d" * 64, - donor_size_bytes=141_323_762, - donor_rows=100_000, - stage2_training_rows=1, - spi_prediction_rows=spi_count, - reviewed_absent_stage2_outputs={ - "incapacity_benefit_reported": "reviewed absent" - }, - ) - return support, imputation - - -def _install_replay_mocks( - monkeypatch: pytest.MonkeyPatch, - dataset: Frame, - tmp_path: Path, - *, - household_weights: tuple[float, ...] = (5.0, 5.0), - gift_aid: tuple[float, ...] | None = None, - charitable_gifts: tuple[float, ...] | None = None, - assessable_income_adjustment: float = 0.0, -) -> tuple[ - list[str], - UKSPISupportResult, - UKSPIIncomeImputationResult, - HMRCIncomeTargetSet, -]: - support, imputation = _support_and_imputation( - dataset, - tmp_path, - household_weights=household_weights, - gift_aid=gift_aid, - charitable_gifts=charitable_gifts, - assessable_income_adjustment=assessable_income_adjustment, - ) - targets = _source_targets(tmp_path) - calls: list[str] = [] - donor_identity = object() - ods_identity = object() - actual_crosswalk = hmrc_restoration.assert_frs_hmrc_auxiliary_crosswalk_available - actual_report_builder = hmrc_restoration.build_conservative_hmrc_replay_report - - def fake_contract() -> None: - calls.append("contract") - - def fake_verify_donor(path: Path) -> object: - assert Path(path).name == "put2223uk.tab" - calls.append("donor_identity") - return donor_identity - - def fake_verify_ods(path: Path) -> object: - assert Path(path).name == "hmrc.ods" - calls.append("ods_identity") - return ods_identity - - def fake_crosswalk(person: pd.DataFrame) -> None: - calls.append("frs_crosswalk") - actual_crosswalk(person) - - def fake_targets(verified: object, *, build_period: str) -> HMRCIncomeTargetSet: - assert verified is ods_identity - assert build_period == HMRC_SPI_BUILD_PERIOD - calls.append("targets") - return targets - - def fake_replace(**kwargs: object) -> UKSPISupportResult: - assert kwargs["person"] is dataset.person - assert kwargs["input_weight_kind"] is WeightKind.DESIGN - calls.append("replace") - return support - - def fake_impute( - actual_support: UKSPISupportResult, - _path: Path, - **kwargs: object, - ) -> UKSPIIncomeImputationResult: - assert actual_support is support - assert kwargs["verified_donor"] is donor_identity - calls.append("impute") - return imputation - - def fake_report(*args: object, **kwargs: object): - calls.append("report") - return actual_report_builder(*args, **kwargs) - - def forbidden_calibration(*_args: object, **_kwargs: object) -> None: - raise AssertionError("The adjudicated replay must never calibrate.") - - monkeypatch.setattr( - hmrc_restoration, - "assert_uk_hmrc_income_source_contract_current", - fake_contract, - ) - monkeypatch.setattr( - hmrc_restoration, - "verify_spi_donor_identity", - fake_verify_donor, - ) - monkeypatch.setattr( - hmrc_restoration, - "verify_hmrc_spi_collated_ods", - fake_verify_ods, - ) - monkeypatch.setattr( - hmrc_restoration, - "assert_frs_hmrc_auxiliary_crosswalk_available", - fake_crosswalk, - ) - monkeypatch.setattr( - hmrc_restoration, - "materialize_hmrc_spi_income_band_targets", - fake_targets, - ) - monkeypatch.setattr( - hmrc_restoration, - "replace_uk_spi_support_tables", - fake_replace, - ) - monkeypatch.setattr( - hmrc_restoration, - "impute_uk_spi_income_support", - fake_impute, - ) - monkeypatch.setattr( - hmrc_restoration, - "build_conservative_hmrc_replay_report", - fake_report, - ) - monkeypatch.setattr( - hmrc_restoration, - "calibrate_uk_hmrc_income", - forbidden_calibration, - raising=False, - ) - monkeypatch.setattr( - hmrc_restoration, - "materialize_uk_hmrc_calibration_frame", - forbidden_calibration, - raising=False, - ) - return calls, support, imputation, targets - - -def _restore( - frame: Frame, - candidate: UKCertifiedCandidateIdentity, - tmp_path: Path, - *, - provenance: UKStagingProvenance | None = None, -): - return restore_uk_hmrc_income_family( - frame, - spi_tab_path=tmp_path / "put2223uk.tab", - hmrc_ods_path=tmp_path / "hmrc.ods", - certified_candidate=candidate, - staging_provenance=provenance, - frs_source_evidence=_FRS_SOURCE_EVIDENCE, - ) - - -def test_hmrc_stage_transform_exposes_last_fit_weight_records(tmp_path: Path) -> None: - transform = UKHMRCIncomeStageTransform( - spi_tab_path=tmp_path / "put2223uk.tab", - hmrc_ods_path=tmp_path / "hmrc.ods", - certified_candidate=_candidate_identity(tmp_path), - ) - records = ( - FitWeightRecord("uk_spi_2022_23_income", "design"), - FitWeightRecord("uk_frs_only_spi_fill", "importance"), - ) - - assert transform.fit_weight_records == () - transform.last_result = SimpleNamespace( - imputation=SimpleNamespace(fit_weight_records=records) - ) - assert transform.fit_weight_records == records - - -def test_certified_candidate_verification_binds_size_sha_and_stable_bytes( - monkeypatch: pytest.MonkeyPatch, - tmp_path: Path, -) -> None: - candidate = tmp_path / CERTIFIED_UK_CANDIDATE_FILENAME - contents = b"certified candidate" - candidate.write_bytes(contents) - monkeypatch.setattr( - hmrc_restoration, - "CERTIFIED_UK_CANDIDATE_SIZE_BYTES", - len(contents), - ) - monkeypatch.setattr( - hmrc_restoration, - "CERTIFIED_UK_CANDIDATE_SHA256", - hashlib.sha256(contents).hexdigest(), - ) - - identity = verify_certified_uk_candidate(candidate) - - assert identity.path == candidate.resolve() - assert identity.tier == "frs" - assert identity.size_bytes == len(contents) - assert identity.sha256 == hashlib.sha256(contents).hexdigest() - assert identity._source_file_fingerprint is not None - - tampered = bytes((contents[0] ^ 1,)) + contents[1:] - candidate.write_bytes(tampered) - with pytest.raises(ValueError, match="sha256 .* does not match"): - verify_certified_uk_candidate(candidate) - - -def test_restoration_binds_loaded_candidate_bytes_before_source_io( - monkeypatch: pytest.MonkeyPatch, - tmp_path: Path, -) -> None: - pytest.importorskip("tables") - pytest.importorskip("h5py") - candidate_path = tmp_path / CERTIFIED_UK_CANDIDATE_FILENAME - write_uk_national_frame(_dataset(), candidate_path) - monkeypatch.setattr( - hmrc_restoration, - "CERTIFIED_UK_CANDIDATE_SIZE_BYTES", - candidate_path.stat().st_size, - ) - monkeypatch.setattr( - hmrc_restoration, - "CERTIFIED_UK_CANDIDATE_SHA256", - hashlib.sha256(candidate_path.read_bytes()).hexdigest(), - ) - monkeypatch.setattr( - hmrc_restoration, - "assert_uk_hmrc_income_source_contract_current", - lambda: None, - ) - identity = verify_certified_uk_candidate(candidate_path) - - base = _dataset() - replacement = uk_national_frame( - person=base.person.assign(gift_aid=1.0), - benunit=base.table("benunit"), - household=base.table("household"), - time_period=HMRC_SPI_BUILD_PERIOD, - household_weights=base.weights_for("household").values, - ) - write_uk_national_frame(replacement, candidate_path) - loaded_replacement, replacement_provenance = load_uk_national_frame(candidate_path) - monkeypatch.setattr( - hmrc_restoration, - "verify_spi_donor_identity", - lambda _path: pytest.fail("source I/O preceded candidate byte binding"), - ) - - with pytest.raises(ValueError, match="changed after SHA-256 verification"): - _restore( - loaded_replacement, - identity, - tmp_path, - provenance=replacement_provenance, - ) - - -def test_restoration_rejects_forged_or_unbound_candidate_identity( - monkeypatch: pytest.MonkeyPatch, - tmp_path: Path, -) -> None: - monkeypatch.setattr( - hmrc_restoration, - "assert_uk_hmrc_income_source_contract_current", - lambda: None, - ) - forged = _candidate_identity(tmp_path, verified=False) - forged_frame, forged_provenance = _dataset_from_source(forged.path) - with pytest.raises(ValueError, match="must come from verify_certified"): - _restore(forged_frame, forged, tmp_path, provenance=forged_provenance) - - verified = _candidate_identity(tmp_path) - with pytest.raises(ValueError, match="loaded from the verified"): - _restore(_dataset(), verified, tmp_path) - - object.__setattr__(verified, "tier", "public") - bound_frame, bound_provenance = _dataset_from_source(verified.path) - with pytest.raises(ValueError, match="base identity does not match"): - _restore(bound_frame, verified, tmp_path, provenance=bound_provenance) - - -def test_source_pair_is_verified_before_parse_or_support_rebuild( - monkeypatch: pytest.MonkeyPatch, - tmp_path: Path, -) -> None: - candidate = _candidate_identity(tmp_path) - dataset, provenance = _dataset_from_source(candidate.path) - calls: list[str] = [] - monkeypatch.setattr( - hmrc_restoration, - "assert_uk_hmrc_income_source_contract_current", - lambda: None, - ) - monkeypatch.setattr( - hmrc_restoration, - "verify_spi_donor_identity", - lambda _path: calls.append("donor_identity") or object(), - ) - - def reject_ods(_path: Path) -> object: - calls.append("ods_identity") - raise RuntimeError("reviewed ODS identity mismatch") - - def forbidden(*_args: object, **_kwargs: object) -> None: - raise AssertionError("source parsing/support ran before paired preflight") - - monkeypatch.setattr( - hmrc_restoration, - "verify_hmrc_spi_collated_ods", - reject_ods, - ) - monkeypatch.setattr( - hmrc_restoration, - "materialize_hmrc_spi_income_band_targets", - forbidden, - ) - monkeypatch.setattr( - hmrc_restoration, - "replace_uk_spi_support_tables", - forbidden, - ) - monkeypatch.setattr( - hmrc_restoration, - "impute_uk_spi_income_support", - forbidden, - ) - - with pytest.raises(RuntimeError, match="reviewed ODS identity mismatch"): - _restore(dataset, candidate, tmp_path, provenance=provenance) - assert calls == ["donor_identity", "ods_identity"] - - -def test_restoration_runs_replay_without_calibration_and_emits_208_facts( - monkeypatch: pytest.MonkeyPatch, - tmp_path: Path, -) -> None: - candidate = _candidate_identity(tmp_path) - dataset, provenance = _dataset_from_source(candidate.path) - calls, support, imputation, targets = _install_replay_mocks( - monkeypatch, - dataset, - tmp_path, - ) - - restored = _restore(dataset, candidate, tmp_path, provenance=provenance) - - assert calls == [ - "contract", - "donor_identity", - "ods_identity", - "frs_crosswalk", - "targets", - "replace", - "impute", - "report", - ] - assert restored.support is support - assert restored.imputation is imputation - assert restored.source_targets is targets - assert uk_household_weight_kind(restored.frame) is WeightKind.IMPORTANCE - assert restored.frame.mass_log == support.mass_log - assert restored.post_draw_identity_rows == 1 - assert restored.distributional_mass_shares == { - "gift_aid": 0.5, - "charitable_investment_gifts": 0.5, - } - assert len(restored.replay_report.facts) == HMRC_SPI_TARGET_RECORD_COUNT == 208 - assert restored.replay_report.summary["excluded_with_fence"] == 208 - assert restored.replay_report.summary["exact_pass"] == 0 - assert restored.replay_report.source_evidence["certified_candidate"]["tier"] == ( - "frs" - ) - evidence = restored.evidence() - assert evidence["calibration"] == { - "performed": False, - "reason": ( - "Complete FRS Total Income band assignment is unavailable; the 208 " - "facts are reviewed exclusions rather than biased calibration " - "constraints." - ), - "output_weight_kind": "importance", - } - assert evidence["post_draw_identity"]["exact"] is True - - -def test_sampled_rung_defers_the_effective_mass_floor_to_the_terminal_gate( - monkeypatch: pytest.MonkeyPatch, - tmp_path: Path, -) -> None: - """A #627 rung build records a thin imputed column instead of aborting. - - Sparse imputed columns can legitimately restore near-zero effective mass - on a small sample; the declared rung defers the mid-stage floor to the - terminal input-coverage gate, whose verdict is receipted. The strict - raise is unchanged without the declaration. - """ - - candidate = _candidate_identity(tmp_path) - dataset, provenance = _dataset_from_source(candidate.path) - _install_replay_mocks(monkeypatch, dataset, tmp_path) - monkeypatch.setattr( - hmrc_restoration, - "_distributional_mass_shares", - lambda _frame: {"charitable_investment_gifts": 1e-7}, - ) - - with pytest.raises(RuntimeError, match="did not restore required effective-mass"): - _restore(dataset, candidate, tmp_path, provenance=provenance) - - restored = restore_uk_hmrc_income_family( - dataset, - spi_tab_path=tmp_path / "put2223uk.tab", - hmrc_ods_path=tmp_path / "hmrc.ods", - certified_candidate=candidate, - staging_provenance=provenance, - frs_source_evidence=_FRS_SOURCE_EVIDENCE, - sampled_rung=True, - ) - # The thin share still reaches the replay report's evidence surface. - assert restored.distributional_mass_shares == {"charitable_investment_gifts": 1e-7} - - -def test_post_draw_total_income_identity_is_exact_not_tolerance_based( - monkeypatch: pytest.MonkeyPatch, - tmp_path: Path, -) -> None: - candidate = _candidate_identity(tmp_path) - dataset, provenance = _dataset_from_source(candidate.path) - _install_replay_mocks( - monkeypatch, - dataset, - tmp_path, - assessable_income_adjustment=np.spacing(15.0), - ) - - with pytest.raises(RuntimeError, match=r"must equal deterministic TEI \+ TII"): - _restore(dataset, candidate, tmp_path, provenance=provenance) - - -@pytest.mark.parametrize( - "thin_column", - ("gift_aid", "charitable_investment_gifts"), -) -def test_distributional_inputs_must_reach_one_ppm_positive_spi_mass( - monkeypatch: pytest.MonkeyPatch, - tmp_path: Path, - thin_column: str, -) -> None: - candidate = _candidate_identity(tmp_path) - dataset, provenance = _dataset_from_source(candidate.path) - household_weights = (9.899991, 0.1, 0.000009) - values = { - "gift_aid": (0.0, 1.0, 0.0), - "charitable_investment_gifts": (0.0, 1.0, 0.0), - } - values[thin_column] = (0.0, 0.0, 1.0) - _install_replay_mocks( - monkeypatch, - dataset, - tmp_path, - household_weights=household_weights, - gift_aid=values["gift_aid"], - charitable_gifts=values["charitable_investment_gifts"], - ) - - with pytest.raises(RuntimeError, match="required effective-mass") as error: - _restore(dataset, candidate, tmp_path, provenance=provenance) - assert thin_column in str(error.value) - assert "e-07" in str(error.value) - - -def test_distributional_one_ppm_floor_is_inclusive( - monkeypatch: pytest.MonkeyPatch, - tmp_path: Path, -) -> None: - candidate = _candidate_identity(tmp_path) - dataset, provenance = _dataset_from_source(candidate.path) - _install_replay_mocks( - monkeypatch, - dataset, - tmp_path, - household_weights=(9.99999, 0.00001), - gift_aid=(0.0, 1.0), - charitable_gifts=(0.0, 1.0), - ) - - restored = _restore(dataset, candidate, tmp_path, provenance=provenance) - - assert set(restored.distributional_mass_shares) == { - "gift_aid", - "charitable_investment_gifts", - } - for share in restored.distributional_mass_shares.values(): - assert share >= DEFAULT_MINIMUM_NONDEFAULT_MASS_SHARE - assert share == pytest.approx(DEFAULT_MINIMUM_NONDEFAULT_MASS_SHARE) - - -def test_stage_transform_requires_retained_leaf_stage_and_forwards_evidence( - monkeypatch: pytest.MonkeyPatch, - tmp_path: Path, -) -> None: - dataset = _dataset() - transform = UKHMRCIncomeStageTransform( - spi_tab_path=tmp_path / "put2223uk.tab", - hmrc_ods_path=tmp_path / "hmrc.ods", - certified_candidate=_candidate_identity(tmp_path), - ) - with pytest.raises(RuntimeError, match="retained-leaves stage"): - transform(dataset) - - # A retained result that cannot prove its content lineage fails closed - # rather than downgrading to a weaker check. - unprovable = SimpleNamespace( - frame=_tampered_dataset(), - evidence=lambda: _FRS_SOURCE_EVIDENCE, - ) - transform.retained_leaves_transform = SimpleNamespace(last_result=unprovable) - with pytest.raises(RuntimeError, match="carries no output_content_identity"): - transform(dataset) - - # A retained result whose recorded output differs from the received - # frame's content is a substitution and is refused. - tampered = _tampered_dataset() - stale_result = SimpleNamespace( - frame=tampered, - evidence=lambda: _FRS_SOURCE_EVIDENCE, - output_content_identity=uk_frame_content_identity(tampered), - ) - transform.retained_leaves_transform = SimpleNamespace(last_result=stale_result) - with pytest.raises(RuntimeError, match="not bound to the frame"): - transform(dataset) - - retained_result = SimpleNamespace( - frame=dataset, - evidence=lambda: _FRS_SOURCE_EVIDENCE, - ) - transform.retained_leaves_transform.last_result = retained_result - expected = SimpleNamespace(frame=dataset) - forwarded: dict[str, object] = {} - - def fake_restore(*_args: object, **kwargs: object): - forwarded.update(kwargs) - return expected - - monkeypatch.setattr( - hmrc_restoration, - "restore_uk_hmrc_income_family", - fake_restore, - ) - - assert transform(dataset) is dataset - assert transform.last_result is expected - assert forwarded["frs_source_evidence"] == _FRS_SOURCE_EVIDENCE - - -def test_stage_transform_binding_is_single_use_and_asserts_descent( - monkeypatch: pytest.MonkeyPatch, - tmp_path: Path, -) -> None: - """The provenance binding carries the descent fence by content identity. - - Binding records the loaded frame's content identity; the stage consumes - the binding on use, so a stale binding can never fence a later run, and - a pipeline whose first stage consumed a content-different frame fails - closed even when a matching load once happened. - """ - - loaded = _dataset() - provenance = UKStagingProvenance( - source_h5=(tmp_path / "populace_uk_2023.h5").resolve(), - fingerprint=_TEST_SOURCE_FINGERPRINT, - ) - transform = UKHMRCIncomeStageTransform( - spi_tab_path=tmp_path / "put2223uk.tab", - hmrc_ods_path=tmp_path / "hmrc.ods", - certified_candidate=_candidate_identity(tmp_path), - ) - forwarded: list[object] = [] - monkeypatch.setattr( - hmrc_restoration, - "restore_uk_hmrc_income_family", - lambda frame, **kwargs: ( - forwarded.append(kwargs["staging_provenance"]), - SimpleNamespace(frame=frame), - )[1], - ) - - # Descent violation: the pipeline's first stage consumed a frame whose - # content differs from the one the driver loaded and bound. - substituted = _tampered_dataset() - transform.retained_leaves_transform = SimpleNamespace( - last_result=SimpleNamespace( - frame=loaded, - evidence=lambda: _FRS_SOURCE_EVIDENCE, - input_content_identity=uk_frame_content_identity(substituted), - output_content_identity=uk_frame_content_identity(loaded), - ), - ) - transform.bind_staging_provenance(provenance, loaded) - with pytest.raises(RuntimeError, match="did not start from the frame"): - transform(loaded) - assert transform.staging_provenance is None - assert transform.bound_input_identity is None - - # Descent-consistent run forwards the bound provenance exactly once... - transform.retained_leaves_transform.last_result.input_content_identity = ( - uk_frame_content_identity(loaded) - ) - transform.bind_staging_provenance(provenance, loaded) - assert transform(loaded) is loaded - assert forwarded == [provenance] - - # ...and a second run without rebinding gets no provenance (the real - # restore then fails closed on staging_provenance=None). - assert transform(loaded) is loaded - assert forwarded == [provenance, None] - - -def test_stage_transform_descent_fence_is_content_addressed( - monkeypatch: pytest.MonkeyPatch, - tmp_path: Path, -) -> None: - """The descent fence survives a process boundary by construction. - - A content-identical frame that is a different Python object — the shape - a checkpoint rehydration produces — passes both fences; a tampered - frame with the same structure fails them. Object identity is only a - fast path, never the guarantee. - """ - - loaded = _dataset() - provenance = UKStagingProvenance( - source_h5=(tmp_path / "populace_uk_2023.h5").resolve(), - fingerprint=_TEST_SOURCE_FINGERPRINT, - ) - transform = UKHMRCIncomeStageTransform( - spi_tab_path=tmp_path / "put2223uk.tab", - hmrc_ods_path=tmp_path / "hmrc.ods", - certified_candidate=_candidate_identity(tmp_path), - ) - monkeypatch.setattr( - hmrc_restoration, - "restore_uk_hmrc_income_family", - lambda frame, **kwargs: SimpleNamespace(frame=frame), - ) - - # Rehydration shape: the received frame and the bound input are fresh, - # content-identical reconstructions, not the original objects. - stage_output = _dataset() - transform.retained_leaves_transform = SimpleNamespace( - last_result=SimpleNamespace( - frame=stage_output, - evidence=lambda: _FRS_SOURCE_EVIDENCE, - input_content_identity=uk_frame_content_identity(_dataset()), - output_content_identity=uk_frame_content_identity(stage_output), - ), - ) - transform.bind_staging_provenance(provenance, loaded) - rehydrated = _dataset() - assert rehydrated is not stage_output - assert transform(rehydrated) is rehydrated - - # Tampered payload with identical structure: fence A refuses it. - transform.bind_staging_provenance(provenance, loaded) - with pytest.raises(RuntimeError, match="not bound to the frame"): - transform(_tampered_dataset()) - - -@pytest.mark.parametrize( - ("override", "value"), - (("donor_sample_size", None), ("spi_prior_mass_share", 0.25)), -) -def test_restoration_rejects_unreviewed_release_parameter_overrides( - monkeypatch: pytest.MonkeyPatch, - tmp_path: Path, - override: str, - value: object, -) -> None: - candidate = _candidate_identity(tmp_path) - monkeypatch.setattr( - hmrc_restoration, - "assert_uk_hmrc_income_source_contract_current", - lambda: None, - ) - kwargs = { - "spi_tab_path": tmp_path / "put2223uk.tab", - "hmrc_ods_path": tmp_path / "hmrc.ods", - "certified_candidate": candidate, - "frs_source_evidence": _FRS_SOURCE_EVIDENCE, - override: value, - } - - frame, provenance = _dataset_from_source(candidate.path) - with pytest.raises(ValueError, match="reviewed source manifest"): - restore_uk_hmrc_income_family( - frame, - staging_provenance=provenance, - **kwargs, - ) - - -def test_checkpoint_metadata_round_trips_the_fit_weight_audit(tmp_path) -> None: - """A resumed SPI stage still feeds the weights audit from its record.""" - - from microcosm.build.gates import FitWeightRecord - - transform = UKHMRCIncomeStageTransform( - spi_tab_path=tmp_path / "put2223uk.tab", - hmrc_ods_path=tmp_path / "hmrc.ods", - certified_candidate=_candidate_identity(tmp_path), - ) - with pytest.raises(RuntimeError, match="completed SPI restoration run"): - transform.checkpoint_metadata() - - records = ( - FitWeightRecord(fit_name="uk_spi_fill_qrf", weight_kind="design"), - FitWeightRecord(fit_name="uk_spi_income_qrf", weight_kind="design"), - ) - evidence = {"stage": "hmrc_spi_income", "post_draw_identity_rows": 3} - replay_payload = {"summary": {"status": "comparisons_passed"}, "facts": {}} - transform.last_result = SimpleNamespace( - frame=_dataset(), - imputation=SimpleNamespace(fit_weight_records=records), - evidence=lambda: dict(evidence), - replay_report=SimpleNamespace(to_payload=lambda: dict(replay_payload)), - ) - metadata = transform.checkpoint_metadata() - assert metadata["output_content_identity"] == uk_frame_content_identity(_dataset()) - - resumed = UKHMRCIncomeStageTransform( - spi_tab_path=tmp_path / "put2223uk.tab", - hmrc_ods_path=tmp_path / "hmrc.ods", - certified_candidate=_candidate_identity(tmp_path), - ) - # A resume consumes the single-use binding: the stage will not run. - resumed.bind_staging_provenance( - UKStagingProvenance( - source_h5=(tmp_path / "populace_uk_2023.h5").resolve(), - fingerprint=_TEST_SOURCE_FINGERPRINT, - ), - _dataset(), - ) - resumed.resume_from_checkpoint(metadata, _dataset()) - assert resumed.fit_weight_records == records - assert resumed.last_result.evidence() == evidence - assert resumed.last_result.replay_payload == replay_payload - assert resumed.staging_provenance is None - assert resumed.bound_input_identity is None - - with pytest.raises(RuntimeError, match="cannot feed the weights audit"): - resumed.resume_from_checkpoint({}, _dataset()) - - # The terminal stage runs the same drift check as the retained stage: - # a frame that does not match the recorded output identity is refused. - with pytest.raises(RuntimeError, match="drifted record"): - resumed.resume_from_checkpoint(metadata, _tampered_dataset()) diff --git a/packages/microcosm-build/tests/test_uk_ladder_rowwise_clone.py b/packages/microcosm-build/tests/test_uk_ladder_rowwise_clone.py index 71101a331..114401945 100644 --- a/packages/microcosm-build/tests/test_uk_ladder_rowwise_clone.py +++ b/packages/microcosm-build/tests/test_uk_ladder_rowwise_clone.py @@ -410,7 +410,7 @@ def test_driver_ladder_route_builds_with_gate(monkeypatch, toy_ladder, tmp_path) assert summary["assigned_constituencies"] >= 4 assert summary["weights"]["household_weight_kind"] == "importance" assert summary["weights"]["mass_conservation"]["passed"] is True - assert (output_dir / "populace_uk_2023_rowwise.h5").exists() + assert (output_dir / "staging_rowwise.h5").exists() def test_driver_ladder_dry_run_matches_real_assignment( @@ -442,7 +442,7 @@ def test_driver_ladder_dry_run_matches_real_assignment( monkeypatch.setattr(sys, "argv", [*base_argv, "--out", str(plan_dir), "--dry-run"]) assert builder.main() == 0 plan = json.loads((plan_dir / builder.DRY_RUN_PLAN_FILENAME).read_text()) - assert not (plan_dir / "populace_uk_2023_rowwise.h5").exists() + assert not (plan_dir / "staging_rowwise.h5").exists() monkeypatch.setattr(sys, "argv", [*base_argv, "--out", str(build_dir)]) assert builder.main() == 0 @@ -453,7 +453,7 @@ def test_driver_ladder_dry_run_matches_real_assignment( row["area_code"]: row["rows"] for row in plan["realized_support"]["constituency"]["bottom"] } - with pd.HDFStore(build_dir / "populace_uk_2023_rowwise.h5", mode="r") as store: + with pd.HDFStore(build_dir / "staging_rowwise.h5", mode="r") as store: household = store["household"] built_counts = household["constituency_code"].value_counts() for code, rows in realized.items(): @@ -547,7 +547,7 @@ def test_ladder_clone_rejects_unknown_weight_kind_h5(toy_ladder, tmp_path) -> No pytest.importorskip("h5py") import h5py - from microcosm.build.uk_runtime.national_build import ( + from microcosm.build.uk_runtime.national_frame import ( UK_HOUSEHOLD_WEIGHT_KIND_ATTR, ) diff --git a/packages/microcosm-build/tests/test_uk_national_build.py b/packages/microcosm-build/tests/test_uk_national_build.py deleted file mode 100644 index a9419c8ed..000000000 --- a/packages/microcosm-build/tests/test_uk_national_build.py +++ /dev/null @@ -1,2004 +0,0 @@ -from __future__ import annotations - -import json -from datetime import date, datetime -from pathlib import Path -from types import SimpleNamespace - -import numpy as np -import pandas as pd -import pytest - -from microcosm.build.country_spec import country_stage_plan, load_country_spec -from microcosm.build.gate_battery import ( - EvidenceContext, - GateBatteryBlockedError, - gate_signing_key_env, -) -from microcosm.build.gates import FitWeightRecord, GateResult -from microcosm.build.ledger_targets import LedgerTargetReference -from microcosm.build.plan import Stage, StagePlan -from microcosm.build.uk_runtime.battery_bindings import ( - UK_GATE_REGISTRY, - UKGateBinding, - _evaluate_calibration_reference_coverage, -) -from microcosm.build.uk_runtime.national_build import ( - UKNationalStage, - build_uk_national_dataset, - load_uk_national_frame, -) -from microcosm.build.uk_runtime.national_calibration import ( - UKNationalCalibrationStage, -) -from microcosm.build.uk_runtime.national_doctrine import UKNationalSolveDoctrine -from microcosm.build.uk_runtime.national_frame import ( - _uk_gate_surface, - uk_household_weight_kind, - uk_national_frame, - uk_time_period, -) -from microcosm.build.uk_runtime.parity_reference import ( - EfrsParityReference, - EfrsParitySource, -) -from microcosm.build.uk_runtime.release_input_coverage import ( - uk_release_input_coverage_gate, -) -from microcosm.calibrate import TargetRegistry, TargetSpec -from microcosm.frame import Frame, MassChangeRecord, WeightKind - -TEST_UK_RELEASE_ID = "populace-uk-2023-frs-k535080" -TEST_UK_CALIBRATION_DIAGNOSTICS_SHA256 = "c" * 64 -TEST_UK_TERMINAL_GATE_SIGNING_KEY = "MDEyMzQ1Njc4OWFiY2RlZjAxMjM0NTY3ODlhYmNkZWY=" -#: A fixed exclusion clock inside the committed register's validity window -#: keeps toy builds deterministic across the suite's lifetime. -TEST_UK_EXCLUSION_CLOCK = date(2026, 9, 1) - - -def _toy_coverage_evaluator(context, parameters): - """Pass the manifest preflight, run the real coverage gate at terminal. - - The manifest-currency check needs the shipped coverage machinery these - toy builds do not carry; the terminal verdict stays the real gate over - the real frame surface, as the legacy seam fixture ran it. - """ - - if parameters.get("check") == "manifest_current": - return GateResult( - name="release_input_coverage", - passed=True, - details={"check": "manifest_current", "toy_preflight": True}, - ) - return uk_release_input_coverage_gate( - _uk_gate_surface(context.frame), context.artifacts["coverage_engine"] - ) - - -def _toy_gate_registry() -> dict[str, UKGateBinding]: - """The seam-test registry: real terminal coverage, pass-through roster. - - These seam tests use toy stages, so the family-roster gate and the - manifest preflight are pass-throughs (both have their own tests) and - every gate without a binding is a named ``evidence_absent`` gap — - non-blocking off the release-candidate posture, exactly the legacy - fixture's effect of reporting only the coverage verdict. The one - exception is the weights audit: its manifest entry declares - ``evidence_absent_blocks`` (an absent audit is not a passing audit, - in every posture), so the seam registry binds it as a pass-through — - the strict-absence behavior has its own tests. - """ - - return { - "weights_audit": UKGateBinding( - name="weights_audit", - evaluator=lambda context, parameters: GateResult( - name="weights_audit", - passed=True, - details={"toy_audit": True}, - ), - needs_frame=False, - ), - "release_input_coverage": UKGateBinding( - name="release_input_coverage", - evaluator=_toy_coverage_evaluator, - parameter_keys=frozenset({"check"}), - artifact_keys=frozenset({"coverage_engine"}), - frame_predicate=( - lambda parameters: parameters.get("check") != "manifest_current" - ), - legacy_name="uk_release_input_coverage", - ), - "source_coverage": UKGateBinding( - name="source_coverage", - evaluator=lambda context, parameters: GateResult( - name="source_coverage", - passed=True, - details={"toy_stage_roster": True}, - ), - needs_frame=False, - ), - } - - -def _run_national_build(**kwargs): - kwargs.setdefault("gate_registry", _toy_gate_registry()) - kwargs.setdefault("now", TEST_UK_EXCLUSION_CLOCK) - return build_uk_national_dataset( - release_id=TEST_UK_RELEASE_ID, - calibration_diagnostics_sha256=TEST_UK_CALIBRATION_DIAGNOSTICS_SHA256, - **kwargs, - ) - - -def _replace_person(frame: Frame, person: pd.DataFrame) -> Frame: - """Rebuild the frame with the person table replaced (mass untouched).""" - - return uk_national_frame( - person=person, - benunit=frame.table("benunit"), - household=frame.table("household"), - time_period=uk_time_period(frame), - weight_kind=uk_household_weight_kind(frame), - household_weights=frame.weights_for("household").values, - mass_log=frame.mass_log, - ) - - -def _assert_same_frame_payload(left: Frame, right: Frame) -> None: - assert left.schema == right.schema - assert left.entities == right.entities - for entity in left.entities: - pd.testing.assert_frame_equal( - left.table(entity), - right.table(entity), - check_exact=True, - check_dtype=True, - ) - assert left.weighted_entities == right.weighted_entities - for entity in left.weighted_entities: - assert left.weights_for(entity).kind is right.weights_for(entity).kind - pd.testing.assert_series_equal( - pd.Series(left.weights_for(entity).values), - pd.Series(right.weights_for(entity).values), - check_exact=True, - check_dtype=True, - ) - pd.testing.assert_series_equal(left.strata, right.strata, check_exact=True) - assert left.mass_log == right.mass_log - assert left.metadata == right.metadata - - -@pytest.fixture(autouse=True) -def _trusted_terminal_gate_signing_key(monkeypatch) -> None: - monkeypatch.setenv( - gate_signing_key_env("uk"), - TEST_UK_TERMINAL_GATE_SIGNING_KEY, - ) - - -def _write_toy_h5(path: Path, *, employment_income: float = 0.0) -> None: - with pd.HDFStore(path) as store: - store.put( - "person", - pd.DataFrame( - { - "person_id": [10], - "person_household_id": [1], - "person_benunit_id": [100], - "employment_income": [employment_income], - } - ), - format="table", - data_columns=True, - ) - store.put( - "benunit", - pd.DataFrame({"benunit_id": [100]}), - format="table", - data_columns=True, - ) - store.put( - "household", - pd.DataFrame( - { - "household_id": [1], - "household_weight": [2.0], - } - ), - format="table", - data_columns=True, - ) - store.put( - "time_period", - pd.Series(["2023"]), - format="table", - data_columns=True, - ) - - -def _write_two_row_h5( - path: Path, - *, - employment_income: tuple[float, float] = (40_000.0, 55_000.0), - include_calibration_columns: bool = False, -) -> None: - n = 100 - household_ids = np.arange(1, n + 1) - person_ids = np.arange(10, 10 + n) - benunit_ids = np.arange(100, 100 + n) - employment = np.resize(np.asarray(employment_income, dtype=float), n) - - def flags(true_count: int) -> list[bool]: - return [index < true_count for index in range(n)] - - with pd.HDFStore(path) as store: - store.put( - "person", - pd.DataFrame( - { - "person_id": person_ids, - "person_household_id": household_ids, - "person_benunit_id": benunit_ids, - "employment_income": employment, - "age": [6 + index % 3 for index in range(n)], - "would_claim_marriage_allowance": flags(50), - "would_claim_scp": flags(85), - "attends_private_school_random_draw": np.linspace(0.01, 0.99, n), - } - ), - format="table", - data_columns=True, - ) - benunit = pd.DataFrame( - { - "benunit_id": benunit_ids, - "would_claim_child_benefit": flags(89), - "child_benefit_opts_out": flags(23), - "would_claim_pc": flags(70), - "would_claim_uc": flags(55), - "would_claim_tfc": flags(88), - "would_claim_extended_childcare": flags(81), - "would_claim_universal_childcare": flags(56), - "would_claim_targeted_childcare": flags(60), - "maximum_extended_childcare_hours_usage": np.linspace(1.0, 30.0, n), - } - ) - if include_calibration_columns: - benunit["universal_credit"] = flags(55) - store.put( - "benunit", - benunit, - format="table", - data_columns=True, - ) - store.put( - "household", - pd.DataFrame( - { - "household_id": household_ids, - "household_weight": np.ones(n), - "household_is_spi_synthetic": [ - index % 2 == 1 for index in range(n) - ], - "household_is_capital_gains_clone": [ - index % 4 >= 2 for index in range(n) - ], - "household_owns_tv": flags(95), - "would_evade_tv_licence_fee": flags(13), - "main_residential_property_purchased_is_first_home": flags(38), - "property_purchased": flags(4), - "brma": [ - "ABERDEEN_AND_SHIRE" if index % 2 == 0 else "ARGYLL_AND_BUTE" - for index in range(n) - ], - } - ), - format="table", - data_columns=True, - ) - store.put( - "time_period", - pd.Series(["2023"]), - format="table", - data_columns=True, - ) - - -def _passing_gate() -> GateResult: - return GateResult( - name="uk_release_input_coverage", - passed=True, - failures=(), - details={"required_columns": 1, "missing": [], "degenerate": []}, - ) - - -def _failing_gate() -> GateResult: - return GateResult( - name="uk_release_input_coverage", - passed=False, - failures=("required column employment_income is default-only",), - details={ - "required_columns": 1, - "missing": [], - "degenerate": ["employment_income"], - }, - ) - - -def _registry_with_coverage(gate_result_factory) -> dict[str, UKGateBinding]: - """The toy registry with the terminal coverage verdict stubbed.""" - - def evaluator(context, parameters): - if parameters.get("check") == "manifest_current": - return GateResult( - name="release_input_coverage", - passed=True, - details={"check": "manifest_current", "toy_preflight": True}, - ) - return gate_result_factory() - - registry = _toy_gate_registry() - registry["release_input_coverage"] = UKGateBinding( - name="release_input_coverage", - evaluator=evaluator, - parameter_keys=frozenset({"check"}), - artifact_keys=frozenset({"coverage_engine"}), - frame_predicate=( - lambda parameters: parameters.get("check") != "manifest_current" - ), - legacy_name="uk_release_input_coverage", - ) - return registry - - -def _registry_with_calibration() -> dict[str, UKGateBinding]: - registry = _registry_with_coverage(_passing_gate) - registry["calibration_reference_coverage"] = UK_GATE_REGISTRY[ - "calibration_reference_coverage" - ] - return registry - - -def _registry_with_calibration_and_parity_trio() -> dict[str, UKGateBinding]: - registry = _registry_with_calibration() - for name in ("export_surface", "target_surface", "target_fit"): - registry[name] = UK_GATE_REGISTRY[name] - return registry - - -def _uc_reference(**overrides) -> LedgerTargetReference: - values = { - "name": "dwp.uc.households", - "ledger_selector": { - "source_name": "dwp", - "source_concept": "dwp.uc_benefit_units", - "geography_level": "country", - }, - "entity": "benunit", - "measure": "dwp/uc/households", - "family": "dwp_uc", - "period": 2025, - "metadata": {"contract_target_id": "dwp.uc.households"}, - } - values.update(overrides) - return LedgerTargetReference(**values) - - -def _calibration_fact(value: float = 60.0) -> dict: - return { - "aggregate_fact_key": "ledger.aggregate_fact.v2:uc-build-fixture", - "aggregation": {"method": "sum"}, - "assertion": "observation", - "geography": {"level": "country", "id": "K02000001"}, - "observed_measure": { - "source_name": "dwp", - "source_concept": "dwp.uc_benefit_units", - "source_measure_id": "total_units", - "unit": "count", - }, - "period": {"type": "month", "value": "2025-12"}, - "value": value, - } - - -def _calibration_registry(value: float = 60.0) -> TargetRegistry: - return TargetRegistry( - [ - TargetSpec( - name="dwp.uc.households", - entity="benunit", - measure="dwp/uc/households", - value=value, - source="test", - metadata={"contract_target_id": "dwp.uc.households"}, - ) - ], - country="uk", - ) - - -def test_driver_validates_the_uk_residue_after_each_stage( - monkeypatch, tmp_path -) -> None: - """The driver's post-stage validate is load-bearing, not decorative. - - A stage can directly construct a kernel-valid Frame carrying the exported - ``household_weight`` column. Only the driver's - ``validate_uk_national_frame`` call can stop that column from returning - to the in-build carrier. - """ - - pytest.importorskip("tables") - - input_h5 = tmp_path / "base.h5" - _write_two_row_h5(input_h5) - - def return_export_column(frame: Frame) -> Frame: - return Frame( - { - "person": frame.table("person"), - "benunit": frame.table("benunit"), - "household": frame.table("household").assign(household_weight=999.0), - }, - frame.schema, - {"household": frame.weights_for("household")}, - metadata=frame.metadata, - mass_log=frame.mass_log, - ) - - with pytest.raises(ValueError, match="must not persist exported weight"): - _run_national_build( - input_h5=input_h5, - staging_h5=tmp_path / "staging.h5", - stages=(UKNationalStage("export_column", return_export_column),), - coverage_engine=object(), - ) - - -class _RecordedFitStage: - fit_weight_records = ( - FitWeightRecord("uk_spi_2022_23_income", "design"), - FitWeightRecord("uk_frs_only_spi_fill", "importance"), - ) - - def __call__(self, frame: Frame) -> Frame: - return frame - - -class _WASRecordedFitStage: - fit_weight_records = ( - FitWeightRecord("uk_was_2018_20_wealth:owned_land", "explicit"), - FitWeightRecord("uk_was_2018_20_wealth:cash_isa", "explicit"), - ) - - def __call__(self, frame: Frame) -> Frame: - return frame - - -def test_stage_fit_weight_records_aggregates_every_fitting_stage() -> None: - from types import SimpleNamespace - - from microcosm.build.uk_runtime.national_build import _stage_fit_weight_records - - plain = SimpleNamespace(name="frs_take_up", transform=lambda frame: frame) - hmrc = SimpleNamespace(name="hmrc_spi_income", transform=_RecordedFitStage()) - was = SimpleNamespace(name="was_wealth", transform=_WASRecordedFitStage()) - - assert _stage_fit_weight_records((plain,)) is None - # A declared fitting stage with a hollow transform owes evidence: the - # failing empty artifact, not a named absence. - assert ( - _stage_fit_weight_records( - (SimpleNamespace(name="was_wealth", transform=lambda frame: frame),) - ) - == () - ) - records = _stage_fit_weight_records((plain, hmrc, was)) - assert [record.fit_name for record in records] == [ - "uk_spi_2022_23_income", - "uk_frs_only_spi_fill", - "uk_was_2018_20_wealth:owned_land", - "uk_was_2018_20_wealth:cash_isa", - ] - - class _EmptyFitStage: - fit_weight_records = () - - def __call__(self, frame: Frame) -> Frame: - return frame - - # A scheduled fitting stage with no records is missing evidence: it must - # force the failing empty artifact, never be absorbed by another stage's - # records (the audit-bypass the adversarial review flagged). - assert ( - _stage_fit_weight_records( - (hmrc, SimpleNamespace(name="was_wealth", transform=_EmptyFitStage())) - ) - == () - ) - - -def test_weights_audit_details_carry_the_was_fit_records() -> None: - from microcosm.build.gate_battery import EvidenceContext - from microcosm.build.uk_runtime.battery_bindings import UK_GATE_REGISTRY - - binding = UK_GATE_REGISTRY["weights_audit"] - combined = ( - *_RecordedFitStage.fit_weight_records, - *_WASRecordedFitStage.fit_weight_records, - ) - result = binding.evaluate( - EvidenceContext(artifacts={"fit_weight_records": combined}), - {}, - ) - assert result.passed - resolved = result.details["resolved_weight_kinds"] - assert resolved["uk_was_2018_20_wealth:owned_land"] == "explicit" - assert resolved["uk_was_2018_20_wealth:cash_isa"] == "explicit" - - -def test_resumed_national_calibration_feeds_reference_coverage_gate( - tmp_path, -) -> None: - from microcosm.build.uk_runtime.national_build import ( - _stage_calibration_evidence, - ) - - pytest.importorskip("tables") - - input_h5 = tmp_path / "base.h5" - _write_two_row_h5(input_h5, include_calibration_columns=True) - frame, _provenance = load_uk_national_frame(input_h5) - stage = UKNationalCalibrationStage( - _calibration_registry(), - period=2025, - doctrine=UKNationalSolveDoctrine(epochs=1), - ) - staged = stage(frame) - metadata = json.loads(json.dumps(stage.checkpoint_metadata())) - resumed = UKNationalCalibrationStage( - _calibration_registry(), - period=2025, - doctrine=UKNationalSolveDoctrine(epochs=1), - ) - - resumed.resume_from_checkpoint(metadata, staged) - evidence = _stage_calibration_evidence( - (SimpleNamespace(name="national_calibration", transform=resumed),) - ) - result = _evaluate_calibration_reference_coverage( - EvidenceContext(artifacts={"national_calibration": evidence}), - {}, - ) - - assert evidence == stage.manifest - assert result.passed - assert result.details == {"activated": 1, "resolved": 1, "matrix": 1} - - -def test_national_build_runs_preflight_stages_gate_then_staging_write( - monkeypatch, tmp_path -) -> None: - pytest.importorskip("tables") - from microcosm.build.uk_runtime import national_build - - input_h5 = tmp_path / "base.h5" - staging_h5 = tmp_path / "staging.h5" - coverage_json = tmp_path / "input_coverage.json" - _write_toy_h5(input_h5) - events: list[str] = [] - - def stage_transform(frame: Frame) -> Frame: - events.append("stage:income") - person = frame.table("person").copy() - person["employment_income"] = 50_000.0 - return _replace_person(frame, person) - - def recording_coverage(context, parameters): - if parameters.get("check") == "manifest_current": - events.append("manifest_preflight") - return GateResult( - name="release_input_coverage", - passed=True, - details={"check": "manifest_current"}, - ) - events.append("final_coverage_gate") - surface = _uk_gate_surface(context.frame) - assert surface.person["employment_income"].tolist() == [50_000.0] - # The battery's evidence surface carries the frame's metadata — the - # coverage gate's hmrc family reads these attrs, and a bare table - # mapping silently fails them to ''/() (caught by the first - # credentialed acceptance build, not by CI's toy stages). - assert surface.time_period == "2023" - assert surface.household_weight_kind is WeightKind.DESIGN - assert surface.mass_log == () - return _passing_gate() - - registry = _toy_gate_registry() - registry["release_input_coverage"] = UKGateBinding( - name="release_input_coverage", - evaluator=recording_coverage, - parameter_keys=frozenset({"check"}), - artifact_keys=frozenset({"coverage_engine"}), - frame_predicate=( - lambda parameters: parameters.get("check") != "manifest_current" - ), - legacy_name="uk_release_input_coverage", - ) - - real_writer = national_build.write_uk_national_frame - - def recording_writer(frame, path): - events.append("staging_write") - return real_writer(frame, path) - - monkeypatch.setattr( - national_build, - "write_uk_national_frame", - recording_writer, - ) - - result = _run_national_build( - input_h5=input_h5, - staging_h5=staging_h5, - stages=(UKNationalStage("income", stage_transform),), - coverage_engine=object(), - input_coverage_path=coverage_json, - gate_registry=registry, - ) - - assert events == [ - "manifest_preflight", - "stage:income", - "final_coverage_gate", - "staging_write", - ] - assert result.sampling_receipt is None - assert result.stage_names == ("income",) - assert result.input_coverage.passed is True - assert result.gate_report["blocked_at_phase"] is None - assert result.gate_report["phases_evaluated"] == ["preflight", "terminal"] - gates = result.gate_report["gates"] - assert gates["uk_release_input_coverage"]["status"] == "passed" - assert result.gate_report["release_evidence"] == { - "calibration_diagnostics_sha256": TEST_UK_CALIBRATION_DIAGNOSTICS_SHA256 - } - assert result.terminal_gate_path == coverage_json.resolve() - assert result.input_coverage_path == result.terminal_gate_path - assert result.provenance.source_h5 == input_h5.resolve() - assert staging_h5.exists() - staged, staged_provenance = load_uk_national_frame(staging_h5) - assert staged_provenance.source_h5 == staging_h5.resolve() - assert staged.person["employment_income"].tolist() == [50_000.0] - assert staged.weights_for("household").values.tolist() == [2.0] - diagnostic = json.loads(coverage_json.read_text()) - assert diagnostic["enforced"] is True - assert diagnostic["input_coverage"]["passed"] is True - - -def test_national_build_accepts_stage_plan_and_records_stage_evidence( - tmp_path, -) -> None: - pytest.importorskip("tables") - - input_h5 = tmp_path / "base.h5" - staging_h5 = tmp_path / "staging.h5" - _write_toy_h5(input_h5) - - def add_bonus(frame: Frame) -> Frame: - person = frame.table("person").copy() - person["bonus_income"] = [125.0] - return _replace_person(frame, person) - - result = _run_national_build( - input_h5=input_h5, - staging_h5=staging_h5, - stages=StagePlan( - ( - Stage( - name="income", - transform=add_bonus, - produces=("bonus_income",), - ), - ) - ), - coverage_engine=object(), - gate_registry=_registry_with_coverage(_passing_gate), - ) - - assert result.stage_names == ("income",) - assert [record.stage for record in result.stage_records] == ["income"] - assert result.stage_records[0].produced == ("bonus_income",) - assert result.stage_records[0].nonzero_share == {"bonus_income": 1.0} - - -def test_deprecated_shim_and_country_stage_plan_paths_are_payload_identical( - tmp_path, -) -> None: - pytest.importorskip("tables") - - input_h5 = tmp_path / "base.h5" - _write_toy_h5(input_h5, employment_income=40_000.0) - spec = load_country_spec("uk") - # Select by name, not position: the manifest's stage order changed when - # frs_spine became the pipeline root, and this test only exercises the - # two national staging stages. - stages_by_name = {stage.stage: stage for stage in spec.sources.stages} - retained_outputs = stages_by_name["frs_hmrc_retained_leaves"].outputs - hmrc_outputs = stages_by_name["hmrc_spi_income"].outputs - - def retained(frame: Frame) -> Frame: - person = frame.table("person").copy() - for index, column in enumerate(retained_outputs, start=1): - person[column] = float(index) - return _replace_person(frame, person) - - def hmrc(frame: Frame) -> Frame: - person = frame.table("person").copy() - for index, column in enumerate(hmrc_outputs, start=1): - person[column] = float(index * 10) - return _replace_person(frame, person) - - legacy = _run_national_build( - input_h5=input_h5, - staging_h5=tmp_path / "legacy.h5", - stages=( - UKNationalStage("frs_hmrc_retained_leaves", retained), - UKNationalStage("hmrc_spi_income", hmrc), - ), - coverage_engine=object(), - gate_registry=_registry_with_coverage(_passing_gate), - ) - shared = _run_national_build( - input_h5=input_h5, - staging_h5=tmp_path / "shared.h5", - stages=country_stage_plan( - spec, - { - "frs_hmrc_retained_leaves": retained, - "hmrc_spi_income": hmrc, - }, - stage_names=("frs_hmrc_retained_leaves", "hmrc_spi_income"), - ), - coverage_engine=object(), - gate_registry=_registry_with_coverage(_passing_gate), - ) - - _assert_same_frame_payload(legacy.frame, shared.frame) - - -def _write_clone_family_h5(path: Path) -> None: - """Four base clone families (canonical + one geography clone each). - - Persons are ``household_id + 200``, so the canonical max is 214 and the - clone multiplier is 1000 — clone ids reverse onto the canonical surface - exactly as the stage fence re-derives them. - """ - - canonical = [11, 12, 13, 14] - regions = ["london", "north", "london", "north"] - rows = [] - for household_id, region in zip(canonical, regions, strict=True): - rows.append((household_id, 0, region)) - rows.append((household_id + 1_000, 1, "scotland")) - rows.sort() - ids = [row[0] for row in rows] - with pd.HDFStore(path) as store: - store.put( - "person", - pd.DataFrame( - { - "person_id": [value + 200 for value in ids], - "person_household_id": ids, - "person_benunit_id": [value + 5_000_000 for value in ids], - } - ), - format="table", - data_columns=True, - ) - store.put( - "benunit", - pd.DataFrame({"benunit_id": [value + 5_000_000 for value in ids]}), - format="table", - data_columns=True, - ) - store.put( - "household", - pd.DataFrame( - { - "household_id": ids, - "household_weight": [2.0] * len(ids), - "clone_index": [row[1] for row in rows], - "region": [row[2] for row in rows], - "household_is_spi_synthetic": [False] * len(ids), - "household_is_capital_gains_clone": [False] * len(ids), - } - ), - format="table", - data_columns=True, - ) - store.put( - "time_period", - pd.Series(["2023"]), - format="table", - data_columns=True, - ) - - -def test_national_build_samples_the_loaded_frame_before_stages(tmp_path) -> None: - pytest.importorskip("tables") - - input_h5 = tmp_path / "base.h5" - staging_h5 = tmp_path / "staging.h5" - coverage_json = tmp_path / "gates.json" - _write_clone_family_h5(input_h5) - stage_household_counts: list[int] = [] - - def stage_transform(frame: Frame) -> Frame: - stage_household_counts.append(len(frame.table("household"))) - return frame - - result = _run_national_build( - input_h5=input_h5, - staging_h5=staging_h5, - stages=(UKNationalStage("income", stage_transform),), - coverage_engine=object(), - input_coverage_path=coverage_json, - sample_fraction=0.5, - sample_seed=3, - gate_registry=_registry_with_coverage(_passing_gate), - ) - - receipt = result.sampling_receipt - assert receipt is not None - # The stages saw the sampled frame — the rung is upstream of stage one. - assert stage_household_counts == [receipt["realized_household_count"]] - assert receipt["realized_household_count"] < 8 - assert receipt["uk_policy"]["sampling_unit"] == "source_frs_family" - # Renormalization: the staged artifact carries the full input mass. - staged, _staged_provenance = load_uk_national_frame(staging_h5) - assert float(staged.weights_for("household").total) == pytest.approx(8 * 2.0) - assert result.gate_report["blocked_at_phase"] is None - - -def test_legacy_input_coverage_alias_is_byte_compatible_with_origin_main( - tmp_path, -) -> None: - pytest.importorskip("tables") - - input_h5 = tmp_path / "base.h5" - staging_h5 = tmp_path / "staging.h5" - legacy_json = tmp_path / "input_coverage.json" - _write_toy_h5(input_h5, employment_income=40_000.0) - _run_national_build( - input_h5=input_h5, - staging_h5=staging_h5, - coverage_engine=object(), - input_coverage_path=legacy_json, - gate_registry=_registry_with_coverage(_passing_gate), - ) - - # Pinned from origin/main's schema-1 serializer for this exact GateResult. - expected = ( - b'{\n "enforced": true,\n "input_coverage": {\n' - b' "details": {\n "degenerate": [],\n "missing": [],\n' - b' "required_columns": 1\n },\n "failures": [],\n' - b' "passed": true\n },\n "schema_version": 1\n}\n' - ) - assert legacy_json.read_bytes() == expected - - -def test_full_scale_build_refuses_to_stage_unsigned(monkeypatch, tmp_path) -> None: - """No full-scale staging artifact without an attested report. - - The battery core records a missing key as ``signing_error`` and carries - on; the national build restores the legacy guarantee for full-scale - builds — the unsigned report is on disk, the H5 is not. - """ - - pytest.importorskip("tables") - - input_h5 = tmp_path / "base.h5" - staging_h5 = tmp_path / "staging.h5" - terminal_json = tmp_path / "terminal_gates.json" - _write_two_row_h5(input_h5) - monkeypatch.delenv(gate_signing_key_env("uk")) - - with pytest.raises(RuntimeError, match="unsigned and this is a full-scale"): - _run_national_build( - input_h5=input_h5, - staging_h5=staging_h5, - coverage_engine=object(), - terminal_gate_path=terminal_json, - gate_registry=_registry_with_coverage(_passing_gate), - ) - - assert not staging_h5.exists() - payload = json.loads(terminal_json.read_text(encoding="utf-8")) - assert payload["schema_version"] == 4 - assert payload["shippable"] is False - assert payload["attestation"]["signature"] is None - assert payload["attestation"]["signing_key_sha256"] is None - assert "signing_error" in payload["attestation"] - - -def test_rung_build_proceeds_unsigned_with_an_honest_report( - monkeypatch, tmp_path -) -> None: - """A rung is structurally non-releasable, so it may run without the key; - its report says so instead of pretending.""" - - pytest.importorskip("tables") - - input_h5 = tmp_path / "base.h5" - staging_h5 = tmp_path / "staging.h5" - terminal_json = tmp_path / "terminal_gates.json" - _write_clone_family_h5(input_h5) - monkeypatch.delenv(gate_signing_key_env("uk")) - - result = _run_national_build( - input_h5=input_h5, - staging_h5=staging_h5, - coverage_engine=object(), - terminal_gate_path=terminal_json, - sample_fraction=0.5, - sample_seed=3, - gate_registry=_registry_with_coverage(_passing_gate), - ) - - assert staging_h5.exists() - assert result.gate_report["shippable"] is False - assert "signing_error" in result.gate_report["attestation"] - - -def test_national_build_gate_failure_writes_diagnostic_not_h5(tmp_path) -> None: - pytest.importorskip("tables") - - input_h5 = tmp_path / "base.h5" - staging_h5 = tmp_path / "staging.h5" - coverage_json = tmp_path / "input_coverage.json" - _write_toy_h5(input_h5) - with pytest.raises(GateBatteryBlockedError, match="Gate battery blocked"): - _run_national_build( - input_h5=input_h5, - staging_h5=staging_h5, - coverage_engine=object(), - input_coverage_path=coverage_json, - gate_registry=_registry_with_coverage(_failing_gate), - ) - - assert not staging_h5.exists() - diagnostic = json.loads(coverage_json.read_text()) - coverage = diagnostic["input_coverage"] - assert coverage["passed"] is False - assert coverage["details"]["degenerate"] == ["employment_income"] - - -def test_default_terminal_report_write_precedes_gate_failure_raise( - monkeypatch, - tmp_path, -) -> None: - pytest.importorskip("tables") - from microcosm.build.uk_runtime import national_build - - input_h5 = tmp_path / "base.h5" - staging_h5 = tmp_path / "staging.h5" - default_terminal_json = staging_h5.with_suffix(".terminal_gates.json") - _write_toy_h5(input_h5) - events: list[str] = [] - real_loader = national_build.load_uk_national_frame - - def load(path): - events.append("load") - return real_loader(path) - - def recording_coverage(context, parameters): - if parameters.get("check") == "manifest_current": - events.append("preflight") - return GateResult( - name="release_input_coverage", - passed=True, - details={"check": "manifest_current"}, - ) - events.append("evaluate") - # The preflight report is already on disk before the frame loads — - # the write-then-block ordering holds per phase, not just at the end. - assert json.loads(default_terminal_json.read_text())["phases_evaluated"] == [ - "preflight" - ] - return _failing_gate() - - def recording_roster(context, parameters): - events.append("stage contract") - return GateResult(name="source_coverage", passed=True, details={}) - - registry = _toy_gate_registry() - registry["release_input_coverage"] = UKGateBinding( - name="release_input_coverage", - evaluator=recording_coverage, - parameter_keys=frozenset({"check"}), - artifact_keys=frozenset({"coverage_engine"}), - frame_predicate=( - lambda parameters: parameters.get("check") != "manifest_current" - ), - legacy_name="uk_release_input_coverage", - ) - registry["source_coverage"] = UKGateBinding( - name="source_coverage", - evaluator=recording_roster, - needs_frame=False, - ) - monkeypatch.setattr(national_build, "load_uk_national_frame", load) - - with pytest.raises(GateBatteryBlockedError, match="Gate battery blocked"): - _run_national_build( - input_h5=input_h5, - staging_h5=staging_h5, - coverage_engine=object(), - terminal_gate_path=None, - gate_registry=registry, - ) - events.append("raise") - - assert events == [ - "preflight", - "stage contract", - "load", - "evaluate", - "raise", - ] - assert default_terminal_json.is_file() - payload = json.loads(default_terminal_json.read_text()) - assert payload["schema_version"] == 4 - assert payload["blocked_at_phase"] == "terminal" - assert payload["gates"]["uk_release_input_coverage"]["status"] == "failed" - assert not staging_h5.exists() - - -def _stub_real_coverage(monkeypatch, gate_result_factory) -> None: - """Point the real registry's coverage binding at a stubbed verdict. - - The bindings resolve the manifest assert and the coverage gate as - module globals at call time, so patching them where the bindings look - them up leaves every other real binding untouched. - """ - - from microcosm.build.uk_runtime import battery_bindings - - monkeypatch.setattr( - battery_bindings, - "assert_uk_release_input_coverage_manifest_current", - lambda **_kwargs: None, - ) - monkeypatch.setattr( - battery_bindings, - "assert_uk_release_input_coverage_build_stages", - lambda _stage_names, manifest=None: None, - ) - monkeypatch.setattr( - battery_bindings, - "uk_release_input_coverage_gate", - lambda _surface, _engine, manifest=None: gate_result_factory(), - ) - - -def test_national_build_real_terminal_batch_blocks_incomplete_qrf_before_staging( - monkeypatch, - tmp_path, -) -> None: - pytest.importorskip("tables") - - input_h5 = tmp_path / "healthy.h5" - staging_h5 = tmp_path / "staging.h5" - terminal_json = tmp_path / "terminal_gates.json" - _write_two_row_h5(input_h5) - _stub_real_coverage(monkeypatch, _passing_gate) - - with pytest.raises(GateBatteryBlockedError) as error: - _run_national_build( - input_h5=input_h5, - staging_h5=staging_h5, - # The audit's absence blocks every posture (evidence_absent_blocks), - # so this supplies the HMRC stage's audit evidence. The real QRF - # gate is spec-armed now and correctly blocks this tiny synthetic - # frame because it lacks the declared QRF output surface. - stages=(UKNationalStage("hmrc_spi_income", _RecordedFitStage()),), - coverage_engine=object(), - terminal_gate_path=terminal_json, - gate_registry=None, # the real UK registry - ) - - assert "[uk_qrf_tail_concentration]" in str(error.value) - assert error.value.phase == "terminal" - assert not staging_h5.exists() - payload = json.loads(terminal_json.read_text(encoding="utf-8")) - assert payload["schema_version"] == 4 - assert payload["blocked_at_phase"] == "terminal" - statuses = {entry_id: gate["status"] for entry_id, gate in payload["gates"].items()} - assert statuses == { - "uk_release_input_coverage_manifest_current": "passed", - "uk_release_family_build_stages": "passed", - "uk_ledger_compile_parity_production_2023": "evidence_absent", - "uk_ledger_compile_parity_incumbent_2025": "evidence_absent", - "uk_release_input_coverage": "passed", - "uk_degenerate_release_surface": "passed", - "uk_weights_audit": "passed", - "uk_nonnegative_columns": "passed", - "uk_support": "passed", - "uk_aggregate_admin": "evidence_absent", - "uk_take_up_signal": "passed", - "uk_student_loan_plan_enum_domain": "failed", - # The legacy report omitted unevidenced gates; the battery names - # every gap — non-blocking off the release-candidate posture. - "uk_export_surface": "evidence_absent", - "uk_target_surface": "evidence_absent", - "uk_input_mass_parity": "evidence_absent", - "uk_qrf_tail_concentration": "failed", - } - # One exclusion clock: the evaluated exclusion gate stamps the injected - # date, never a per-gate default. - degenerate = payload["gates"]["uk_degenerate_release_surface"] - assert ( - degenerate["details"]["exclusions_evaluated_on"] - == TEST_UK_EXCLUSION_CLOCK.isoformat() - ) - assert payload["release_evidence"] == { - "calibration_diagnostics_sha256": TEST_UK_CALIBRATION_DIAGNOSTICS_SHA256 - } - - -def test_national_build_real_terminal_batch_writes_all_findings_before_raise( - monkeypatch, - tmp_path, -) -> None: - pytest.importorskip("tables") - - input_h5 = tmp_path / "defective.h5" - staging_h5 = tmp_path / "staging.h5" - terminal_json = tmp_path / "terminal_gates.json" - _write_two_row_h5(input_h5, employment_income=(0.0, 0.0)) - _stub_real_coverage(monkeypatch, _failing_gate) - - with pytest.raises(GateBatteryBlockedError) as error: - _run_national_build( - input_h5=input_h5, - staging_h5=staging_h5, - stages=(UKNationalStage("hmrc_spi_income", lambda dataset: dataset),), - coverage_engine=object(), - terminal_gate_path=terminal_json, - gate_registry=None, # the real UK registry - ) - - assert "[uk_release_input_coverage]" in str(error.value) - assert "[uk_degenerate_release_surface]" in str(error.value) - assert "[uk_weights_audit]" in str(error.value) - assert error.value.phase == "terminal" - assert terminal_json.is_file() - payload = json.loads(terminal_json.read_text(encoding="utf-8")) - assert payload["blocked_at_phase"] == "terminal" - assert payload["shippable"] is False - assert payload["gates"]["uk_release_input_coverage"]["status"] == "failed" - assert payload["gates"]["uk_degenerate_release_surface"]["status"] == "failed" - weights_audit = payload["gates"]["uk_weights_audit"] - assert weights_audit["status"] == "failed" - assert weights_audit["details"] == { - "evidence_missing": True, - "fits_checked": 0, - } - assert weights_audit["failures"] == [ - "A production fit stage ran but emitted no FitWeightRecord evidence; " - "an absent audit is not a passing audit." - ] - assert not staging_h5.exists() - - -def test_national_build_parity_trio_is_evidence_absent( - monkeypatch, - tmp_path, -) -> None: - pytest.importorskip("tables") - - input_h5 = tmp_path / "healthy.h5" - _write_two_row_h5(input_h5) - _stub_real_coverage(monkeypatch, _passing_gate) - - terminal_json = tmp_path / "terminal_gates.json" - with pytest.raises(GateBatteryBlockedError): - _run_national_build( - input_h5=input_h5, - staging_h5=tmp_path / "staging.h5", - stages=(UKNationalStage("hmrc_spi_income", _RecordedFitStage()),), - coverage_engine=object(), - terminal_gate_path=terminal_json, - gate_registry=None, - ) - - gates = json.loads(terminal_json.read_text(encoding="utf-8"))["gates"] - assert gates["uk_weights_audit"]["status"] == "passed" - assert gates["uk_weights_audit"]["details"]["resolved_weight_kinds"] == { - "uk_frs_only_spi_fill": "importance", - "uk_spi_2022_23_income": "design", - } - for entry_id in ("uk_export_surface", "uk_target_surface"): - assert gates[entry_id]["status"] == "evidence_absent", entry_id - assert gates[entry_id]["reason"] == "missing evidence: parity_evidence" - assert "uk_target_fit" not in gates - assert gates["uk_input_mass_parity"]["status"] == "evidence_absent" - assert gates["uk_qrf_tail_concentration"]["status"] == "failed" - - -def _fixture_parity_reference(input_h5) -> EfrsParityReference: - """A synthetic frozen instrument matching the fixture's export surface. - - Reference and candidate stay independently derived in production; the - test constructs the reference from the *input* artifact, before any - stage runs, so a stage that leaked scratch columns would still fail - the export-surface comparison. - """ - - frame, _provenance = load_uk_national_frame(input_h5) - input_entities = { - str(column): entity - for entity in frame.entities - for column in frame.table(entity).columns - } - return EfrsParityReference( - source=EfrsParitySource( - repo_id="example/synthetic", - repo_type="model", - filename="synthetic_reference.h5", - revision="0" * 40, - sha256="0" * 64, - url="https://example.invalid/synthetic_reference.h5", - vintage="2024_25", - period="2024", - size_bytes=1, - ), - nonzero_shares={name: 1.0 for name in input_entities}, - input_entities=input_entities, - ) - - -def test_national_build_parity_trio_evaluates_for_armed_calibration( - tmp_path, -) -> None: - pytest.importorskip("tables") - - input_h5 = tmp_path / "healthy.h5" - terminal_json = tmp_path / "terminal_gates.json" - _write_two_row_h5(input_h5, include_calibration_columns=True) - - _run_national_build( - input_h5=input_h5, - staging_h5=tmp_path / "staging.h5", - stages=( - Stage( - name="national_calibration", - transform=UKNationalCalibrationStage( - _calibration_registry(55.0), - period=2025, - doctrine=UKNationalSolveDoctrine(epochs=1), - ), - ), - ), - coverage_engine=object(), - terminal_gate_path=terminal_json, - gate_registry=_registry_with_calibration_and_parity_trio(), - parity_reference=_fixture_parity_reference(input_h5), - ) - - gates = json.loads(terminal_json.read_text(encoding="utf-8"))["gates"] - for entry_id in ("uk_export_surface", "uk_target_surface"): - assert gates[entry_id]["status"] == "passed", entry_id - assert "uk_target_fit" not in gates - assert "uk_calibration_reference_coverage" not in gates - - -def test_armed_calibration_without_parity_reference_stays_evidence_absent( - tmp_path, -) -> None: - """No frozen instrument, no trio evidence — never a copied reference.""" - - pytest.importorskip("tables") - - input_h5 = tmp_path / "healthy.h5" - terminal_json = tmp_path / "terminal_gates.json" - _write_two_row_h5(input_h5, include_calibration_columns=True) - - _run_national_build( - input_h5=input_h5, - staging_h5=tmp_path / "staging.h5", - stages=( - Stage( - name="national_calibration", - transform=UKNationalCalibrationStage( - _calibration_registry(55.0), - period=2025, - doctrine=UKNationalSolveDoctrine(epochs=1), - ), - ), - ), - coverage_engine=object(), - terminal_gate_path=terminal_json, - gate_registry=_registry_with_calibration_and_parity_trio(), - ) - - gates = json.loads(terminal_json.read_text(encoding="utf-8"))["gates"] - for entry_id in ("uk_export_surface", "uk_target_surface"): - assert gates[entry_id]["status"] == "evidence_absent", entry_id - assert gates[entry_id]["reason"] == "missing evidence: parity_evidence" - assert "uk_target_fit" not in gates - assert "uk_calibration_reference_coverage" not in gates - - -def test_national_build_rejects_both_gate_path_names_and_h5_collisions( - tmp_path, -) -> None: - pytest.importorskip("tables") - input_h5 = tmp_path / "base.h5" - _write_toy_h5(input_h5) - - with pytest.raises(ValueError, match="mutually exclusive"): - _run_national_build( - input_h5=input_h5, - staging_h5=tmp_path / "staging.h5", - coverage_engine=object(), - terminal_gate_path=tmp_path / "terminal.json", - input_coverage_path=tmp_path / "coverage.json", - ) - - with pytest.raises(ValueError, match="must differ"): - _run_national_build( - input_h5=input_h5, - staging_h5=tmp_path / "staging.h5", - coverage_engine=object(), - terminal_gate_path=input_h5, - ) - - -def test_national_build_rejects_duplicate_stage_names_before_running( - tmp_path, -) -> None: - pytest.importorskip("tables") - - input_h5 = tmp_path / "base.h5" - _write_toy_h5(input_h5) - called = False - - def transform(frame: Frame) -> Frame: - nonlocal called - called = True - return frame - - with pytest.raises(ValueError, match="Duplicate UK national stage"): - _run_national_build( - input_h5=input_h5, - staging_h5=tmp_path / "staging.h5", - stages=( - UKNationalStage("income", transform), - UKNationalStage("income", transform), - ), - coverage_engine=object(), - ) - - assert called is False - - -def test_national_build_manifest_failure_blocks_before_stages_with_a_report( - tmp_path, -) -> None: - """Preflight drift blocks before any stage — and now leaves a report. - - The legacy assertions raised bare, deleting the stale outputs and - writing nothing; the battery persists the refusal as a schema-4 report - with the terminal entries honestly ``unreached``. - """ - - pytest.importorskip("tables") - - input_h5 = tmp_path / "base.h5" - staging_h5 = tmp_path / "staging.h5" - coverage_json = tmp_path / "input_coverage.json" - _write_toy_h5(input_h5) - staging_h5.write_bytes(b"stale-success") - coverage_json.write_text('{"stale_success": true}\n') - stage_called = False - - def stage_transform(frame: Frame) -> Frame: - nonlocal stage_called - stage_called = True - return frame - - def drifting_coverage(context, parameters): - if parameters.get("check") == "manifest_current": - raise ValueError("manifest drift") - return _passing_gate() - - registry = _toy_gate_registry() - registry["release_input_coverage"] = UKGateBinding( - name="release_input_coverage", - evaluator=drifting_coverage, - parameter_keys=frozenset({"check"}), - artifact_keys=frozenset({"coverage_engine"}), - frame_predicate=( - lambda parameters: parameters.get("check") != "manifest_current" - ), - legacy_name="uk_release_input_coverage", - ) - - with pytest.raises(GateBatteryBlockedError, match="manifest drift") as error: - _run_national_build( - input_h5=input_h5, - staging_h5=staging_h5, - stages=(UKNationalStage("should_not_run", stage_transform),), - coverage_engine=object(), - input_coverage_path=coverage_json, - gate_registry=registry, - ) - - assert error.value.phase == "preflight" - assert stage_called is False - assert not staging_h5.exists() - payload = json.loads(coverage_json.read_text()) - assert payload["schema_version"] == 4 - assert payload["blocked_at_phase"] == "preflight" - assert ( - payload["gates"]["uk_release_input_coverage_manifest_current"]["status"] - == "failed" - ) - assert payload["gates"]["uk_release_input_coverage"]["status"] == "unreached" - assert payload["gates"]["uk_release_input_coverage"]["status"] == "unreached" - - -def test_national_build_rejects_stage_that_breaks_entity_links(tmp_path) -> None: - pytest.importorskip("tables") - - input_h5 = tmp_path / "base.h5" - _write_toy_h5(input_h5) - - def break_links(frame: Frame) -> Frame: - person = frame.table("person").copy() - person["person_household_id"] = 999 - return _replace_person(frame, person) - - # Frame construction inside the stage is where the invariant now lives. - with pytest.raises(ValueError, match="absent from the table"): - _run_national_build( - input_h5=input_h5, - staging_h5=tmp_path / "staging.h5", - stages=(UKNationalStage("bad", break_links),), - coverage_engine=object(), - ) - - -@pytest.mark.parametrize( - ("stage_name", "transform", "message"), - [ - ( - "missing_period", - lambda frame: uk_national_frame( - person=frame.table("person"), - benunit=frame.table("benunit"), - household=frame.table("household"), - time_period=None, - household_weights=frame.weights_for("household").values, - ), - "time_period must be a non-empty string", - ), - ( - "zero_population", - lambda frame: uk_national_frame( - person=frame.table("person"), - benunit=frame.table("benunit"), - household=frame.table("household").assign(household_weight=0.0), - time_period=uk_time_period(frame), - ), - "Weights cannot be all zero", - ), - ], -) -def test_national_build_rejects_invalid_stage_population_metadata( - tmp_path, stage_name, transform, message -) -> None: - pytest.importorskip("tables") - - input_h5 = tmp_path / "base.h5" - _write_toy_h5(input_h5) - - with pytest.raises(ValueError, match=message): - _run_national_build( - input_h5=input_h5, - staging_h5=tmp_path / "staging.h5", - stages=(UKNationalStage(stage_name, transform),), - coverage_engine=object(), - ) - - -def test_national_build_refuses_to_overwrite_its_input(tmp_path) -> None: - pytest.importorskip("tables") - - input_h5 = tmp_path / "base.h5" - _write_toy_h5(input_h5) - - with pytest.raises(ValueError, match="must differ"): - _run_national_build( - input_h5=input_h5, - staging_h5=input_h5, - coverage_engine=object(), - ) - - -def test_national_build_accepts_hugging_face_style_h5_symlink(tmp_path) -> None: - pytest.importorskip("tables") - - cached_blob = tmp_path / "content-addressed-blob" - input_h5 = tmp_path / "populace_uk_2023.h5" - staging_h5 = tmp_path / "staging.h5" - _write_toy_h5(cached_blob, employment_income=40_000.0) - input_h5.symlink_to(cached_blob) - - result = _run_national_build( - input_h5=input_h5, - staging_h5=staging_h5, - coverage_engine=object(), - gate_registry=_registry_with_coverage(_passing_gate), - ) - - assert result.input_h5 == cached_blob.resolve() - assert result.provenance.source_h5 == cached_blob.resolve() - assert staging_h5.is_file() - - -@pytest.mark.requires_uk -def test_national_staging_h5_loads_through_policyengine_uk(tmp_path) -> None: - pytest.importorskip("tables") - policyengine_data = pytest.importorskip("policyengine_uk.data") - from microcosm.build.uk_runtime.national_build import write_uk_national_frame - - input_h5 = tmp_path / "base.h5" - staging_h5 = tmp_path / "staging.h5" - _write_toy_h5(input_h5, employment_income=40_000.0) - frame, provenance = load_uk_national_frame(input_h5) - assert provenance.source_h5 == input_h5.resolve() - frame = uk_national_frame( - person=frame.table("person"), - benunit=frame.table("benunit"), - household=frame.table("household"), - time_period=uk_time_period(frame), - weight_kind=WeightKind.IMPORTANCE, - household_weights=frame.weights_for("household").values, - mass_log=( - MassChangeRecord( - entity="household", - old_total=2.0, - new_total=2.0, - declared_factor=1.0, - reason="test reviewed support-channel mass allocation", - ), - ), - ) - - write_uk_national_frame(frame, staging_h5) - - round_tripped, _staging_provenance = load_uk_national_frame(staging_h5) - assert uk_household_weight_kind(round_tripped) is WeightKind.IMPORTANCE - assert round_tripped.mass_log == frame.mass_log - - loaded = policyengine_data.UKSingleYearDataset(file_path=str(staging_h5)) - assert loaded.time_period == "2023" - assert loaded.person["employment_income"].tolist() == [40_000.0] - assert loaded.household["household_weight"].tolist() == [2.0] - - -def test_atomic_writer_cleans_temporary_h5_after_write_failure( - monkeypatch, tmp_path -) -> None: - pytest.importorskip("tables") - from microcosm.build.uk_runtime import national_frame - - input_h5 = tmp_path / "base.h5" - staging_h5 = tmp_path / "staging.h5" - _write_toy_h5(input_h5, employment_income=40_000.0) - frame, _provenance = load_uk_national_frame(input_h5) - staging_h5.write_bytes(b"previous-good-artifact") - - def fail_store(path, *_args, **_kwargs): - Path(path).write_bytes(b"partial") - raise OSError("simulated HDF write failure") - - monkeypatch.setattr(national_frame.pd, "HDFStore", fail_store) - - with pytest.raises(OSError, match="simulated HDF write failure"): - national_frame.write_uk_national_frame(frame, staging_h5) - - assert staging_h5.read_bytes() == b"previous-good-artifact" - assert list(tmp_path.glob(".staging.h5.*.tmp.h5")) == [] - - -def _counting_stage(name: str, calls: list[str] | None = None) -> UKNationalStage: - def transform(frame: Frame) -> Frame: - if calls is not None: - calls.append(name) - person = frame.table("person").copy() - person["employment_income"] = person["employment_income"] + 1.0 - return _replace_person(frame, person) - - return UKNationalStage(name=name, transform=transform) - - -def _counting_cleanup_stage( - name: str, - calls: list[str] | None = None, -) -> UKNationalStage: - def transform(frame: Frame) -> Frame: - if calls is not None: - calls.append(name) - person = frame.table("person").copy() - person["employment_income"] = person["employment_income"] + 1.0 - benunit = frame.table("benunit").drop( - columns=["dwp/uc/households"], - errors="ignore", - ) - return uk_national_frame( - person=person, - benunit=benunit, - household=frame.table("household"), - time_period=uk_time_period(frame), - weight_kind=uk_household_weight_kind(frame), - household_weights=frame.weights_for("household").values, - mass_log=frame.mass_log, - ) - - return UKNationalStage(name=name, transform=transform) - - -class _CountingCalibrationStage: - def __init__(self, calls: list[str]) -> None: - self.calls = calls - self.inner = UKNationalCalibrationStage( - _calibration_registry(), - period=2025, - doctrine=UKNationalSolveDoctrine(epochs=1), - ) - - @property - def manifest(self) -> dict[str, object] | None: - return self.inner.manifest - - @property - def diagnostics(self) -> tuple[dict[str, object], ...]: - return self.inner.diagnostics - - def __call__(self, frame: Frame) -> Frame: - self.calls.append("national_calibration") - return self.inner(frame) - - def checkpoint_metadata(self) -> dict[str, object]: - return dict(self.inner.checkpoint_metadata()) - - def resume_from_checkpoint( - self, - metadata: dict[str, object], - frame: Frame, - ) -> None: - self.inner.resume_from_checkpoint(metadata, frame) - - -def _assert_same_staging_payload(left: Path, right: Path) -> None: - left_frame, _ = load_uk_national_frame(left) - right_frame, _ = load_uk_national_frame(right) - from microcosm.build.uk_runtime import uk_frame_content_identity - - assert uk_frame_content_identity(left_frame) == uk_frame_content_identity( - right_frame - ) - - -def test_checkpointed_build_matches_the_monolith(monkeypatch, tmp_path) -> None: - """The checkpointed mode is the monolith plus receipts, not a variant. - - Same input, same stages: the staged build's output is content-identical - to the monolith's, and each stage boundary leaves a resumable checkpoint. - """ - - pytest.importorskip("tables") - pytest.importorskip("h5py") - - registry = _registry_with_coverage(_passing_gate) - input_h5 = tmp_path / "base.h5" - _write_two_row_h5(input_h5) - run_config = {"input_sha256": "a" * 64, "seed": 42} - - _run_national_build( - coverage_engine=object(), - input_h5=input_h5, - staging_h5=tmp_path / "mono.h5", - stages=(_counting_stage("one"), _counting_stage("two")), - gate_registry=registry, - ) - calls: list[str] = [] - _run_national_build( - coverage_engine=object(), - input_h5=input_h5, - staging_h5=tmp_path / "staged.h5", - stages=(_counting_stage("one", calls), _counting_stage("two", calls)), - checkpoint_dir=tmp_path / "checkpoints", - run_config=run_config, - gate_registry=registry, - ) - assert calls == ["one", "two"] - _assert_same_staging_payload(tmp_path / "mono.h5", tmp_path / "staged.h5") - context = json.loads( - (tmp_path / "checkpoints" / "stage_run_context.json").read_text() - ) - assert context["completed"] == ["one", "two"] - - # A full resume re-runs no transform and reproduces the same payload. - resumed_calls: list[str] = [] - _run_national_build( - coverage_engine=object(), - input_h5=input_h5, - staging_h5=tmp_path / "resumed.h5", - stages=( - _counting_stage("one", resumed_calls), - _counting_stage("two", resumed_calls), - ), - checkpoint_dir=tmp_path / "checkpoints", - run_config=run_config, - gate_registry=registry, - ) - assert resumed_calls == [] - _assert_same_staging_payload(tmp_path / "mono.h5", tmp_path / "resumed.h5") - - -def test_checkpointed_build_resumes_past_a_crash(monkeypatch, tmp_path) -> None: - """A stage crash leaves the completed prefix; the rerun picks up after it.""" - - pytest.importorskip("tables") - pytest.importorskip("h5py") - - registry = _registry_with_coverage(_passing_gate) - input_h5 = tmp_path / "base.h5" - _write_two_row_h5(input_h5) - run_config = {"input_sha256": "a" * 64, "seed": 42} - - def exploding(frame: Frame) -> Frame: - raise RuntimeError("boom") - - with pytest.raises(RuntimeError, match="boom"): - _run_national_build( - coverage_engine=object(), - input_h5=input_h5, - staging_h5=tmp_path / "crashed.h5", - stages=( - _counting_stage("one"), - UKNationalStage(name="two", transform=exploding), - ), - checkpoint_dir=tmp_path / "checkpoints", - run_config=run_config, - gate_registry=registry, - ) - - calls: list[str] = [] - _run_national_build( - coverage_engine=object(), - input_h5=input_h5, - staging_h5=tmp_path / "recovered.h5", - stages=(_counting_stage("one", calls), _counting_stage("two", calls)), - checkpoint_dir=tmp_path / "checkpoints", - run_config=run_config, - gate_registry=registry, - ) - assert calls == ["two"] - - -def test_checkpointed_build_resumes_completed_calibration_evidence( - tmp_path, -) -> None: - pytest.importorskip("tables") - pytest.importorskip("h5py") - - registry = _registry_with_calibration() - input_h5 = tmp_path / "base.h5" - _write_two_row_h5(input_h5, include_calibration_columns=True) - run_config = {"input_sha256": "a" * 64, "seed": 42} - - def exploding(frame: Frame) -> Frame: - raise RuntimeError("boom") - - calibration_calls: list[str] = [] - with pytest.raises(RuntimeError, match="boom"): - _run_national_build( - coverage_engine=object(), - input_h5=input_h5, - staging_h5=tmp_path / "crashed.h5", - stages=( - UKNationalStage( - "national_calibration", - _CountingCalibrationStage(calibration_calls), - ), - UKNationalStage(name="after", transform=exploding), - ), - checkpoint_dir=tmp_path / "checkpoints", - run_config=run_config, - gate_registry=registry, - ) - assert calibration_calls == ["national_calibration"] - - resumed_calibration_calls: list[str] = [] - after_calls: list[str] = [] - result = _run_national_build( - coverage_engine=object(), - input_h5=input_h5, - staging_h5=tmp_path / "recovered.h5", - stages=( - UKNationalStage( - "national_calibration", - _CountingCalibrationStage(resumed_calibration_calls), - ), - _counting_cleanup_stage("after", after_calls), - ), - checkpoint_dir=tmp_path / "checkpoints", - run_config=run_config, - gate_registry=registry, - terminal_gate_path=tmp_path / "terminal_gates.json", - ) - - assert resumed_calibration_calls == [] - assert after_calls == ["after"] - assert "uk_calibration_reference_coverage" not in result.gate_report["gates"] - - -def test_checkpointed_build_pins_the_run_config(tmp_path) -> None: - """Resuming under a different configuration is refused, never blended.""" - - pytest.importorskip("tables") - pytest.importorskip("h5py") - - registry = _registry_with_coverage(_passing_gate) - input_h5 = tmp_path / "base.h5" - _write_two_row_h5(input_h5) - - with pytest.raises(ValueError, match="requires run_config"): - _run_national_build( - coverage_engine=object(), - input_h5=input_h5, - staging_h5=tmp_path / "unpinned.h5", - stages=(_counting_stage("one"),), - checkpoint_dir=tmp_path / "checkpoints", - gate_registry=registry, - ) - - _run_national_build( - coverage_engine=object(), - input_h5=input_h5, - staging_h5=tmp_path / "first.h5", - stages=(_counting_stage("one"),), - checkpoint_dir=tmp_path / "checkpoints", - run_config={"input_sha256": "a" * 64, "seed": 42}, - gate_registry=registry, - ) - with pytest.raises(ValueError, match="new checkpoint directory"): - _run_national_build( - coverage_engine=object(), - input_h5=input_h5, - staging_h5=tmp_path / "drifted.h5", - stages=(_counting_stage("one"),), - checkpoint_dir=tmp_path / "checkpoints", - run_config={"input_sha256": "b" * 64, "seed": 42}, - gate_registry=registry, - ) - - -def test_release_candidate_blocks_on_named_evidence_gaps(tmp_path) -> None: - """The chartered semantics live: a candidate cannot excuse absent - evidence, a dev build records the same gaps and continues.""" - - pytest.importorskip("tables") - - input_h5 = tmp_path / "base.h5" - _write_toy_h5(input_h5) - registry = _registry_with_coverage(_passing_gate) - - dev = _run_national_build( - input_h5=input_h5, - staging_h5=tmp_path / "dev.h5", - coverage_engine=object(), - terminal_gate_path=tmp_path / "dev_gates.json", - gate_registry=registry, - ) - assert dev.gate_report["blocked_at_phase"] is None - absent = { - entry_id - for entry_id, gate in dev.gate_report["gates"].items() - if gate["status"] == "evidence_absent" - } - assert "uk_export_surface" in absent # unbound in the toy registry - - with pytest.raises(GateBatteryBlockedError) as error: - _run_national_build( - input_h5=input_h5, - staging_h5=tmp_path / "candidate.h5", - coverage_engine=object(), - terminal_gate_path=tmp_path / "candidate_gates.json", - gate_registry=registry, - release_candidate=True, - ) - assert error.value.phase == "preflight" - assert "[uk_ledger_compile_parity_production_2023]" in str(error.value) - assert not (tmp_path / "candidate.h5").exists() - - -def test_release_candidate_is_refused_on_a_rung_before_any_unlink( - tmp_path, -) -> None: - pytest.importorskip("tables") - - input_h5 = tmp_path / "base.h5" - _write_toy_h5(input_h5) - terminal_json = tmp_path / "terminal_gates.json" - terminal_json.write_text('{"previous_report": true}\n') - - with pytest.raises(ValueError, match="structurally non-releasable"): - _run_national_build( - input_h5=input_h5, - staging_h5=tmp_path / "staging.h5", - coverage_engine=object(), - terminal_gate_path=terminal_json, - sample_fraction=0.5, - release_candidate=True, - ) - - # Configuration refusals precede the sidecar unlinks: the contradictory - # request must not destroy the previous run's report. - assert terminal_json.read_text() == '{"previous_report": true}\n' - - -@pytest.mark.parametrize( - ("bad_arguments", "match"), - [ - ({"release_id": ""}, "release_id"), - ({"calibration_diagnostics_sha256": ""}, "release_evidence"), - ({"now": datetime(2026, 9, 1, 12, 0)}, "date"), - ( - {"release_candidate": True, "use_alias_path": True}, - "mutually exclusive", - ), - ], - ids=["empty-release-id", "empty-diagnostics-sha", "datetime-clock", "alias"], -) -def test_every_identity_refusal_precedes_the_sidecar_unlinks( - tmp_path, bad_arguments, match -) -> None: - """No destructive step precedes argument validation — for every - validation, including the ones the battery construction owns.""" - - pytest.importorskip("tables") - - input_h5 = tmp_path / "base.h5" - _write_toy_h5(input_h5) - staging_h5 = tmp_path / "staging.h5" - terminal_json = tmp_path / "terminal_gates.json" - staging_h5.write_bytes(b"previous-artifact") - terminal_json.write_text('{"previous_report": true}\n') - arguments: dict = { - "input_h5": input_h5, - "staging_h5": staging_h5, - "release_id": TEST_UK_RELEASE_ID, - "calibration_diagnostics_sha256": TEST_UK_CALIBRATION_DIAGNOSTICS_SHA256, - "coverage_engine": object(), - "now": TEST_UK_EXCLUSION_CLOCK, - "gate_registry": _toy_gate_registry(), - "terminal_gate_path": terminal_json, - } - arguments.update(bad_arguments) - if arguments.pop("use_alias_path", False): - arguments["input_coverage_path"] = arguments.pop("terminal_gate_path") - - with pytest.raises((ValueError, TypeError), match=match): - build_uk_national_dataset(**arguments) - - assert staging_h5.read_bytes() == b"previous-artifact" - assert terminal_json.read_text() == '{"previous_report": true}\n' diff --git a/packages/microcosm-build/tests/test_uk_national_build_driver.py b/packages/microcosm-build/tests/test_uk_national_build_driver.py deleted file mode 100644 index 1f793c12e..000000000 --- a/packages/microcosm-build/tests/test_uk_national_build_driver.py +++ /dev/null @@ -1,1628 +0,0 @@ -from __future__ import annotations - -import hashlib -import importlib.util -import json -import sys -from itertools import combinations -from pathlib import Path -from types import SimpleNamespace - -import pandas as pd -import pytest - -from microcosm.build.gate_battery import GateBatteryBlockedError -from microcosm.build.logbook import LOGBOOK_ROW_FIELDS, load_spool_rows -from microcosm.build.logbook_adoption import role_pins_digest -from microcosm.build.uk_runtime.national_frame import ( - UKStagingProvenance, - _uk_source_file_fingerprint, - uk_national_frame, -) -from microcosm.frame import MassChangeRecord, WeightKind - - -def _toy_result_frame(): - """A real one-household frame satisfying the driver's evidence reads.""" - - return uk_national_frame( - person=pd.DataFrame( - { - "person_id": [1, 2], - "person_benunit_id": [1, 1], - "person_household_id": [1, 1], - } - ), - benunit=pd.DataFrame({"benunit_id": [1]}), - household=pd.DataFrame({"household_id": [1], "household_weight": [2.0]}), - time_period="2023", - weight_kind=WeightKind.IMPORTANCE, - mass_log=( - MassChangeRecord( - entity="household", - old_total=2.0, - new_total=2.0, - declared_factor=1.0, - reason="reviewed test mass allocation", - ), - ), - ) - - -_PATH_ARGUMENTS = ( - "evidence_path", - "replay_path", - "terminal_gate_path", - "input_h5", - "staging_h5", - "spi_tab", - "hmrc_ods", - "cgt_ods", - "adult_tab", - "benefits_tab", - "build_record_path", - "input_mass_reference_path", - "input_mass_exclusions_path", - "qrf_tail_exclusions_path", - "degenerate_exclusions_path", - "rung_abort_path", -) -_IDENTITY_CLI_ARGUMENTS = ( - "--release-id", - "populace-uk-2023-frs-k535080", - "--calibration-diagnostics-sha256", - "c" * 64, -) - - -@pytest.fixture(autouse=True) -def _spool_only_by_default(monkeypatch: pytest.MonkeyPatch) -> None: - """Driver tests never inherit operator Logbook configuration.""" - - monkeypatch.delenv("POPULACE_LEDGER_URL", raising=False) - monkeypatch.delenv("POPULACE_LEDGER_KEY", raising=False) - monkeypatch.delenv("POPULACE_LEDGER_API_KEY", raising=False) - monkeypatch.delenv("POPULACE_LOGBOOK_PREV_ROW_DIGEST", raising=False) - - -def _spool_rows(tmp_path: Path): - rows = load_spool_rows(tmp_path / "logbook-spool") - for row in rows: - assert frozenset(row.to_mapping()) == LOGBOOK_ROW_FIELDS - return rows - - -def _local_ref(path: Path) -> str: - return f"local://{path.resolve().as_posix().lstrip('/')}" - - -def _gate_result(*, passed: bool) -> SimpleNamespace: - return SimpleNamespace( - passed=passed, - failures=() if passed else ("seeded coverage failure",), - details={"required_columns": 145}, - ) - - -def _fake_gate_report(input_coverage: SimpleNamespace) -> dict: - """A schema-4-shaped payload as the build result now carries it.""" - - return { - "schema_version": 4, - "blocked_at_phase": None, - "shippable": False, - "release_evidence": {"calibration_diagnostics_sha256": "c" * 64}, - "gates": { - "uk_release_input_coverage": { - "status": "passed" if input_coverage.passed else "failed", - "failures": list(input_coverage.failures), - "details": dict(input_coverage.details), - }, - "uk_weight_ess": { - "status": "passed", - "failures": [], - "details": {"ess_fraction": 0.5}, - }, - }, - } - - -def _load_builder_module(): - root = Path(__file__).resolve().parents[3] - path = root / "tools" / "build_uk_national_dataset.py" - spec = importlib.util.spec_from_file_location("build_uk_national_dataset", path) - module = importlib.util.module_from_spec(spec) - assert spec.loader is not None - spec.loader.exec_module(module) - return module - - -def test_national_build_driver_uses_standalone_national_seam( - monkeypatch, tmp_path, capsys -) -> None: - builder = _load_builder_module() - input_h5 = tmp_path / "base.h5" - staging_h5 = tmp_path / "staging.h5" - spi_tab = tmp_path / "put2223uk.tab" - hmrc_ods = tmp_path / "hmrc.ods" - cgt_ods = tmp_path / "cgt.ods" - frs_raw_dir = tmp_path / "frs_2023_24" - build_record_path = tmp_path / "national_staging_build_record.json" - adult_tab = frs_raw_dir / "adult.tab" - benefits_tab = frs_raw_dir / "benefits.tab" - frs_raw_dir.mkdir() - input_h5.write_bytes(b"base") - spi_tab.write_bytes(b"spi") - hmrc_ods.write_bytes(b"hmrc") - cgt_ods.write_bytes(b"cgt") - adult_tab.write_bytes(b"adult") - benefits_tab.write_bytes(b"benefits") - calls = [] - replay_writes = [] - - def fake_build(**kwargs): - calls.append(kwargs) - stages = kwargs["stages"] - stages[0].transform.last_result = SimpleNamespace( - evidence=lambda: {"stage": "frs_hmrc_retained_leaves"} - ) - stages[1].transform.last_result = SimpleNamespace( - evidence=lambda: {"stage": "hmrc_spi_income"}, - replay_report=SimpleNamespace(summary={"excluded_with_fence": 208}), - ) - staging_h5.write_bytes(b"staged") - kwargs["terminal_gate_path"].write_text('{"passed": true}\n') - input_coverage = _gate_result(passed=True) - return SimpleNamespace( - frame=_toy_result_frame(), - provenance=UKStagingProvenance( - source_h5=input_h5.resolve(), - fingerprint=_uk_source_file_fingerprint(input_h5.resolve()), - ), - input_h5=input_h5.resolve(), - staging_h5=staging_h5.resolve(), - stage_names=( - "frs_hmrc_retained_leaves", - "hmrc_spi_income", - ), - phase_reports=(), - gate_report=_fake_gate_report(input_coverage), - input_coverage=input_coverage, - sampling_receipt=None, - ) - - monkeypatch.setattr(builder, "build_uk_national_dataset", fake_build) - monkeypatch.setattr( - builder, - "write_hmrc_replay_report", - lambda report, path: ( - replay_writes.append((report, Path(path))), - Path(path).write_text('{"excluded_with_fence": 208}\n'), - )[1], - ) - monkeypatch.setattr( - builder, - "verify_certified_uk_candidate", - lambda path: SimpleNamespace( - path=Path(path).resolve(), - filename="populace_uk_2023.h5", - tier="frs", - revision="test-revision", - sha256="a" * 64, - size_bytes=4, - ), - ) - monkeypatch.setattr( - sys, - "argv", - [ - "build_uk_national_dataset.py", - *_IDENTITY_CLI_ARGUMENTS, - "--input-h5", - str(input_h5), - "--staging-h5", - str(staging_h5), - "--frs-raw-dir", - str(frs_raw_dir), - "--spi-tab", - str(spi_tab), - "--hmrc-ods", - str(hmrc_ods), - "--cgt-ods", - str(cgt_ods), - "--build-record-json", - str(build_record_path), - ], - ) - - assert builder.main() == 0 - - assert len(calls) == 1 - assert calls[0]["input_h5"] == input_h5 - assert calls[0]["staging_h5"] == staging_h5 - assert calls[0]["release_id"] == "populace-uk-2023-frs-k535080" - assert calls[0]["calibration_diagnostics_sha256"] == "c" * 64 - stages = calls[0]["stages"] - assert len(stages) == 3 - assert stages[0].name == "frs_hmrc_retained_leaves" - retained_transform = stages[0].transform - assert retained_transform.adult_tab_path == adult_tab - assert retained_transform.benefits_tab_path == benefits_tab - assert stages[1].name == "hmrc_spi_income" - hmrc_transform = stages[1].transform - assert hmrc_transform.spi_tab_path == spi_tab - assert hmrc_transform.hmrc_ods_path == hmrc_ods - assert hmrc_transform.certified_candidate.revision == "test-revision" - assert hmrc_transform.retained_leaves_transform is retained_transform - assert calls[0]["stages"][2].name == "hmrc_cgt_gains" - assert calls[0]["terminal_gate_path"] == staging_h5.with_suffix( - ".terminal_gates.json" - ) - assert calls[0]["release_candidate"] is False - # No --degenerate-exclusions: the artifact channel stays empty so the - # binding resolves the committed register itself and the run never - # self-describes as an override (the register is still preflighted). - assert calls[0]["reviewed_degenerate_exclusions"] is None - captured = capsys.readouterr() - payload = json.loads(captured.out) - assert "Wrote Logbook row:" in captured.err - assert payload["schema_version"] == 5 - assert payload["build_kind"] == "uk_national_staging_dataset" - assert payload["stages"] == [ - "frs_hmrc_retained_leaves", - "hmrc_spi_income", - ] - assert payload["input_coverage"]["passed"] is True - assert payload["terminal_gates"]["schema_version"] == 4 - assert payload["terminal_gates"]["blocked_at_phase"] is None - assert payload["terminal_gates"]["gates"]["uk_weight_ess"]["status"] == "passed" - assert payload["hmrc_replay"]["summary"] == {"excluded_with_fence": 208} - assert payload["artifacts"]["staging_h5"]["sha256"] - evidence_path = staging_h5.with_suffix(".hmrc_income.json") - evidence = json.loads(evidence_path.read_text(encoding="utf-8")) - assert evidence["base_candidate"]["tier"] == "frs" - assert evidence["base_candidate"]["revision"] == "test-revision" - assert evidence["retained_leaves"]["stage"] == "frs_hmrc_retained_leaves" - assert evidence["family"]["stage"] == "hmrc_spi_income" - assert payload["artifacts"]["hmrc_evidence"]["sha256"] - replay_path = staging_h5.with_suffix(".hmrc_replay.json") - assert replay_writes == [(hmrc_transform.last_result.replay_report, replay_path)] - assert payload["artifacts"]["hmrc_replay"]["sha256"] - assert payload["artifacts"]["frs_adult"]["sha256"] - assert payload["artifacts"]["frs_benefits"]["sha256"] - assert payload["artifacts"]["terminal_gates"]["sha256"] - assert payload["artifacts"]["build_record"]["sha256"] - record = json.loads(build_record_path.read_text(encoding="utf-8")) - assert record["schema_version"] == 3 - assert record["status"] == "passed" - assert record["calibration_diagnostics_sha256"] == "c" * 64 - assert record["terminal_gates"]["gates"]["uk_weight_ess"]["status"] == "passed" - assert record["dataset"] == { - "entity_rows": {"benunit": 1, "household": 1, "person": 2}, - "household_weight_kind": "importance", - "household_weight_total": 2.0, - "mass_changes": [ - { - "declared_factor": 1.0, - "entity": "household", - "new_total": 2.0, - "old_total": 2.0, - "reason": "reviewed test mass allocation", - } - ], - "time_period": "2023", - } - assert record["artifacts"]["staging_h5"]["retention"] == "local_untracked" - assert all("path" not in artifact for artifact in record["artifacts"].values()) - rows = _spool_rows(tmp_path) - assert len(rows) == 1 - row = rows[0] - assert row.pipeline == "uk-frs-staging" - assert row.rung == "f100" - assert row.seed == 578 - assert row.disposition == "iterating" - assert row.artifact_location == _local_ref(staging_h5) - assert row.phases_reached == ( - "attempt_started", - "configured", - "candidate_verified", - "inputs_pinned", - "build_completed", - "stage_reports_written", - "build_record_written", - ) - assert row.gate_verdicts == { - "uk_release_input_coverage": { - "verdict": "passed", - "receipt": f"{_local_ref(staging_h5.with_suffix('.terminal_gates.json'))}#/gates/uk_release_input_coverage", - }, - "uk_weight_ess": { - "verdict": "passed", - "receipt": f"{_local_ref(staging_h5.with_suffix('.terminal_gates.json'))}#/gates/uk_weight_ess", - }, - } - - -def test_national_driver_threads_logbook_predecessor_between_runs( - monkeypatch, - tmp_path, -) -> None: - builder = _load_builder_module() - input_h5 = tmp_path / "base.h5" - spi_tab = tmp_path / "put2223uk.tab" - hmrc_ods = tmp_path / "hmrc.ods" - cgt_ods = tmp_path / "cgt.ods" - frs_raw_dir = tmp_path / "frs_2023_24" - frs_raw_dir.mkdir() - for path, content in ( - (input_h5, b"base"), - (spi_tab, b"spi"), - (hmrc_ods, b"hmrc"), - (cgt_ods, b"cgt"), - (frs_raw_dir / "adult.tab", b"adult"), - (frs_raw_dir / "benefits.tab", b"benefits"), - ): - path.write_bytes(content) - - monkeypatch.setattr( - builder, - "verify_certified_uk_candidate", - lambda path: SimpleNamespace( - path=Path(path).resolve(), - filename="populace_uk_2023.h5", - tier="frs", - revision="test-revision", - sha256="a" * 64, - size_bytes=4, - ), - ) - monkeypatch.setattr( - builder, - "write_hmrc_replay_report", - lambda report, path: ( - Path(path).write_text('{"excluded_with_fence": 208}\n'), - Path(path), - )[1], - ) - - def fake_build(**kwargs): - kwargs["stages"][0].transform.last_result = SimpleNamespace( - evidence=lambda: {"stage": "frs_hmrc_retained_leaves"} - ) - kwargs["stages"][1].transform.last_result = SimpleNamespace( - evidence=lambda: {"stage": "hmrc_spi_income"}, - replay_report=SimpleNamespace(summary={"excluded_with_fence": 208}), - ) - kwargs["staging_h5"].write_bytes(b"staged") - kwargs["terminal_gate_path"].write_text('{"passed": true}\n') - input_coverage = _gate_result(passed=True) - return SimpleNamespace( - frame=_toy_result_frame(), - provenance=UKStagingProvenance( - source_h5=input_h5.resolve(), - fingerprint=_uk_source_file_fingerprint(input_h5.resolve()), - ), - input_h5=input_h5.resolve(), - staging_h5=kwargs["staging_h5"].resolve(), - stage_names=("frs_hmrc_retained_leaves", "hmrc_spi_income"), - phase_reports=(), - gate_report=_fake_gate_report(input_coverage), - input_coverage=input_coverage, - sampling_receipt=None, - ) - - monkeypatch.setattr(builder, "build_uk_national_dataset", fake_build) - - def argv(staging_name: str, *extra: str) -> list[str]: - return [ - *_IDENTITY_CLI_ARGUMENTS, - "--input-h5", - str(input_h5), - "--staging-h5", - str(tmp_path / staging_name), - "--frs-raw-dir", - str(frs_raw_dir), - "--spi-tab", - str(spi_tab), - "--hmrc-ods", - str(hmrc_ods), - "--cgt-ods", - str(cgt_ods), - *extra, - ] - - assert builder.main(argv("staging-1.h5")) == 0 - first = _spool_rows(tmp_path)[0] - - assert ( - builder.main( - argv("staging-2.h5", "--logbook-prev-row-digest", first.row_digest) - ) - == 0 - ) - rows = _spool_rows(tmp_path) - assert [row.prev_row_digest for row in rows] == [None, first.row_digest] - - -def test_national_driver_writes_aggregate_reports_before_reraising_final_gate( - monkeypatch, - tmp_path, -) -> None: - builder = _load_builder_module() - input_h5 = tmp_path / "base.h5" - staging_h5 = tmp_path / "staging.h5" - spi_tab = tmp_path / "put2223uk.tab" - hmrc_ods = tmp_path / "hmrc.ods" - cgt_ods = tmp_path / "cgt.ods" - frs_raw_dir = tmp_path / "frs_2023_24" - frs_raw_dir.mkdir() - for path, content in ( - (input_h5, b"base"), - (spi_tab, b"spi"), - (hmrc_ods, b"hmrc"), - (cgt_ods, b"cgt"), - (frs_raw_dir / "adult.tab", b"adult"), - (frs_raw_dir / "benefits.tab", b"benefits"), - ): - path.write_bytes(content) - - replay_report = object() - - def fake_build(**kwargs): - stages = kwargs["stages"] - stages[0].transform.last_result = SimpleNamespace( - evidence=lambda: {"stage": "frs_hmrc_retained_leaves"} - ) - stages[1].transform.last_result = SimpleNamespace( - evidence=lambda: {"stage": "hmrc_spi_income"}, - replay_report=replay_report, - ) - kwargs["terminal_gate_path"].write_text( - json.dumps( - { - "schema_version": 4, - "blocked_at_phase": "terminal", - "gates": { - "uk_release_input_coverage": { - "status": "failed", - "failures": ["gift_aid remains reviewed"], - }, - "uk_weight_ess": { - "status": "passed", - "failures": [], - }, - }, - }, - sort_keys=True, - ) - + "\n" - ) - raise GateBatteryBlockedError( - "terminal", - [ - "[uk_release_input_coverage] gift_aid remains a reviewed " - "exclusion with positive effective-mass signal" - ], - kwargs["terminal_gate_path"], - ) - - monkeypatch.setattr(builder, "build_uk_national_dataset", fake_build) - monkeypatch.setattr( - builder, - "verify_certified_uk_candidate", - lambda path: SimpleNamespace( - path=Path(path).resolve(), - filename="populace_uk_2023.h5", - tier="frs", - revision="test-revision", - sha256="a" * 64, - size_bytes=4, - ), - ) - replay_calls = [] - - def fake_write_replay(report, path): - replay_calls.append((report, Path(path))) - Path(path).write_text('{"excluded_with_fence": 208}\n') - return Path(path) - - monkeypatch.setattr(builder, "write_hmrc_replay_report", fake_write_replay) - monkeypatch.setattr( - sys, - "argv", - [ - "build_uk_national_dataset.py", - *_IDENTITY_CLI_ARGUMENTS, - "--input-h5", - str(input_h5), - "--staging-h5", - str(staging_h5), - "--frs-raw-dir", - str(frs_raw_dir), - "--spi-tab", - str(spi_tab), - "--hmrc-ods", - str(hmrc_ods), - "--cgt-ods", - str(cgt_ods), - ], - ) - - with pytest.raises(GateBatteryBlockedError, match="Gate battery blocked"): - builder.main() - - evidence = json.loads( - staging_h5.with_suffix(".hmrc_income.json").read_text(encoding="utf-8") - ) - assert evidence["retained_leaves"]["stage"] == ("frs_hmrc_retained_leaves") - assert evidence["family"]["stage"] == "hmrc_spi_income" - assert replay_calls == [ - (replay_report, staging_h5.with_suffix(".hmrc_replay.json")) - ] - assert staging_h5.with_suffix(".terminal_gates.json").is_file() - assert not staging_h5.exists() - assert not staging_h5.with_suffix(".build.json").exists() - rows = _spool_rows(tmp_path) - assert len(rows) == 1 - row = rows[0] - assert row.disposition == "failed" - assert row.pipeline == "uk-frs-staging" - assert row.gate_verdicts == { - "uk_release_input_coverage": { - "verdict": "failed", - "receipt": f"{_local_ref(staging_h5.with_suffix('.terminal_gates.json'))}#/gates/uk_release_input_coverage", - }, - "uk_weight_ess": { - "verdict": "passed", - "receipt": f"{_local_ref(staging_h5.with_suffix('.terminal_gates.json'))}#/gates/uk_weight_ess", - }, - } - assert "pipeline_error" not in row.gate_verdicts - - -def test_national_driver_writes_no_stage_reports_for_a_preflight_block( - monkeypatch, - tmp_path, -) -> None: - """A preflight block ran no stage; aggregate reports would be fiction.""" - - builder = _load_builder_module() - input_h5 = tmp_path / "base.h5" - staging_h5 = tmp_path / "staging.h5" - spi_tab = tmp_path / "put2223uk.tab" - hmrc_ods = tmp_path / "hmrc.ods" - cgt_ods = tmp_path / "cgt.ods" - frs_raw_dir = tmp_path / "frs_2023_24" - frs_raw_dir.mkdir() - for path in ( - input_h5, - spi_tab, - hmrc_ods, - cgt_ods, - frs_raw_dir / "adult.tab", - frs_raw_dir / "benefits.tab", - ): - path.write_bytes(b"source") - - def fake_build(**kwargs): - kwargs["terminal_gate_path"].write_text( - json.dumps( - { - "schema_version": 4, - "blocked_at_phase": "preflight", - "gates": { - "uk_release_input_coverage_manifest_current": { - "status": "blocked", - "failures": ["manifest drift"], - } - }, - }, - sort_keys=True, - ) - + "\n" - ) - raise GateBatteryBlockedError( - "preflight", - ["[uk_release_input_coverage_manifest_current] manifest drift"], - kwargs["terminal_gate_path"], - ) - - monkeypatch.setattr(builder, "build_uk_national_dataset", fake_build) - monkeypatch.setattr( - builder, - "verify_certified_uk_candidate", - lambda path: SimpleNamespace( - path=Path(path).resolve(), - filename="populace_uk_2023.h5", - tier="frs", - revision="test-revision", - sha256="a" * 64, - size_bytes=6, - ), - ) - monkeypatch.setattr( - builder, - "write_hmrc_replay_report", - lambda *_args: pytest.fail("preflight blocks must not emit replay reports"), - ) - monkeypatch.setattr( - sys, - "argv", - [ - "build_uk_national_dataset.py", - *_IDENTITY_CLI_ARGUMENTS, - "--input-h5", - str(input_h5), - "--staging-h5", - str(staging_h5), - "--frs-raw-dir", - str(frs_raw_dir), - "--spi-tab", - str(spi_tab), - "--hmrc-ods", - str(hmrc_ods), - "--cgt-ods", - str(cgt_ods), - ], - ) - - with pytest.raises(GateBatteryBlockedError): - builder.main() - - assert not staging_h5.with_suffix(".hmrc_income.json").exists() - assert not staging_h5.with_suffix(".hmrc_replay.json").exists() - assert not staging_h5.with_suffix(".build.json").exists() - rows = _spool_rows(tmp_path) - assert len(rows) == 1 - row = rows[0] - assert row.disposition == "failed" - assert row.gate_verdicts == { - "uk_release_input_coverage_manifest_current": { - "verdict": "blocked", - "receipt": f"{_local_ref(staging_h5.with_suffix('.terminal_gates.json'))}#/gates/uk_release_input_coverage_manifest_current", - } - } - - -def test_national_driver_does_not_write_reports_for_stage_failure( - monkeypatch, - tmp_path, -) -> None: - builder = _load_builder_module() - input_h5 = tmp_path / "base.h5" - staging_h5 = tmp_path / "staging.h5" - spi_tab = tmp_path / "put2223uk.tab" - hmrc_ods = tmp_path / "hmrc.ods" - cgt_ods = tmp_path / "cgt.ods" - frs_raw_dir = tmp_path / "frs_2023_24" - frs_raw_dir.mkdir() - for path in ( - input_h5, - spi_tab, - hmrc_ods, - cgt_ods, - frs_raw_dir / "adult.tab", - frs_raw_dir / "benefits.tab", - ): - path.write_bytes(b"source") - - monkeypatch.setattr( - builder, - "build_uk_national_dataset", - lambda **_kwargs: (_ for _ in ()).throw( - RuntimeError("SPI donor identity mismatch") - ), - ) - monkeypatch.setattr( - builder, - "verify_certified_uk_candidate", - lambda path: SimpleNamespace( - path=Path(path).resolve(), - filename="populace_uk_2023.h5", - tier="frs", - revision="test-revision", - sha256="a" * 64, - size_bytes=6, - ), - ) - monkeypatch.setattr( - builder, - "write_hmrc_replay_report", - lambda *_args: pytest.fail("stage errors must not emit replay reports"), - ) - monkeypatch.setattr( - sys, - "argv", - [ - "build_uk_national_dataset.py", - *_IDENTITY_CLI_ARGUMENTS, - "--input-h5", - str(input_h5), - "--staging-h5", - str(staging_h5), - "--frs-raw-dir", - str(frs_raw_dir), - "--spi-tab", - str(spi_tab), - "--hmrc-ods", - str(hmrc_ods), - "--cgt-ods", - str(cgt_ods), - ], - ) - - with pytest.raises(RuntimeError, match="SPI donor identity mismatch"): - builder.main() - - assert not staging_h5.with_suffix(".hmrc_income.json").exists() - assert not staging_h5.with_suffix(".hmrc_replay.json").exists() - assert not staging_h5.with_suffix(".build.json").exists() - rows = _spool_rows(tmp_path) - assert len(rows) == 1 - row = rows[0] - assert row.disposition == "failed" - assert row.pipeline == "uk-frs-staging" - assert row.gate_verdicts["pipeline_error"]["verdict"] == "error" - assert row.gate_verdicts["pipeline_error"]["receipt"].endswith("#/error_type") - assert "error" in row.phases_reached - - -@pytest.mark.parametrize( - ("left", "right"), - list(combinations(_PATH_ARGUMENTS, 2)), - ids=lambda value: value, -) -def test_national_driver_requires_every_input_output_path_to_be_distinct( - tmp_path, - left, - right, -) -> None: - builder = _load_builder_module() - paths = {name: tmp_path / f"{name}.artifact" for name in _PATH_ARGUMENTS} - collision = tmp_path / "collision.artifact" - paths[left] = collision - paths[right] = collision - - with pytest.raises(ValueError, match="pairwise distinct") as error: - builder._validate_distinct_paths(**paths) - - message = str(error.value) - assert collision.as_posix() in message - - -def test_national_driver_rejects_case_only_path_aliases(tmp_path) -> None: - builder = _load_builder_module() - candidate = tmp_path / "Candidate.H5" - candidate.write_bytes(b"certified base") - paths = {name: tmp_path / f"{name}.artifact" for name in _PATH_ARGUMENTS} - paths["input_h5"] = candidate - paths["terminal_gate_path"] = tmp_path / "candidate.h5" - - with pytest.raises(ValueError, match="pairwise distinct"): - builder._validate_distinct_paths(**paths) - - assert candidate.read_bytes() == b"certified base" - - -def test_national_driver_rejects_existing_hardlink_aliases(tmp_path) -> None: - builder = _load_builder_module() - candidate = tmp_path / "candidate.h5" - alias = tmp_path / "coverage.json" - candidate.write_bytes(b"certified base") - alias.hardlink_to(candidate) - paths = {name: tmp_path / f"{name}.artifact" for name in _PATH_ARGUMENTS} - paths["input_h5"] = candidate - paths["terminal_gate_path"] = alias - - with pytest.raises(ValueError, match="pairwise distinct"): - builder._validate_distinct_paths(**paths) - - -def test_national_driver_rejects_source_sidecar_collision_before_unlink( - monkeypatch, - tmp_path, -) -> None: - builder = _load_builder_module() - input_h5 = tmp_path / "base.h5" - staging_h5 = tmp_path / "staging.h5" - spi_tab = tmp_path / "put2223uk.tab" - hmrc_ods = tmp_path / "hmrc.ods" - cgt_ods = tmp_path / "cgt.ods" - frs_raw_dir = tmp_path / "frs_2023_24" - evidence = tmp_path / "evidence.json" - frs_raw_dir.mkdir() - input_h5.write_bytes(b"certified base") - staging_h5.write_bytes(b"previous staging") - spi_tab.write_bytes(b"licensed donor") - hmrc_ods.write_bytes(b"official surface") - (frs_raw_dir / "adult.tab").write_bytes(b"raw adult") - (frs_raw_dir / "benefits.tab").write_bytes(b"raw benefits") - evidence.write_bytes(b"previous evidence") - monkeypatch.setattr( - builder, - "verify_certified_uk_candidate", - lambda _path: pytest.fail("path validation must precede candidate hashing"), - ) - monkeypatch.setattr( - builder, - "build_uk_national_dataset", - lambda **_kwargs: pytest.fail("a colliding path must not start the build"), - ) - monkeypatch.setattr( - sys, - "argv", - [ - "build_uk_national_dataset.py", - *_IDENTITY_CLI_ARGUMENTS, - "--input-h5", - str(input_h5), - "--staging-h5", - str(staging_h5), - "--frs-raw-dir", - str(frs_raw_dir), - "--spi-tab", - str(spi_tab), - "--hmrc-ods", - str(hmrc_ods), - "--cgt-ods", - str(cgt_ods), - "--input-coverage-json", - str(spi_tab), - "--hmrc-evidence-json", - str(evidence), - ], - ) - - with pytest.raises(ValueError, match="pairwise distinct"): - builder.main() - - assert input_h5.read_bytes() == b"certified base" - assert staging_h5.read_bytes() == b"previous staging" - assert spi_tab.read_bytes() == b"licensed donor" - assert hmrc_ods.read_bytes() == b"official surface" - assert evidence.read_bytes() == b"previous evidence" - rows = _spool_rows(tmp_path) - assert len(rows) == 1 - row = rows[0] - assert row.disposition == "failed" - assert row.code_pin == "unresolved-local-git-code-pin" - assert row.gate_verdicts["pipeline_error"]["verdict"] == "error" - - -def test_national_driver_accepts_legacy_input_coverage_path_alias( - monkeypatch, - tmp_path, -) -> None: - builder = _load_builder_module() - legacy_path = tmp_path / "legacy-coverage.json" - monkeypatch.setattr( - sys, - "argv", - [ - "build_uk_national_dataset.py", - *_IDENTITY_CLI_ARGUMENTS, - "--input-h5", - "base.h5", - "--staging-h5", - "staging.h5", - "--frs-raw-dir", - "frs_2023_24", - "--spi-tab", - "put2223uk.tab", - "--hmrc-ods", - "hmrc.ods", - "--cgt-ods", - "cgt.ods", - "--input-coverage-json", - str(legacy_path), - "--logbook-prev-row-digest", - "d" * 64, - ], - ) - - args = builder._parse_args() - - assert args.input_coverage_json == legacy_path - assert args.terminal_gates_json is None - assert args.release_id == "populace-uk-2023-frs-k535080" - assert args.calibration_diagnostics_sha256 == "c" * 64 - assert args.logbook_prev_row_digest == "d" * 64 - - -def test_national_driver_accepts_staging_candidate_input_sha( - monkeypatch, - tmp_path, -) -> None: - builder = _load_builder_module() - declared_sha = "a" * 64 - monkeypatch.setattr( - sys, - "argv", - [ - "build_uk_national_dataset.py", - *_IDENTITY_CLI_ARGUMENTS, - "--input-h5", - "base.h5", - "--staging-h5", - "staging.h5", - "--staging-candidate-input-sha256", - declared_sha, - "--frs-raw-dir", - "frs_2023_24", - "--spi-tab", - "put2223uk.tab", - "--hmrc-ods", - "hmrc.ods", - "--cgt-ods", - "cgt.ods", - ], - ) - - args = builder._parse_args() - - assert args.staging_candidate_input_sha256 == declared_sha - - -def test_national_driver_forwards_legacy_output_to_compatibility_serializer( - monkeypatch, - tmp_path, -) -> None: - builder = _load_builder_module() - legacy_path = tmp_path / "legacy-coverage.json" - frs_raw_dir = tmp_path / "frs_2023_24" - frs_raw_dir.mkdir() - for path in ( - tmp_path / "base.h5", - tmp_path / "put2223uk.tab", - tmp_path / "hmrc.ods", - tmp_path / "cgt.ods", - frs_raw_dir / "adult.tab", - frs_raw_dir / "benefits.tab", - ): - path.write_bytes(b"source") - calls = [] - - class StopAfterForwardingError(Exception): - pass - - def fake_build(**kwargs): - calls.append(kwargs) - raise StopAfterForwardingError - - monkeypatch.setattr(builder, "build_uk_national_dataset", fake_build) - monkeypatch.setattr( - builder, - "verify_certified_uk_candidate", - lambda path: SimpleNamespace( - path=Path(path).resolve(), - filename="populace_uk_2023.h5", - tier="frs", - revision="test-revision", - sha256="a" * 64, - size_bytes=4, - ), - ) - monkeypatch.setattr( - sys, - "argv", - [ - "build_uk_national_dataset.py", - *_IDENTITY_CLI_ARGUMENTS, - "--input-h5", - str(tmp_path / "base.h5"), - "--staging-h5", - str(tmp_path / "staging.h5"), - "--frs-raw-dir", - str(tmp_path / "frs_2023_24"), - "--spi-tab", - str(tmp_path / "put2223uk.tab"), - "--hmrc-ods", - str(tmp_path / "hmrc.ods"), - "--cgt-ods", - str(tmp_path / "cgt.ods"), - "--input-coverage-json", - str(legacy_path), - ], - ) - - with pytest.raises(StopAfterForwardingError): - builder.main() - - assert len(calls) == 1 - assert calls[0]["input_coverage_path"] == legacy_path - assert "terminal_gate_path" not in calls[0] - - -def test_national_driver_rejects_both_terminal_gate_cli_names( - monkeypatch, - tmp_path, -) -> None: - builder = _load_builder_module() - monkeypatch.setattr( - sys, - "argv", - [ - "build_uk_national_dataset.py", - *_IDENTITY_CLI_ARGUMENTS, - "--input-h5", - "base.h5", - "--staging-h5", - "staging.h5", - "--frs-raw-dir", - "frs_2023_24", - "--spi-tab", - "put2223uk.tab", - "--hmrc-ods", - "hmrc.ods", - "--cgt-ods", - "cgt.ods", - "--terminal-gates-json", - str(tmp_path / "terminal.json"), - "--input-coverage-json", - str(tmp_path / "legacy.json"), - ], - ) - - with pytest.raises(SystemExit): - builder._parse_args() - - -@pytest.mark.parametrize( - "removed_flag", - [ - "--spi-donor-sample-size", - "--max-weight-ratio", - "--maximum-abs-relative-error", - "--calibration-epochs", - "--calibration-learning-rate", - ], -) -def test_national_driver_rejects_unreviewed_release_overrides( - monkeypatch, - removed_flag, -) -> None: - builder = _load_builder_module() - monkeypatch.setattr( - sys, - "argv", - [ - "build_uk_national_dataset.py", - *_IDENTITY_CLI_ARGUMENTS, - "--input-h5", - "base.h5", - "--staging-h5", - "staging.h5", - "--frs-raw-dir", - "frs_2023_24", - "--spi-tab", - "put2223uk.tab", - "--hmrc-ods", - "hmrc.ods", - "--cgt-ods", - "cgt.ods", - removed_flag, - "10", - ], - ) - - with pytest.raises(SystemExit): - builder._parse_args() - - -def test_stage_reports_survive_a_checkpoint_resumed_spi_stage(tmp_path): - """The adversarial-review blocker: a resumed run must still write sidecars. - - A checkpoint-resumed SPI transform carries the rehydrated evidence - surface instead of a live report object; the driver's stage reports - must consume it, and the resumed replay sidecar must be byte-identical - to what the payload serializes to. - """ - - import json - from types import SimpleNamespace - - builder = _load_builder_module() - from microcosm.build.uk_runtime.frs_hmrc_leaves import ( - UKFRSHMRCRetainedLeavesStageTransform, - _ResumedRetainedLeaves, - ) - from microcosm.build.uk_runtime.hmrc_restoration import ( - UKHMRCIncomeStageTransform, - ) - - retained = UKFRSHMRCRetainedLeavesStageTransform( - adult_tab_path=tmp_path / "adult.tab", - benefits_tab_path=tmp_path / "benefits.tab", - ) - retained.last_result = _ResumedRetainedLeaves( - frame=None, - evidence_payload={"stage": "frs_hmrc_retained_leaves"}, - input_content_identity="a" * 64, - output_content_identity="b" * 64, - ) - hmrc = UKHMRCIncomeStageTransform( - spi_tab_path=tmp_path / "put2223uk.tab", - hmrc_ods_path=tmp_path / "hmrc.ods", - certified_candidate=SimpleNamespace(), - ) - from microcosm.build.uk_runtime.content_identity import ( - uk_frame_content_identity, - ) - - resumed_frame = _toy_result_frame() - replay_payload = {"summary": {"status": "comparisons_passed"}, "facts": {}} - hmrc.resume_from_checkpoint( - { - "fit_weight_records": [ - {"fit_name": "uk_spi_fill_qrf", "weight_kind": "design"} - ], - "evidence": {"stage": "hmrc_spi_income"}, - "replay_payload": replay_payload, - "output_content_identity": uk_frame_content_identity(resumed_frame), - }, - resumed_frame, - ) - - evidence_path = tmp_path / "evidence.json" - replay_path = tmp_path / "replay.json" - builder._write_stage_reports( - evidence_path=evidence_path, - replay_path=replay_path, - candidate=SimpleNamespace( - path=tmp_path / "candidate.h5", - filename="candidate.h5", - tier="frs", - revision="r", - sha256="c" * 64, - size_bytes=1, - ), - retained_leaves_transform=retained, - hmrc_transform=hmrc, - ) - written = json.loads(replay_path.read_text()) - assert written == replay_payload - evidence = json.loads(evidence_path.read_text()) - assert evidence["family"] == {"stage": "hmrc_spi_income"} - assert builder._replay_summary(hmrc.last_result) == replay_payload["summary"] - - -def test_national_driver_rejects_non_rung_sample_fractions(monkeypatch) -> None: - builder = _load_builder_module() - monkeypatch.setattr( - sys, - "argv", - [ - "build_uk_national_dataset.py", - *_IDENTITY_CLI_ARGUMENTS, - "--input-h5", - "base.h5", - "--staging-h5", - "staging.h5", - "--frs-raw-dir", - "frs_2023_24", - "--spi-tab", - "put2223uk.tab", - "--hmrc-ods", - "hmrc.ods", - "--cgt-ods", - "cgt.ods", - "--sample-fraction", - "0.2", - ], - ) - - with pytest.raises(SystemExit): - builder._parse_args() - - -def test_national_driver_refuses_canonical_release_ids_for_rung_builds( - monkeypatch, capsys -) -> None: - builder = _load_builder_module() - # _IDENTITY_CLI_ARGUMENTS carries the canonical populace-uk-...-k id; a - # sampled rung build must refuse it — rung artifacts are receipts, never - # releases (#627). - monkeypatch.setattr( - sys, - "argv", - [ - "build_uk_national_dataset.py", - *_IDENTITY_CLI_ARGUMENTS, - "--input-h5", - "base.h5", - "--staging-h5", - "staging.h5", - "--frs-raw-dir", - "frs_2023_24", - "--spi-tab", - "put2223uk.tab", - "--hmrc-ods", - "hmrc.ods", - "--cgt-ods", - "cgt.ods", - "--sample-fraction", - "0.01", - ], - ) - - with pytest.raises(SystemExit): - builder._parse_args() - assert "non-releasable" in capsys.readouterr().err - - -def test_national_driver_refuses_release_candidate_on_a_rung( - monkeypatch, capsys -) -> None: - builder = _load_builder_module() - monkeypatch.setattr( - sys, - "argv", - [ - "build_uk_national_dataset.py", - "--release-id", - "uk-dev-rung-check", - "--calibration-diagnostics-sha256", - "c" * 64, - "--input-h5", - "base.h5", - "--staging-h5", - "staging.h5", - "--frs-raw-dir", - "frs_2023_24", - "--spi-tab", - "put2223uk.tab", - "--hmrc-ods", - "hmrc.ods", - "--cgt-ods", - "cgt.ods", - "--sample-fraction", - "0.10", - "--release-candidate", - ], - ) - - with pytest.raises(SystemExit): - builder._parse_args() - assert "non-releasable" in capsys.readouterr().err - - -def test_national_driver_refuses_release_candidate_with_the_legacy_alias( - monkeypatch, capsys -) -> None: - builder = _load_builder_module() - monkeypatch.setattr( - sys, - "argv", - [ - "build_uk_national_dataset.py", - *_IDENTITY_CLI_ARGUMENTS, - "--input-h5", - "base.h5", - "--staging-h5", - "staging.h5", - "--frs-raw-dir", - "frs_2023_24", - "--spi-tab", - "put2223uk.tab", - "--hmrc-ods", - "hmrc.ods", - "--cgt-ods", - "cgt.ods", - "--input-coverage-json", - "coverage.json", - "--release-candidate", - ], - ) - - with pytest.raises(SystemExit): - builder._parse_args() - assert "signed schema-4 report" in capsys.readouterr().err - - -def test_national_driver_refuses_release_candidate_with_staging_candidate_input( - monkeypatch, - capsys, -) -> None: - builder = _load_builder_module() - monkeypatch.setattr( - sys, - "argv", - [ - "build_uk_national_dataset.py", - *_IDENTITY_CLI_ARGUMENTS, - "--input-h5", - "base.h5", - "--staging-h5", - "staging.h5", - "--staging-candidate-input-sha256", - "a" * 64, - "--frs-raw-dir", - "frs_2023_24", - "--spi-tab", - "put2223uk.tab", - "--hmrc-ods", - "hmrc.ods", - "--cgt-ods", - "cgt.ods", - "--release-candidate", - ], - ) - - with pytest.raises(SystemExit): - builder._parse_args() - assert "non-certified spine" in capsys.readouterr().err - - -def test_staging_candidate_input_verifier_requires_declared_sha(tmp_path) -> None: - builder = _load_builder_module() - source = tmp_path / "synthetic-spine.h5" - source.write_bytes(b"synthetic") - digest = hashlib.sha256(b"synthetic").hexdigest() - - identity = builder.verify_staging_candidate_uk_input( - source, - expected_sha256=digest, - ) - - assert identity.tier == "staging_candidate" - assert identity.sha256 == digest - with pytest.raises(ValueError, match="does not match declared"): - builder.verify_staging_candidate_uk_input( - source, - expected_sha256="0" * 64, - ) - - -def test_staging_run_config_pins_the_sampling_identity(monkeypatch, tmp_path) -> None: - builder = _load_builder_module() - for name in ("adult.tab", "benefits.tab", "put2223uk.tab", "hmrc.ods", "cgt.ods"): - (tmp_path / name).write_bytes(b"x") - import microcosm.build.code_identity as code_identity_module - - monkeypatch.setattr( - code_identity_module, - "builder_code_identity", - lambda *args, **kwargs: {"stub": True}, - ) - args = SimpleNamespace( - release_id="uk-dev-rung", - calibration_diagnostics_sha256="c" * 64, - seed=42, - qrf_estimators=100, - sample_fraction=0.01, - sample_seed=7, - cgt_ods=tmp_path / "cgt.ods", - ) - retained = SimpleNamespace( - adult_tab_path=tmp_path / "adult.tab", - benefits_tab_path=tmp_path / "benefits.tab", - ) - hmrc = SimpleNamespace( - spi_tab_path=tmp_path / "put2223uk.tab", - hmrc_ods_path=tmp_path / "hmrc.ods", - ) - candidate = SimpleNamespace(sha256="a" * 64, size_bytes=3) - - config = builder._staging_run_config( - args, - candidate=candidate, - retained_leaves_transform=retained, - hmrc_transform=hmrc, - ) - - # The fraction is a string on purpose: run-config equality is exact over - # canonical JSON, and float normalization is exactly the ambiguity a run - # identity must not carry. Two rungs on one checkpoint directory refuse - # instead of cross-resuming. - assert config["sampling"] == { - "sample_fraction": "0.01", - "sample_seed": 7, - "rung_token": "f001", - } - - -def _rung_abort_argv(tmp_path: Path, *, fraction: str) -> list[str]: - frs_raw_dir = tmp_path / "frs_2023_24" - frs_raw_dir.mkdir(exist_ok=True) - for name in ("adult.tab", "benefits.tab"): - (frs_raw_dir / name).write_bytes(b"x") - for name in ("base.h5", "put2223uk.tab", "hmrc.ods", "cgt.ods"): - (tmp_path / name).write_bytes(b"x") - return [ - "build_uk_national_dataset.py", - "--release-id", - "uk-dev-rung", - "--calibration-diagnostics-sha256", - "c" * 64, - "--input-h5", - str(tmp_path / "base.h5"), - "--staging-h5", - str(tmp_path / "staging.h5"), - "--frs-raw-dir", - str(frs_raw_dir), - "--spi-tab", - str(tmp_path / "put2223uk.tab"), - "--hmrc-ods", - str(tmp_path / "hmrc.ods"), - "--cgt-ods", - str(tmp_path / "cgt.ods"), - "--sample-fraction", - fraction, - ] - - -def _named_edge_error() -> ValueError: - return ValueError( - "The least populated classes in y have only 1 member, which is too " - "few. The minimum number of groups for any class cannot be less " - "than 2. Classes with too few members are: [0.0]" - ) - - -def _install_rung_abort_seams(builder, monkeypatch, error: Exception) -> None: - monkeypatch.setattr( - builder, - "verify_certified_uk_candidate", - lambda path: SimpleNamespace( - path=Path(path).resolve(), - filename="populace_uk_2023.h5", - tier="frs", - revision="test-revision", - sha256="a" * 64, - size_bytes=1, - ), - ) - - def raising_build(**_kwargs): - raise error - - monkeypatch.setattr(builder, "build_uk_national_dataset", raising_build) - - -def test_rung_named_edge_aborts_with_a_receipt(monkeypatch, tmp_path) -> None: - """The one named dev-scale edge (#657) receipts instead of crashing.""" - - builder = _load_builder_module() - _install_rung_abort_seams(builder, monkeypatch, _named_edge_error()) - monkeypatch.setattr(sys, "argv", _rung_abort_argv(tmp_path, fraction="0.10")) - - assert builder.main() == builder._RUNG_ABORT_EXIT_CODE - - receipt = json.loads((tmp_path / "staging.rung_abort.json").read_text()) - assert receipt["named_edge"] == "spi_split_singleton_class" - assert receipt["disposition"] == "aborted_with_receipt" - assert receipt["sampling"]["rung_token"] == "f010" - assert "least populated classes" in receipt["error"] - rows = _spool_rows(tmp_path) - assert len(rows) == 1 - row = rows[0] - assert row.disposition == "discarded" - assert row.rung == "f010" - assert row.gate_verdicts == { - "uk_rung_abort": { - "verdict": "aborted", - "receipt": f"{_local_ref(tmp_path / 'staging.rung_abort.json')}#/named_edge", - } - } - assert "rung_aborted" in row.phases_reached - - -def test_full_scale_named_edge_still_crashes(monkeypatch, tmp_path) -> None: - builder = _load_builder_module() - _install_rung_abort_seams(builder, monkeypatch, _named_edge_error()) - monkeypatch.setattr(sys, "argv", _rung_abort_argv(tmp_path, fraction="1.0")) - - with pytest.raises(ValueError, match="least populated classes"): - builder.main() - assert not (tmp_path / "staging.rung_abort.json").exists() - rows = _spool_rows(tmp_path) - assert len(rows) == 1 - assert rows[0].disposition == "failed" - assert rows[0].gate_verdicts["pipeline_error"]["verdict"] == "error" - - -def test_rung_unknown_exception_still_crashes(monkeypatch, tmp_path) -> None: - """Only the named edge is receipted — the path cannot absorb defects.""" - - builder = _load_builder_module() - _install_rung_abort_seams( - builder, monkeypatch, ValueError("some entirely different failure") - ) - monkeypatch.setattr(sys, "argv", _rung_abort_argv(tmp_path, fraction="0.10")) - - with pytest.raises(ValueError, match="entirely different"): - builder.main() - assert not (tmp_path / "staging.rung_abort.json").exists() - rows = _spool_rows(tmp_path) - assert len(rows) == 1 - assert rows[0].disposition == "failed" - assert rows[0].gate_verdicts["pipeline_error"]["verdict"] == "error" - - -@pytest.mark.parametrize( - ("cli_digest", "env_value", "match"), - [ - pytest.param(None, "not-a-digest", "lowercase SHA-256", id="malformed-env"), - pytest.param( - "a" * 64, - "b" * 64, - "disagrees with POPULACE_LOGBOOK_PREV_ROW_DIGEST", - id="cli-env-conflict", - ), - ], -) -def test_invalid_logbook_predecessor_refuses_before_sidecar_cleanup( - monkeypatch, tmp_path, cli_digest, env_value, match -) -> None: - """Broken chain config aborts before any prior sidecar is unlinked. - - Adversarial-review finding on #666: the predecessor used to resolve - after the driver deleted the previous attempt's evidence sidecars, so a - config typo destroyed local evidence and then crashed. Config refusals - must leave the output directory untouched and record no row. - """ - - builder = _load_builder_module() - _install_rung_abort_seams(builder, monkeypatch, _named_edge_error()) - argv = _rung_abort_argv(tmp_path, fraction="0.10") - if cli_digest is not None: - argv += ["--logbook-prev-row-digest", cli_digest] - monkeypatch.setattr(sys, "argv", argv) - monkeypatch.setenv("POPULACE_LOGBOOK_PREV_ROW_DIGEST", env_value) - staging_h5 = tmp_path / "staging.h5" - sidecars = [ - staging_h5.with_suffix(".hmrc_income.json"), - staging_h5.with_suffix(".hmrc_replay.json"), - staging_h5.with_suffix(".build.json"), - staging_h5.with_suffix(".rung_abort.json"), - ] - for sidecar in sidecars: - sidecar.write_text('{"stale": true}\n') - - with pytest.raises(ValueError, match=match): - builder.main() - - for sidecar in sidecars: - assert sidecar.read_text() == '{"stale": true}\n' - assert not (tmp_path / "logbook-spool").exists() - - -@pytest.mark.parametrize("feed_layout", ["file", "directory"]) -def test_ledger_facts_role_pin_is_digestible(tmp_path, feed_layout) -> None: - """The armed run's Ledger role pin must survive role_pins_digest. - - First-armed-run finding (2026-08-23): _source_pins stored the full Ledger - provenance block under the 'ledger_facts' role, but role_pins_digest - accepts exactly {sha256, size_bytes}, so every armed build raised - ValueError at input_pins_digest before its first stage. Neither PR branch - fired it alone because only --ledger-facts reaches this path. - """ - - builder = _load_builder_module() - facts_body = b'{"fact": 1}\n' - if feed_layout == "file": - feed_path = tmp_path / "consumer_facts.jsonl" - feed_path.write_bytes(facts_body) - else: - feed_path = tmp_path / "feed" - feed_path.mkdir() - (feed_path / "consumer_facts.jsonl").write_bytes(facts_body) - artifact = SimpleNamespace( - path=feed_path, - facts_sha256="a" * 64, - provenance=lambda: { - "facts_sha256": "a" * 64, - "manifest_sha256": "b" * 64, - "profile": "uk-national", - }, - ) - - pin = builder._ledger_facts_pin(artifact) - - assert set(pin) == {"sha256", "size_bytes"} - assert pin["sha256"] == "a" * 64 - assert pin["size_bytes"] == len(facts_body) - digest = role_pins_digest({"ledger_facts": pin}) - assert len(digest) == 64 - with pytest.raises(ValueError): - role_pins_digest({"ledger_facts": dict(artifact.provenance())}) diff --git a/packages/microcosm-build/tests/test_uk_national_calibration.py b/packages/microcosm-build/tests/test_uk_national_calibration.py index 3006f087a..1886242e2 100644 --- a/packages/microcosm-build/tests/test_uk_national_calibration.py +++ b/packages/microcosm-build/tests/test_uk_national_calibration.py @@ -28,13 +28,15 @@ UKLedgerTargetCompilation, _uk_contract_targets, ) -from microcosm.build.uk_runtime.national_build import write_uk_national_frame from microcosm.build.uk_runtime.national_calibration import ( UKNationalCalibrationStage, _post_solve_calibration_record, national_calibration_mass_reason, ) -from microcosm.build.uk_runtime.national_frame import validate_uk_national_frame +from microcosm.build.uk_runtime.national_frame import ( + validate_uk_national_frame, + write_uk_national_frame, +) from microcosm.calibrate import TargetRegistry, TargetSpec from microcosm.frame import EntitySchema, Frame, WeightKind, Weights diff --git a/packages/microcosm-build/tests/test_uk_national_frame.py b/packages/microcosm-build/tests/test_uk_national_frame.py index 6fcf7d38c..95b85e785 100644 --- a/packages/microcosm-build/tests/test_uk_national_frame.py +++ b/packages/microcosm-build/tests/test_uk_national_frame.py @@ -20,16 +20,15 @@ import pandas as pd import pytest -from microcosm.build.uk_runtime.national_build import ( - load_uk_national_frame, - write_uk_national_frame, -) from microcosm.build.uk_runtime.national_frame import ( UK_NATIONAL_SCHEMA, + UKNationalStage, + load_uk_national_frame, uk_household_weight_kind, uk_national_frame, uk_time_period, validate_uk_national_frame, + write_uk_national_frame, ) from microcosm.build.uk_runtime.rowwise_dataset import ( read_uk_single_year_weight_metadata, @@ -110,6 +109,23 @@ def test_construction_accessors_and_residue_validation() -> None: validate_uk_national_frame(frame) +def test_national_stage_runs_frame_transform() -> None: + frame = _frame() + stage = UKNationalStage("identity", lambda candidate: candidate) + + assert stage.run(frame) is frame + + +def test_national_stage_requires_named_callable_frame_transform() -> None: + with pytest.raises(ValueError, match="non-empty"): + UKNationalStage("", lambda candidate: candidate) + with pytest.raises(TypeError, match="callable"): + UKNationalStage("bad", "not-callable") # type: ignore[arg-type] + + with pytest.raises(TypeError, match="must return a microcosm Frame"): + UKNationalStage("bad-output", lambda candidate: object()).run(_frame()) + + def test_construction_requires_the_exported_weight_column() -> None: with pytest.raises(ValueError, match="household_weight"): _frame(household=household_frame().drop(columns=["household_weight"])) diff --git a/packages/microcosm-build/tests/test_uk_rowwise_build_driver.py b/packages/microcosm-build/tests/test_uk_rowwise_build_driver.py index 4ea9c6c53..2ece175f6 100644 --- a/packages/microcosm-build/tests/test_uk_rowwise_build_driver.py +++ b/packages/microcosm-build/tests/test_uk_rowwise_build_driver.py @@ -301,7 +301,7 @@ def test_build_uk_rowwise_dataset_counts_blank_geography(monkeypatch, tmp_path): def test_build_uk_rowwise_dataset_infers_source_year_from_h5(monkeypatch, tmp_path): pytest.importorskip("tables") builder = _load_builder_module() - input_h5 = tmp_path / "populace_uk_2024.h5" + input_h5 = tmp_path / "microcosm_uk_2024.h5" crosswalk_path = tmp_path / "crosswalk.csv.gz" output_dir = tmp_path / "out" _write_toy_h5(input_h5, time_period="2024") @@ -329,8 +329,8 @@ def test_build_uk_rowwise_dataset_infers_source_year_from_h5(monkeypatch, tmp_pa manifest = json.loads((output_dir / builder.MANIFEST_FILENAME).read_text()) assert manifest["parameters"]["source_year"] == 2024 assert manifest["rowwise_dataset"]["time_period"] == "2024" - assert (output_dir / "populace_uk_2024_rowwise.h5").exists() - with pd.HDFStore(output_dir / "populace_uk_2024_rowwise.h5", mode="r") as store: + assert (output_dir / "microcosm_uk_2024_rowwise.h5").exists() + with pd.HDFStore(output_dir / "microcosm_uk_2024_rowwise.h5", mode="r") as store: household = store["household"] assert household["source_year"].unique().tolist() == [2024] assert household["source_household_key"].tolist() == [ @@ -489,6 +489,7 @@ def test_dataset_output_path_rejects_paths_and_reserved_names( builder._dataset_output_path( tmp_path, dataset_filename=dataset_filename, + input_stem="microcosm_uk_2024", source_year=2023, ) @@ -596,12 +597,14 @@ def test_build_uk_rowwise_dataset_rejects_overwriting_input(monkeypatch, tmp_pat "argv", [ "build_uk_rowwise_dataset.py", - "--input-h5", - str(input_h5), - "--out", - str(tmp_path), - ], - ) + "--input-h5", + str(input_h5), + "--out", + str(tmp_path), + "--dataset-filename", + input_h5.name, + ], + ) with pytest.raises(ValueError, match="must differ"): builder.main() diff --git a/packages/microcosm-build/tests/test_uk_rowwise_dry_run.py b/packages/microcosm-build/tests/test_uk_rowwise_dry_run.py index b83f902aa..9d1b2a2fb 100644 --- a/packages/microcosm-build/tests/test_uk_rowwise_dry_run.py +++ b/packages/microcosm-build/tests/test_uk_rowwise_dry_run.py @@ -468,10 +468,10 @@ def test_driver_full_build_records_weight_chain_and_lineage( assert lineage["immediate"] is None assert manifest["base_dataset"]["distinct_source_households"] is None - output_h5 = output_dir / "populace_uk_2023_rowwise.h5" + output_h5 = output_dir / "pool_rowwise.h5" import h5py - from microcosm.build.uk_runtime.national_build import ( + from microcosm.build.uk_runtime.national_frame import ( UK_HOUSEHOLD_WEIGHT_KIND_ATTR, ) diff --git a/packages/microcosm-build/tests/test_uk_rowwise_weight_metadata.py b/packages/microcosm-build/tests/test_uk_rowwise_weight_metadata.py index 117a9d6b0..b9f6ba84c 100644 --- a/packages/microcosm-build/tests/test_uk_rowwise_weight_metadata.py +++ b/packages/microcosm-build/tests/test_uk_rowwise_weight_metadata.py @@ -29,7 +29,7 @@ write_uk_national_frame, write_uk_rowwise_dataset, ) -from microcosm.build.uk_runtime.national_build import ( +from microcosm.build.uk_runtime.national_frame import ( UK_HOUSEHOLD_WEIGHT_KIND_ATTR, ) from microcosm.frame import Frame, MassChangeRecord, WeightKind diff --git a/packages/microcosm-frame/src/microcosm/frame/adapters/policyengine_uk.py b/packages/microcosm-frame/src/microcosm/frame/adapters/policyengine_uk.py index aacf23777..f84d206a9 100644 --- a/packages/microcosm-frame/src/microcosm/frame/adapters/policyengine_uk.py +++ b/packages/microcosm-frame/src/microcosm/frame/adapters/policyengine_uk.py @@ -105,11 +105,11 @@ def write_dataset( path: str | Path, period: int | str, ) -> None: - """UK dataset export remains on the national-build writer in E1.""" + """UK dataset export remains on the national-frame writer in E1.""" raise NotImplementedError( "PolicyEngine-UK dataset export is not implemented in the frame " - "adapter yet; use microcosm.build.uk_runtime.national_build." + "adapter yet; use microcosm.build.uk_runtime.national_frame." "write_uk_national_frame." ) diff --git a/tools/build_uk_frs_spine.py b/tools/build_uk_frs_spine.py index c945beeaf..a4a248f98 100644 --- a/tools/build_uk_frs_spine.py +++ b/tools/build_uk_frs_spine.py @@ -74,8 +74,10 @@ from microcosm.build.uk_runtime.lcfs_consumption import ( UKLCFSConsumptionStageTransform, ) -from microcosm.build.uk_runtime.national_build import write_uk_national_frame -from microcosm.build.uk_runtime.national_frame import uk_household_weight_kind +from microcosm.build.uk_runtime.national_frame import ( + uk_household_weight_kind, + write_uk_national_frame, +) from microcosm.build.uk_runtime.national_sampling import ( UK_SAMPLE_RUNG_TOKENS, UK_SAMPLE_SEED_DEFAULT, diff --git a/tools/build_uk_national_dataset.py b/tools/build_uk_national_dataset.py deleted file mode 100644 index a600bf05d..000000000 --- a/tools/build_uk_national_dataset.py +++ /dev/null @@ -1,1482 +0,0 @@ -"""Build the national UK staging file with the guarded HMRC/SPI family.""" - -from __future__ import annotations - -import argparse -import hashlib -import json -import re -import sys -import time -import uuid -from datetime import UTC, datetime -from importlib import resources as importlib_resources -from itertools import combinations -from pathlib import Path - -from microcosm.build.country_spec import country_stage_plan, load_country_spec -from microcosm.build.gate_battery import GateBatteryBlockedError -from microcosm.build.ledger_artifact import ( - add_ledger_artifact_args, - resolve_ledger_artifact, -) -from microcosm.build.logbook import canonical_json_bytes -from microcosm.build.logbook_adoption import ( - AttemptState, - append_phase, - apply_error_verdict, - error_receipt_path, - git_code_pin, - local_artifact_reference, - preflight_digest, - record_terminal_attempt, - resolve_predecessor, - role_pins_digest, - sha256_argument, - write_error_receipt, -) -from microcosm.build.plan import Stage as PlanStage -from microcosm.build.uk_runtime.cgt_imputation import ( - uk_capital_gains_imputation_stage, -) -from microcosm.build.uk_runtime.diagnostics import write_uk_calibration_diagnostics -from microcosm.build.uk_runtime.frs_hmrc_leaves import ( - UKFRSHMRCRetainedLeavesStageTransform, -) -from microcosm.build.uk_runtime.frs_release import load_uk_frs_release -from microcosm.build.uk_runtime.hmrc_replay import write_hmrc_replay_report -from microcosm.build.uk_runtime.hmrc_restoration import ( - UKHMRCIncomeStageTransform, - verify_certified_uk_candidate, - verify_staging_candidate_uk_input, -) -from microcosm.build.uk_runtime.ledger_targets import compile_uk_target_registry -from microcosm.build.uk_runtime.national_build import build_uk_national_dataset -from microcosm.build.uk_runtime.national_calibration import ( - UKNationalCalibrationStage, -) -from microcosm.build.uk_runtime.national_frame import ( - uk_household_weight_kind, - uk_time_period, -) -from microcosm.build.uk_runtime.national_sampling import ( - UK_SAMPLE_RUNG_TOKENS, - UK_SAMPLE_SEED_DEFAULT, -) -from microcosm.build.uk_runtime.parity_reference import ( - load_efrs_parity_reference, -) -from microcosm.build.uk_runtime.release_identity import UK_RELEASE_TIERS -from microcosm.build.uk_runtime.source_runtime import uk_stage_implementations -from microcosm.build.uk_runtime.terminal_gates import ( - uk_default_degenerate_reviewed_exclusions, -) -from microcosm.build.uk_runtime.weighted_integrity import ( - UK_DEGENERATE_EXCLUSION_REGISTER_RESOURCE, - UK_INPUT_MASS_EXCLUSION_REGISTER_RESOURCE, - UK_QRF_TAIL_EXCLUSION_REGISTER_RESOURCE, - load_uk_input_mass_reference, - load_uk_reference_scoped_exclusion_register, - load_uk_reviewed_exclusion_register, - uk_default_input_mass_reviewed_exclusions, - uk_default_qrf_tail_reviewed_exclusions, -) - -#: Canonical UK release ids (and the grandfathered June id) name shippable -#: artifacts; a sampled rung build must never carry one. Mirrors the -#: microcosm-data contract's release-identity check without importing the -#: data shard into the build tool. The durable coupling is the gate -#: battery's ``release_candidate`` flag (wired below: ``--release-candidate`` -#: is refused on a rung); this fence stays as defense in depth over the id -#: namespace itself. -# Year and count widths mirror the microcosm-data contract's release-identity -# regex ([1-9][0-9]*), and the tier alternation is built from the build -# shard's ratified UK_RELEASE_TIERS so a newly ratified tier is fenced -# automatically (adversarial-review finding). -_CANONICAL_UK_RELEASE_ID = re.compile( - r"populace-uk-[1-9][0-9]*-(?:" - + "|".join(sorted(re.escape(tier) for tier in UK_RELEASE_TIERS)) - + r")-k[1-9][0-9]*" -) -_UK_JUNE_RELEASE_ID = "populace-uk-2023-dd68c73-4aa4b14-20260619T023711Z" - -#: The one named dev-scale statistical edge receipted on a rung (#657, -#: closed without code): sklearn's stratified split inside the SPI imputation -#: refuses a singleton class on an unlucky small-sample composition. The -#: computation is never altered — the rung build aborts, but with a receipt -#: naming the edge instead of a bare traceback, and the remedy is re-rolling -#: ``--seed``. Only this named edge is receipted; unknown exceptions crash -#: loudly, so the receipt path can never absorb a real defect. -_RUNG_NAMED_EDGE_SIGNATURE = "The least populated classes in y have only 1 member" -_RUNG_ABORT_EXIT_CODE = 3 -_UK_NATIONAL_PIPELINE = "uk-frs-staging" -_REPOSITORY = Path(__file__).resolve().parents[1] - - -def _rung_sample_fraction(value: str) -> float: - """CLI rung policy (#624) over the permissive library validator.""" - - try: - fraction = float(value) - except ValueError as error: - raise argparse.ArgumentTypeError( - f"sample fraction must be a number; got {value!r}." - ) from error - if fraction not in UK_SAMPLE_RUNG_TOKENS: - raise argparse.ArgumentTypeError( - "sample fraction must be one of 0.01, 0.10, or 1.0 (the #624 rungs)." - ) - return fraction - - -def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: - parser = argparse.ArgumentParser() - parser.add_argument( - "--input-h5", - type=Path, - required=True, - help="Compact UK single-year H5 supplying the national base tables.", - ) - parser.add_argument( - "--staging-candidate-input-sha256", - type=sha256_argument, - help=( - "Declare --input-h5 as a non-certified staging-candidate spine and " - "bind it to this SHA-256. Refused with --release-candidate." - ), - ) - parser.add_argument( - "--staging-h5", - type=Path, - required=True, - help="Caller-owned path for the gated national staging H5.", - ) - parser.add_argument( - "--release-id", - required=True, - help="Canonical release id to bind into the signed terminal report.", - ) - parser.add_argument( - "--calibration-diagnostics-sha256", - required=True, - help=( - "Lowercase SHA-256 of the exact calibration_diagnostics.json bytes " - "that will ship with this release." - ), - ) - parser.add_argument( - "--national-calibration-diagnostics-json", - type=Path, - help="Per-target diagnostics emitted by the national calibration stage.", - ) - parser.add_argument( - "--frs-raw-dir", - type=Path, - required=True, - help=( - "Raw FRS 2024-25 directory containing adult.tab and benefits.tab " - "for source-faithful retained HMRC leaves." - ), - ) - parser.add_argument( - "--spi-tab", - type=Path, - required=True, - help="Licensed UKDS SPI 2022-23 donor named put2223uk.tab.", - ) - parser.add_argument( - "--hmrc-ods", - type=Path, - required=True, - help="Official HMRC Personal Incomes 2023-24 collated ODS.", - ) - parser.add_argument( - "--cgt-ods", - type=Path, - required=True, - help=( - "Official HMRC Capital Gains Tax statistics table 3 ODS (size of " - "gain by taxable income); fingerprint-verified before it is read." - ), - ) - gate_output = parser.add_mutually_exclusive_group() - gate_output.add_argument( - "--terminal-gates-json", - type=Path, - help=( - "Consolidated terminal-gate report path. Defaults beside " - "--staging-h5 with suffix '.terminal_gates.json'." - ), - ) - gate_output.add_argument( - "--input-coverage-json", - type=Path, - help=( - "Legacy schema-1 input-coverage diagnostic path. Cannot be " - "supplied with the preferred terminal-gate option." - ), - ) - parser.add_argument( - "--hmrc-evidence-json", - type=Path, - help=( - "HMRC stage evidence path. Defaults beside --staging-h5 with " - "suffix '.hmrc_income.json'." - ), - ) - parser.add_argument( - "--hmrc-replay-json", - type=Path, - help=( - "Aggregate-only 208-fact replay report path. Defaults beside " - "--staging-h5 with suffix '.hmrc_replay.json'." - ), - ) - parser.add_argument( - "--build-record-json", - type=Path, - help=( - "Aggregate, path-free staging build record. Defaults beside " - "--staging-h5 with suffix '.build.json'." - ), - ) - parser.add_argument("--seed", type=int, default=42) - parser.add_argument("--qrf-estimators", type=int, default=100) - parser.add_argument( - "--sample-fraction", - type=_rung_sample_fraction, - default=1.0, - help=( - "Scale-ladder rung (#627): 0.01 smoke, 0.10 dev, or 1.0 full. " - "Below 1.0 the loaded compact is sampled at clone-family grain, " - "renormalized to full household mass, and refused a canonical " - "release id — rung artifacts are receipts, never releases." - ), - ) - parser.add_argument( - "--sample-seed", - type=int, - default=UK_SAMPLE_SEED_DEFAULT, - help=( - "Whole-clone-family survey sampling seed (default: " - f"{UK_SAMPLE_SEED_DEFAULT}). Separate from --seed so dev-scale " - "sweeps vary one draw at a time." - ), - ) - parser.add_argument( - "--checkpoint-dir", - type=Path, - help=( - "Persist a lossless Frame checkpoint at every stage boundary and " - "resume completed stages from it (#612 increment 3). The run is " - "pinned by a content-addressed run config (input digest, seed, " - "QRF estimators, raw-source digests); rerunning the same command " - "against the same directory resumes, a changed configuration is " - "refused. Omit for the destructive single-process build." - ), - ) - parser.add_argument( - "--input-mass-reference-json", - type=Path, - help=( - "Frozen weighted per-column reference totals (schema_version 1: " - "identity + totals) emitted by the #609 measurement tooling. " - "Supplying it provides the licensed evidence for the spec-armed " - "input_mass_parity gate; absent, the gate records evidence_absent " - "and blocks release candidates." - ), - ) - parser.add_argument( - "--input-mass-exclusions", - type=Path, - help=( - "Reviewed input-mass exclusion register overriding the committed " - f"{UK_INPUT_MASS_EXCLUSION_REGISTER_RESOURCE}. The override is " - "schema-3 and scoped per named reference. Stale entries fail the " - "gate; dormant entries are reported." - ), - ) - parser.add_argument( - "--qrf-tail-exclusions", - type=Path, - help=( - "Reviewed QRF tail-concentration exclusion register overriding " - f"the committed {UK_QRF_TAIL_EXCLUSION_REGISTER_RESOURCE}. Stale " - "entries fail the gate; dormant entries are reported." - ), - ) - parser.add_argument( - "--degenerate-exclusions", - type=Path, - help=( - "Reviewed degenerate-release-surface exclusion register " - f"overriding the committed {UK_DEGENERATE_EXCLUSION_REGISTER_RESOURCE} " - "(#630). Stale entries fail the gate; dormant entries are " - "reported. The gate is always armed; the override is digested " - "into the report's evidence_sha256, so an overridden run " - "self-describes against the committed register." - ), - ) - parser.add_argument( - "--release-candidate", - action="store_true", - help=( - "Arm the battery's release-candidate posture: every " - "evidence_absent gap blocks instead of being recorded. Refused " - "on a sampled rung — a rung is structurally non-releasable " - "(#627). Default off: the staging build records its gaps " - "honestly and continues." - ), - ) - parser.add_argument( - "--logbook-prev-row-digest", - type=sha256_argument, - help=( - "Optional current Logbook chain head. If omitted, " - "POPULACE_LOGBOOK_PREV_ROW_DIGEST is used, then genesis null." - ), - ) - add_ledger_artifact_args(parser) - args = parser.parse_args(argv) - if args.release_candidate and args.sample_fraction != 1.0: - parser.error( - "--release-candidate is refused on a sampled rung; a rung build " - "is structurally non-releasable (#627)." - ) - if args.release_candidate and args.input_coverage_json is not None: - parser.error( - "--release-candidate is refused with --input-coverage-json; the " - "schema-1 alias is last-written over the report path and a " - "candidate must keep its signed schema-4 report." - ) - if args.release_candidate and args.staging_candidate_input_sha256 is not None: - parser.error( - "--release-candidate requires the certified input posture; " - "--staging-candidate-input-sha256 declares a non-certified spine." - ) - if args.sample_seed < 0: - parser.error("sample seed must be a non-negative integer.") - if args.sample_fraction != 1.0 and ( - _CANONICAL_UK_RELEASE_ID.fullmatch(args.release_id) - or args.release_id == _UK_JUNE_RELEASE_ID - ): - parser.error( - "a sampled build (--sample-fraction below 1.0) must not carry a " - "canonical release id; rung artifacts are structurally " - "non-releasable (#627)." - ) - return args - - -def _weighted_integrity_arguments(args: argparse.Namespace) -> dict[str, object]: - """Assemble weighted-integrity evidence and optional review overrides.""" - - def parser_error(message: str) -> None: - raise SystemExit(f"error: {message}") - - arguments: dict[str, object] = {} - if args.input_mass_reference_json is not None: - arguments["input_mass_reference"] = load_uk_input_mass_reference( - args.input_mass_reference_json - ) - if args.input_mass_exclusions is not None: - arguments["reviewed_input_mass_exclusions"] = ( - load_uk_reference_scoped_exclusion_register( - args.input_mass_exclusions, - resource=UK_INPUT_MASS_EXCLUSION_REGISTER_RESOURCE, - ) - ) - else: - uk_default_input_mass_reviewed_exclusions() - elif args.input_mass_exclusions is not None: - parser_error("--input-mass-exclusions requires --input-mass-reference-json.") - else: - uk_default_input_mass_reviewed_exclusions() - if args.qrf_tail_exclusions is not None: - arguments["reviewed_qrf_tail_exclusions"] = load_uk_reviewed_exclusion_register( - args.qrf_tail_exclusions, - resource=UK_QRF_TAIL_EXCLUSION_REGISTER_RESOURCE, - ) - else: - uk_default_qrf_tail_reviewed_exclusions() - return arguments - - -def _new_national_attempt_id(*, timestamp: datetime) -> str: - instant = timestamp.astimezone(UTC) - return ( - "uk-national-attempt-" - f"{instant.strftime('%Y%m%dT%H%M%SZ')}-{uuid.uuid4().hex[:8]}" - ) - - -def _new_national_build_id( - *, - rung: str, - sample_seed: int, - seed: int, - timestamp: datetime, -) -> str: - instant = timestamp.astimezone(UTC) - return ( - f"uk-national-{rung}-ss{sample_seed}-s{seed}-" - f"{instant.strftime('%Y%m%dT%H%M%SZ')}-{uuid.uuid4().hex[:8]}" - ) - - -def _artifact_pin(path: str | Path) -> dict[str, object]: - info = _artifact_info(path) - return {"sha256": info["sha256"], "size_bytes": info["size_bytes"]} - - -def _source_pins( - *, - candidate: object, - retained_leaves_transform: UKFRSHMRCRetainedLeavesStageTransform, - hmrc_transform: UKHMRCIncomeStageTransform, - cgt_ods_path: Path, - ledger_artifact: object | None = None, -) -> dict[str, dict[str, object]]: - pins = { - "certified_candidate": { - "sha256": str(candidate.sha256), - "size_bytes": int(candidate.size_bytes), - }, - "adult_tab": _artifact_pin(retained_leaves_transform.adult_tab_path), - "benefits_tab": _artifact_pin(retained_leaves_transform.benefits_tab_path), - "spi_tab": _artifact_pin(hmrc_transform.spi_tab_path), - "hmrc_ods": _artifact_pin(hmrc_transform.hmrc_ods_path), - "cgt_ods": _artifact_pin(cgt_ods_path), - } - if ledger_artifact is not None: - pins["ledger_facts"] = _ledger_facts_pin(ledger_artifact) - return pins - - -def _ledger_facts_pin(ledger_artifact: object) -> dict[str, object]: - """Pin the consumer feed by content and size. - - Logbook role pins are exactly ``sha256`` and ``size_bytes``; the richer - Ledger identity block travels separately in ``safe_artifacts`` and - ``source_vintages``. The feed digest is already verified against the - manifest and the CLI pin at load, so it is reused rather than recomputed - over a multi-hundred-megabyte file. - """ - - path = Path(ledger_artifact.path) - facts_path = path / "consumer_facts.jsonl" if path.is_dir() else path - return { - "sha256": str(ledger_artifact.facts_sha256), - "size_bytes": int(facts_path.stat().st_size), - } - - -def _input_posture(candidate: object) -> dict[str, object]: - tier = str(getattr(candidate, "tier", "frs")) - return { - "posture": "staging_candidate" if tier == "staging_candidate" else "certified", - "filename": str(getattr(candidate, "filename", "")), - "tier": tier, - "revision": str(getattr(candidate, "revision", "")), - "sha256": str(candidate.sha256), - "size_bytes": int(candidate.size_bytes), - } - - -def _gate_verdicts_from_report( - report: dict[str, object], - *, - gate_output_path: Path, -) -> dict[str, dict[str, object]]: - gates = report.get("gates") - if not isinstance(gates, dict): - raise ValueError("terminal gate report must contain a gates object.") - reference = local_artifact_reference(gate_output_path, repository_hint=_REPOSITORY) - return { - str(entry_id): { - "verdict": str(entry["status"]), - "receipt": f"{reference}#/gates/{entry_id}", - } - for entry_id, entry in gates.items() - if isinstance(entry, dict) and "status" in entry - } - - -def _record_national_attempt( - *, - state: AttemptState, - started_at: float, - started_ts: datetime, - rung: str, - seed: int | None, - code_pin: str, - disposition: str, - predecessor: str | None, - spool_dir: Path, -) -> Path: - return record_terminal_attempt( - state=state, - started_at=started_at, - started_ts=started_ts, - pipeline=_UK_NATIONAL_PIPELINE, - rung=rung, - seed=seed, - code_pin=code_pin, - disposition=disposition, - predecessor=predecessor, - spool_dir=spool_dir, - ) - - -def _record_failed_exception( - *, - error: BaseException, - state: AttemptState, - started_at: float, - started_ts: datetime, - rung: str, - seed: int | None, - code_pin: str, - predecessor: str | None, - receipt_base_dir: Path, - spool_dir: Path, -) -> None: - error_path = write_error_receipt( - error_receipt_path(receipt_base_dir, build_id=state.build_id), - state=state, - pipeline=_UK_NATIONAL_PIPELINE, - error=error, - ) - apply_error_verdict( - state, - f"{local_artifact_reference(error_path, repository_hint=_REPOSITORY)}#/error_type", - ) - _record_national_attempt( - state=state, - started_at=started_at, - started_ts=started_ts, - rung=rung, - seed=seed, - code_pin=code_pin, - disposition="failed", - predecessor=predecessor, - spool_dir=spool_dir, - ) - - -def _rung_abort_receipt( - args: argparse.Namespace, - *, - rung: str, - error: BaseException, -) -> dict[str, object]: - return { - "schema_version": 1, - "artifact_kind": "uk_rung_abort_receipt", - "build_kind": "uk_national_staging_dataset", - "release_id": str(args.release_id), - "sampling": { - "sample_fraction": float(args.sample_fraction), - "sample_seed": int(args.sample_seed), - "rung_token": rung, - }, - "seed": int(args.seed), - "named_edge": "spi_split_singleton_class", - "stage": "hmrc_spi_income", - "error": str(error), - "disposition": "aborted_with_receipt", - "remedy": ( - "Re-roll --seed; accepted dev-scale statistical edge " - "(microcosm#657, closed). The computation is never altered to avoid it." - ), - } - - -def main(argv: list[str] | None = None) -> int: - args = _parse_args(argv) - started_at = time.perf_counter() - started_ts = datetime.now(UTC) - rung = UK_SAMPLE_RUNG_TOKENS[args.sample_fraction] - code_pin = "unresolved-local-git-code-pin" - # Logbook chain configuration is validated before any side effect: a - # malformed or conflicting predecessor refuses the run here, before the - # build can unlink the prior attempt's sidecars (#666 adversarial-review - # finding). Config refusals record no row, like argparse refusals. - predecessor = resolve_predecessor(args.logbook_prev_row_digest) - attempt_context: dict[str, object] = { - "code_pin": code_pin, - "predecessor": predecessor, - } - stage_context: dict[str, object] = {} - logbook_seed: int | None = args.sample_seed - receipt_base_dir = args.staging_h5.parent - spool_dir = args.staging_h5.parent / "logbook-spool" - digest = preflight_digest(_UK_NATIONAL_PIPELINE) - state = AttemptState( - build_id=_new_national_attempt_id(timestamp=started_ts), - identity_digest=digest, - input_pins_digest=digest, - phases_reached=["attempt_started"], - gate_verdicts={ - "pipeline": { - "verdict": "running", - "receipt": "pending-build-scoped-terminal-receipt", - } - }, - ) - try: - return _main_recording( - args=args, - state=state, - started_at=started_at, - started_ts=started_ts, - rung=rung, - logbook_seed=logbook_seed, - attempt_context=attempt_context, - stage_context=stage_context, - spool_dir=spool_dir, - ) - except ValueError as error: - if args.sample_fraction != 1.0 and _RUNG_NAMED_EDGE_SIGNATURE in str(error): - rung_abort_path = args.staging_h5.with_suffix(".rung_abort.json") - receipt = _rung_abort_receipt( - args, - rung=rung, - error=error, - ) - _write_json(rung_abort_path, receipt) - state.gate_verdicts = { - "uk_rung_abort": { - "verdict": "aborted", - "receipt": ( - f"{local_artifact_reference(rung_abort_path, repository_hint=_REPOSITORY)}" - "#/named_edge" - ), - } - } - append_phase(state, "rung_aborted") - _record_national_attempt( - state=state, - started_at=started_at, - started_ts=started_ts, - rung=rung, - seed=logbook_seed, - code_pin=str(attempt_context["code_pin"]), - disposition="discarded", - predecessor=attempt_context["predecessor"], - spool_dir=spool_dir, - ) - print(json.dumps(receipt, indent=2, sort_keys=True)) - return _RUNG_ABORT_EXIT_CODE - _record_failed_exception( - error=error, - state=state, - started_at=started_at, - started_ts=started_ts, - rung=rung, - seed=logbook_seed, - code_pin=str(attempt_context["code_pin"]), - predecessor=attempt_context["predecessor"], - receipt_base_dir=receipt_base_dir, - spool_dir=spool_dir, - ) - raise - except GateBatteryBlockedError as error: - retained_leaves_transform = stage_context.get("retained_leaves_transform") - hmrc_transform = stage_context.get("hmrc_transform") - candidate = stage_context.get("candidate") - evidence_path = stage_context.get("evidence_path") - replay_path = stage_context.get("replay_path") - if ( - error.phase == "terminal" - and retained_leaves_transform is not None - and hmrc_transform is not None - and candidate is not None - and evidence_path is not None - and replay_path is not None - and retained_leaves_transform.last_result is not None - and hmrc_transform.last_result is not None - ): - _write_stage_reports( - evidence_path=evidence_path, - replay_path=replay_path, - candidate=candidate, - retained_leaves_transform=retained_leaves_transform, - hmrc_transform=hmrc_transform, - ) - gate_report = json.loads(error.report_path.read_text(encoding="utf-8")) - state.gate_verdicts = _gate_verdicts_from_report( - gate_report, - gate_output_path=error.report_path, - ) - append_phase(state, "gate_battery_blocked") - _record_national_attempt( - state=state, - started_at=started_at, - started_ts=started_ts, - rung=rung, - seed=logbook_seed, - code_pin=str(attempt_context["code_pin"]), - disposition="failed", - predecessor=attempt_context["predecessor"], - spool_dir=spool_dir, - ) - raise - except Exception as error: - _record_failed_exception( - error=error, - state=state, - started_at=started_at, - started_ts=started_ts, - rung=rung, - seed=logbook_seed, - code_pin=str(attempt_context["code_pin"]), - predecessor=attempt_context["predecessor"], - receipt_base_dir=receipt_base_dir, - spool_dir=spool_dir, - ) - raise - - -def _main_recording( - *, - args: argparse.Namespace, - state: AttemptState, - started_at: float, - started_ts: datetime, - rung: str, - logbook_seed: int | None, - attempt_context: dict[str, object], - stage_context: dict[str, object], - spool_dir: Path, -) -> int: - legacy_input_coverage_path = args.input_coverage_json - terminal_gate_path = ( - None - if legacy_input_coverage_path is not None - else ( - args.terminal_gates_json - or args.staging_h5.with_suffix(".terminal_gates.json") - ) - ) - gate_output_path = legacy_input_coverage_path or terminal_gate_path - assert gate_output_path is not None - evidence_path = args.hmrc_evidence_json or args.staging_h5.with_suffix( - ".hmrc_income.json" - ) - replay_path = args.hmrc_replay_json or args.staging_h5.with_suffix( - ".hmrc_replay.json" - ) - build_record_path = args.build_record_json or args.staging_h5.with_suffix( - ".build.json" - ) - stage_context.update( - { - "evidence_path": evidence_path, - "replay_path": replay_path, - } - ) - rung_abort_path = args.staging_h5.with_suffix(".rung_abort.json") - retained_leaves_transform = ( - UKFRSHMRCRetainedLeavesStageTransform.from_raw_frs_directory( - args.frs_raw_dir, - # A rung build's base deliberately carries a sampled subset of - # source families; the stage receipts the dropped raw surface - # instead of failing its completeness fence (#627). - sampled_rung=args.sample_fraction != 1.0, - ) - ) - _validate_distinct_paths( - evidence_path=evidence_path, - replay_path=replay_path, - terminal_gate_path=gate_output_path, - input_h5=args.input_h5, - staging_h5=args.staging_h5, - spi_tab=args.spi_tab, - hmrc_ods=args.hmrc_ods, - cgt_ods=args.cgt_ods, - adult_tab=retained_leaves_transform.adult_tab_path, - benefits_tab=retained_leaves_transform.benefits_tab_path, - build_record_path=build_record_path, - input_mass_reference_path=args.input_mass_reference_json, - input_mass_exclusions_path=args.input_mass_exclusions, - qrf_tail_exclusions_path=args.qrf_tail_exclusions, - degenerate_exclusions_path=args.degenerate_exclusions, - rung_abort_path=rung_abort_path, - ) - # Read-only gate inputs are materialized before any sidecar unlink so a - # path collision cannot consume a just-deleted file — and so a typo'd - # register path dies here, before it can destroy a prior build's - # sidecars. The default register is preflighted for the same reason: a - # corrupted committed register must not surface hours later at - # terminal-gate time. - weighted_integrity_arguments = _weighted_integrity_arguments(args) - if args.degenerate_exclusions is None: - # Preflight the committed register without passing it: a corrupted - # register dies here, while the absent artifact leaves the binding - # resolving the same policy of record itself — the artifact stays - # the review-time override channel, so a default run never - # self-describes as an override. - uk_default_degenerate_reviewed_exclusions() - reviewed_degenerate_exclusions = None - else: - reviewed_degenerate_exclusions = load_uk_reviewed_exclusion_register( - args.degenerate_exclusions, - resource=UK_DEGENERATE_EXCLUSION_REGISTER_RESOURCE, - ) - ledger_artifact = resolve_ledger_artifact(args) - ledger_compilations = ( - None - if ledger_artifact is None - else { - period: compile_uk_target_registry( - ledger_artifact.facts, target_period=period - ) - for period in (2023, 2025) - } - ) - if args.staging_candidate_input_sha256 is None: - candidate = verify_certified_uk_candidate(args.input_h5) - else: - candidate = verify_staging_candidate_uk_input( - args.input_h5, - expected_sha256=args.staging_candidate_input_sha256, - ) - evidence_path.unlink(missing_ok=True) - replay_path.unlink(missing_ok=True) - build_record_path.unlink(missing_ok=True) - # A prior rung abort must never sit beside a fresh build's artifacts - # (adversarial-review finding: a stale receipt contradicted a later - # successful run at the same staging path). - rung_abort_path.unlink(missing_ok=True) - hmrc_transform = UKHMRCIncomeStageTransform( - spi_tab_path=args.spi_tab, - hmrc_ods_path=args.hmrc_ods, - certified_candidate=candidate, - retained_leaves_transform=retained_leaves_transform, - seed=args.seed, - qrf_estimators=args.qrf_estimators, - sampled_rung=args.sample_fraction != 1.0, - ) - stage_context.update( - { - "retained_leaves_transform": retained_leaves_transform, - "hmrc_transform": hmrc_transform, - "candidate": candidate, - } - ) - source_pins = _source_pins( - candidate=candidate, - retained_leaves_transform=retained_leaves_transform, - hmrc_transform=hmrc_transform, - cgt_ods_path=args.cgt_ods, - ledger_artifact=ledger_artifact, - ) - run_config = _staging_run_config( - args, - candidate=candidate, - retained_leaves_transform=retained_leaves_transform, - hmrc_transform=hmrc_transform, - source_pins=source_pins, - ) - attempt_context["code_pin"] = git_code_pin(_REPOSITORY) - state.build_id = _new_national_build_id( - rung=rung, - sample_seed=args.sample_seed, - seed=args.seed, - timestamp=started_ts, - ) - state.input_pins_digest = role_pins_digest(source_pins) - state.identity_digest = hashlib.sha256(canonical_json_bytes(run_config)).hexdigest() - append_phase(state, "configured") - append_phase(state, "candidate_verified") - append_phase(state, "inputs_pinned") - # Without Ledger facts, this staging path has no real target-surface or - # target-fit evidence; the schema-4 battery records the missing evidence - # explicitly. Armed calibration builds add target evidence from the solve. - # Input-mass evidence joins only when the caller supplies the licensed - # frozen reference sidecar; QRF-tail is spec-armed and runs whenever the - # frame evidence is present. - gate_path_argument = ( - {"input_coverage_path": legacy_input_coverage_path} - if legacy_input_coverage_path is not None - else {"terminal_gate_path": terminal_gate_path} - ) - checkpoint_arguments: dict[str, object] = {} - if args.checkpoint_dir is not None: - checkpoint_arguments = { - "checkpoint_dir": args.checkpoint_dir, - "run_config": run_config, - } - if (args.ledger_facts is None) != ( - args.national_calibration_diagnostics_json is None - ): - raise ValueError( - "--ledger-facts and --national-calibration-diagnostics-json must " - "be supplied together." - ) - if args.release_candidate and args.ledger_facts is None: - raise ValueError("a release candidate requires the national calibration stage.") - calibration_transform = None - calibration_stages: tuple[PlanStage, ...] = () - if args.ledger_facts is not None: - assert ledger_artifact is not None # resolved and pin-checked above - assert ledger_compilations is not None - calibration_year = load_uk_frs_release().calibration_year - calibration_transform = UKNationalCalibrationStage( - ledger_compilations[calibration_year], - period=calibration_year, - ) - calibration_stages = ( - PlanStage( - name="national_calibration", - transform=calibration_transform, - ), - ) - result = build_uk_national_dataset( - input_h5=args.input_h5, - staging_h5=args.staging_h5, - release_id=args.release_id, - calibration_diagnostics_sha256=args.calibration_diagnostics_sha256, - reviewed_degenerate_exclusions=reviewed_degenerate_exclusions, - stages=( - *country_stage_plan( - load_country_spec("uk"), - uk_stage_implementations( - retained_leaves_transform=retained_leaves_transform, - hmrc_income_transform=hmrc_transform, - ), - # The manifest also declares the frs_spine pipeline root; - # the national staging pipeline selects its own stages. - stage_names=("frs_hmrc_retained_leaves", "hmrc_spi_income"), - ).stages, - # Runs after the SPI restoration so the taxable-income proxy - # sees the restored income surface. Declared today in the - # bespoke uk/cgt_source_stages.json; absorbing it into the - # canonical source_stages.json is WS-E follow-up work. - uk_capital_gains_imputation_stage(args.cgt_ods), - *calibration_stages, - ), - **gate_path_argument, - **weighted_integrity_arguments, - **checkpoint_arguments, - sample_fraction=args.sample_fraction, - sample_seed=args.sample_seed, - release_candidate=args.release_candidate, - ledger_target_registry=( - None - if ledger_compilations is None - else { - period: compilation.registry - for period, compilation in ledger_compilations.items() - } - ), - # The frozen instrument supplies the parity trio's reference side; - # the candidate side comes from the staged frame and the solve. - parity_reference=( - load_efrs_parity_reference() if calibration_transform is not None else None - ), - ) - if calibration_transform is not None: - if calibration_transform.solve_result is None: - raise RuntimeError( - "national calibration diagnostics require the intact solve " - "result; checkpoint-resumed runs must rerun calibration before " - "writing calibration_diagnostics.json." - ) - assert ledger_artifact is not None - write_uk_calibration_diagnostics( - calibration_transform.solve_result, - args.national_calibration_diagnostics_json, - result.frame, - target_geography_levels=_uk_target_geography_levels( - calibration_transform.registry - ), - target_registry=calibration_transform.registry, - build={ - "build_id": state.build_id, - "ledger_facts": ledger_artifact.provenance(), - "code_pin": attempt_context["code_pin"], - "source_pins": source_pins, - "input_posture": _input_posture(candidate), - "score_vs_enhanced_frs": None, - }, - ) - append_phase(state, "build_completed") - _write_stage_reports( - evidence_path=evidence_path, - replay_path=replay_path, - candidate=candidate, - retained_leaves_transform=retained_leaves_transform, - hmrc_transform=hmrc_transform, - ) - append_phase(state, "stage_reports_written") - assert hmrc_transform.last_result is not None # guarded by report writer - hmrc_evidence = { - "passed": True, - "summary": _replay_summary(hmrc_transform.last_result), - } - artifact_paths = { - "input_h5": result.input_h5, - "staging_h5": result.staging_h5, - "terminal_gates": gate_output_path, - "hmrc_evidence": evidence_path, - "hmrc_replay": replay_path, - "spi_donor": args.spi_tab, - "hmrc_surface": args.hmrc_ods, - "frs_adult": retained_leaves_transform.adult_tab_path, - "frs_benefits": retained_leaves_transform.benefits_tab_path, - } - artifacts = {role: _artifact_info(path) for role, path in artifact_paths.items()} - build_record = _aggregate_build_record( - result=result, - artifacts=artifacts, - retained_evidence=retained_leaves_transform.last_result.evidence(), - family_evidence=hmrc_transform.last_result.evidence(), - seed=args.seed, - qrf_estimators=args.qrf_estimators, - sample_fraction=args.sample_fraction, - sample_seed=args.sample_seed, - degenerate_exclusions_override=args.degenerate_exclusions is not None, - ledger_artifact_provenance=( - None if ledger_artifact is None else ledger_artifact.provenance() - ), - input_posture=_input_posture(candidate), - ) - _write_json(build_record_path, build_record) - append_phase(state, "build_record_written") - payload = { - "schema_version": 5, - "build_kind": "uk_national_staging_dataset", - "sampling": { - "sample_fraction": float(args.sample_fraction), - "sample_seed": int(args.sample_seed), - "rung_token": UK_SAMPLE_RUNG_TOKENS[args.sample_fraction], - }, - "stages": list(result.stage_names), - "terminal_gates": dict(result.gate_report), - "input_coverage": { - "passed": result.input_coverage.passed, - "failures": list(result.input_coverage.failures), - "details": dict(result.input_coverage.details), - }, - "artifacts": { - **artifacts, - "build_record": _artifact_info(build_record_path), - }, - "hmrc_replay": hmrc_evidence, - } - state.gate_verdicts = _gate_verdicts_from_report( - dict(result.gate_report), - gate_output_path=gate_output_path, - ) - state.artifact_location = local_artifact_reference( - result.staging_h5, - repository_hint=_REPOSITORY, - ) - spool_path = _record_national_attempt( - state=state, - started_at=started_at, - started_ts=started_ts, - rung=rung, - seed=logbook_seed, - code_pin=str(attempt_context["code_pin"]), - disposition="iterating", - predecessor=attempt_context["predecessor"], - spool_dir=spool_dir, - ) - print(json.dumps(payload, indent=2, sort_keys=True)) - print(f"Wrote Logbook row: {spool_path}", file=sys.stderr) - return 0 - - -def _staging_run_config( - args: argparse.Namespace, - *, - candidate: object, - retained_leaves_transform: UKFRSHMRCRetainedLeavesStageTransform, - hmrc_transform: UKHMRCIncomeStageTransform, - source_pins: dict[str, dict[str, object]] | None = None, -) -> dict[str, object]: - """The content-addressed identity of a checkpointed staging run. - - Everything that determines the stage outputs is pinned by content, not - by path or stat: the verified certified-candidate digest, the raw-source - digests (read from the transforms' own resolved paths, so the pinned - files are exactly the files the stages consume), the seeds, the release - coordinates, and the builder code identity (packaged sources plus the - numeric-dependency versions). The stage runtime refuses to resume a - checkpoint directory under a different config, so a drifted input, - parameter, code change, or environment upgrade can never blend into an - old run's prefix. - """ - - from microcosm.build.code_identity import builder_code_identity - - pins = source_pins or _source_pins( - candidate=candidate, - retained_leaves_transform=retained_leaves_transform, - hmrc_transform=hmrc_transform, - cgt_ods_path=args.cgt_ods, - ) - - return { - "build_kind": "uk_national_staging_dataset", - "release_id": str(args.release_id), - "calibration_diagnostics_sha256": str(args.calibration_diagnostics_sha256), - "seed": int(args.seed), - "qrf_estimators": int(args.qrf_estimators), - "sampling": { - # Pinned as a string: run-config equality is exact over canonical - # JSON, and float normalization across serializers is exactly the - # ambiguity a run identity must not carry. Two rungs pointed at - # one checkpoint directory refuse instead of cross-resuming. - "sample_fraction": str(float(args.sample_fraction)), - "sample_seed": int(args.sample_seed), - "rung_token": UK_SAMPLE_RUNG_TOKENS[args.sample_fraction], - }, - "certified_candidate": dict(pins["certified_candidate"]), - "input_posture": _input_posture(candidate), - "sources": { - "adult_tab": dict(pins["adult_tab"]), - "benefits_tab": dict(pins["benefits_tab"]), - "spi_tab": dict(pins["spi_tab"]), - "hmrc_ods": dict(pins["hmrc_ods"]), - "cgt_ods": dict(pins["cgt_ods"]), - }, - "code_identity": builder_code_identity( - Path(__file__).resolve().parents[1], - tool_path=Path(__file__).resolve(), - distributions=( - "h5py", - "numpy", - "pandas", - "quantile-forest", - "scikit-learn", - "tables", - ), - ), - } - - -def _artifact_info(path: str | Path) -> dict[str, str | int]: - artifact = Path(path).resolve() - digest = hashlib.sha256() - with artifact.open("rb") as file: - for chunk in iter(lambda: file.read(1024 * 1024), b""): - digest.update(chunk) - return { - "path": str(artifact), - "sha256": digest.hexdigest(), - "size_bytes": artifact.stat().st_size, - } - - -def _aggregate_build_record( - *, - result: object, - artifacts: dict[str, dict[str, str | int]], - retained_evidence: dict[str, object], - family_evidence: dict[str, object], - seed: int, - qrf_estimators: int, - sample_fraction: float = 1.0, - sample_seed: int = UK_SAMPLE_SEED_DEFAULT, - degenerate_exclusions_override: bool = False, - ledger_artifact_provenance: dict[str, object] | None = None, - input_posture: dict[str, object] | None = None, -) -> dict[str, object]: - """Return commit-safe aggregate evidence for one successful staging build.""" - - details = dict(result.input_coverage.details) - safe_artifacts = { - role: { - "sha256": str(info["sha256"]), - "size_bytes": int(info["size_bytes"]), - } - for role, info in artifacts.items() - } - safe_artifacts["staging_h5"]["retention"] = "local_untracked" - if ledger_artifact_provenance is not None: - safe_artifacts["ledger_facts"] = ledger_artifact_provenance - retained_sources = dict(retained_evidence.get("sources", {})) - family_sources = dict(family_evidence.get("sources", {})) - mass_changes = [ - { - "entity": record.entity, - "old_total": float(record.old_total), - "new_total": float(record.new_total), - "declared_factor": ( - None - if record.declared_factor is None - else float(record.declared_factor) - ), - "reason": record.reason, - } - for record in result.frame.mass_log - ] - release_evidence = dict(result.gate_report["release_evidence"]) - source_vintages = dict(family_evidence.get("source_vintages", {})) - source_vintages["frs"] = load_uk_frs_release().vintage - if ledger_artifact_provenance is not None: - source_vintages["ledger_facts"] = ledger_artifact_provenance - return { - "schema_version": 3, - "build_kind": "uk_national_staging_dataset", - "status": "passed", - "calibration_diagnostics_sha256": release_evidence[ - "calibration_diagnostics_sha256" - ], - "stages": list(result.stage_names), - "parameters": { - "seed": int(seed), - "qrf_estimators": int(qrf_estimators), - "sample_fraction": float(sample_fraction), - "sample_seed": int(sample_seed), - "rung_token": UK_SAMPLE_RUNG_TOKENS[sample_fraction], - # Answers "did the operator invoke the override path" — the - # operator-action record, kept path-free by contract. The signed - # report's evidence answers the different question "which - # register content governed" (``exclusions_policy``); a review - # file byte-identical to the committed register makes the two - # honestly disagree, which is why they carry distinct names. - "degenerate_exclusions_override_supplied": (degenerate_exclusions_override), - }, - "input_posture": dict(input_posture or {}), - "sampling": ( - None if result.sampling_receipt is None else dict(result.sampling_receipt) - ), - "dataset": { - "time_period": uk_time_period(result.frame), - "entity_rows": { - "person": len(result.frame.table("person")), - "benunit": len(result.frame.table("benunit")), - "household": len(result.frame.table("household")), - }, - "household_weight_kind": uk_household_weight_kind(result.frame).value, - "household_weight_total": float( - result.frame.weights_for("household").total - ), - "mass_changes": mass_changes, - }, - "source_rows": { - "frs_adult": int(dict(retained_sources.get("adult", {})).get("rows", 0)), - "frs_benefits": int( - dict(retained_sources.get("benefits", {})).get("rows", 0) - ), - "spi_donor_used": int( - dict(family_sources.get("spi_donor", {})).get("rows_used", 0) - ), - }, - "source_vintages": source_vintages, - "terminal_gates": dict(result.gate_report), - "input_coverage": { - "passed": bool(result.input_coverage.passed), - "failures": list(result.input_coverage.failures), - "required_columns": int(details.get("required_columns", 0)), - "reviewed_exclusion_columns": len( - dict(details.get("reviewed_exclusions", {})) - ), - "missing": list(details.get("missing", ())), - "degenerate_required": list(details.get("degenerate_required", ())), - "insufficient_effective_mass": list( - details.get("insufficient_effective_mass", ()) - ), - "stale_exclusions": list(details.get("stale_exclusions", ())), - "effective_mass_policy": dict(details.get("effective_mass_policy", {})), - "family_effective_mass": dict(details.get("family_effective_mass", {})), - "family_build_state": dict(details.get("family_build_state", {})), - }, - "hmrc_replay": { - "summary": dict( - dict(family_evidence.get("targets", {})).get("classification", {}) - ), - "post_draw_identity": dict(family_evidence.get("post_draw_identity", {})), - }, - "artifacts": safe_artifacts, - } - - -def _write_json(path: str | Path, payload: dict[str, object]) -> Path: - output = Path(path).resolve() - output.parent.mkdir(parents=True, exist_ok=True) - temporary = output.with_name(f".{output.name}.{uuid.uuid4().hex}.tmp") - try: - temporary.write_text( - json.dumps(payload, indent=2, sort_keys=True) + "\n", - encoding="utf-8", - ) - temporary.replace(output) - finally: - temporary.unlink(missing_ok=True) - return output - - -def _write_stage_reports( - *, - evidence_path: Path, - replay_path: Path, - candidate: object, - retained_leaves_transform: UKFRSHMRCRetainedLeavesStageTransform, - hmrc_transform: UKHMRCIncomeStageTransform, -) -> None: - retained_result = retained_leaves_transform.last_result - hmrc_result = hmrc_transform.last_result - if retained_result is None or hmrc_result is None: - raise RuntimeError( - "UK national HMRC stages did not both complete; refusing to write " - "partial or stale aggregate evidence." - ) - payload = { - "schema_version": 2, - "base_candidate": { - "path": str(candidate.path), - "filename": candidate.filename, - "tier": candidate.tier, - "revision": candidate.revision, - "sha256": candidate.sha256, - "size_bytes": candidate.size_bytes, - }, - "retained_leaves": retained_result.evidence(), - "family": hmrc_result.evidence(), - } - _write_json(evidence_path, payload) - # A checkpoint-resumed SPI stage carries no report object, only the - # payload its real report produced at completion time; _write_json and - # write_hmrc_replay_report share the exact serialization (indent=2, - # sort_keys, trailing newline), so the resumed sidecar is byte-identical. - replay_report = getattr(hmrc_result, "replay_report", None) - if replay_report is not None: - write_hmrc_replay_report(replay_report, replay_path) - else: - _write_json(replay_path, dict(_resumed_replay_payload(hmrc_result))) - - -def _resumed_replay_payload(hmrc_result: object) -> dict[str, object]: - payload = getattr(hmrc_result, "replay_payload", None) - if not isinstance(payload, dict): - raise RuntimeError( - "resumed SPI restoration carries no replay payload; the " - "checkpoint record cannot feed the driver's stage reports." - ) - return payload - - -def _replay_summary(hmrc_result: object) -> dict[str, object]: - report = getattr(hmrc_result, "replay_report", None) - if report is not None: - return dict(report.summary) - summary = _resumed_replay_payload(hmrc_result).get("summary") - if not isinstance(summary, dict): - raise RuntimeError("resumed SPI replay payload carries no summary block.") - return dict(summary) - - -def _validate_distinct_paths( - *, - evidence_path: Path, - replay_path: Path, - terminal_gate_path: Path, - input_h5: Path, - staging_h5: Path, - spi_tab: Path, - hmrc_ods: Path, - cgt_ods: Path, - adult_tab: Path, - benefits_tab: Path, - build_record_path: Path, - input_mass_reference_path: Path | None, - input_mass_exclusions_path: Path | None, - qrf_tail_exclusions_path: Path | None, - degenerate_exclusions_path: Path | None, - rung_abort_path: Path, -) -> None: - paths = { - "--input-h5": input_h5.resolve(), - "--staging-h5": staging_h5.resolve(), - "--spi-tab": spi_tab.resolve(), - "--hmrc-ods": hmrc_ods.resolve(), - "--cgt-ods": cgt_ods.resolve(), - "--frs-raw-dir/adult.tab": adult_tab.resolve(), - "--frs-raw-dir/benefits.tab": benefits_tab.resolve(), - "--build-record-json": build_record_path.resolve(), - "--terminal-gates-json/--input-coverage-json": terminal_gate_path.resolve(), - "--hmrc-evidence-json": evidence_path.resolve(), - "--hmrc-replay-json": replay_path.resolve(), - "rung-abort receipt (derived from --staging-h5)": rung_abort_path.resolve(), - } - paths.update( - (label, path.resolve()) - for label, path in { - "--input-mass-reference-json": input_mass_reference_path, - "--input-mass-exclusions": input_mass_exclusions_path, - "--qrf-tail-exclusions": qrf_tail_exclusions_path, - "--degenerate-exclusions": degenerate_exclusions_path, - }.items() - if path is not None - ) - collisions = [ - (left_label, right_label, left_path, right_path) - for (left_label, left_path), (right_label, right_path) in combinations( - paths.items(), - 2, - ) - if _paths_alias(left_path, right_path) - ] - if collisions: - details = "; ".join( - f"{left_label}, {right_label} -> {left_path} == {right_path}" - for left_label, right_label, left_path, right_path in collisions - ) - raise ValueError( - "UK national build input, staging, and sidecar paths must be " - f"pairwise distinct: {details}." - ) - - -def _uk_target_geography_levels(registry) -> dict[str, str]: - payload = json.loads( - importlib_resources.files("microcosm.build.uk") - .joinpath("uk_national_targets.json") - .read_text(encoding="utf-8") - ) - targets = {row["target_id"]: row for row in payload["targets"]} - levels: dict[str, str] = {} - for spec in registry.specs: - target_id = spec.metadata.get("contract_target_id") - target = targets.get(str(target_id)) - if target is None: - raise ValueError( - f"UK calibration target {spec.name!r} references unknown " - f"contract target {target_id!r}." - ) - geography_levels = tuple(target.get("geography_levels") or ()) - if not geography_levels: - raise ValueError( - f"UK calibration target {spec.name!r} has no geography level." - ) - levels[spec.to_target().row_name] = str(geography_levels[0]) - return levels - - -def _paths_alias(left: Path, right: Path) -> bool: - """Conservatively identify aliases before any build path is unlinked.""" - - if left == right: - return True - try: - if left.exists() and right.exists() and left.samefile(right): - return True - except OSError: - # The case-folded resolved identity below remains a safe fallback for - # a path that changes between the existence and samefile checks. - pass - # macOS volumes are commonly case-insensitive even though Path.resolve() - # preserves caller casing. Reject case-only distinctions everywhere: a - # destructive build tool has no legitimate need for them, and doing so - # also protects outputs that do not exist yet. - return str(left).casefold() == str(right).casefold() - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/tools/build_uk_rowwise_dataset.py b/tools/build_uk_rowwise_dataset.py index c62d7b65a..0b6fb7541 100644 --- a/tools/build_uk_rowwise_dataset.py +++ b/tools/build_uk_rowwise_dataset.py @@ -66,7 +66,7 @@ from microcosm.frame import engine_tables CROSSWALK_FILENAME = "uk_official_geography_crosswalk.csv.gz" -DATASET_FILENAME_TEMPLATE = "populace_uk_{source_year}_rowwise.h5" +DATASET_FILENAME_TEMPLATE = "{input_stem}_rowwise.h5" MANIFEST_FILENAME = "rowwise_build_manifest.json" COVERAGE_FILENAME = "geography_coverage_summary.csv" DRY_RUN_PLAN_FILENAME = "rowwise_dry_run_plan.json" @@ -135,7 +135,8 @@ def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: parser.add_argument( "--dataset-filename", help=( - f"Output H5 filename within --out. Defaults to {DATASET_FILENAME_TEMPLATE}." + "Output H5 filename within --out. Defaults to the input H5 stem " + "plus '_rowwise.h5'." ), ) parser.add_argument( @@ -356,6 +357,7 @@ def _main_impl( output_h5 = _dataset_output_path( args.out, dataset_filename=args.dataset_filename, + input_stem=input_h5.stem, source_year=source_year, ) _validate_output_paths(input_h5=input_h5, output_h5=output_h5, args=args) @@ -544,10 +546,12 @@ def _dataset_output_path( out_dir: Path, *, dataset_filename: str | None, + input_stem: str, source_year: int, ) -> Path: filename = dataset_filename or DATASET_FILENAME_TEMPLATE.format( - source_year=source_year + input_stem=input_stem, + source_year=source_year, ) path = Path(filename) if path.is_absolute() or path.name != filename or path.name in {"", ".", ".."}: diff --git a/tools/ci_test_groups.py b/tools/ci_test_groups.py index 34fb2810d..6c2d71280 100644 --- a/tools/ci_test_groups.py +++ b/tools/ci_test_groups.py @@ -62,6 +62,7 @@ def stray_nested_test_files(flat: tuple[str, ...]) -> tuple[str, ...]: line for line in result.stdout.splitlines() if "/tests/" in line + and (ROOT / line).is_file() and basename(line).startswith("test_") and line.endswith(".py") and line not in known @@ -77,7 +78,13 @@ def tracked_test_files() -> tuple[str, ...]: text=True, stdout=subprocess.PIPE, ) - return tuple(sorted(line for line in result.stdout.splitlines() if line)) + return tuple( + sorted( + line + for line in result.stdout.splitlines() + if line and (ROOT / line).is_file() + ) + ) def package(path: str) -> str: diff --git a/tools/emit_uk_brma_distribution.py b/tools/emit_uk_brma_distribution.py index 735f1e01f..0f8e9c89f 100644 --- a/tools/emit_uk_brma_distribution.py +++ b/tools/emit_uk_brma_distribution.py @@ -25,8 +25,10 @@ assign_brma_by_cell, load_brma_count_resource, ) -from microcosm.build.uk_runtime.national_build import load_uk_national_frame -from microcosm.build.uk_runtime.national_frame import uk_time_period +from microcosm.build.uk_runtime.national_frame import ( + load_uk_national_frame, + uk_time_period, +) def brma_cell_distribution( diff --git a/tools/measure_uk_weighted_integrity_baselines.py b/tools/measure_uk_weighted_integrity_baselines.py index b147f9239..f07ac4a0f 100644 --- a/tools/measure_uk_weighted_integrity_baselines.py +++ b/tools/measure_uk_weighted_integrity_baselines.py @@ -47,7 +47,7 @@ from microcosm.build.uk_runtime.hmrc_source_contract import ( uk_hmrc_weighted_qrf_output_columns, ) -from microcosm.build.uk_runtime.national_build import load_uk_national_frame +from microcosm.build.uk_runtime.national_frame import load_uk_national_frame from microcosm.build.uk_runtime.was_wealth import UK_WAS_WEALTH_HOUSEHOLD_OUTPUT_COLUMNS from microcosm.build.uk_runtime.weighted_integrity import ( uk_input_mass_totals, diff --git a/tools/score_uk_national_candidate.py b/tools/score_uk_national_candidate.py index 451b4af23..df75a54c6 100644 --- a/tools/score_uk_national_candidate.py +++ b/tools/score_uk_national_candidate.py @@ -45,7 +45,7 @@ def score_uk_national_candidate( target_registry: TargetRegistry, calibration_year: int, measure_resolver_factory: Callable[[Path, Any], Any] | None = None, - candidate_label: str = "populace_uk_2023", + candidate_label: str | None = None, incumbent_label: str = "enhanced_frs_2024_25", ) -> dict[str, Any]: """Return the #578 rule-1 score block on a shared target registry. @@ -58,6 +58,9 @@ def score_uk_national_candidate( if target_registry.country != "uk" or not target_registry.specs: raise ValueError("UK candidate scoring requires a non-empty UK registry.") + candidate_path = Path(candidate_h5) + if candidate_label is None: + candidate_label = candidate_path.stem candidate_pin = _verify_artifact("candidate", candidate_h5, candidate_sha256) incumbent_pin = _verify_artifact("incumbent", incumbent_h5, incumbent_sha256) candidate_frame, candidate_resolution = _scored_frame( @@ -293,6 +296,15 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument("--registry-json", required=True, type=Path) parser.add_argument("--output-json", required=True, type=Path) parser.add_argument("--calibration-year", type=int) + parser.add_argument( + "--candidate-label", + help="Override the candidate label; by default it is the candidate H5 stem.", + ) + parser.add_argument( + "--incumbent-label", + default="enhanced_frs_2024_25", + help="Override the incumbent/reference label.", + ) parser.add_argument( "--no-measure-resolution", action="store_true", @@ -324,6 +336,8 @@ def main(argv: list[str] | None = None) -> int: target_registry=registry, calibration_year=int(calibration_year), measure_resolver_factory=factory, + candidate_label=args.candidate_label, + incumbent_label=args.incumbent_label, ) _write_json(args.output_json, {"score_vs_enhanced_frs": score}) return 0 diff --git a/tools/verify_uk_identity_stability.py b/tools/verify_uk_identity_stability.py index e92ba9e8f..00925635c 100644 --- a/tools/verify_uk_identity_stability.py +++ b/tools/verify_uk_identity_stability.py @@ -31,8 +31,10 @@ aggregate_person_reported_to_benunit, derive_frs_take_up, ) -from microcosm.build.uk_runtime.national_build import load_uk_national_frame -from microcosm.build.uk_runtime.national_frame import uk_time_period +from microcosm.build.uk_runtime.national_frame import ( + load_uk_national_frame, + uk_time_period, +) from microcosm.build.uk_runtime.regional_uprating import ( load_regional_land_values_resource, uprate_household_property_by_region, From 45865198e0d9d3365aff1f4724d16cf6b723c675 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Wed, 26 Aug 2026 15:14:28 +0200 Subject: [PATCH 05/14] Refuse the manufactured parity verdicts, and test the E6 contract that exists MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The post-merge audit of the swap instruments (#770-#775) demonstrated four ways to manufacture an acceptance verdict: quote a register entry against a stale incumbent value, match a signed entry with the wrong direction or an absent count, widen the share band under strict, and compare weighted totals on the intersection with anonymous artifacts. No gate gets signed against instruments with known bugs, so all four doors close. The register now carries structured quantitative expectations beside its prose: per-column incumbent share, direction, and magnitude bound, and exact entity count deltas. The verifier enforces them — the audit's own reproductions now land as defect: counts omitted or set to one stop matching the count entry, and water moving down by 0.78 stops matching a candidate_above entry bounded at 0.10. A hermetic check pins every quoted incumbent share to the packaged parity reference, which is what catches the stale 1.56.14 main-residence quote the audit found. Strict acceptance is pinned to the contract share band; any other band refuses outright, and a widened-band run can only ever say diagnostic, with defect still winning over diagnostic so a wide band cannot launder one. Weighted totals are closed-world under strict — every one-sided key is an unsigned difference — and both sidecars must carry content identities that match the artifacts they describe. The E6 identity check recomputes NHS at the age surface the stage actually saw, min(age, UK_AGE_TOP_CODE) — an exact inversion, since age_tail refuses inputs above the top code and rewrites only persons at exactly it, upward. The receipt declares the basis, and an integration regression through the real etb_services -> age_tail ordering fails if anyone reverts the instrument to final-age semantics. The production stages are untouched: stored NHS values are stage-time-derived by design, final age is calibration support, and the better-modeling alternative is filed as #785 to land after the swap certifies. Closes the code half of #770, #772, #773, #774, #775; the licensed receipts (#770 acceptance, #771 regeneration) follow on this branch. Co-Authored-By: Claude Fable 5 --- .../757-parity-instrument-hardening.fixed.md | 1 + changelog.d/770-uk-e6-stage-time-age.fixed.md | 1 + .../uk/spine_swap_signed_differences.json | 222 +++++++++- .../build/uk_runtime/signed_differences.py | 143 +++++- .../test_uk_identity_stability_receipts.py | 163 +++++++ .../tests/test_uk_signed_differences.py | 86 ++++ .../tests/test_uk_spine_parity_instrument.py | 414 +++++++++++++++++- tools/verify_uk_identity_stability.py | 16 +- tools/verify_uk_spine_parity.py | 212 ++++++++- 9 files changed, 1219 insertions(+), 39 deletions(-) create mode 100644 changelog.d/757-parity-instrument-hardening.fixed.md create mode 100644 changelog.d/770-uk-e6-stage-time-age.fixed.md diff --git a/changelog.d/757-parity-instrument-hardening.fixed.md b/changelog.d/757-parity-instrument-hardening.fixed.md new file mode 100644 index 000000000..2c016d26e --- /dev/null +++ b/changelog.d/757-parity-instrument-hardening.fixed.md @@ -0,0 +1 @@ +The UK spine parity instruments now refuse the manufactured verdicts the #747 post-merge audit demonstrated. The signed-differences register carries structured quantitative expectations — per-column incumbent share, direction, and magnitude bound, and exact entity-count deltas — that the verifier enforces, so a share moving the wrong way, an out-of-bound magnitude, an absent count, or an implausible one stops matching its signed entry and lands as a defect; a hermetic check pins every quoted incumbent share to the packaged parity reference, which also corrects the stale 1.56.14 main-residence quote. Strict acceptance is pinned to the contract share band: any other band refuses outright, and a widened-band run can only ever emit a diagnostic verdict, never an acceptance one, with the effective and contract bands bound into the receipt. Weighted-total comparison is closed-world — every one-sided key is an unsigned difference under strict — and both sidecars must carry content identities that match the artifacts they describe. diff --git a/changelog.d/770-uk-e6-stage-time-age.fixed.md b/changelog.d/770-uk-e6-stage-time-age.fixed.md new file mode 100644 index 000000000..6ad4e92be --- /dev/null +++ b/changelog.d/770-uk-e6-stage-time-age.fixed.md @@ -0,0 +1 @@ +Repair the UK E6 identity verifier so NHS allocations are checked against the stage-time top-coded age surface used by `etb_services`, not the later age-tail-disaggregated final age. diff --git a/packages/microcosm-build/src/microcosm/build/uk/spine_swap_signed_differences.json b/packages/microcosm-build/src/microcosm/build/uk/spine_swap_signed_differences.json index 8e05de966..8b9fc9610 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/spine_swap_signed_differences.json +++ b/packages/microcosm-build/src/microcosm/build/uk/spine_swap_signed_differences.json @@ -18,7 +18,16 @@ "magnitude_evidence": "The spine's unweighted nonzero share exceeds the incumbent's by about +0.10 on the whole file. FRS 2024-25 retired CWATAMT/CSEWAMT: the headers survive but carry no data in any of the 16,288 households. The incumbent adds the retired CSEWAMT before filling, so NaN propagates and the charge is zeroed for every Scottish household that has one; the spine fills per column and they stand. Reproduced on the raw tab as 12,644 nonzero households for the incumbent formula against 14,307 for ours, a +0.1021 share gap whose 1,663 differing households are all Scottish, with England, Wales and Northern Ireland identical under both. Latent since at least 2023-24, where 378 Scottish households were already affected. The defect is on the incumbent side and survives at 1.56.16.", "evidence": "experiments/686-uk-spine-swap-receipts.md#r1--scottish-water-and-sewerage-charges-736-item-13", "adjudicator": "juaristi22", - "adjudicated_on": "2026-08-22" + "adjudicated_on": "2026-08-22", + "quantitative": { + "shares": { + "water_and_sewerage_charges": { + "incumbent_share": 0.776937, + "direction": "candidate_above", + "max_abs_delta": 0.100897 + } + } + } }, { "id": "scottish-water-sewerage-successor-level", @@ -37,7 +46,15 @@ "magnitude_evidence": "The Scottish charge is assembled from the successors FRS 2024-25 published for the cells it retired, so its level rises against both the incumbent and our own earlier build. CWATAMTD is the water charge alone; CSEWAMT1 supplies the sewerage side and is discounted at the household's own observed CWATAMTD/CWATAMT1 factor, which keeps the retired cells' after-discount meaning rather than switching to a gross basis. Weighted annual per Scottish household moves from about GBP 185 on water alone to about GBP 395, against roughly GBP 490 for England and Wales on WATSEWRT; the incumbent sits at zero because of the separate NaN defect. The same amount is netted from council_tax, so that column moves by the same construction. The nonzero share is unaffected, so this entry deliberately does not sign the share surface.", "evidence": "experiments/686-uk-spine-swap-receipts.md#r1--scottish-water-and-sewerage-charges-736-item-13", "adjudicator": "juaristi22", - "adjudicated_on": "2026-08-22" + "adjudicated_on": "2026-08-22", + "quantitative": { + "weighted_totals": { + "expected_columns": [ + "water_and_sewerage_charges", + "council_tax" + ] + } + } }, { "id": "lcfs-consumption-regime-gated-incidence", @@ -63,7 +80,56 @@ "magnitude_evidence": "The spine draws LCFS consumption through a regime-gated QRF that carries the donor's zero mass as a modelled incidence, where the incumbent's plain QRF regresses a zero-inflated target toward its conditional mean. On these nine columns the spine's unweighted share is closer to the LCFS 2023-24 survey-weighted donor share than the incumbent's is, measured on the stage's own cleaned donor frame of 4,202 households against the pinned 1.56.16 artifact (sha256 e433e532): education_consumption donor 0.0476 against incumbent 0.1256 and ours 0.0170; restaurants_and_hotels donor 0.7651 against 0.6324 and 0.7903; miscellaneous donor 0.9728 against 0.9003 and 0.9918; domestic_energy donor 0.9835 against 0.9549 and 0.9953, with its electricity and gas components moving the same way. Population mean per household is closer for the spine on eight of the nine, the exception being communication_consumption at 1.20x the donor against the incumbent's 1.15x; the incumbent runs 1.7x to 4.8x the donor mean on the rest. Every cell is above the disclosure floor, the thinnest being 193 donor carriers on education_consumption.", "evidence": "experiments/686-uk-spine-comparison-ledger.md#e6--consumption--signed", "adjudicator": "juaristi22", - "adjudicated_on": "2026-08-24" + "adjudicated_on": "2026-08-24", + "quantitative": { + "shares": { + "communication_consumption": { + "incumbent_share": 0.79743, + "direction": "candidate_above", + "max_abs_delta": 0.08396999999999999 + }, + "domestic_energy_consumption": { + "incumbent_share": 0.954869, + "direction": "candidate_above", + "max_abs_delta": 0.040430999999999995 + }, + "education_consumption": { + "incumbent_share": 0.125591, + "direction": "candidate_below", + "max_abs_delta": 0.10859100000000001 + }, + "electricity_consumption": { + "incumbent_share": 0.86209, + "direction": "candidate_above", + "max_abs_delta": 0.10231000000000001 + }, + "gas_consumption": { + "incumbent_share": 0.953241, + "direction": "candidate_above", + "max_abs_delta": 0.04195899999999997 + }, + "health_consumption": { + "incumbent_share": 0.512849, + "direction": "candidate_above", + "max_abs_delta": 0.02575099999999997 + }, + "household_furnishings_consumption": { + "incumbent_share": 0.813553, + "direction": "candidate_above", + "max_abs_delta": 0.108047 + }, + "miscellaneous_consumption": { + "incumbent_share": 0.900333, + "direction": "candidate_above", + "max_abs_delta": 0.09146699999999996 + }, + "restaurants_and_hotels_consumption": { + "incumbent_share": 0.632441, + "direction": "candidate_above", + "max_abs_delta": 0.15785899999999997 + } + } + } }, { "id": "lcfs-fuel-consumption-incidence-gate", @@ -82,7 +148,21 @@ "magnitude_evidence": "These two are signed with the evidence pointing the other way on incidence, and that is the point of scoping them apart from the rest of the LCFS class. The spine gates fuel spending on a has_fuel draw, so it places incidence on fewer households than either the donor or the incumbent: petrol donor 0.3911 against incumbent 0.4446 and ours 0.3002, diesel donor 0.2040 against 0.1910 and 0.1580. The incumbent is closer on both shares. On level the ordering reverses decisively - population mean per household is 0.85x the donor for petrol and 0.81x for diesel against the incumbent's 2.05x and 2.31x - so the gate is under-placing incidence while the incumbent is over-stating amounts by roughly a factor of two. Signed as the accepted cost of the fuel gate, not as a claim that the spine is closer here; the incidence rate of the gate is the pre-registered lever if this surface needs to move.", "evidence": "experiments/686-uk-spine-comparison-ledger.md#e6--consumption--signed", "adjudicator": "juaristi22", - "adjudicated_on": "2026-08-24" + "adjudicated_on": "2026-08-24", + "quantitative": { + "shares": { + "diesel_spending": { + "incumbent_share": 0.191027, + "direction": "candidate_below", + "max_abs_delta": 0.033027 + }, + "petrol_spending": { + "incumbent_share": 0.444632, + "direction": "candidate_below", + "max_abs_delta": 0.144432 + } + } + } }, { "id": "lcfs-aggregate-incidence-incumbent-closer", @@ -101,7 +181,21 @@ "magnitude_evidence": "Scoped apart from the rest of the LCFS class because on these two the incumbent is closer on the share and the spine is closer on the level, so a class verdict would misstate both. transport_consumption: donor 0.8702 against incumbent 0.8668 and ours 0.8934, so the spine overshoots by 0.0232 where the incumbent undershoots by 0.0034; on level the spine is at 1.37x the donor population mean per household against the incumbent's 1.62x. alcohol_and_tobacco_consumption: donor 0.5383 against incumbent 0.5603 and ours 0.5150, so the incumbent is off by 0.0220 and the spine by 0.0233 - close enough that it read as a tie against the previous 1.56.14 pin and resolves to the incumbent against the pinned 1.56.16 artifact; on level the two are within a point of each other, 1.24x for the spine against 1.26x. Signed as accepted incidence costs of the regime-gated draw, with the direction recorded so each can be re-examined on its own rather than under a class verdict it does not share.", "evidence": "experiments/686-uk-spine-comparison-ledger.md#e6--consumption--signed", "adjudicator": "juaristi22", - "adjudicated_on": "2026-08-24" + "adjudicated_on": "2026-08-24", + "quantitative": { + "shares": { + "alcohol_and_tobacco_consumption": { + "incumbent_share": 0.560288, + "direction": "candidate_below", + "max_abs_delta": 0.045287999999999995 + }, + "transport_consumption": { + "incumbent_share": 0.866802, + "direction": "candidate_above", + "max_abs_delta": 0.02659800000000001 + } + } + } }, { "id": "etb-services-regime-gated-incidence", @@ -120,7 +214,21 @@ "magnitude_evidence": "The incumbent's state-education column is degenerate and the spine's is not, which makes this a defect fix on the incumbent side rather than a method preference. Measured on the ETB services stage's own cleaned donor frame \u2014 SN 8856, year 2023, complete cases on the thirteen-column services subset, 4,199 rows, weighted by hhold_adj_weight \u2014 dfe_education_spending has a donor share of 0.2794 weighted (0.2546 unweighted, which reproduces the E6 acceptance receipt's figure exactly) and a donor population mean of GBP 3,461 per household. The incumbent carries 14 nonzero households out of 52,846, a share of 0.000265 and a population mean of GBP 2 per household; the spine carries 11,934, a share of 0.2258 and GBP 3,111, or 0.90x the donor. bus_subsidy_spending moves the same way: donor share 0.5255 weighted and GBP 87 per household, against incumbent 0.3167 and GBP 113 (1.30x) and ours 0.5554 and GBP 89 (1.02x). The spine is closer on both the share and the level of both columns. This resolves the ETB weight-basis question that was previously recorded as blocking these rows: the stage's convention is the household grossing weight, and the verdict holds on either basis.", "evidence": "experiments/686-uk-spine-comparison-ledger.md#etb--the-weight-basis-question-is-closed", "adjudicator": "juaristi22", - "adjudicated_on": "2026-08-24" + "adjudicated_on": "2026-08-24", + "quantitative": { + "shares": { + "bus_subsidy_spending": { + "incumbent_share": 0.316675, + "direction": "candidate_above", + "max_abs_delta": 0.23872500000000002 + }, + "dfe_education_spending": { + "incumbent_share": 0.000265, + "direction": "candidate_above", + "max_abs_delta": 0.225535 + } + } + } }, { "id": "was-wealth-qrf-incidence", @@ -139,10 +247,39 @@ ] }, "expectation": "column_differs", - "magnitude_evidence": "Carries forward the E5 adjudication of 2026-08-19 \u2014 that the wealth stage is not required to reproduce the incumbent's inflated totals \u2014 now scoped to the five household columns that actually diverge beyond the band, and re-measured against WAS Round 8 on the stage's own cleaning of the pinned tab (15,128 rows, weighted by R8xshhwgt). The spine is closer than the incumbent on all five survey-weighted donor shares: savings donor 0.6072 against incumbent 0.6620 and ours 0.6107; other_residential_property_value donor 0.0363 against 0.0763 and 0.0367; property_wealth donor 0.6433 against 0.7081 and 0.6607; main_residence_value donor 0.6236 against 0.6761 and 0.6356; corporate_wealth donor 0.7629 against 0.8222 and 0.7792. The level evidence behind the original adjudication reproduces and is the more dramatic surface: population mean per household runs 5.66x the donor for the incumbent's savings against 1.97x for ours, and 6.12x against 1.26x for other residential property. main_residence_value is the one column where the incumbent's level is closer, at 1.02x against our 0.88x. Note that the unweighted donor shares tell the opposite story on incidence, because WAS oversamples wealth-holders by design; the weighted basis is the population one and is the basis quoted here throughout.", + "magnitude_evidence": "Carries forward the E5 adjudication of 2026-08-19 \u2014 that the wealth stage is not required to reproduce the incumbent's inflated totals \u2014 now scoped to the five household columns that actually diverge beyond the band, and re-measured against WAS Round 8 on the stage's own cleaning of the pinned tab (15,128 rows, weighted by R8xshhwgt). The spine is closer than the incumbent on all five survey-weighted donor shares: savings donor 0.6072 against incumbent 0.6620 and ours 0.6107; other_residential_property_value donor 0.0363 against 0.0763 and 0.0367; property_wealth donor 0.6433 against 0.7081 and 0.6607; main_residence_value donor 0.6236 against 0.6747 and 0.6356; corporate_wealth donor 0.7629 against 0.8222 and 0.7792. The level evidence behind the original adjudication reproduces and is the more dramatic surface: population mean per household runs 5.66x the donor for the incumbent's savings against 1.97x for ours, and 6.12x against 1.26x for other residential property. main_residence_value is the one column where the incumbent's level is closer, at 1.02x against our 0.88x. Note that the unweighted donor shares tell the opposite story on incidence, because WAS oversamples wealth-holders by design; the weighted basis is the population one and is the basis quoted here throughout.", "evidence": "experiments/686-uk-spine-comparison-ledger.md#e5--wealth--signed-carried-forward", "adjudicator": "juaristi22", - "adjudicated_on": "2026-08-24" + "adjudicated_on": "2026-08-24", + "quantitative": { + "shares": { + "corporate_wealth": { + "incumbent_share": 0.822181, + "direction": "candidate_below", + "max_abs_delta": 0.04298100000000005 + }, + "main_residence_value": { + "incumbent_share": 0.674658, + "direction": "candidate_below", + "max_abs_delta": 0.039057999999999926 + }, + "other_residential_property_value": { + "incumbent_share": 0.076316, + "direction": "candidate_below", + "max_abs_delta": 0.03961599999999999 + }, + "property_wealth": { + "incumbent_share": 0.708057, + "direction": "candidate_below", + "max_abs_delta": 0.04735700000000009 + }, + "savings": { + "incumbent_share": 0.661999, + "direction": "candidate_below", + "max_abs_delta": 0.051298999999999984 + } + } + } }, { "id": "was-student-loan-balance-fold", @@ -160,7 +297,16 @@ "magnitude_evidence": "Scoped apart from the benchmarked wealth columns because it has no donor benchmark to quote: the column is a fold of two WAS aggregates (total loans less total loans excluding Student Loans Company debt) and is a person-entity column where the wealth columns beside it are household-entity, so no like-for-like donor share exists on the stage's cleaned frame. The observed divergence is +0.0296 on the unweighted share, incumbent 0.0197 against ours 0.0493 \u2014 the spine places student debt on about two and a half times as many carriers. Signed under the standing E5 adjudication as part of the same correlated-rank draw, with the absence of a benchmark stated rather than papered over; if the wealth stage is revisited, this is the column whose direction is unevidenced.", "evidence": "experiments/686-uk-spine-comparison-ledger.md#e5--wealth--signed-carried-forward", "adjudicator": "juaristi22", - "adjudicated_on": "2026-08-24" + "adjudicated_on": "2026-08-24", + "quantitative": { + "shares": { + "student_loan_balance": { + "incumbent_share": 0.019707, + "direction": "candidate_above", + "max_abs_delta": 0.029592999999999998 + } + } + } }, { "id": "spi-channel-qrf-incidence", @@ -180,7 +326,26 @@ "magnitude_evidence": "The three columns rewritten on the SPI channel, each measured against its own source of truth rather than against one another, which is what closes the open item #717 left. savings_interest_income against the SPI donor INCBBS, FACT-weighted: truth 0.3960, incumbent 0.4250, ours 0.3946. tax_free_savings_income against the raw FRS at the frs_spine stage: truth 0.1540, incumbent 0.1897, ours 0.1352. employer_pension_contributions against the 3x derive at frs_hmrc_spine_leaves: truth 0.2587, incumbent 0.3149, ours 0.2682. The spine is closer on all three, so the divergence #717 recorded as uniformly one-way and unexplained is uniformly toward the source. Attribution is to the last stage that rewrites, not the stage that produces: the first two originate in frs_spine and are rewritten by hmrc_spi_income_spine, and attributing them to their producer would report them as raw-mapping defects, which is the one signature that indicates a genuine port defect.", "evidence": "experiments/686-uk-spine-comparison-ledger.md#e7--spi-channel--evidence-gap-closed-favours-the-spine", "adjudicator": "juaristi22", - "adjudicated_on": "2026-08-24" + "adjudicated_on": "2026-08-24", + "quantitative": { + "shares": { + "employer_pension_contributions": { + "incumbent_share": 0.314865, + "direction": "candidate_below", + "max_abs_delta": 0.04666500000000001 + }, + "savings_interest_income": { + "incumbent_share": 0.424963, + "direction": "candidate_below", + "max_abs_delta": 0.030362999999999973 + }, + "tax_free_savings_income": { + "incumbent_share": 0.189664, + "direction": "candidate_below", + "max_abs_delta": 0.05446400000000001 + } + } + } }, { "id": "salary-sacrifice-conversion-depth", @@ -198,7 +363,16 @@ "magnitude_evidence": "Signed at #684 and transcribed here against the re-pinned reference. The spine converts salary-sacrificed pension contributions at the depth the mechanism specifies, where the incumbent's conversion step was inert, so contributions that should have moved out of the employee column stayed in it. Unweighted share 0.2735 for the incumbent against 0.2246 for the spine, a difference of -0.0488 on the person entity. The counterpart column pension_contributions_via_salary_sacrifice sits inside the acceptance band at -0.0035 and is therefore not signed.", "evidence": "experiments/686-uk-spine-comparison-ledger.md#e8-and-entity-counts--signed-at-684", "adjudicator": "juaristi22", - "adjudicated_on": "2026-08-24" + "adjudicated_on": "2026-08-24", + "quantitative": { + "shares": { + "employee_pension_contributions": { + "incumbent_share": 0.273489, + "direction": "candidate_below", + "max_abs_delta": 0.04888899999999999 + } + } + } }, { "id": "donor-selection-rng-entity-counts", @@ -218,7 +392,13 @@ "magnitude_evidence": "The CGT band-donor selection draws over id-sorted candidate households, so the 270 donors it picks are not the 270 the incumbent picked, and the two sets carry different numbers of people and benefit units. Persons 113,617 in the reference against 113,649 in the spine, a difference of +32; benefit units 61,223 against 61,211, a difference of -12. Households are 52,846 on both sides and match exactly, which is what proves this is a selection difference rather than a miscount: the record-count identity (16,288 raw FRS plus 10,000 SPI) times two for the capital-gains clone, plus 270 band donors, closes on the nose. This entry is deliberately scoped to the two entities that differ rather than written surface-wide, so that any future divergence in the household count is still a defect.", "evidence": "experiments/686-uk-spine-swap-receipts.md#r3--the-spine-rebuilt-and-the-l2-adjudication-queue", "adjudicator": "juaristi22", - "adjudicated_on": "2026-08-24" + "adjudicated_on": "2026-08-24", + "quantitative": { + "expected_deltas": { + "benunit": -12, + "person": 32 + } + } }, { "id": "num-bedrooms-net-new-column", @@ -236,7 +416,14 @@ "magnitude_evidence": "The spine populates num_bedrooms at the frs_spine stage from the raw household tape; the pinned incumbent does not populate it at all, so the column is present in the candidate and absent from the reference. This is coverage the spine adds rather than a divergence in a shared column, and it cannot be measured as a share difference. Its predictor quality is a separate question tracked on #145, not a parity matter.", "evidence": "experiments/686-uk-spine-comparison-ledger.md#column-coverage", "adjudicator": "juaristi22", - "adjudicated_on": "2026-08-24" + "adjudicated_on": "2026-08-24", + "quantitative": { + "structural": { + "expected_columns": [ + "num_bedrooms" + ] + } + } }, { "id": "other-investment-income-net-new-column", @@ -254,7 +441,14 @@ "magnitude_evidence": "The spine populates other_investment_income at the hmrc_spi_income_spine stage. The column is declared by the incumbent's own national restoration but is not populated in the pinned artifact, so the spine is ahead of the reference here rather than diverging from it. Present in the candidate, absent from the reference, and not measurable as a share difference.", "evidence": "experiments/686-uk-spine-comparison-ledger.md#column-coverage", "adjudicator": "juaristi22", - "adjudicated_on": "2026-08-24" + "adjudicated_on": "2026-08-24", + "quantitative": { + "structural": { + "expected_columns": [ + "other_investment_income" + ] + } + } } ] } diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/signed_differences.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/signed_differences.py index fc0fd654b..bfd7b756d 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/signed_differences.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/signed_differences.py @@ -29,6 +29,7 @@ from dataclasses import dataclass from importlib.resources import files from pathlib import Path +from typing import Any __all__ = [ "UK_SPINE_SWAP_SIGNED_DIFFERENCES_RESOURCE", @@ -83,6 +84,10 @@ } ) +SIGNED_DIFFERENCE_SHARE_DIRECTIONS = frozenset( + {"candidate_above", "candidate_below"} +) + _ID = re.compile(r"^[a-z0-9]+(?:-[a-z0-9]+)*$") _ISO_DATE = re.compile(r"^\d{4}-\d{2}-\d{2}$") @@ -120,6 +125,7 @@ class UKSignedDifference: evidence: str adjudicator: str adjudicated_on: str + quantitative: Mapping[str, Any] | None = None def covers( self, @@ -274,6 +280,122 @@ def _require_str_tuple( ) +def _require_number( + value: object, + *, + field_name: str, + resource: str, + minimum: float | None = None, + maximum: float | None = None, +) -> float: + if isinstance(value, bool) or not isinstance(value, int | float): + raise ValueError(f"{resource}: field {field_name!r} must be a number.") + number = float(value) + if not (float("-inf") < number < float("inf")): + raise ValueError(f"{resource}: field {field_name!r} must be finite.") + if minimum is not None and number < minimum: + raise ValueError( + f"{resource}: field {field_name!r} must be >= {minimum}, got {number}." + ) + if maximum is not None and number > maximum: + raise ValueError( + f"{resource}: field {field_name!r} must be <= {maximum}, got {number}." + ) + return number + + +def _require_quantitative( + value: object, + *, + surface: str, + expectation: str, + columns: tuple[str, ...], + resource: str, + where: str, +) -> Mapping[str, Any] | None: + if value is None: + return None + if not isinstance(value, Mapping): + raise ValueError(f"{resource}: {where}.quantitative must be an object.") + + quantitative = dict(value) + if surface == "nonzero_shares" and expectation == "column_differs": + raw_shares = quantitative.get("shares") + if not isinstance(raw_shares, Mapping): + raise ValueError( + f"{resource}: {where}.quantitative.shares must be an object." + ) + missing = sorted(set(columns) - set(raw_shares)) + extra = sorted(set(raw_shares) - set(columns)) + if missing or extra: + raise ValueError( + f"{resource}: {where}.quantitative.shares must exactly cover " + f"the scoped columns; missing={missing}, extra={extra}." + ) + shares: dict[str, dict[str, float | str]] = {} + for column in columns: + raw_entry = raw_shares[column] + if not isinstance(raw_entry, Mapping): + raise ValueError( + f"{resource}: {where}.quantitative.shares[{column!r}] " + "must be an object." + ) + direction = _require_member( + raw_entry.get("direction"), + allowed=SIGNED_DIFFERENCE_SHARE_DIRECTIONS, + field_name=f"{where}.quantitative.shares[{column!r}].direction", + resource=resource, + ) + shares[column] = { + "incumbent_share": _require_number( + raw_entry.get("incumbent_share"), + field_name=( + f"{where}.quantitative.shares[{column!r}]." + "incumbent_share" + ), + resource=resource, + minimum=0.0, + maximum=1.0, + ), + "direction": direction, + "max_abs_delta": _require_number( + raw_entry.get("max_abs_delta"), + field_name=( + f"{where}.quantitative.shares[{column!r}].max_abs_delta" + ), + resource=resource, + minimum=0.0, + ), + } + quantitative["shares"] = shares + elif surface == "entity_counts" and expectation == "count_differs": + raw_deltas = quantitative.get("expected_deltas") + if not isinstance(raw_deltas, Mapping): + raise ValueError( + f"{resource}: {where}.quantitative.expected_deltas must be an " + "object." + ) + expected_entities = set(columns) + missing = sorted(expected_entities - set(raw_deltas)) + extra = sorted(set(raw_deltas) - expected_entities) + if missing or extra: + raise ValueError( + f"{resource}: {where}.quantitative.expected_deltas must exactly " + f"cover the scoped entities; missing={missing}, extra={extra}." + ) + deltas: dict[str, int] = {} + for entity in columns: + delta = raw_deltas[entity] + if isinstance(delta, bool) or not isinstance(delta, int): + raise ValueError( + f"{resource}: {where}.quantitative.expected_deltas[{entity!r}] " + "must be an integer." + ) + deltas[entity] = delta + quantitative["expected_deltas"] = deltas + return quantitative + + def load_uk_spine_swap_signed_differences( resource: str = UK_SPINE_SWAP_SIGNED_DIFFERENCES_RESOURCE, ) -> UKSignedDifferenceRegister: @@ -343,6 +465,12 @@ def load_uk_spine_swap_signed_differences( field_name=f"{where}.scope.surface", resource=resource, ) + expectation = _require_member( + raw.get("expectation"), + allowed=SIGNED_DIFFERENCE_EXPECTATIONS, + field_name=f"{where}.expectation", + resource=resource, + ) columns = _require_str_tuple( raw_scope.get("columns"), field_name=f"{where}.scope.columns", @@ -382,12 +510,7 @@ def load_uk_spine_swap_signed_differences( resource=resource, ), surface=surface, - expectation=_require_member( - raw.get("expectation"), - allowed=SIGNED_DIFFERENCE_EXPECTATIONS, - field_name=f"{where}.expectation", - resource=resource, - ), + expectation=expectation, columns=columns, entities=entities, magnitude_evidence=_require_str( @@ -406,6 +529,14 @@ def load_uk_spine_swap_signed_differences( resource=resource, ), adjudicated_on=adjudicated_on, + quantitative=_require_quantitative( + raw.get("quantitative"), + surface=surface, + expectation=expectation, + columns=columns, + resource=resource, + where=where, + ), ) ) diff --git a/packages/microcosm-build/tests/test_uk_identity_stability_receipts.py b/packages/microcosm-build/tests/test_uk_identity_stability_receipts.py index 90fa0438e..cc99d5684 100644 --- a/packages/microcosm-build/tests/test_uk_identity_stability_receipts.py +++ b/packages/microcosm-build/tests/test_uk_identity_stability_receipts.py @@ -13,10 +13,22 @@ import importlib.util from pathlib import Path +from types import SimpleNamespace +import numpy as np import pandas as pd import pytest +from microcosm.build.source_manifest import SourceOperationSpec, SourceStageSpec +from microcosm.build.uk_runtime.age_tail import ( + UK_AGE_TOP_CODE, + disaggregate_uk_age_top_code, +) +from microcosm.build.uk_runtime.etb_services import ( + UK_NHS_OUTPUT_COLUMNS, + UKETBServicesStageTransform, + allocate_nhs_by_age_gender, +) from microcosm.build.uk_runtime.national_frame import uk_national_frame from microcosm.frame import WeightKind @@ -125,3 +137,154 @@ def test_a_corrupted_stored_channel_is_caught(self) -> None: household.loc[household.index[-1], "household_support_channel"] = "frs" receipt = tool.e7_identity_receipt(frame, permutation_seed=7) assert receipt["matches_stored_columns"] is False + + +class TestE6Receipt: + def test_nhs_receipt_uses_stage_time_top_coded_age( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + class _FakeModel: + def __init__(self, *, n_estimators, seed): + del n_estimators, seed + + def start_chain(self, donor, predictors, targets, *, weights): + del donor, predictors, weights + return {"targets": list(targets)} + + def fit_draw_next(self, donor, recipient_base, raw, *, state, weights): + del recipient_base, raw, weights + target = state["targets"][0] + state = {"targets": state["targets"][1:]} + return SimpleNamespace( + raw_draw=pd.Series( + [float(donor[target].iloc[0])], index=donor.index[:1] + ), + weight_kind="explicit", + state=state, + ) + + import microcosm.fit as fit_module + + monkeypatch.setattr(fit_module, "RegimeGatedQRF", _FakeModel) + + class _FakeEngine: + country = "uk" + + _entities = { + "is_adult": "person", + "is_child": "person", + "is_SP_age": "person", + "dla": "person", + "pip": "person", + "hbai_household_net_income": "household", + "current_education": "person", + } + + def variable_metadata(self, name): + return SimpleNamespace(entity=self._entities[name]) + + def materialize(self, frame, variables, period): + del period + person_rows = len(frame.table("person")) + household_rows = len(frame.table("household")) + values = { + "is_adult": np.ones(person_rows), + "is_child": np.zeros(person_rows), + "is_SP_age": np.ones(person_rows), + "dla": np.zeros(person_rows), + "pip": np.zeros(person_rows), + "hbai_household_net_income": np.full(household_rows, 100.0), + "current_education": np.full( + person_rows, "NOT_IN_EDUCATION", dtype=object + ), + } + return {variable: values[variable] for variable in variables} + + frame = uk_national_frame( + person=pd.DataFrame( + { + "person_id": [1], + "person_benunit_id": [10], + "person_household_id": [100], + "person_source_id": ["source-1"], + "age": [float(UK_AGE_TOP_CODE)], + "gender": ["FEMALE"], + } + ), + benunit=pd.DataFrame({"benunit_id": [10], "benunit_household_id": [100]}), + household=pd.DataFrame( + {"household_id": [100], "household_weight": [1.0]} + ), + time_period="2024", + weight_kind=WeightKind.DESIGN, + ) + stage = SourceStageSpec( + stage="etb_services", + survey="etb", + source="fixture", + grain="household", + artifacts=(), + operations=( + SourceOperationSpec(kind="derive", parameters={}), + SourceOperationSpec( + kind="fit_weighted_qrf_chain", parameters={"seed": 0} + ), + ), + outputs=(), + ) + donor = pd.DataFrame( + { + "year": [2024], + "adults": [1], + "childs": [0], + "disinc": [100.0], + "educ": [1.0], + "rail": [1.0], + "bussub": [1.0], + "hhold_adj_weight": [1.0], + "noretd": [1], + "primed": [0], + "secoed": [0], + "furted": [0], + "disliv": [0.0], + "pips": [0.0], + } + ) + frame = UKETBServicesStageTransform( + stage=stage, engine=_FakeEngine(), donor=donor + )(frame) + disaggregate_uk_age_top_code( + frame, + band_populations={ + ("MALE", "80_84"): 1.0, + ("MALE", "85_89"): 1.0, + ("MALE", "90_plus"): 1.0, + ("FEMALE", "80_84"): 1.0, + ("FEMALE", "85_89"): 1e9, + ("FEMALE", "90_plus"): 1.0, + }, + ) + + person = frame.table("person") + household = frame.table("household") + final_age_nhs = allocate_nhs_by_age_gender( + person, + household_weights=frame.weights_for("household").values, + household=household, + nhs_table=None, + ) + stored = person.set_index("person_id")[list(UK_NHS_OUTPUT_COLUMNS)] + final_age_nhs.index = stored.index + assert any( + not np.allclose( + stored[column].to_numpy(dtype=float), + final_age_nhs[column].to_numpy(dtype=float), + ) + for column in UK_NHS_OUTPUT_COLUMNS + ) + + receipt = _load_tool().e6_identity_receipt(frame, permutation_seed=7) + assert receipt["nhs_age_basis"] == "stage_time_top_coded" + assert receipt["nhs_age_top_code"] == UK_AGE_TOP_CODE + assert receipt["matches_stored_columns"] is True + assert receipt["stored_column_mismatches"] == {} diff --git a/packages/microcosm-build/tests/test_uk_signed_differences.py b/packages/microcosm-build/tests/test_uk_signed_differences.py index d5733aad2..88aec675c 100644 --- a/packages/microcosm-build/tests/test_uk_signed_differences.py +++ b/packages/microcosm-build/tests/test_uk_signed_differences.py @@ -9,6 +9,7 @@ import json import re +from decimal import ROUND_HALF_UP, Decimal from importlib.resources import files from pathlib import Path @@ -69,6 +70,26 @@ def _github_anchors(path: Path) -> set[str]: return anchors +_DECIMAL = r"(?P\d+\.\d{4,})" + + +def _incumbent_share_quotes(text: str, column: str) -> list[str]: + """Incumbent share figures quoted beside a scoped column name.""" + + column_pattern = re.escape(column) + patterns = [ + rf"{column_pattern}[^;]*?incumbent(?:'s)?(?:\s+\w+)?\s+{_DECIMAL}", + rf"{column_pattern}[^;]*?donor\s+\d+\.\d{{4,}}\s+against\s+{_DECIMAL}\s+and", + rf"{column_pattern}[^;]*?incumbent\s+carries[^;]*?share\s+of\s+{_DECIMAL}", + rf"{column_pattern}[\s\S]{{0,220}}?Unweighted\s+share\s+{_DECIMAL}\s+for\s+the\s+incumbent", + ] + quotes: list[str] = [] + for pattern in patterns: + for matched in re.finditer(pattern, text): + quotes.append(matched.group("value")) + return quotes + + class TestCommittedRegister: def test_committed_register_loads(self) -> None: register = load_uk_spine_swap_signed_differences() @@ -172,6 +193,71 @@ def test_no_committed_entry_expires(self) -> None: for entry in payload["differences"]: assert "expires_on" not in entry + def test_committed_quantitative_blocks_match_the_reference(self) -> None: + reference = json.loads( + files("microcosm.build.uk") + .joinpath("efrs_parity_reference.json") + .read_text(encoding="utf-8") + ) + for difference in load_uk_spine_swap_signed_differences().differences: + assert difference.quantitative is not None, ( + f"{difference.id} has no quantitative block; the verifier " + "would have no bound to check against." + ) + if ( + difference.surface == "nonzero_shares" + and difference.expectation == "column_differs" + ): + shares = difference.quantitative["shares"] + assert set(shares) == set(difference.columns) + for column in difference.columns: + measured = shares[column] + incumbent = reference["nonzero_shares"][column] + assert measured["incumbent_share"] == incumbent + assert measured["direction"] in { + "candidate_above", + "candidate_below", + } + assert measured["max_abs_delta"] > 0.0 + elif difference.surface == "entity_counts": + assert difference.quantitative["expected_deltas"] == { + "benunit": -12, + "person": 32, + } + + def test_committed_incumbent_share_quotes_match_the_reference(self) -> None: + reference = json.loads( + files("microcosm.build.uk") + .joinpath("efrs_parity_reference.json") + .read_text(encoding="utf-8") + ) + checked: list[tuple[str, str, str]] = [] + for difference in load_uk_spine_swap_signed_differences().differences: + if ( + difference.surface != "nonzero_shares" + or difference.expectation != "column_differs" + ): + continue + for column in difference.columns: + for quote in _incumbent_share_quotes( + difference.magnitude_evidence, column + ): + checked.append((difference.id, column, quote)) + places = len(quote.rsplit(".", 1)[1]) + expected = Decimal(str(reference["nonzero_shares"][column])) + quantized = expected.quantize( + Decimal("1").scaleb(-places), rounding=ROUND_HALF_UP + ) + assert Decimal(quote) == quantized, ( + f"{difference.id} quotes {column} incumbent share as " + f"{quote}, but the packaged reference rounds to " + f"{quantized} at {places} decimals." + ) + assert len(checked) >= 10, ( + "The prose quote check matched too little evidence; it is likely " + "not enforcing the committed register." + ) + class TestLookup: def test_matching_finds_the_signing_entry(self) -> None: diff --git a/packages/microcosm-build/tests/test_uk_spine_parity_instrument.py b/packages/microcosm-build/tests/test_uk_spine_parity_instrument.py index 6ba3c8567..35550aafa 100644 --- a/packages/microcosm-build/tests/test_uk_spine_parity_instrument.py +++ b/packages/microcosm-build/tests/test_uk_spine_parity_instrument.py @@ -78,8 +78,10 @@ def _entry( surface: str, columns: list[str], expectation: str = "column_differs", + direction: str = "candidate_above", + max_abs_delta: float = 1.0, ) -> dict: - return { + entry = { "id": identifier, "class": "mechanism_change", "scope": {"surface": surface, "columns": columns, "entities": ["household"]}, @@ -89,6 +91,25 @@ def _entry( "adjudicator": "juaristi22", "adjudicated_on": "2026-08-22", } + if surface == "nonzero_shares" and expectation == "column_differs": + reference = _reference_payload()["nonzero_shares"] + entry["quantitative"] = { + "shares": { + column: { + "incumbent_share": reference.get(column, 0.0), + "direction": direction, + "max_abs_delta": max_abs_delta, + } + for column in columns + } + } + elif surface == "entity_counts" and expectation == "count_differs": + entry["quantitative"] = {"expected_deltas": {column: 1 for column in columns}} + elif surface == "weighted_totals": + entry["quantitative"] = {"weighted_totals": {"expected_columns": columns}} + else: + entry["quantitative"] = {"structural": {"expected_columns": columns}} + return entry def _first_household_column() -> str: @@ -99,6 +120,18 @@ def _first_household_column() -> str: raise AssertionError("reference carries no household column") +def _weighted_identity(sha256: str) -> dict[str, str]: + return {"filename": "artifact.h5", "sha256": sha256} + + +def _reference_weighted_identity() -> dict[str, str]: + return _weighted_identity(_reference_payload()["source"]["sha256"]) + + +def _candidate_weighted_identity() -> dict[str, str]: + return _weighted_identity("a" * 64) + + class TestVerdicts: def test_matching_candidate_is_parity(self, tmp_path: Path) -> None: tool = _load_tool() @@ -343,9 +376,13 @@ def test_dormancy_is_not_a_loophole_once_the_surface_is_compared( # register rot again. tool = _load_tool() candidate = _write(tmp_path / "c.json", _candidate_from_reference()) - left = _write(tmp_path / "ref.json", {"identity": {}, "totals": {"col": 100.0}}) + left = _write( + tmp_path / "ref.json", + {"identity": _reference_weighted_identity(), "totals": {"col": 100.0}}, + ) right = _write( - tmp_path / "cand.json", {"identity": {}, "totals": {"col": 100.0}} + tmp_path / "cand.json", + {"identity": _candidate_weighted_identity(), "totals": {"col": 100.0}}, ) register = _register( tmp_path, @@ -477,9 +514,13 @@ def test_relative_deltas_are_reported_without_absolute_values( def test_unsigned_weighted_divergence_is_a_defect(self, tmp_path: Path) -> None: tool = _load_tool() candidate = _write(tmp_path / "c.json", _candidate_from_reference()) - left = _write(tmp_path / "ref.json", {"identity": {}, "totals": {"col": 100.0}}) + left = _write( + tmp_path / "ref.json", + {"identity": _reference_weighted_identity(), "totals": {"col": 100.0}}, + ) right = _write( - tmp_path / "cand.json", {"identity": {}, "totals": {"col": 125.0}} + tmp_path / "cand.json", + {"identity": _candidate_weighted_identity(), "totals": {"col": 125.0}}, ) assert ( @@ -498,6 +539,145 @@ def test_unsigned_weighted_divergence_is_a_defect(self, tmp_path: Path) -> None: == 1 ) + def test_strict_weighted_totals_reference_only_key_is_a_defect( + self, tmp_path: Path + ) -> None: + tool = _load_tool() + candidate = _write(tmp_path / "c.json", _candidate_from_reference()) + left = _write( + tmp_path / "ref.json", + { + "identity": _reference_weighted_identity(), + "totals": {"kept": 100.0, "omitted": 1.0}, + }, + ) + right = _write( + tmp_path / "cand.json", + {"identity": _candidate_weighted_identity(), "totals": {"kept": 100.0}}, + ) + receipt = tmp_path / "receipt.json" + + code = tool.main( + [ + "--candidate-json", + str(candidate), + "--register", + str(_register(tmp_path)), + "--reference-weighted-totals", + str(left), + "--candidate-weighted-totals", + str(right), + "--strict", + "--receipt-json", + str(receipt), + ] + ) + + assert code == 1 + report = json.loads(receipt.read_text(encoding="utf-8")) + assert report["verdict"] == "defect" + assert report["weighted_totals"]["only_in_reference"] == ["omitted"] + assert "omitted" in report["unsigned_differences"] + + def test_strict_weighted_totals_candidate_only_key_is_a_defect( + self, tmp_path: Path + ) -> None: + tool = _load_tool() + candidate = _write(tmp_path / "c.json", _candidate_from_reference()) + left = _write( + tmp_path / "ref.json", + {"identity": _reference_weighted_identity(), "totals": {"kept": 100.0}}, + ) + right = _write( + tmp_path / "cand.json", + { + "identity": _candidate_weighted_identity(), + "totals": {"extra": 1.0, "kept": 100.0}, + }, + ) + receipt = tmp_path / "receipt.json" + + code = tool.main( + [ + "--candidate-json", + str(candidate), + "--register", + str(_register(tmp_path)), + "--reference-weighted-totals", + str(left), + "--candidate-weighted-totals", + str(right), + "--strict", + "--receipt-json", + str(receipt), + ] + ) + + assert code == 1 + report = json.loads(receipt.read_text(encoding="utf-8")) + assert report["verdict"] == "defect" + assert report["weighted_totals"]["only_in_candidate"] == ["extra"] + assert "extra" in report["unsigned_differences"] + + def test_strict_weighted_totals_missing_candidate_identity_is_refused( + self, tmp_path: Path + ) -> None: + tool = _load_tool() + candidate = _write(tmp_path / "c.json", _candidate_from_reference()) + left = _write( + tmp_path / "ref.json", + {"identity": _reference_weighted_identity(), "totals": {"kept": 100.0}}, + ) + right = _write(tmp_path / "cand.json", {"totals": {"kept": 100.0}}) + + assert ( + tool.main( + [ + "--candidate-json", + str(candidate), + "--register", + str(_register(tmp_path)), + "--reference-weighted-totals", + str(left), + "--candidate-weighted-totals", + str(right), + "--strict", + ] + ) + == 2 + ) + + def test_strict_weighted_totals_cross_artifact_identity_is_refused( + self, tmp_path: Path + ) -> None: + tool = _load_tool() + candidate = _write(tmp_path / "c.json", _candidate_from_reference()) + left = _write( + tmp_path / "ref.json", + {"identity": _reference_weighted_identity(), "totals": {"kept": 100.0}}, + ) + right = _write( + tmp_path / "cand.json", + {"identity": _weighted_identity("b" * 64), "totals": {"kept": 100.0}}, + ) + + assert ( + tool.main( + [ + "--candidate-json", + str(candidate), + "--register", + str(_register(tmp_path)), + "--reference-weighted-totals", + str(left), + "--candidate-weighted-totals", + str(right), + "--strict", + ] + ) + == 2 + ) + class TestAcceptanceBand: """The band decides what must be adjudicated, never what is reported.""" @@ -626,6 +806,76 @@ def test_an_out_of_range_band_yields_no_verdict(self, tmp_path: Path) -> None: == 2 ) + def test_strict_refuses_a_non_contract_band(self, tmp_path: Path) -> None: + tool = _load_tool() + candidate = _write(tmp_path / "c.json", _candidate_from_reference()) + + assert ( + tool.main( + [ + "--candidate-json", + str(candidate), + "--register", + str(_register(tmp_path)), + "--strict", + "--share-band", + "0.9", + ] + ) + == 2 + ) + + def test_diagnostic_non_contract_band_is_not_parity( + self, tmp_path: Path + ) -> None: + tool = _load_tool() + candidate = _write(tmp_path / "c.json", _candidate_from_reference()) + receipt = tmp_path / "receipt.json" + + code = tool.main( + [ + "--candidate-json", + str(candidate), + "--register", + str(_register(tmp_path)), + "--share-band", + "0.9", + "--receipt-json", + str(receipt), + ] + ) + + assert code == 0 + report = json.loads(receipt.read_text(encoding="utf-8")) + assert report["verdict"] == "diagnostic" + assert report["share_band"] == {"contract": 0.02, "effective": 0.9} + + def test_strict_contract_band_behaviour_is_unchanged( + self, tmp_path: Path + ) -> None: + tool = _load_tool() + candidate = _write(tmp_path / "c.json", _candidate_from_reference()) + receipt = tmp_path / "receipt.json" + + code = tool.main( + [ + "--candidate-json", + str(candidate), + "--register", + str(_register(tmp_path)), + "--strict", + "--share-band", + "0.02", + "--receipt-json", + str(receipt), + ] + ) + + assert code == 0 + report = json.loads(receipt.read_text(encoding="utf-8")) + assert report["verdict"] == "parity" + assert report["strict_failure"] is False + class TestReviewFindings: """Regressions for the review findings on the proof machinery (#747). @@ -771,6 +1021,152 @@ def test_a_structural_expectation_does_not_sign_a_value_divergence( == 1 ) + def test_signed_count_omitted_from_candidate_is_a_defect( + self, tmp_path: Path + ) -> None: + tool = _load_tool() + payload = _candidate_from_reference() + del payload["entity_stats"]["person"] + candidate = _write(tmp_path / "c.json", payload) + register = _register( + tmp_path, + _entry( + "counts-signed", + surface="entity_counts", + columns=["person"], + expectation="count_differs", + ), + ) + receipt = tmp_path / "receipt.json" + + code = tool.main( + [ + "--candidate-json", + str(candidate), + "--register", + str(register), + "--receipt-json", + str(receipt), + ] + ) + + assert code == 1 + report = json.loads(receipt.read_text(encoding="utf-8")) + assert report["verdict"] == "defect" + assert report["entity_counts"]["person"]["signed_id"] is None + assert "person" in report["unsigned_differences"] + + def test_signed_count_with_wrong_delta_is_a_defect(self, tmp_path: Path) -> None: + tool = _load_tool() + payload = _candidate_from_reference() + payload["entity_stats"]["person"]["records"] = 1 + candidate = _write(tmp_path / "c.json", payload) + register = _register( + tmp_path, + _entry( + "counts-signed", + surface="entity_counts", + columns=["person"], + expectation="count_differs", + ), + ) + receipt = tmp_path / "receipt.json" + + code = tool.main( + [ + "--candidate-json", + str(candidate), + "--register", + str(register), + "--receipt-json", + str(receipt), + ] + ) + + assert code == 1 + report = json.loads(receipt.read_text(encoding="utf-8")) + assert report["verdict"] == "defect" + assert report["entity_counts"]["person"]["signed_id"] is None + assert "person" in report["unsigned_differences"] + + def test_signed_share_with_reversed_direction_is_a_defect( + self, tmp_path: Path + ) -> None: + tool = _load_tool() + column = "water_and_sewerage_charges" + payload = _candidate_from_reference() + payload["nonzero_shares"][column] = ( + _reference_payload()["nonzero_shares"][column] - 0.1 + ) + candidate = _write(tmp_path / "c.json", payload) + register = _register( + tmp_path, + _entry( + "water-signed", + surface="nonzero_shares", + columns=[column], + direction="candidate_above", + max_abs_delta=0.100897, + ), + ) + receipt = tmp_path / "receipt.json" + + code = tool.main( + [ + "--candidate-json", + str(candidate), + "--register", + str(register), + "--receipt-json", + str(receipt), + ] + ) + + assert code == 1 + report = json.loads(receipt.read_text(encoding="utf-8")) + assert report["verdict"] == "defect" + assert report["nonzero_shares"]["differing"][column]["signed_id"] is None + assert column in report["unsigned_differences"] + + def test_signed_share_beyond_magnitude_is_a_defect( + self, tmp_path: Path + ) -> None: + tool = _load_tool() + column = "water_and_sewerage_charges" + payload = _candidate_from_reference() + payload["nonzero_shares"][column] = ( + _reference_payload()["nonzero_shares"][column] + 0.2 + ) + candidate = _write(tmp_path / "c.json", payload) + register = _register( + tmp_path, + _entry( + "water-signed", + surface="nonzero_shares", + columns=[column], + direction="candidate_above", + max_abs_delta=0.100897, + ), + ) + receipt = tmp_path / "receipt.json" + + code = tool.main( + [ + "--candidate-json", + str(candidate), + "--register", + str(register), + "--receipt-json", + str(receipt), + ] + ) + + assert code == 1 + report = json.loads(receipt.read_text(encoding="utf-8")) + assert report["verdict"] == "defect" + assert report["nonzero_shares"]["differing"][column]["signed_id"] is None + assert column in report["unsigned_differences"] + class TestWeightedTotalsRegisterAccounting: """A totals-scoped entry counts as matched, not as register rot. @@ -785,9 +1181,13 @@ def test_a_totals_entry_that_matched_is_not_reported_unused( ) -> None: tool = _load_tool() candidate = _write(tmp_path / "c.json", _candidate_from_reference()) - left = _write(tmp_path / "ref.json", {"identity": {}, "totals": {"col": 100.0}}) + left = _write( + tmp_path / "ref.json", + {"identity": _reference_weighted_identity(), "totals": {"col": 100.0}}, + ) right = _write( - tmp_path / "cand.json", {"identity": {}, "totals": {"col": 125.0}} + tmp_path / "cand.json", + {"identity": _candidate_weighted_identity(), "totals": {"col": 125.0}}, ) register = _register( tmp_path, diff --git a/tools/verify_uk_identity_stability.py b/tools/verify_uk_identity_stability.py index 00925635c..ef3eb5764 100644 --- a/tools/verify_uk_identity_stability.py +++ b/tools/verify_uk_identity_stability.py @@ -17,6 +17,7 @@ import numpy as np import pandas as pd +from microcosm.build.uk_runtime.age_tail import UK_AGE_TOP_CODE from microcosm.build.uk_runtime.frs_brma import ( UK_BRMA_DECLARED_SEEDS, _benunit_regions, @@ -340,6 +341,10 @@ def e6_identity_receipt( Covered: the domestic-energy fold (elec + gas), the rail_usage ratio, petrol/diesel zeroing idempotence for non-fuel households, and the NHS age-gender person allocation recomputed from the committed resource. + The production contract is stage-time-age-derived: ``etb_services`` runs + on the top-coded FRS age surface, then ``age_tail`` disaggregates the + top-code later as calibration support. Stored NHS columns are therefore + checked against ``min(final_age, UK_AGE_TOP_CODE)`` rather than final age. The QRF chain draws and the NEED raking outcome are covered by twin-build determinism and the aggregate_admin NEED-margin receipt respectively (raking inputs are consumed by the stage and are not @@ -378,8 +383,15 @@ def recompute(person_t, benunit_t, household_t) -> dict[str, pd.DataFrame]: no_fuel, 0.0, household[column].to_numpy(dtype=float) ) if {"age", "gender"} <= set(person_t.columns): + nhs_person = person_t.copy() + nhs_person["age"] = np.minimum( + pd.to_numeric(nhs_person["age"], errors="coerce") + .fillna(0) + .to_numpy(dtype=float), + UK_AGE_TOP_CODE, + ) nhs = allocate_nhs_by_age_gender( - person_t, + nhs_person, household_weights=household["household_weight"].to_numpy(dtype=float), household=household, nhs_table=None, @@ -460,6 +472,8 @@ def recompute(person_t, benunit_t, household_t) -> dict[str, pd.DataFrame]: return { "check": "uk_e6_identity_stability", "permutation_seed": permutation_seed, + "nhs_age_basis": "stage_time_top_coded", + "nhs_age_top_code": UK_AGE_TOP_CODE, "identical_under_permutation": not mismatches, "permutation_mismatches": mismatches, "matches_stored_columns": not stored_mismatches, diff --git a/tools/verify_uk_spine_parity.py b/tools/verify_uk_spine_parity.py index 482429ca1..234d62b98 100644 --- a/tools/verify_uk_spine_parity.py +++ b/tools/verify_uk_spine_parity.py @@ -52,6 +52,7 @@ load_efrs_parity_reference, ) from microcosm.build.uk_runtime.signed_differences import ( # noqa: E402 + UKSignedDifference, UKSignedDifferenceRegister, load_uk_spine_swap_signed_differences, ) @@ -76,6 +77,109 @@ VERDICT_PARITY = "parity" VERDICT_SIGNED_PARITY = "signed_parity" VERDICT_DEFECT = "defect" +VERDICT_DIAGNOSTIC = "diagnostic" + + +def _require_quantitative(entry: UKSignedDifference) -> Mapping[str, Any]: + if entry.quantitative is None: + raise ValueError( + f"signed-difference entry {entry.id!r} has no quantitative block; " + "the register cannot verify its signed magnitude." + ) + return entry.quantitative + + +def _signed_count_entry( + *, + register: UKSignedDifferenceRegister, + entity: str, + reference_records: Any, + candidate_records: Any, +) -> UKSignedDifference | None: + signed = register.matching( + surface="entity_counts", + column=entity, + expectation="count_differs", + entity=entity, + ) + if signed is None: + return None + quantitative = _require_quantitative(signed) + expected_deltas = quantitative.get("expected_deltas") + if not isinstance(expected_deltas, Mapping): + raise ValueError( + f"signed-difference entry {signed.id!r} has no expected_deltas block." + ) + expected_delta = expected_deltas.get(entity) + if not isinstance(expected_delta, int): + raise ValueError( + f"signed-difference entry {signed.id!r} has no expected delta for " + f"{entity!r}." + ) + if not isinstance(reference_records, int) or not isinstance( + candidate_records, int + ): + return None + return ( + signed + if candidate_records - reference_records == expected_delta + else None + ) + + +def _signed_share_entry( + *, + register: UKSignedDifferenceRegister, + column: str, + entity: str | None, + reference_share: float, + delta: float, +) -> UKSignedDifference | None: + signed = register.matching( + surface="nonzero_shares", + column=column, + expectation="column_differs", + entity=entity, + ) + if signed is None: + return None + quantitative = _require_quantitative(signed) + shares = quantitative.get("shares") + if not isinstance(shares, Mapping): + raise ValueError( + f"signed-difference entry {signed.id!r} has no shares block." + ) + share = shares.get(column) + if not isinstance(share, Mapping): + raise ValueError( + f"signed-difference entry {signed.id!r} has no share block for " + f"{column!r}." + ) + if share.get("incumbent_share") != reference_share: + raise ValueError( + f"signed-difference entry {signed.id!r} records incumbent_share " + f"{share.get('incumbent_share')!r} for {column!r}, but the " + f"packaged reference is {reference_share!r}." + ) + direction = share.get("direction") + if direction == "candidate_above": + direction_matches = delta > 0.0 + elif direction == "candidate_below": + direction_matches = delta < 0.0 + else: + raise ValueError( + f"signed-difference entry {signed.id!r} records invalid direction " + f"{direction!r} for {column!r}." + ) + max_abs_delta = share.get("max_abs_delta") + if not isinstance(max_abs_delta, int | float): + raise ValueError( + f"signed-difference entry {signed.id!r} records invalid " + f"max_abs_delta {max_abs_delta!r} for {column!r}." + ) + if not direction_matches or abs(delta) > float(max_abs_delta): + return None + return signed def _load_json(path: Path) -> Mapping[str, Any]: @@ -121,6 +225,58 @@ def _candidate_identity(payload: Mapping[str, Any]) -> dict[str, Any]: } +def _weighted_totals_identity( + payload: Mapping[str, Any], *, label: str +) -> dict[str, Any]: + identity = payload.get("identity") + if not isinstance(identity, Mapping): + raise ValueError( + f"{label} weighted-totals sidecar carries no identity block; " + "strict comparison requires content identity for both artifacts." + ) + sha256 = identity.get("sha256") + if ( + not isinstance(sha256, str) + or len(sha256) != 64 + or any(character not in "0123456789abcdef" for character in sha256) + ): + raise ValueError( + f"{label} weighted-totals sidecar identity carries no lowercase " + "sha256 content digest." + ) + return { + key: identity.get(key) + for key in ("filename", "revision", "sha256", "size_bytes", "vintage", "period") + if identity.get(key) is not None + } + + +def _assert_weighted_totals_identities_bound( + *, + reference_sidecar: Mapping[str, Any], + candidate_sidecar: Mapping[str, Any], + reference_sha256: str, + candidate_sha256: str, +) -> tuple[dict[str, Any], dict[str, Any]]: + reference_identity = _weighted_totals_identity( + reference_sidecar, label="reference" + ) + candidate_identity = _weighted_totals_identity( + candidate_sidecar, label="candidate" + ) + if reference_identity["sha256"] != reference_sha256: + raise ValueError( + "reference weighted-totals sidecar describes a different artifact: " + f"{reference_identity['sha256']} != {reference_sha256}." + ) + if candidate_identity["sha256"] != candidate_sha256: + raise ValueError( + "candidate weighted-totals sidecar describes a different artifact: " + f"{candidate_identity['sha256']} != {candidate_sha256}." + ) + return reference_identity, candidate_identity + + def _compare_entity_counts( reference_stats: Mapping[str, Any], candidate_stats: Mapping[str, Any], @@ -134,11 +290,11 @@ def _compare_entity_counts( equal = expected == observed entry = {"reference": expected, "candidate": observed, "equal": equal} if not equal: - signed = register.matching( - surface="entity_counts", - column=entity, - expectation="count_differs", + signed = _signed_count_entry( + register=register, entity=entity, + reference_records=expected, + candidate_records=observed, ) entry["signed_id"] = signed.id if signed else None if signed is None: @@ -170,11 +326,12 @@ def _compare_shares( "candidate": observed, "delta": delta, } - signed = register.matching( - surface="nonzero_shares", + signed = _signed_share_entry( + register=register, column=column, - expectation="column_differs", entity=entities.get(column), + reference_share=expected, + delta=delta, ) if abs(delta) <= band: # Reported, never dropped: the band decides what must be @@ -232,9 +389,15 @@ def _compare_weighted_totals( candidate_totals: Mapping[str, float], register: UKSignedDifferenceRegister, entities: Mapping[str, str] | None = None, + strict: bool = False, ) -> tuple[dict[str, Any], list[str]]: unsigned: list[str] = [] differing: dict[str, Any] = {} + only_in_reference = sorted(set(reference_totals) - set(candidate_totals)) + only_in_candidate = sorted(set(candidate_totals) - set(reference_totals)) + if strict: + unsigned.extend(only_in_reference) + unsigned.extend(only_in_candidate) compared = sorted(set(reference_totals) & set(candidate_totals)) for column in compared: expected = float(reference_totals[column]) @@ -267,8 +430,8 @@ def _compare_weighted_totals( { "compared": len(compared), "differing": differing, - "only_in_reference": sorted(set(reference_totals) - set(candidate_totals)), - "only_in_candidate": sorted(set(candidate_totals) - set(reference_totals)), + "only_in_reference": only_in_reference, + "only_in_candidate": only_in_candidate, }, unsigned, ) @@ -346,6 +509,10 @@ def verify_uk_spine_parity( report: dict[str, Any] = { "check": "uk_whole_spine_parity", "schema_version": 1, + "share_band": { + "contract": SHARE_PARITY_BAND, + "effective": share_band, + }, "reference": { "resource": "efrs_parity_reference.json", "source": { @@ -365,6 +532,19 @@ def verify_uk_spine_parity( if reference_weighted_totals is not None and candidate_weighted_totals is not None: left = _load_json(reference_weighted_totals) right = _load_json(candidate_weighted_totals) + if strict: + ( + reference_weighted_identity, + candidate_weighted_identity, + ) = _assert_weighted_totals_identities_bound( + reference_sidecar=left, + candidate_sidecar=right, + reference_sha256=reference.source.sha256, + candidate_sha256=candidate_identity["sha256"], + ) + else: + reference_weighted_identity = left.get("identity") + candidate_weighted_identity = right.get("identity") left_totals = left.get("totals") right_totals = right.get("totals") if not isinstance(left_totals, Mapping) or not isinstance( @@ -376,9 +556,10 @@ def verify_uk_spine_parity( {k: float(v) for k, v in right_totals.items()}, register, reference.input_entities, + strict=strict, ) - totals_report["reference_identity"] = left.get("identity") - totals_report["candidate_identity"] = right.get("identity") + totals_report["reference_identity"] = reference_weighted_identity + totals_report["candidate_identity"] = candidate_weighted_identity report["weighted_totals"] = totals_report unsigned.extend(totals_unsigned) matched_ids.update( @@ -417,6 +598,8 @@ def verify_uk_spine_parity( if report["unsigned_differences"]: report["verdict"] = VERDICT_DEFECT + elif share_band != SHARE_PARITY_BAND: + report["verdict"] = VERDICT_DIAGNOSTIC elif matched_ids: report["verdict"] = VERDICT_SIGNED_PARITY else: @@ -502,6 +685,13 @@ def main(argv: list[str] | None = None) -> int: file=sys.stderr, ) return 2 + if args.strict and args.share_band != SHARE_PARITY_BAND: + print( + "error: strict_share_band_mismatch: --strict requires the " + f"contract share band {SHARE_PARITY_BAND}.", + file=sys.stderr, + ) + return 2 totals = (args.reference_weighted_totals, args.candidate_weighted_totals) if any(totals) and not all(totals): From 0574841979a4cd07b4f821df180c3bac7cbf7ba4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Wed, 26 Aug 2026 16:03:20 +0200 Subject: [PATCH 06/14] Certify the 25-stage candidate: ladder green, bounds re-cut, receipt bound MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The licensed runs the hardened instruments demanded, and what they caught. The e4-e8 identity ladder ran on the real 25-stage candidate. e6 passed under the repaired stage-time contract — and e8 failed, 255 donors swapped: its donor-selection recompute picked oldest-adult carriers from final age, the same defect one check over from the one #770 named. Both recompute sites now clamp to min(age, UK_AGE_TOP_CODE), the receipt declares donor_age_basis, and a tie-flip regression pins the mechanism — two adults tied at the top code flip carriers under an unclamped recompute the moment age_tail lifts one of them. The ladder is green end to end. The hardened strict parity run returned defect on its first licensed outing: fifteen register entries failed on magnitude by two to five hundred-thousandths each, every direction correct. The bounds had been populated from the comparison ledger's rounded quotes, measured on the 24-stage pre-SPI-zero-fix build — water's bound equalled its historical delta exactly, zero headroom. All twenty-six share bounds are re-measured on this candidate at a declared 1e-4 grain with per-entry provenance naming the candidate sha and the superseded basis. Directions re-verified; strict parity returns signed_parity with zero unsigned differences, under the instrument that refuses manufactured verdicts. The acceptance evidence is committed and bound (#771): the receipt carries the candidate sha, the full 25-stage roster, the cross-commit twin-payload identity that doubles as the evidence-layer inertness proof, all five ladder outcomes with their declared age bases, and the strict-parity verdict. A binder test fails CI if the roster ever drifts from the plan the driver executes — the committed evidence can no longer describe a build that no longer exists. Declaring the receipt moved the UK spec bundle sha; re-pinned from the live measurement. Completes the licensed halves of #770 and #771; with the previous commit this addresses all six findings of the #747 post-merge audit, and #785 carries the age-first NHS derivation forward as its own adjudicated method change. Co-Authored-By: Claude Fable 5 --- .../771-spine-acceptance-recut.added.md | 1 + .../microcosm/build/uk/country_package.json | 5 ++ .../build/uk/spine_candidate_acceptance.json | 83 +++++++++++++++++++ .../uk/spine_swap_signed_differences.json | 79 ++++++++++-------- .../tests/test_country_spec.py | 2 + .../tests/test_spec_engine_country_bundles.py | 2 +- .../test_uk_identity_stability_receipts.py | 41 +++++++++ .../tests/test_uk_spine_acceptance_receipt.py | 60 ++++++++++++++ tools/verify_uk_identity_stability.py | 20 ++++- 9 files changed, 256 insertions(+), 37 deletions(-) create mode 100644 changelog.d/771-spine-acceptance-recut.added.md create mode 100644 packages/microcosm-build/src/microcosm/build/uk/spine_candidate_acceptance.json create mode 100644 packages/microcosm-build/tests/test_uk_spine_acceptance_receipt.py diff --git a/changelog.d/771-spine-acceptance-recut.added.md b/changelog.d/771-spine-acceptance-recut.added.md new file mode 100644 index 000000000..97ec2fdc1 --- /dev/null +++ b/changelog.d/771-spine-acceptance-recut.added.md @@ -0,0 +1 @@ +The UK spine acceptance evidence now describes the exact 25-stage candidate and is bound so it cannot drift again. The committed acceptance receipt carries the candidate's content sha, its full stage roster, the cross-commit twin-payload identity, all five identity-ladder outcomes, and the strict-parity verdict; a binder test fails CI if the roster ever differs from the plan the spine driver executes, or the verdicts differ from the accepted ones. The e8 identity check joins e6 on the stage-time age contract — the donor stage picked its oldest-adult carriers before age_tail ran, so the recompute clamps to the top code, which is what turned the 255-donor mismatch on the first 25-stage ladder run back into an exact identity. The register's magnitude bounds are re-measured on this candidate at declared 1e-4 grain with per-entry provenance, replacing the comparison ledger's rounded quotes measured on the 24-stage pre-fix build; every direction re-verified, and strict parity returns signed_parity with zero unsigned differences. diff --git a/packages/microcosm-build/src/microcosm/build/uk/country_package.json b/packages/microcosm-build/src/microcosm/build/uk/country_package.json index aeacb95f0..d09b020b6 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/country_package.json +++ b/packages/microcosm-build/src/microcosm/build/uk/country_package.json @@ -182,6 +182,11 @@ "kind": "legacy_json", "schema_id": "legacy_json" }, + { + "path": "spine_candidate_acceptance.json", + "kind": "legacy_json", + "schema_id": "legacy_json" + }, { "path": "ledger_compile_parity_incumbent_2025_signed_differences.json", "kind": "legacy_json", diff --git a/packages/microcosm-build/src/microcosm/build/uk/spine_candidate_acceptance.json b/packages/microcosm-build/src/microcosm/build/uk/spine_candidate_acceptance.json new file mode 100644 index 000000000..ad2aaca64 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk/spine_candidate_acceptance.json @@ -0,0 +1,83 @@ +{ + "artifact_kind": "uk_spine_candidate_acceptance", + "candidate": { + "entity_row_counts": { + "benunit": 61211, + "household": 52846, + "person": 113649 + }, + "name": "spine-e", + "sha256": "3c8799970851c409e4cb8578d33a180acb30ae600f4bd99ca3a190f9c5eb870a", + "sidecar_sha256": "5fa5f27065140a3a7d2fd56923150a9f89c5da52504648aebfb70e18da100d32", + "stage_count": 25, + "stage_roster": [ + "frs_spine", + "frs_employment", + "frs_council_tax", + "frs_disability", + "frs_education", + "frs_legacy_proxies", + "frs_education_grant_split", + "frs_take_up", + "frs_person_draws", + "frs_household_draws", + "frs_brma", + "was_wealth", + "regional_property_uprating", + "lcfs_consumption", + "etb_vat", + "etb_services", + "frs_hmrc_spine_leaves", + "spi_support_channel", + "hmrc_spi_income_spine", + "cgt_incidence_clone", + "cgt_band_donors", + "hmrc_cgt_gains_spine", + "salary_sacrifice", + "student_loans", + "age_tail" + ] + }, + "identity_ladder": { + "e4": { + "identical_under_permutation": true, + "matches_stored_columns": true + }, + "e5": { + "identical_under_permutation": true, + "matches_stored_columns": true + }, + "e6": { + "identical_under_permutation": true, + "matches_stored_columns": true, + "nhs_age_basis": "stage_time_top_coded" + }, + "e7": { + "identical_under_permutation": true, + "matches_stored_columns": true + }, + "e8": { + "donor_age_basis": "stage_time_top_coded", + "identical_under_permutation": true, + "matches_stored_columns": true + } + }, + "measured_on": "2026-08-26", + "schema_version": 1, + "strict_parity": { + "receipt_sha256": "b9e5118ef5785b4fb16f4aa13db9f302d73429e118d51d7bff6f60e7afe2513f", + "share_band": { + "contract": 0.02, + "effective": 0.02 + }, + "strict_failure": false, + "unsigned_differences": 0, + "verdict": "signed_parity" + }, + "twin": { + "name": "spine-d", + "note": "Cross-commit twin: spine-d built pre-Run-1 evidence layer, spine-e post; payload_identical across all tables, keys and root attrs, so the pair is simultaneously the twin-determinism receipt and the Run-1 payload-inertness receipt.", + "payload_identical": true, + "sha256": "f3f96805102d2e147408ee03dc3a5702a55b8232df417d7dc13109ac3caf75d1" + } +} diff --git a/packages/microcosm-build/src/microcosm/build/uk/spine_swap_signed_differences.json b/packages/microcosm-build/src/microcosm/build/uk/spine_swap_signed_differences.json index 8b9fc9610..3f1bf1922 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/spine_swap_signed_differences.json +++ b/packages/microcosm-build/src/microcosm/build/uk/spine_swap_signed_differences.json @@ -24,9 +24,10 @@ "water_and_sewerage_charges": { "incumbent_share": 0.776937, "direction": "candidate_above", - "max_abs_delta": 0.100897 + "max_abs_delta": 0.1009 } - } + }, + "magnitude_provenance": "max_abs_delta re-measured 2026-08-26 on the 25-stage candidate spine-e (sha256 3c8799970851c409e4cb8578d33a180acb30ae600f4bd99ca3a190f9c5eb870a) against the packaged 1.56.16 reference, rounded up at 1e-4 grain; the prior bounds were the comparison ledger's rounded quotes measured on the 24-stage pre-SPI-zero-fix build." } }, { @@ -86,49 +87,50 @@ "communication_consumption": { "incumbent_share": 0.79743, "direction": "candidate_above", - "max_abs_delta": 0.08396999999999999 + "max_abs_delta": 0.084 }, "domestic_energy_consumption": { "incumbent_share": 0.954869, "direction": "candidate_above", - "max_abs_delta": 0.040430999999999995 + "max_abs_delta": 0.0404 }, "education_consumption": { "incumbent_share": 0.125591, "direction": "candidate_below", - "max_abs_delta": 0.10859100000000001 + "max_abs_delta": 0.1087 }, "electricity_consumption": { "incumbent_share": 0.86209, "direction": "candidate_above", - "max_abs_delta": 0.10231000000000001 + "max_abs_delta": 0.1024 }, "gas_consumption": { "incumbent_share": 0.953241, "direction": "candidate_above", - "max_abs_delta": 0.04195899999999997 + "max_abs_delta": 0.042 }, "health_consumption": { "incumbent_share": 0.512849, "direction": "candidate_above", - "max_abs_delta": 0.02575099999999997 + "max_abs_delta": 0.0258 }, "household_furnishings_consumption": { "incumbent_share": 0.813553, "direction": "candidate_above", - "max_abs_delta": 0.108047 + "max_abs_delta": 0.1081 }, "miscellaneous_consumption": { "incumbent_share": 0.900333, "direction": "candidate_above", - "max_abs_delta": 0.09146699999999996 + "max_abs_delta": 0.0915 }, "restaurants_and_hotels_consumption": { "incumbent_share": 0.632441, "direction": "candidate_above", - "max_abs_delta": 0.15785899999999997 + "max_abs_delta": 0.1579 } - } + }, + "magnitude_provenance": "max_abs_delta re-measured 2026-08-26 on the 25-stage candidate spine-e (sha256 3c8799970851c409e4cb8578d33a180acb30ae600f4bd99ca3a190f9c5eb870a) against the packaged 1.56.16 reference, rounded up at 1e-4 grain; the prior bounds were the comparison ledger's rounded quotes measured on the 24-stage pre-SPI-zero-fix build." } }, { @@ -154,14 +156,15 @@ "diesel_spending": { "incumbent_share": 0.191027, "direction": "candidate_below", - "max_abs_delta": 0.033027 + "max_abs_delta": 0.0331 }, "petrol_spending": { "incumbent_share": 0.444632, "direction": "candidate_below", - "max_abs_delta": 0.144432 + "max_abs_delta": 0.1445 } - } + }, + "magnitude_provenance": "max_abs_delta re-measured 2026-08-26 on the 25-stage candidate spine-e (sha256 3c8799970851c409e4cb8578d33a180acb30ae600f4bd99ca3a190f9c5eb870a) against the packaged 1.56.16 reference, rounded up at 1e-4 grain; the prior bounds were the comparison ledger's rounded quotes measured on the 24-stage pre-SPI-zero-fix build." } }, { @@ -187,14 +190,15 @@ "alcohol_and_tobacco_consumption": { "incumbent_share": 0.560288, "direction": "candidate_below", - "max_abs_delta": 0.045287999999999995 + "max_abs_delta": 0.0453 }, "transport_consumption": { "incumbent_share": 0.866802, "direction": "candidate_above", - "max_abs_delta": 0.02659800000000001 + "max_abs_delta": 0.0266 } - } + }, + "magnitude_provenance": "max_abs_delta re-measured 2026-08-26 on the 25-stage candidate spine-e (sha256 3c8799970851c409e4cb8578d33a180acb30ae600f4bd99ca3a190f9c5eb870a) against the packaged 1.56.16 reference, rounded up at 1e-4 grain; the prior bounds were the comparison ledger's rounded quotes measured on the 24-stage pre-SPI-zero-fix build." } }, { @@ -220,14 +224,15 @@ "bus_subsidy_spending": { "incumbent_share": 0.316675, "direction": "candidate_above", - "max_abs_delta": 0.23872500000000002 + "max_abs_delta": 0.2388 }, "dfe_education_spending": { "incumbent_share": 0.000265, "direction": "candidate_above", - "max_abs_delta": 0.225535 + "max_abs_delta": 0.2256 } - } + }, + "magnitude_provenance": "max_abs_delta re-measured 2026-08-26 on the 25-stage candidate spine-e (sha256 3c8799970851c409e4cb8578d33a180acb30ae600f4bd99ca3a190f9c5eb870a) against the packaged 1.56.16 reference, rounded up at 1e-4 grain; the prior bounds were the comparison ledger's rounded quotes measured on the 24-stage pre-SPI-zero-fix build." } }, { @@ -256,29 +261,30 @@ "corporate_wealth": { "incumbent_share": 0.822181, "direction": "candidate_below", - "max_abs_delta": 0.04298100000000005 + "max_abs_delta": 0.0431 }, "main_residence_value": { "incumbent_share": 0.674658, "direction": "candidate_below", - "max_abs_delta": 0.039057999999999926 + "max_abs_delta": 0.0391 }, "other_residential_property_value": { "incumbent_share": 0.076316, "direction": "candidate_below", - "max_abs_delta": 0.03961599999999999 + "max_abs_delta": 0.0397 }, "property_wealth": { "incumbent_share": 0.708057, "direction": "candidate_below", - "max_abs_delta": 0.04735700000000009 + "max_abs_delta": 0.0474 }, "savings": { "incumbent_share": 0.661999, "direction": "candidate_below", - "max_abs_delta": 0.051298999999999984 + "max_abs_delta": 0.0513 } - } + }, + "magnitude_provenance": "max_abs_delta re-measured 2026-08-26 on the 25-stage candidate spine-e (sha256 3c8799970851c409e4cb8578d33a180acb30ae600f4bd99ca3a190f9c5eb870a) against the packaged 1.56.16 reference, rounded up at 1e-4 grain; the prior bounds were the comparison ledger's rounded quotes measured on the 24-stage pre-SPI-zero-fix build." } }, { @@ -303,9 +309,10 @@ "student_loan_balance": { "incumbent_share": 0.019707, "direction": "candidate_above", - "max_abs_delta": 0.029592999999999998 + "max_abs_delta": 0.0297 } - } + }, + "magnitude_provenance": "max_abs_delta re-measured 2026-08-26 on the 25-stage candidate spine-e (sha256 3c8799970851c409e4cb8578d33a180acb30ae600f4bd99ca3a190f9c5eb870a) against the packaged 1.56.16 reference, rounded up at 1e-4 grain; the prior bounds were the comparison ledger's rounded quotes measured on the 24-stage pre-SPI-zero-fix build." } }, { @@ -332,19 +339,20 @@ "employer_pension_contributions": { "incumbent_share": 0.314865, "direction": "candidate_below", - "max_abs_delta": 0.04666500000000001 + "max_abs_delta": 0.0467 }, "savings_interest_income": { "incumbent_share": 0.424963, "direction": "candidate_below", - "max_abs_delta": 0.030362999999999973 + "max_abs_delta": 0.0305 }, "tax_free_savings_income": { "incumbent_share": 0.189664, "direction": "candidate_below", - "max_abs_delta": 0.05446400000000001 + "max_abs_delta": 0.0545 } - } + }, + "magnitude_provenance": "max_abs_delta re-measured 2026-08-26 on the 25-stage candidate spine-e (sha256 3c8799970851c409e4cb8578d33a180acb30ae600f4bd99ca3a190f9c5eb870a) against the packaged 1.56.16 reference, rounded up at 1e-4 grain; the prior bounds were the comparison ledger's rounded quotes measured on the 24-stage pre-SPI-zero-fix build." } }, { @@ -369,9 +377,10 @@ "employee_pension_contributions": { "incumbent_share": 0.273489, "direction": "candidate_below", - "max_abs_delta": 0.04888899999999999 + "max_abs_delta": 0.0489 } - } + }, + "magnitude_provenance": "max_abs_delta re-measured 2026-08-26 on the 25-stage candidate spine-e (sha256 3c8799970851c409e4cb8578d33a180acb30ae600f4bd99ca3a190f9c5eb870a) against the packaged 1.56.16 reference, rounded up at 1e-4 grain; the prior bounds were the comparison ledger's rounded quotes measured on the 24-stage pre-SPI-zero-fix build." } }, { diff --git a/packages/microcosm-build/tests/test_country_spec.py b/packages/microcosm-build/tests/test_country_spec.py index f3691a1a5..12eb7fa73 100644 --- a/packages/microcosm-build/tests/test_country_spec.py +++ b/packages/microcosm-build/tests/test_country_spec.py @@ -311,6 +311,7 @@ def test_spi_spine_adds_no_country_package_resources(self) -> None: "take_up_contract.json", "input_mass_reviewed_exclusions.json", "spine_swap_signed_differences.json", + "spine_candidate_acceptance.json", "ledger_compile_parity_incumbent_2025_signed_differences.json", "ledger_compile_parity_production_2023_signed_differences.json", "national_staging_build_record.json", @@ -387,6 +388,7 @@ def test_uk_package_loads(self) -> None: "take_up_contract.json", "input_mass_reviewed_exclusions.json", "spine_swap_signed_differences.json", + "spine_candidate_acceptance.json", "ledger_compile_parity_incumbent_2025_signed_differences.json", "ledger_compile_parity_production_2023_signed_differences.json", "national_staging_build_record.json", diff --git a/packages/microcosm-build/tests/test_spec_engine_country_bundles.py b/packages/microcosm-build/tests/test_spec_engine_country_bundles.py index cc672a159..9d028071f 100644 --- a/packages/microcosm-build/tests/test_spec_engine_country_bundles.py +++ b/packages/microcosm-build/tests/test_spec_engine_country_bundles.py @@ -42,7 +42,7 @@ ), ( "uk", - "0c85845b4d463638ae3e5c5a25e17de8b720794e3653c5991dc4f069d95762d3", + "4793baa59a90930956faf8f5b422ece57da3445c7ed0030622ef62ff35dfb142", { "benunit.benunit_id", "household.household_id", diff --git a/packages/microcosm-build/tests/test_uk_identity_stability_receipts.py b/packages/microcosm-build/tests/test_uk_identity_stability_receipts.py index cc99d5684..cc663745b 100644 --- a/packages/microcosm-build/tests/test_uk_identity_stability_receipts.py +++ b/packages/microcosm-build/tests/test_uk_identity_stability_receipts.py @@ -288,3 +288,44 @@ def materialize(self, frame, variables, period): assert receipt["nhs_age_top_code"] == UK_AGE_TOP_CODE assert receipt["matches_stored_columns"] is True assert receipt["stored_column_mismatches"] == {} + + +def test_e8_carrier_recompute_is_invariant_to_the_age_tail_rewrite(): + """The donor stage picked carriers before age_tail ran; the recompute must too. + + Two adults tied at the top code at stage time: the stage's carrier is the + stable-order winner of that tie. After age_tail lifts one of them to 90, + an unclamped recompute flips the tie to the lifted person — the mechanism + behind the 255-donor mismatch on the first 25-stage ladder run — while the + stage-time clamp reproduces the stage's own choice exactly. + """ + + from microcosm.build.uk_runtime.age_tail import UK_AGE_TOP_CODE as TOP + from microcosm.build.uk_runtime.cgt_structure import _oldest_adult_indices + + stage_time = pd.DataFrame( + { + "person_id": [0, 1], + "person_household_id": [7, 7], + "age": [float(TOP), float(TOP)], + } + ) + after_age_tail = stage_time.assign(age=[float(TOP), 90.0]) + + stage_choice = _oldest_adult_indices(stage_time, household_ids={7}) + + unclamped = _oldest_adult_indices(after_age_tail, household_ids={7}) + assert unclamped.tolist() != stage_choice.tolist() + + clamped = after_age_tail.assign( + age=np.minimum( + pd.to_numeric(after_age_tail["age"], errors="coerce").to_numpy( + dtype=float + ), + float(TOP), + ) + ) + assert ( + _oldest_adult_indices(clamped, household_ids={7}).tolist() + == stage_choice.tolist() + ) diff --git a/packages/microcosm-build/tests/test_uk_spine_acceptance_receipt.py b/packages/microcosm-build/tests/test_uk_spine_acceptance_receipt.py new file mode 100644 index 000000000..361b5346a --- /dev/null +++ b/packages/microcosm-build/tests/test_uk_spine_acceptance_receipt.py @@ -0,0 +1,60 @@ +"""The committed spine acceptance receipt binds to the production plan. + +microcosm#771: the previous acceptance evidence quietly described a 24-stage +build after the plan had grown to 25. This binder makes that class of drift a +CI failure: the receipt's stage roster must equal the roster the spine driver +actually executes, its verdicts must be the accepted ones, and its identity +bases must name the stage-time contract the instruments verify. +""" + +from __future__ import annotations + +import importlib.util +import json +from importlib.resources import files +from pathlib import Path + +_DRIVER_PATH = ( + Path(__file__).resolve().parents[3] / "tools" / "build_uk_frs_spine.py" +) + + +def _receipt() -> dict: + return json.loads( + files("microcosm.build.uk") + .joinpath("spine_candidate_acceptance.json") + .read_text() + ) + + +def _driver_stage_names() -> tuple[str, ...]: + spec = importlib.util.spec_from_file_location("build_uk_frs_spine", _DRIVER_PATH) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return tuple(module._STAGE_NAMES) + + +def test_receipt_roster_is_the_production_plan(): + receipt = _receipt() + roster = tuple(receipt["candidate"]["stage_roster"]) + assert roster == _driver_stage_names() + assert receipt["candidate"]["stage_count"] == len(roster) + + +def test_receipt_identity_and_verdicts_are_the_accepted_ones(): + receipt = _receipt() + assert len(receipt["candidate"]["sha256"]) == 64 + assert int(receipt["candidate"]["entity_row_counts"]["household"]) == 52846 + assert receipt["twin"]["payload_identical"] is True + ladder = receipt["identity_ladder"] + assert set(ladder) == {"e4", "e5", "e6", "e7", "e8"} + for check, row in ladder.items(): + assert row["identical_under_permutation"] is True, check + assert row["matches_stored_columns"] is True, check + assert ladder["e6"]["nhs_age_basis"] == "stage_time_top_coded" + assert ladder["e8"]["donor_age_basis"] == "stage_time_top_coded" + parity = receipt["strict_parity"] + assert parity["verdict"] == "signed_parity" + assert parity["unsigned_differences"] == 0 + assert parity["strict_failure"] is False + assert parity["share_band"]["effective"] == parity["share_band"]["contract"] diff --git a/tools/verify_uk_identity_stability.py b/tools/verify_uk_identity_stability.py index ef3eb5764..2b9246aa9 100644 --- a/tools/verify_uk_identity_stability.py +++ b/tools/verify_uk_identity_stability.py @@ -726,12 +726,21 @@ def e8_identity_receipt( problems["clone_half_masses"] = [float(left.sum()), float(right.sum())] # (2) Band-donor selection recomputed from the committed resources. + # Same contract as the E6 NHS check: the donor stage ran on the top-coded + # FRS age surface, four stages before age_tail disaggregated it, so its + # oldest-adult carriers are stage-time-age-derived. Recompute on + # min(age, top_code) — exact, because age_tail refuses inputs above the + # top code and rewrites only persons at exactly it, upward. distribution = load_advani_summers_distribution() bands = _retained_size_bands(load_hmrc_cgt_size_bands()) non_donor_ids = set(non_donor["household_id"].tolist()) nd_person = person.loc[ person["person_household_id"].isin(non_donor_ids) ].reset_index(drop=True) + nd_person["age"] = np.minimum( + pd.to_numeric(nd_person["age"], errors="coerce").to_numpy(dtype=float), + float(UK_AGE_TOP_CODE), + ) nd_benunit = benunit.loc[ benunit["benunit_id"].isin(set(nd_person["person_benunit_id"].tolist())) ].reset_index(drop=True) @@ -801,7 +810,15 @@ def select_donors(person_t: pd.DataFrame) -> np.ndarray: problems["donor_stored_weights"] = True donor_person = person.loc[ person["person_household_id"].isin(set(stored_donors["household_id"])) - ] + ].copy() + # Stage-time age basis again: the stage picked each donor household's + # carrier before age_tail ran. + donor_person["age"] = np.minimum( + pd.to_numeric(donor_person["age"], errors="coerce").to_numpy( + dtype=float + ), + float(UK_AGE_TOP_CODE), + ) carrier_rows = _oldest_adult_indices( donor_person, household_ids=set(stored_donors["household_id"]) ) @@ -847,6 +864,7 @@ def select_donors(person_t: pd.DataFrame) -> np.ndarray: structural_ok = not problems return { "check": "uk_e8_identity_stability", + "donor_age_basis": "stage_time_top_coded", "permutation_seed": permutation_seed, "identical_under_permutation": bool( "donor_selection_permutation" not in problems From b768788fe870181a176268215113f3662f34df86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Wed, 26 Aug 2026 16:53:33 +0200 Subject: [PATCH 07/14] Give reviewed exclusions a lifetime, and re-sign owned_land on fresh evidence MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The calibration measure-exclusion register migrates to the weighted-integrity record shape: approver, adjudication, canonical ISO approval and expiry dates, with the window enforced when exclusions apply. Outside it the run refuses with a correct-or-renew message naming the tracked gap, so a narrowing of the target surface neither lapses silently nor lives forever — the gap the #743 audit named. The five salary-sacrifice entries carry Maria's three-month window under that adjudication. The owned_land input-mass exclusion was due to expire 2026-09-20 on E5-era evidence. The stability instrument re-ran on the 25-stage candidate — the E5 method, adapted to strip post-wealth stage columns, drop the stacked SPI and CGT rows, and clamp age to the stage-time top code — and the instability persists: 53.8 percent national and 96.7 percent worst-region owned_land swing between adjacent seeds, the uk-data#448 realization-variance class. The exclusion re-signs on the fresh receipt with its one-month expiry and the end-of-workstream revisit intact; the input-mass evidence pin and the contract-test mirror follow the re-signed record. Co-Authored-By: Claude Fable 5 --- changelog.d/757-exclusion-windows.changed.md | 1 + .../uk/calibration_measure_exclusions.json | 32 ++++-- .../uk/input_mass_reviewed_exclusions.json | 38 ++++---- .../build/uk_runtime/measure_simulation.py | 91 ++++++++++++++--- .../tests/test_uk_measure_simulation.py | 97 ++++++++++++++----- .../src/microcosm/data/contract.py | 2 +- .../microcosm-data/tests/test_contract.py | 22 +---- 7 files changed, 198 insertions(+), 85 deletions(-) create mode 100644 changelog.d/757-exclusion-windows.changed.md diff --git a/changelog.d/757-exclusion-windows.changed.md b/changelog.d/757-exclusion-windows.changed.md new file mode 100644 index 000000000..59b573ebd --- /dev/null +++ b/changelog.d/757-exclusion-windows.changed.md @@ -0,0 +1 @@ +Reviewed exclusions on the calibration path now carry the full adjudication record and a lifetime. The measure-exclusion register migrates to the weighted-integrity record shape — approver, adjudication, canonical ISO approval and expiry dates — and the window is enforced when exclusions apply: outside it the run refuses with a correct-or-renew message naming the tracked gap, so a narrowing of the target surface neither lapses silently nor lives forever. The five salary-sacrifice entries carry a three-month window under the #743-audit adjudication. The owned_land input-mass exclusion is re-signed on a fresh seed-stability receipt measured on the 25-stage candidate — 53.8 percent national and 96.7 percent worst-region swing between adjacent seeds, the uk-data#448 realization-variance class reproduced with the E5 instrument's method — keeping its one-month expiry and the end-of-workstream revisit registered on #145. diff --git a/packages/microcosm-build/src/microcosm/build/uk/calibration_measure_exclusions.json b/packages/microcosm-build/src/microcosm/build/uk/calibration_measure_exclusions.json index 087d6637a..21d0ac057 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/calibration_measure_exclusions.json +++ b/packages/microcosm-build/src/microcosm/build/uk/calibration_measure_exclusions.json @@ -1,30 +1,50 @@ { - "schema_version": 1, + "schema_version": 2, "exclusions": [ { "name": "hmrc.salary_sacrifice.it_relief_basic_rate", "reason": "The frame adapter reads only a precomputed slash-named delta column; no artifact ships a live counterfactual route.", - "tracking": "microcosm#623" + "tracking": "microcosm#623", + "approved_by": "juaristi22", + "adjudication": "microcosm#757 (the #743-audit adjudication, issue comment 5413502559)", + "approved_on": "2026-08-25", + "expires_on": "2026-11-25" }, { "name": "hmrc.salary_sacrifice.it_relief_higher_rate", "reason": "The frame adapter reads only a precomputed slash-named delta column; no artifact ships a live counterfactual route.", - "tracking": "microcosm#623" + "tracking": "microcosm#623", + "approved_by": "juaristi22", + "adjudication": "microcosm#757 (the #743-audit adjudication, issue comment 5413502559)", + "approved_on": "2026-08-25", + "expires_on": "2026-11-25" }, { "name": "hmrc.salary_sacrifice.it_relief_additional_rate", "reason": "The frame adapter reads only a precomputed slash-named delta column; no artifact ships a live counterfactual route.", - "tracking": "microcosm#623" + "tracking": "microcosm#623", + "approved_by": "juaristi22", + "adjudication": "microcosm#757 (the #743-audit adjudication, issue comment 5413502559)", + "approved_on": "2026-08-25", + "expires_on": "2026-11-25" }, { "name": "hmrc.salary_sacrifice.nics_relief_employee", "reason": "The frame adapter reads only a precomputed slash-named delta column; no artifact ships a live counterfactual route.", - "tracking": "microcosm#623" + "tracking": "microcosm#623", + "approved_by": "juaristi22", + "adjudication": "microcosm#757 (the #743-audit adjudication, issue comment 5413502559)", + "approved_on": "2026-08-25", + "expires_on": "2026-11-25" }, { "name": "hmrc.salary_sacrifice.nics_relief_employer", "reason": "The frame adapter reads only a precomputed slash-named delta column; no artifact ships a live counterfactual route.", - "tracking": "microcosm#623" + "tracking": "microcosm#623", + "approved_by": "juaristi22", + "adjudication": "microcosm#757 (the #743-audit adjudication, issue comment 5413502559)", + "approved_on": "2026-08-25", + "expires_on": "2026-11-25" } ] } diff --git a/packages/microcosm-build/src/microcosm/build/uk/input_mass_reviewed_exclusions.json b/packages/microcosm-build/src/microcosm/build/uk/input_mass_reviewed_exclusions.json index 995d4824a..3a7d346bb 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/input_mass_reviewed_exclusions.json +++ b/packages/microcosm-build/src/microcosm/build/uk/input_mass_reviewed_exclusions.json @@ -1,22 +1,22 @@ { - "schema_version": 3, - "description": "Reviewed input-mass parity exclusions for the UK terminal battery, scoped per named comparison reference (microcosm#630): an exclusion suppresses only when its reference is the armed one, so a column excluded on a channel-blind reference is compared normally against a channel-aware one with no register change. Each entry is a complete approval receipt (schema-2 fields per #610). The gate reports dormant entries, FAILS stale ones, and FAILS expired ones until the adjudication is renewed, so this register cannot rot.", - "references": { - "efrs-post-calibration": { - "charitable_investment_gifts": { - "reason": "SPI-channel-exclusive column on a channel-blind reference: the efrs-post-calibration incumbent structurally lacks the SPI clone channel, so its reference mass is survey-side scraps while the staged candidate's mass is the admin-captured SPI channel functioning as designed (microcosm#630 case 2). Compared normally against any future channel-aware reference.", - "approved_by": "juaristi22", - "adjudication": "microcosm#630", - "approved_on": "2026-08-20", - "expires_on": "2027-02-20" - }, - "owned_land": { - "reason": "Sparse heavy-tailed WAS donor column (0.7 percent weighted nonzero share) whose weighted total is dominated by a handful of large farm/estate records: the E5 stability receipt (data/ukds/acceptance/e5/owned_land_stability_receipt.json) measures a 37.7 percent national and 2.41x London swing between adjacent seeds, the same realization-variance class the archived incumbent data repo records at uk-data#448 (4.6x Wales swing across releases). Register parity at this grain is not meaningful until the whole-spine comparison; the one-month expiry enforces the end-of-workstream revisit registered on microcosm#145 (winsorised donor or separate land imputation are the candidate remedies).", - "approved_by": "juaristi22", - "adjudication": "microcosm#714", - "approved_on": "2026-08-20", - "expires_on": "2026-09-20" - } + "schema_version": 3, + "description": "Reviewed input-mass parity exclusions for the UK terminal battery, scoped per named comparison reference (microcosm#630): an exclusion suppresses only when its reference is the armed one, so a column excluded on a channel-blind reference is compared normally against a channel-aware one with no register change. Each entry is a complete approval receipt (schema-2 fields per #610). The gate reports dormant entries, FAILS stale ones, and FAILS expired ones until the adjudication is renewed, so this register cannot rot.", + "references": { + "efrs-post-calibration": { + "charitable_investment_gifts": { + "reason": "SPI-channel-exclusive column on a channel-blind reference: the efrs-post-calibration incumbent structurally lacks the SPI clone channel, so its reference mass is survey-side scraps while the staged candidate's mass is the admin-captured SPI channel functioning as designed (microcosm#630 case 2). Compared normally against any future channel-aware reference.", + "approved_by": "juaristi22", + "adjudication": "microcosm#630", + "approved_on": "2026-08-20", + "expires_on": "2027-02-20" + }, + "owned_land": { + "reason": "Sparse heavy-tailed WAS donor column (0.7 percent weighted nonzero share) whose weighted total is dominated by a handful of large farm/estate records: the spine-e stability receipt (data/ukds/acceptance/757-swap/owned_land_stability_receipt_spine_e.json) measures a 53.8 percent national and 96.7 percent West Midlands swing between adjacent seeds on the 25-stage candidate \u2014 the realization-variance class the archived incumbent data repo records at uk-data#448 (4.6x Wales swing across releases), reproduced from the E5 instrument's method. Register parity at this grain stays not meaningful; the one-month expiry keeps the end-of-workstream revisit registered on microcosm#145 live (winsorised donor or separate land imputation are the candidate remedies).", + "approved_by": "juaristi22", + "adjudication": "microcosm#714", + "approved_on": "2026-08-26", + "expires_on": "2026-09-26" + } + } } - } } diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/measure_simulation.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/measure_simulation.py index b28beee1f..c2210865e 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/measure_simulation.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/measure_simulation.py @@ -3,6 +3,7 @@ from __future__ import annotations import json +from datetime import date from importlib import resources as importlib_resources from pathlib import Path from typing import Any @@ -14,6 +15,9 @@ load_uk_national_frame, write_uk_national_frame, ) +from microcosm.build.uk_runtime.weighted_integrity import ( + exclusion_evaluation_date, +) from microcosm.calibrate import TargetRegistry _ENTITY_LINK = {"benunit": "person_benunit_id", "household": "person_household_id"} @@ -156,6 +160,22 @@ def receipt(self) -> dict[str, str]: return dict(self._receipt) +#: Every field a reviewed measure exclusion must carry (the +#: ``weighted_integrity`` reviewed-exclusion record shape, microcosm#757 / +#: the #743-audit adjudication): an exclusion narrows the calibrated target +#: surface, so it names who approved the narrowing, under which adjudication, +#: and for how long — nothing lapses silently and nothing lives forever. +_UK_MEASURE_EXCLUSION_FIELDS = ( + "name", + "reason", + "tracking", + "approved_by", + "adjudication", + "approved_on", + "expires_on", +) + + def load_uk_calibration_measure_exclusions( path: Path | None = None, ) -> tuple[dict[str, str], ...]: @@ -174,8 +194,8 @@ def load_uk_calibration_measure_exclusions( unknown = sorted(set(payload) - allowed_top) if unknown: raise ValueError(f"unknown top-level exclusion key(s): {unknown}") - if payload.get("schema_version") != 1: - raise ValueError("UK calibration measure exclusions schema_version must be 1.") + if payload.get("schema_version") != 2: + raise ValueError("UK calibration measure exclusions schema_version must be 2.") exclusions = payload.get("exclusions") if not isinstance(exclusions, list): raise ValueError("UK calibration measure exclusions must contain a list.") @@ -184,33 +204,71 @@ def load_uk_calibration_measure_exclusions( for entry in exclusions: if not isinstance(entry, dict): raise ValueError("UK calibration measure exclusion entries must be objects.") - name = str(entry.get("name", "")) - reason = str(entry.get("reason", "")) - tracking = str(entry.get("tracking", "")) + unknown_fields = sorted(set(entry) - set(_UK_MEASURE_EXCLUSION_FIELDS)) + if unknown_fields: + raise ValueError( + "unknown UK calibration measure exclusion field(s) " + f"{unknown_fields} on {entry.get('name')!r}." + ) + record = {field: str(entry.get(field, "")) for field in _UK_MEASURE_EXCLUSION_FIELDS} + name = record["name"] if name in seen: raise ValueError(f"duplicate UK calibration measure exclusion {name!r}.") - if not reason.strip(): - raise ValueError(f"UK calibration measure exclusion {name!r} has empty reason.") - if not tracking.strip(): + for field in _UK_MEASURE_EXCLUSION_FIELDS: + if not record[field].strip(): + raise ValueError( + f"UK calibration measure exclusion {name!r} has empty {field}; " + "a narrowed target surface names why, where it is being " + "resolved, who approved it, and for how long." + ) + for field in ("approved_on", "expires_on"): + try: + parsed = date.fromisoformat(record[field]) + except ValueError: + parsed = None + if parsed is None or parsed.isoformat() != record[field]: + raise ValueError( + f"UK calibration measure exclusion {name!r} {field} must be " + f"canonical ISO (YYYY-MM-DD), got {record[field]!r}." + ) + if record["expires_on"] <= record["approved_on"]: raise ValueError( - f"UK calibration measure exclusion {name!r} has empty tracking; " - "a narrowed target surface must name where it is being resolved." + f"UK calibration measure exclusion {name!r} expires_on must be " + "after approved_on." ) seen.add(name) - loaded.append({"name": name, "reason": reason, "tracking": tracking}) + loaded.append(record) return tuple(loaded) def apply_uk_calibration_measure_exclusions( - registry: TargetRegistry, exclusions: tuple[dict[str, str], ...] + registry: TargetRegistry, + exclusions: tuple[dict[str, str], ...], + *, + now: date | None = None, ) -> tuple[TargetRegistry, dict[str, dict[str, str]]]: """Remove reviewed excluded references from a UK target registry. The receipt carries every field the register declares, tracking included: an exclusion narrows the calibrated target surface, so the run evidence - must say where each narrowing is being resolved, not only why. + must say where each narrowing is being resolved, not only why. The window + is enforced at apply time — outside ``approved_on``..``expires_on`` the + run fails with a correct-or-renew message rather than the narrowing + lapsing silently or living forever. """ + evaluated_on = exclusion_evaluation_date(now) + for entry in exclusions: + approved = date.fromisoformat(entry["approved_on"]) + expires = date.fromisoformat(entry["expires_on"]) + if not approved <= evaluated_on <= expires: + raise ValueError( + f"UK calibration measure exclusion {entry['name']!r} is outside " + f"its reviewed window ({entry['approved_on']}..{entry['expires_on']}, " + f"evaluated {evaluated_on.isoformat()}): correct the underlying " + f"gap ({entry['tracking']}) or renew the adjudication with a new " + "approval and expiry." + ) declared = {entry["name"]: entry for entry in exclusions} matched = {spec.name for spec in registry.specs if spec.name in declared} stale = sorted(set(declared) - matched) @@ -222,11 +280,14 @@ def apply_uk_calibration_measure_exclusions( kept = [spec for spec in registry.specs if spec.name not in declared] receipt = { name: { - "reason": declared[name]["reason"], - "tracking": declared[name]["tracking"], + field: declared[name][field] + for field in _UK_MEASURE_EXCLUSION_FIELDS + if field != "name" } for name in sorted(matched) } + for record in receipt.values(): + record["evaluated_on"] = evaluated_on.isoformat() return TargetRegistry(kept, country=registry.country), receipt diff --git a/packages/microcosm-build/tests/test_uk_measure_simulation.py b/packages/microcosm-build/tests/test_uk_measure_simulation.py index ae7bb44b6..9054d78d6 100644 --- a/packages/microcosm-build/tests/test_uk_measure_simulation.py +++ b/packages/microcosm-build/tests/test_uk_measure_simulation.py @@ -130,17 +130,26 @@ def _write_exclusions(path: Path, payload: dict) -> Path: return path -def test_exclusion_loader_refusals(tmp_path: Path): - base = { - "schema_version": 1, - "exclusions": [ - {"name": "a", "reason": "reviewed reason", "tracking": "microcosm#623"} - ], +def _entry(**overrides) -> dict: + entry = { + "name": "a", + "reason": "reviewed reason", + "tracking": "microcosm#623", + "approved_by": "juaristi22", + "adjudication": "microcosm#757", + "approved_on": "2026-08-25", + "expires_on": "2026-11-25", } + entry.update(overrides) + return entry + + +def test_exclusion_loader_refusals(tmp_path: Path): + base = {"schema_version": 2, "exclusions": [_entry()]} with pytest.raises(ValueError, match="schema_version"): load_uk_calibration_measure_exclusions( - _write_exclusions(tmp_path / "bad-version.json", {**base, "schema_version": 2}) + _write_exclusions(tmp_path / "bad-version.json", {**base, "schema_version": 1}) ) with pytest.raises(ValueError, match="unknown top-level"): load_uk_calibration_measure_exclusions( @@ -150,20 +159,42 @@ def test_exclusion_loader_refusals(tmp_path: Path): load_uk_calibration_measure_exclusions( _write_exclusions( tmp_path / "empty.json", - {**base, "exclusions": [{"name": "a", "reason": "", "tracking": "x"}]}, + {**base, "exclusions": [_entry(reason="")]}, + ) + ) + with pytest.raises(ValueError, match="empty approved_by"): + load_uk_calibration_measure_exclusions( + _write_exclusions( + tmp_path / "unapproved.json", + {**base, "exclusions": [_entry(approved_by="")]}, + ) + ) + with pytest.raises(ValueError, match="canonical ISO"): + load_uk_calibration_measure_exclusions( + _write_exclusions( + tmp_path / "sloppy-date.json", + {**base, "exclusions": [_entry(approved_on="2026-8-25")]}, + ) + ) + with pytest.raises(ValueError, match="after approved_on"): + load_uk_calibration_measure_exclusions( + _write_exclusions( + tmp_path / "inverted.json", + {**base, "exclusions": [_entry(expires_on="2026-08-25")]}, + ) + ) + with pytest.raises(ValueError, match="unknown UK calibration measure exclusion field"): + load_uk_calibration_measure_exclusions( + _write_exclusions( + tmp_path / "extra-field.json", + {**base, "exclusions": [_entry(sneaky="value")]}, ) ) with pytest.raises(ValueError, match="duplicate"): load_uk_calibration_measure_exclusions( _write_exclusions( tmp_path / "duplicate.json", - { - **base, - "exclusions": [ - {"name": "a", "reason": "one", "tracking": "x"}, - {"name": "a", "reason": "two", "tracking": "x"}, - ], - }, + {**base, "exclusions": [_entry(), _entry(reason="two")]}, ) ) @@ -177,19 +208,38 @@ def test_exclusion_applier_returns_pruned_registry_and_receipt(): country="uk", ) + from datetime import date + + window = _entry(name="drop", reason="reviewed") pruned, receipt = apply_uk_calibration_measure_exclusions( - registry, - ({"name": "drop", "reason": "reviewed", "tracking": "microcosm#623"},), + registry, (window,), now=date(2026, 9, 1) ) assert [spec.name for spec in pruned.specs] == ["keep"] assert receipt == { - "drop": {"reason": "reviewed", "tracking": "microcosm#623"} + "drop": { + "reason": "reviewed", + "tracking": "microcosm#623", + "approved_by": "juaristi22", + "adjudication": "microcosm#757", + "approved_on": "2026-08-25", + "expires_on": "2026-11-25", + "evaluated_on": "2026-09-01", + } } with pytest.raises(ValueError, match="matched zero"): apply_uk_calibration_measure_exclusions( - registry, - ({"name": "stale", "reason": "reviewed", "tracking": "microcosm#623"},), + registry, (_entry(name="stale"),), now=date(2026, 9, 1) + ) + # Outside the reviewed window the run refuses with correct-or-renew — + # the narrowing neither lapses silently nor lives forever. + with pytest.raises(ValueError, match="correct the underlying gap"): + apply_uk_calibration_measure_exclusions( + registry, (window,), now=date(2026, 11, 26) + ) + with pytest.raises(ValueError, match="correct the underlying gap"): + apply_uk_calibration_measure_exclusions( + registry, (window,), now=date(2026, 8, 24) ) @@ -300,11 +350,6 @@ def test_exclusion_loader_requires_tracking(tmp_path: Path): load_uk_calibration_measure_exclusions( _write_exclusions( tmp_path / "untracked.json", - { - "schema_version": 1, - "exclusions": [ - {"name": "a", "reason": "reviewed reason", "tracking": ""} - ], - }, + {"schema_version": 2, "exclusions": [_entry(tracking="")]}, ) ) diff --git a/packages/microcosm-data/src/microcosm/data/contract.py b/packages/microcosm-data/src/microcosm/data/contract.py index 12e81ff7d..95bdef0b9 100644 --- a/packages/microcosm-data/src/microcosm/data/contract.py +++ b/packages/microcosm-data/src/microcosm/data/contract.py @@ -499,7 +499,7 @@ # canonical hash; this pins the wrapped digest so the entry's evidence line # still binds the enhanced-FRS incumbent totals. _UK_GATE_BATTERY_INPUT_MASS_EVIDENCE_SHA256 = ( - "16093e8605ac4bf9cf63fd66967c7b50fa80e29761443e8c6d37551e2d3b1fee" + "c9211cbb923e13f4850b834b5bdb1ff1de87fe9237c332b5de63f01ed417aa2d" ) # The degenerate binding's evidence payload digests the resolved exclusion # records; for a release that must be the committed register, so its digest diff --git a/packages/microcosm-data/tests/test_contract.py b/packages/microcosm-data/tests/test_contract.py index 5f712df02..7d96a0bce 100644 --- a/packages/microcosm-data/tests/test_contract.py +++ b/packages/microcosm-data/tests/test_contract.py @@ -98,25 +98,11 @@ "expires_on": "2027-02-20", }, "owned_land": { - "reason": ( - "Sparse heavy-tailed WAS donor column (0.7 percent weighted " - "nonzero share) whose weighted total is dominated by a handful " - "of large farm/estate records: the E5 stability receipt " - "(data/ukds/acceptance/e5/owned_land_stability_receipt.json) " - "measures a 37.7 percent national and 2.41x London swing " - "between adjacent seeds, the same realization-variance class " - "the archived incumbent data repo records at uk-data#448 (4.6x " - "Wales swing across releases). Register parity at this grain " - "is not meaningful " - "until the whole-spine comparison; the " - "one-month expiry enforces the end-of-workstream revisit " - "registered on microcosm#145 (winsorised donor or separate " - "land imputation are the candidate remedies)." - ), + "reason": "Sparse heavy-tailed WAS donor column (0.7 percent weighted nonzero share) whose weighted total is dominated by a handful of large farm/estate records: the spine-e stability receipt (data/ukds/acceptance/757-swap/owned_land_stability_receipt_spine_e.json) measures a 53.8 percent national and 96.7 percent West Midlands swing between adjacent seeds on the 25-stage candidate \u2014 the realization-variance class the archived incumbent data repo records at uk-data#448 (4.6x Wales swing across releases), reproduced from the E5 instrument's method. Register parity at this grain stays not meaningful; the one-month expiry keeps the end-of-workstream revisit registered on microcosm#145 live (winsorised donor or separate land imputation are the candidate remedies).", "approved_by": "juaristi22", "adjudication": "microcosm#714", - "approved_on": "2026-08-20", - "expires_on": "2026-09-20", + "approved_on": "2026-08-26", + "expires_on": "2026-09-26", }, } GIT_COMMIT = "5fa48f07436a806ad75ff76fd22cfb8613bddbe0" @@ -160,7 +146,7 @@ def _trusted_terminal_gate_signing_key(monkeypatch) -> None: "d0d024043132fa07c378c393dbe2b24fe99bf19e876bcc39997d2c80cc9bd4f6" ) UK_GATE_BATTERY_INPUT_MASS_EVIDENCE_SHA256 = ( - "16093e8605ac4bf9cf63fd66967c7b50fa80e29761443e8c6d37551e2d3b1fee" + "c9211cbb923e13f4850b834b5bdb1ff1de87fe9237c332b5de63f01ed417aa2d" ) #: Spec entry id -> (neutral gate name, phase, legacy detail-schema name). UK_GATE_BATTERY_ENTRIES = { From 0e38b8adf65a8a6de33a45b30315cc77a3d4be9d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Wed, 26 Aug 2026 17:17:02 +0200 Subject: [PATCH 08/14] Bind four targets to the facts they publish MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The audit partition over the first live seam run traced four of the worst target-fit failures to bindings measuring something other than their published fact, not to the data. The OBR total-NICs line bound ni_employee only — the proof was its final estimate sitting bit-identical to the separate employee-component target while employer NICs, seventy-three percent of the missing mass, went unmeasured. The binding's own note called it a suspected defect carried for parity. It now binds total_national_insurance; the near-redundancy with the three separately-calibrated component lines is documented in the note, kept so the published total binds honestly, and the residual is Class 1A/1B on benefits in kind, which the model does not carry. Three ONS household-composition categories lacked the restriction that makes them ONS categories. The family binds one MECE partition of all UK households, but the bindings' union counted 38.6 million households on a 28.5 million frame: a two-adult sharer household landed in four categories at once. Couple-no-children counted every household containing a childless couple and gains the exactly-two-persons restriction that makes it the one-family category; the two lone-person categories gain person count of one, which also turns their age-sum reduction into the person's own age, making the head-age proxy exact rather than approximate. On raw weights the corrected couple-no-children lands within ten percent of the fact — the sixty-four percent miss was the optimizer trading away a target no reweighting could satisfy. The unrelated-adult and lone-parent-non-dependent split stays unfixed by design: the two bindings are provably the same set seventeen households apart, and separating them needs a relationship-to-head surface the frame does not carry. That pair, and multi-family, go to signed exclusions pending that column rather than to bindings that pretend to measure the difference. Co-Authored-By: Claude Fable 5 --- changelog.d/757-target-binding-fixes.fixed.md | 1 + .../build/uk/uk_national_targets.json | 38 +++++++++++++++---- 2 files changed, 31 insertions(+), 8 deletions(-) create mode 100644 changelog.d/757-target-binding-fixes.fixed.md diff --git a/changelog.d/757-target-binding-fixes.fixed.md b/changelog.d/757-target-binding-fixes.fixed.md new file mode 100644 index 000000000..bc8ffca39 --- /dev/null +++ b/changelog.d/757-target-binding-fixes.fixed.md @@ -0,0 +1 @@ +Four UK calibration targets measured something other than their published fact, found by the microcosm#757 audit partition over the first live seam run. The OBR total-NICs line bound ni_employee only — its estimate was bit-identical to the separate employee-component target while employer NICs went unmeasured — and now binds total_national_insurance, with the known redundancy against the three component lines noted in the binding. Three ONS household-composition categories lacked their one-family or one-person restrictions, so a household could be counted in up to four categories at once and the family's union over-covered the frame by ten million households: couple-no-children gains the exactly-two-persons restriction and the two lone-person categories gain person count of one, which also makes their age-sum head-age proxy exact. diff --git a/packages/microcosm-build/src/microcosm/build/uk/uk_national_targets.json b/packages/microcosm-build/src/microcosm/build/uk/uk_national_targets.json index 4cff07bef..b1380ab9d 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/uk_national_targets.json +++ b/packages/microcosm-build/src/microcosm/build/uk/uk_national_targets.json @@ -677,8 +677,8 @@ "hmrc.salary_sacrifice.it_relief_basic_rate": "New declaration backed by chronicle#139 salary-sacrifice facts; the incumbent source CSV is still 410-Gone at 12a1e028 and its rows are silently suppressed by the bare-except drop in build_loss_matrix.py:376-394.", "hmrc.salary_sacrifice.it_relief_higher_rate": "New declaration backed by chronicle#139 salary-sacrifice facts; the incumbent source CSV is still 410-Gone at 12a1e028 and its rows are silently suppressed by the bare-except drop in build_loss_matrix.py:376-394.", "hmrc.salary_sacrifice.it_relief_additional_rate": "New declaration backed by chronicle#139 salary-sacrifice facts; the incumbent source CSV is still 410-Gone at 12a1e028 and its rows are silently suppressed by the bare-except drop in build_loss_matrix.py:376-394.", - "ons.population.female_90_plus": "María ruling 2026-08-21: split the incumbent terminal sex bands into uniform five-year 85_89 bands plus 90_plus tails. The incumbent six-year 85_90 terminal rows leave ages 91+ unconstrained; Microcosm keeps the uniform five-year bands and constrains the tail. The 85_89-vs-85_90 value gap is the single-age-90 share and is a signed semantic difference.", - "ons.population.male_90_plus": "María ruling 2026-08-21: split the incumbent terminal sex bands into uniform five-year 85_89 bands plus 90_plus tails. The incumbent six-year 85_90 terminal rows leave ages 91+ unconstrained; Microcosm keeps the uniform five-year bands and constrains the tail. The 85_89-vs-85_90 value gap is the single-age-90 share and is a signed semantic difference." + "ons.population.female_90_plus": "Mar\u00eda ruling 2026-08-21: split the incumbent terminal sex bands into uniform five-year 85_89 bands plus 90_plus tails. The incumbent six-year 85_90 terminal rows leave ages 91+ unconstrained; Microcosm keeps the uniform five-year bands and constrains the tail. The 85_89-vs-85_90 value gap is the single-age-90 share and is a signed semantic difference.", + "ons.population.male_90_plus": "Mar\u00eda ruling 2026-08-21: split the incumbent terminal sex bands into uniform five-year 85_89 bands plus 90_plus tails. The incumbent six-year 85_90 terminal rows leave ages 91+ unconstrained; Microcosm keeps the uniform five-year bands and constrains the tail. The 85_89-vs-85_90 value gap is the single-age-90 share and is a signed semantic difference." }, "suppressed_ancestors": { "hmrc/salary_sacrifice_it_relief_basic_rate": { @@ -708,7 +708,7 @@ "salary_sacrifice_obr_near_equivalents": "The contract separately maps OBR near-equivalents obr/salary_sacrifice_employee_ni_relief and obr/salary_sacrifice_employer_ni_relief onto hmrc.salary_sacrifice.nics_relief_employee and hmrc.salary_sacrifice.nics_relief_employer.", "two_child_limit_year_surface": "Chronicle's 15 dwp/uc/two_child_limit/* facts are the April-2025 publication (period month 2025-04) and are the only TCL facts; there are no 2026 Ledger facts. At 12a1e028, the incumbent pins the same published numbers at {2026}, so at calibration year 2025 Ledger activates all 15 TCL targets while the incumbent drops them from the 652-row union to the 637-row effective surface; the 2026 effective surface remains 652. The same 12a1e028 refresh adds the new incumbent dwp/uc/households row. Fixture B treats the 15 TCL rows as enumerated ledger-side additions.", "geography_level_order_canonicalization": "The 9 voa.council_tax_stock.* rows use country, region order to match the rest of the contract; consumers use membership only.", - "ons_terminal_sex_band_split": "María ruling 2026-08-21: split the incumbent terminal sex bands into uniform five-year 85_89 bands plus 90_plus tails. The incumbent six-year 85_90 terminal rows leave ages 91+ unconstrained; Microcosm keeps the uniform five-year bands and constrains the tail. The 85_89-vs-85_90 value gap is the single-age-90 share and is a signed semantic difference." + "ons_terminal_sex_band_split": "Mar\u00eda ruling 2026-08-21: split the incumbent terminal sex bands into uniform five-year 85_89 bands plus 90_plus tails. The incumbent six-year 85_90 terminal rows leave ages 91+ unconstrained; Microcosm keeps the uniform five-year bands and constrains the tail. The 85_89-vs-85_90 value gap is the single-age-90 share and is a signed semantic difference." } }, "targets": [ @@ -756,10 +756,10 @@ "bindings": { "policyengine": { "metric_name": "obr/ni", - "value_variable": "ni_employee", + "value_variable": "total_national_insurance", "from_entity": "person", "map_to": "household", - "notes": "uk-data binds this EFO total-NI line to ni_employee only, while separate employee/employer/self-employed lines are also calibrated - suspected binding defect carried as-is for parity; adjudicate in microcosm#622." + "notes": "The EFO line is total NICs receipts; binds total_national_insurance (class 1 employee + employer + classes 2-4). The prior ni_employee-only binding was the uk-data defect carried for parity \u2014 final estimate bit-identical to obr.ni_employee \u2014 fixed under the microcosm#757 audit partition (2026-08-26). Nearly redundant with the three separately-calibrated component lines by construction; kept so the published total binds honestly. Residual gap is Class 1A/1B on benefits in kind, which the model does not carry." }, "axiom": { "metric_name": "obr/ni", @@ -7842,10 +7842,17 @@ "reduce": "sum", "operator": "<", "value": 65 + }, + { + "variable": "age", + "entity": "person", + "reduce": "count", + "operator": "==", + "value": 1 } ], "value_variable": "household_count", - "notes": "uk-data reads ctx.pe('age') - person ages summed to household - as the head's age; exact only for single-person households (compute/households.py:12)." + "notes": "ONS 'one person household': person count == 1 makes the category exact, and the age-sum reduction becomes the person's own age, so the head-age proxy note no longer applies. Without the restriction every all-adult SINGLE-benunit sharer household was also counted here (microcosm#757 audit partition, 2026-08-26)." }, "axiom": { "metric_name": "ons/lone_households_under_65", @@ -7897,10 +7904,17 @@ "reduce": "sum", "operator": ">=", "value": 65 + }, + { + "variable": "age", + "entity": "person", + "reduce": "count", + "operator": "==", + "value": 1 } ], "value_variable": "household_count", - "notes": "uk-data reads ctx.pe('age') - person ages summed to household - as the head's age; exact only for single-person households (compute/households.py:12)." + "notes": "ONS 'one person household': person count == 1 makes the category exact, and the age-sum reduction becomes the person's own age, so the head-age proxy note no longer applies. Without the restriction every all-adult SINGLE-benunit sharer household was also counted here (microcosm#757 audit partition, 2026-08-26)." }, "axiom": { "metric_name": "ons/lone_households_over_65", @@ -7992,9 +8006,17 @@ "reduce": "any", "operator": "==", "value": "COUPLE_NO_CHILDREN" + }, + { + "variable": "age", + "entity": "person", + "reduce": "count", + "operator": "==", + "value": 2 } ], - "value_variable": "household_count" + "value_variable": "household_count", + "notes": "ONS Table 7 'one family only: couple, no children' is the household being exactly one childless couple and nobody else; with the childless-couple benunit present, person count == 2 is equivalent to the one-family restriction. Without it the binding counted every household *containing* such a couple \u2014 a 1.84m overlap with the couple-non-dependent-children and multi-family categories (microcosm#757 audit partition, 2026-08-26)." }, "axiom": { "metric_name": "ons/couple_no_children_households", From 28a701d85930da0b8414c113b5e1b98739607656 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Wed, 26 Aug 2026 17:24:31 +0200 Subject: [PATCH 09/14] Offer each battery boundary only the stages that have run MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The first licensed spine build under the battery died at the assembled boundary: the stage-evidence provider consulted all twenty-five implementations, and an un-run stage's checkpoint hook correctly refuses to invent evidence for a run that has not happened. The hermetic battery tests covered the gates, the scopes, and the seam's refusal, but never drove the driver's boundary wiring — the licensed build was the first to. Each boundary now offers exactly its executed prefix: the provider takes the executed stage names and the driver passes the stages the plan has actually run at that point. The un-run-stage refusal stays as it is — asking for evidence of a run that has not happened should raise; the fix is not asking. A regression test pins the contract from both sides. Co-Authored-By: Claude Fable 5 --- .../tests/test_uk_frs_spine.py | 37 +++++++++++++++++++ tools/build_uk_frs_spine.py | 17 +++++++-- 2 files changed, 50 insertions(+), 4 deletions(-) diff --git a/packages/microcosm-build/tests/test_uk_frs_spine.py b/packages/microcosm-build/tests/test_uk_frs_spine.py index 64270ff0b..8919b6b44 100644 --- a/packages/microcosm-build/tests/test_uk_frs_spine.py +++ b/packages/microcosm-build/tests/test_uk_frs_spine.py @@ -1837,3 +1837,40 @@ def test_in_kind_benefits_map_from_the_raw_person_tapes(tmp_path: Path) -> None: ): assert (adults[column] == 0).all() assert person[column].notna().all() + + +def test_boundary_evidence_asks_only_the_stages_that_have_run() -> None: + """The first licensed battery run failed at the assembled boundary because + the evidence provider consulted all 25 implementations, and an un-run + stage's checkpoint hook (correctly) refuses. Each boundary must offer only + its executed prefix — an un-run stage being consulted is the regression. + """ + + tool = _load_tool() + + class _RefusesUntilRun: + def __init__(self) -> None: + self.ran = False + + def checkpoint_metadata(self) -> dict[str, object]: + if not self.ran: + raise RuntimeError( + "checkpoint metadata requires a completed stage run." + ) + return {"evidence": {"stage": "late_stage", "ok": True}} + + late = _RefusesUntilRun() + implementations = {"early_stage": SimpleNamespace(), "late_stage": late} + + # The assembled-boundary call: only the executed prefix is offered, so the + # un-run late stage is never consulted and nothing raises. + assembled = tool._collect_stage_evidence( + stage_names=("early_stage",), implementations=implementations + ) + assert assembled == {} + + late.ran = True + transferred = tool._collect_stage_evidence( + stage_names=("early_stage", "late_stage"), implementations=implementations + ) + assert transferred == {"late_stage": {"stage": "late_stage", "ok": True}} diff --git a/tools/build_uk_frs_spine.py b/tools/build_uk_frs_spine.py index a4a248f98..477765bfa 100644 --- a/tools/build_uk_frs_spine.py +++ b/tools/build_uk_frs_spine.py @@ -644,25 +644,34 @@ def _run_plan_with_spine_sampling( return spine_frame, spine_records, sampling assembled_end = min(11, len(plan.stages)) frame, assembled_records = StagePlan(plan.stages[1:assembled_end]).run(spine_frame) + # Each boundary offers only the stages that have actually run: asking a + # later stage for checkpoint evidence would (correctly) raise, and the + # first licensed battery run did exactly that at the assembled boundary. + executed = tuple(stage.name for stage in plan.stages[:assembled_end]) if spine_battery is not None: _run_spine_gate_phase( spine_battery, "assembled", frame=frame, stage_evidence=( - stage_evidence_provider() if stage_evidence_provider is not None else {} + stage_evidence_provider(executed) + if stage_evidence_provider is not None + else {} ), ) if assembled_end == len(plan.stages): return frame, (*spine_records, *assembled_records), sampling frame, tail_records = StagePlan(plan.stages[assembled_end:]).run(frame) + executed = tuple(stage.name for stage in plan.stages) if spine_battery is not None: _run_spine_gate_phase( spine_battery, "transferred", frame=frame, stage_evidence=( - stage_evidence_provider() if stage_evidence_provider is not None else {} + stage_evidence_provider(executed) + if stage_evidence_provider is not None + else {} ), ) return frame, (*spine_records, *assembled_records, *tail_records), sampling @@ -977,8 +986,8 @@ def main(argv: list[str] | None = None) -> int: sample_fraction=args.sample_fraction, sample_seed=args.sample_seed, spine_battery=spine_battery, - stage_evidence_provider=lambda: _collect_stage_evidence( - stage_names=_STAGE_NAMES, + stage_evidence_provider=lambda executed: _collect_stage_evidence( + stage_names=executed, implementations=implementations, ), ) From 87fca7095d099142cf4b0bacd246177d55ae9270 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Wed, 26 Aug 2026 17:27:42 +0200 Subject: [PATCH 10/14] Give the boundary battery the rules engine the enum gate resolves against MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Second finding from the first licensed battery run, one gate further in: the BRMA enum-domain gate resolves its domain from the live rules engine, which the national terminal battery supplied as a context artifact and the spine boundary did not — so the gate failed closed with the battery's own fail-closed KeyError, and write-then-block left the report saying exactly that. The boundary context now carries the driver's engine alongside the stage evidence, the same convention the terminal battery uses. Co-Authored-By: Claude Fable 5 --- tools/build_uk_frs_spine.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/tools/build_uk_frs_spine.py b/tools/build_uk_frs_spine.py index 477765bfa..2f5a8be76 100644 --- a/tools/build_uk_frs_spine.py +++ b/tools/build_uk_frs_spine.py @@ -625,6 +625,7 @@ def _run_plan_with_spine_sampling( sample_seed: int, spine_battery: GateBatteryRun | None = None, stage_evidence_provider=None, + gate_artifacts: Mapping[str, object] | None = None, ) -> tuple[object, tuple[object, ...], dict[str, object] | None]: if not plan.stages or plan.stages[0].name != "frs_spine": frame, records = plan.run(uk_frs_spine_seed_frame()) @@ -658,6 +659,7 @@ def _run_plan_with_spine_sampling( if stage_evidence_provider is not None else {} ), + gate_artifacts=gate_artifacts, ) if assembled_end == len(plan.stages): return frame, (*spine_records, *assembled_records), sampling @@ -673,6 +675,7 @@ def _run_plan_with_spine_sampling( if stage_evidence_provider is not None else {} ), + gate_artifacts=gate_artifacts, ) return frame, (*spine_records, *assembled_records, *tail_records), sampling @@ -683,13 +686,15 @@ def _run_spine_gate_phase( *, frame, stage_evidence: Mapping[str, object], + gate_artifacts: Mapping[str, object] | None = None, ) -> None: + artifacts: dict[str, object] = {"stage_evidence": dict(stage_evidence)} + # The enum-domain gate resolves its domain from the live rules engine, + # exactly as the national terminal battery supplied it. + artifacts.update(dict(gate_artifacts or {})) battery.run_phase( phase, - EvidenceContext( - frame=frame, - artifacts={"stage_evidence": dict(stage_evidence)}, - ), + EvidenceContext(frame=frame, artifacts=artifacts), ) battery.enforce(phase, mode=BlockingMode.BLOCKS_ARTIFACT) @@ -990,6 +995,7 @@ def main(argv: list[str] | None = None) -> int: stage_names=executed, implementations=implementations, ), + gate_artifacts={"rules_engine": engine}, ) if spine_battery is not None: append_phase(state, "spine_gates_evaluated") From ae5e9805c3eb34b5920e7bd1cbb3c7b8af7fab84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Wed, 26 Aug 2026 17:56:49 +0200 Subject: [PATCH 11/14] Close the checks that could not fail, from the review and the shakedown at once MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Vahid's #787 review and the spine battery's first licensed runs converged on one class from two directions: checks that pass without testing what they name. The support-clip gates declared empty allowances, so a stage clipping every row would have passed a release-blocking check. The allowances are pinned per column at the receipted spine baseline — zero rows clipped anywhere — so any clipping is a real signal against a clean measured baseline, and the gate now fails closed on a missing allowance with a pin-or-exempt message so the class cannot re-enter through a future entry. The LCFS exempt columns join the declared column set: they were disjoint from it, which left the exempt-marking assertion unable to fire at all. The assembled boundary was a positional index — min(11, len(stages)) — that a stage-plan change in another file would silently move, the same position-for-key shape as policyengine-uk-data#468. It is now the declared name of the boundary stage, and an armed battery refuses a plan that lacks it: a plan change is either correct or loud. Release-candidate strictness is an explicit driver flag again instead of being inferred from full scale, so a developer's full-size build is not a release candidate unless the caller says so. From the shakedown's transferred phase: the source-signal gate's reviewed list now carries the E7 reviewed-absent incapacity column, so the battery agrees with that standing adjudication instead of re-litigating it every build, and the CGT spine stage re-stamps the shared imputation summary with its own stage name — the gate was rightly enforcing the distinct-receipts-per-family rule against a receipt claiming the certified family's name. Battery digests re-cut from the live producer. Co-Authored-By: Claude Fable 5 --- changelog.d/757-vahid-review-787.fixed.md | 1 + .../src/microcosm/build/uk/gates.json | 107 +++++++++++++++--- .../build/uk_runtime/cgt_imputation.py | 8 +- .../build/uk_runtime/stage_health.py | 17 ++- .../tests/test_uk_stage_health.py | 40 ++++++- .../src/microcosm/data/contract.py | 6 +- .../microcosm-data/tests/test_contract.py | 6 +- tools/build_uk_frs_spine.py | 29 ++++- 8 files changed, 187 insertions(+), 27 deletions(-) create mode 100644 changelog.d/757-vahid-review-787.fixed.md diff --git a/changelog.d/757-vahid-review-787.fixed.md b/changelog.d/757-vahid-review-787.fixed.md new file mode 100644 index 000000000..641df290b --- /dev/null +++ b/changelog.d/757-vahid-review-787.fixed.md @@ -0,0 +1 @@ +The #787 review and the spine battery's licensed shakedown converged on the same class from two directions: checks that pass without testing what they name. The support-clip gates declared empty allowances, so a stage clipping every row passed a release-blocking check — the allowances are now pinned per column at the receipted spine baseline of zero rows clipped, a missing allowance fails closed with a pin-or-exempt message, and the LCFS exempt columns join the declared column set so the exempt-marking assertion actually bites. The assembled/transferred boundary was a positional index that a stage-plan change elsewhere would silently move — it is now the declared name of the boundary stage, and an armed battery refuses a plan that lacks it. Release-candidate strictness is an explicit driver flag again rather than inferred from full scale. From the shakedown itself: the E7 reviewed-absent incapacity column is declared in the source-signal gate's reviewed list so the battery agrees with that adjudication instead of re-litigating it, and the CGT spine stage re-stamps the shared imputation summary with its own stage name per the distinct-receipts-per-family rule. diff --git a/packages/microcosm-build/src/microcosm/build/uk/gates.json b/packages/microcosm-build/src/microcosm/build/uk/gates.json index 67203f8a4..099d2142f 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/gates.json +++ b/packages/microcosm-build/src/microcosm/build/uk/gates.json @@ -75,10 +75,38 @@ "stocks_and_shares_isa", "student_loan_balance" ], - "max_clipped_low_rows_by_column": {}, - "max_clipped_high_rows_by_column": {} + "max_clipped_low_rows_by_column": { + "cash_isa": 0, + "corporate_wealth": 0, + "gross_financial_wealth": 0, + "main_residence_value": 0, + "net_financial_wealth": 0, + "non_residential_property_value": 0, + "num_vehicles": 0, + "other_residential_property_value": 0, + "owned_land": 0, + "property_wealth": 0, + "savings": 0, + "stocks_and_shares_isa": 0, + "student_loan_balance": 0 + }, + "max_clipped_high_rows_by_column": { + "cash_isa": 0, + "corporate_wealth": 0, + "gross_financial_wealth": 0, + "main_residence_value": 0, + "net_financial_wealth": 0, + "non_residential_property_value": 0, + "num_vehicles": 0, + "other_residential_property_value": 0, + "owned_land": 0, + "property_wealth": 0, + "savings": 0, + "stocks_and_shares_isa": 0, + "student_loan_balance": 0 + } }, - "notes": "Stage-time support-clip health gate over the WAS wealth receipt; raw FRS mapping stages have no imputation semantics and intentionally have no empty health gate." + "notes": "Stage-time support-clip health gate over the WAS wealth receipt; raw FRS mapping stages have no imputation semantics and intentionally have no empty health gate. Allowances pinned at the receipted spine-e baseline (2026-08-26): zero rows clipped on every non-exempt column, so any clipping is a real signal against a clean baseline; a missing allowance fails closed rather than skipping the comparison." }, { "id": "uk_stage_lcfs_consumption_support", @@ -104,17 +132,52 @@ "petrol_spending", "recreation_consumption", "restaurants_and_hotels_consumption", - "transport_consumption" + "transport_consumption", + "domestic_energy_consumption", + "electricity_consumption", + "gas_consumption" ], "exempt_columns": [ "domestic_energy_consumption", "electricity_consumption", "gas_consumption" ], - "max_clipped_low_rows_by_column": {}, - "max_clipped_high_rows_by_column": {} + "max_clipped_low_rows_by_column": { + "alcohol_and_tobacco_consumption": 0, + "bus_fare_spending": 0, + "clothing_and_footwear_consumption": 0, + "communication_consumption": 0, + "diesel_spending": 0, + "education_consumption": 0, + "food_and_non_alcoholic_beverages_consumption": 0, + "health_consumption": 0, + "household_furnishings_consumption": 0, + "housing_water_and_electricity_consumption": 0, + "miscellaneous_consumption": 0, + "petrol_spending": 0, + "recreation_consumption": 0, + "restaurants_and_hotels_consumption": 0, + "transport_consumption": 0 + }, + "max_clipped_high_rows_by_column": { + "alcohol_and_tobacco_consumption": 0, + "bus_fare_spending": 0, + "clothing_and_footwear_consumption": 0, + "communication_consumption": 0, + "diesel_spending": 0, + "education_consumption": 0, + "food_and_non_alcoholic_beverages_consumption": 0, + "health_consumption": 0, + "household_furnishings_consumption": 0, + "housing_water_and_electricity_consumption": 0, + "miscellaneous_consumption": 0, + "petrol_spending": 0, + "recreation_consumption": 0, + "restaurants_and_hotels_consumption": 0, + "transport_consumption": 0 + } }, - "notes": "Stage-time support-clip health gate over the LCFS receipt. The exempt energy columns are deliberately named here because they are bridged through the NEED/WAS path rather than clipped to LCFS donor support." + "notes": "Stage-time support-clip health gate over the LCFS receipt. The exempt energy columns are deliberately named here because they are bridged through the NEED/WAS path rather than clipped to LCFS donor support. Allowances pinned at the receipted spine-e baseline (2026-08-26): zero rows clipped on every non-exempt column, so any clipping is a real signal against a clean baseline; a missing allowance fails closed rather than skipping the comparison." }, { "id": "uk_stage_etb_vat_support", @@ -128,10 +191,14 @@ "columns": [ "full_rate_vat_expenditure_rate" ], - "max_clipped_low_rows_by_column": {}, - "max_clipped_high_rows_by_column": {} + "max_clipped_low_rows_by_column": { + "full_rate_vat_expenditure_rate": 0 + }, + "max_clipped_high_rows_by_column": { + "full_rate_vat_expenditure_rate": 0 + } }, - "notes": "Stage-time support-clip health gate over the ETB VAT receipt." + "notes": "Stage-time support-clip health gate over the ETB VAT receipt. Allowances pinned at the receipted spine-e baseline (2026-08-26): zero rows clipped on every non-exempt column, so any clipping is a real signal against a clean baseline; a missing allowance fails closed rather than skipping the comparison." }, { "id": "uk_stage_etb_services_support", @@ -147,10 +214,18 @@ "dfe_education_spending", "rail_subsidy_spending" ], - "max_clipped_low_rows_by_column": {}, - "max_clipped_high_rows_by_column": {} + "max_clipped_low_rows_by_column": { + "bus_subsidy_spending": 0, + "dfe_education_spending": 0, + "rail_subsidy_spending": 0 + }, + "max_clipped_high_rows_by_column": { + "bus_subsidy_spending": 0, + "dfe_education_spending": 0, + "rail_subsidy_spending": 0 + } }, - "notes": "Stage-time support-clip health gate over the ETB services receipt." + "notes": "Stage-time support-clip health gate over the ETB services receipt. Allowances pinned at the receipted spine-e baseline (2026-08-26): zero rows clipped on every non-exempt column, so any clipping is a real signal against a clean baseline; a missing allowance fails closed rather than skipping the comparison." }, { "id": "uk_stage_frs_hmrc_spine_leaves_signal", @@ -162,9 +237,11 @@ "stage": "frs_hmrc_spine_leaves", "check": "source_signal", "minimum_signal_rows": 1, - "structural_zero_columns": [] + "structural_zero_columns": [ + "hmrc_spi_incapacity_benefit_income" + ] }, - "notes": "Stage-time source-signal gate over the retained FRS/HMRC leaves receipt." + "notes": "Stage-time source-signal gate over the retained FRS/HMRC leaves receipt. hmrc_spi_incapacity_benefit_income is the E7 reviewed-absent adjudication (microcosm#683, 2026-08-18): the FRS 2024-25 source carries no incapacity-benefit signal, the stage's own fence raises at build time if a future vintage does, and the reviewed list here makes the battery agree with that adjudication instead of re-litigating it every build." }, { "id": "uk_stage_spi_support_channel_mass", diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_imputation.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_imputation.py index ca634f9ef..b82dfa643 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_imputation.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_imputation.py @@ -709,7 +709,13 @@ def __call__(self, frame: Frame) -> Frame: def checkpoint_metadata(self) -> dict[str, object]: if self.last_result is None: raise RuntimeError("checkpoint metadata requires a completed stage run.") - return {"evidence": self.last_result.evidence()} + evidence = self.last_result.evidence() + # The shared summary stamps the certified family's stage name; this + # receipt belongs to the spine stage that produced it (the E8 + # distinct-receipts-per-family rule), and the stage-health gate + # rightly refuses a receipt claiming another stage. + evidence["stage"] = self.stage.stage + return {"evidence": evidence} def _assert_cgt_spine_stage_parameters(stage: SourceStageSpec) -> None: diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/stage_health.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/stage_health.py index 3a6c2378a..a22e4a38b 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/stage_health.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/stage_health.py @@ -135,11 +135,24 @@ def _support_clip_gate( high_rows = int(receipt.get("clipped_high_rows", -1)) allowed_low = max_low.get(column) allowed_high = max_high.get(column) - if allowed_low is not None and low_rows > int(allowed_low): + # A missing allowance is not permission: without it the gate asserts + # receipt shape only, and a stage clipping every row would pass a + # release-blocking check — a green reflecting the absence of a check. + if allowed_low is None: + failures.append( + f"{stage}: {column!r} declares no clipped_low_rows allowance; " + "pin one at the receipted baseline or exempt the column." + ) + elif low_rows > int(allowed_low): failures.append( f"{stage}: {column!r} clipped_low_rows {low_rows} exceeds {allowed_low}." ) - if allowed_high is not None and high_rows > int(allowed_high): + if allowed_high is None: + failures.append( + f"{stage}: {column!r} declares no clipped_high_rows allowance; " + "pin one at the receipted baseline or exempt the column." + ) + elif high_rows > int(allowed_high): failures.append( f"{stage}: {column!r} clipped_high_rows {high_rows} exceeds {allowed_high}." ) diff --git a/packages/microcosm-build/tests/test_uk_stage_health.py b/packages/microcosm-build/tests/test_uk_stage_health.py index fa5370407..8ec28f067 100644 --- a/packages/microcosm-build/tests/test_uk_stage_health.py +++ b/packages/microcosm-build/tests/test_uk_stage_health.py @@ -28,7 +28,7 @@ def test_support_clip_gate_requires_receipted_columns_and_wires_thresholds() -> "check": "support_clip", "columns": ["cash_isa"], "max_clipped_low_rows_by_column": {"cash_isa": 1}, - "max_clipped_high_rows_by_column": {}, + "max_clipped_high_rows_by_column": {"cash_isa": 0}, } assert _passed( @@ -351,3 +351,41 @@ def test_cgt_summary_minimum_rows_parameter_is_live() -> None: "minimum_band_rows": 2, }, ).passed + + +def test_support_clip_gate_fails_closed_on_a_missing_allowance() -> None: + """An undeclared allowance skipped the comparison entirely, so a stage + clipping every row passed a release-blocking gate — the green-by-absence + class the #787 review named. A non-exempt column now needs both bounds + pinned, or the gate says so. + """ + + evidence = { + "stage": "was_wealth", + "support_clip": { + "columns": { + "cash_isa": { + "donor_min": 0.0, + "donor_max": 100.0, + "clipped_low_rows": 0, + "clipped_high_rows": 0, + "rows_considered": 2, + } + } + }, + } + result = uk_stage_health_gate( + evidence=evidence, + stage="was_wealth", + check="support_clip", + parameters={ + "stage": "was_wealth", + "check": "support_clip", + "columns": ["cash_isa"], + "max_clipped_low_rows_by_column": {}, + "max_clipped_high_rows_by_column": {}, + }, + ) + assert not _passed(result) + assert any("no clipped_low_rows allowance" in f for f in result.failures) + assert any("no clipped_high_rows allowance" in f for f in result.failures) diff --git a/packages/microcosm-data/src/microcosm/data/contract.py b/packages/microcosm-data/src/microcosm/data/contract.py index 95bdef0b9..d6b295e8f 100644 --- a/packages/microcosm-data/src/microcosm/data/contract.py +++ b/packages/microcosm-data/src/microcosm/data/contract.py @@ -373,13 +373,13 @@ # fingerprint derives from the manifest digest. Editing the spec moves all # three here in the same reviewed change. _UK_GATE_BATTERY_POLICY_SHA256 = ( - "31c79de22ea90d5766d015f0df5e1416ee21647f07b351f2971faa86f3a133c0" + "0b215cad96263fc8ee937facd189212b0f60639bb317ecdf6d19d7c7004689d9" ) _UK_GATE_BATTERY_GATES_MANIFEST_SHA256 = ( - "f9225c546b706cbf06d3a80dd97f8db82927f3bbcf9baebe7f5d694eac7fc730" + "fe580e1f39924c40f22c9826c21df8a0d02273cf0660dccf13039d173fadee85" ) _UK_GATE_BATTERY_SPEC_FINGERPRINT = ( - "e7176207973ebd6731fe1e8e906da6fc6ef0f5ed79956542332ee21bf184486b" + "c6b43744bdc2ac3187f503d719aea12d764a521d24382f0e0390bf7b92a2bd5f" ) #: Spec entry id -> the legacy gate name whose observable detail checks #: apply unchanged (the battery re-keys the report by entry id; the gate diff --git a/packages/microcosm-data/tests/test_contract.py b/packages/microcosm-data/tests/test_contract.py index 7d96a0bce..c25f31484 100644 --- a/packages/microcosm-data/tests/test_contract.py +++ b/packages/microcosm-data/tests/test_contract.py @@ -134,13 +134,13 @@ def _trusted_terminal_gate_signing_key(monkeypatch) -> None: UK_GATE_BATTERY_PRODUCER = "microcosm.build.gate_battery" UK_GATE_BATTERY_SIGNING_KEY_ENV = "MICROCOSM_UK_TERMINAL_GATE_SIGNING_KEY" UK_GATE_BATTERY_POLICY_SHA256 = ( - "31c79de22ea90d5766d015f0df5e1416ee21647f07b351f2971faa86f3a133c0" + "0b215cad96263fc8ee937facd189212b0f60639bb317ecdf6d19d7c7004689d9" ) UK_GATE_BATTERY_GATES_MANIFEST_SHA256 = ( - "f9225c546b706cbf06d3a80dd97f8db82927f3bbcf9baebe7f5d694eac7fc730" + "fe580e1f39924c40f22c9826c21df8a0d02273cf0660dccf13039d173fadee85" ) UK_GATE_BATTERY_SPEC_FINGERPRINT = ( - "e7176207973ebd6731fe1e8e906da6fc6ef0f5ed79956542332ee21bf184486b" + "c6b43744bdc2ac3187f503d719aea12d764a521d24382f0e0390bf7b92a2bd5f" ) UK_GATE_BATTERY_DEGENERATE_EVIDENCE_SHA256 = ( "d0d024043132fa07c378c393dbe2b24fe99bf19e876bcc39997d2c80cc9bd4f6" diff --git a/tools/build_uk_frs_spine.py b/tools/build_uk_frs_spine.py index 2f5a8be76..27b2c9922 100644 --- a/tools/build_uk_frs_spine.py +++ b/tools/build_uk_frs_spine.py @@ -100,6 +100,11 @@ _REPOSITORY = Path(__file__).resolve().parents[1] _RUNG_NAMED_EDGE_SIGNATURE = "The least populated classes in y have only 1 member" _RUNG_ABORT_EXIT_CODE = 3 +#: The last stage of the assembled checkpoint: everything through the base +#: FRS mapping and the stochastic draws. A name, not an index — a position +#: standing in for a key is correct only while two independently-maintained +#: orderings happen to agree (the policyengine-uk-data#468 class). +UK_SPINE_ASSEMBLED_FINAL_STAGE = "frs_brma" _STAGE_NAMES = ( "frs_spine", "frs_employment", @@ -197,6 +202,16 @@ def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: "renormalized to full household mass, and treated as a receipt." ), ) + parser.add_argument( + "--release-candidate", + action="store_true", + help=( + "Evaluate the spine battery at release-candidate strictness: " + "evidence_absent gaps block instead of being tolerated. Explicit " + "by design - a full-scale developer build is not a release " + "candidate unless the caller says so." + ), + ) parser.add_argument( "--sample-seed", type=int, @@ -643,7 +658,17 @@ def _run_plan_with_spine_sampling( ) if len(plan.stages) == 1: return spine_frame, spine_records, sampling - assembled_end = min(11, len(plan.stages)) + names = tuple(stage.name for stage in plan.stages) + if UK_SPINE_ASSEMBLED_FINAL_STAGE in names: + assembled_end = names.index(UK_SPINE_ASSEMBLED_FINAL_STAGE) + 1 + elif spine_battery is not None: + raise RuntimeError( + "spine battery is armed but the declared assembled-boundary stage " + f"{UK_SPINE_ASSEMBLED_FINAL_STAGE!r} is not in the plan; a stage " + "plan change must move the boundary declaration with it." + ) + else: + assembled_end = len(plan.stages) frame, assembled_records = StagePlan(plan.stages[1:assembled_end]).run(spine_frame) # Each boundary offers only the stages that have actually run: asking a # later stage for checkpoint evidence would (correctly) raise, and the @@ -980,7 +1005,7 @@ def main(argv: list[str] | None = None) -> int: spine_gate_manifest, release_id=state.build_id, report_path=spine_gate_path, - release_candidate=args.sample_fraction == 1.0, + release_candidate=args.release_candidate, registry=UK_GATE_REGISTRY, ) if spine_gate_manifest is not None From 0c6f4206ec66642f41ada6c0f5eb169578add0ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Wed, 26 Aug 2026 18:04:04 +0200 Subject: [PATCH 12/14] Certify the battery-built candidate, and bind the battery into the receipt MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit spine-g is the first UK spine built end to end under the fully armed self-policing regime: fourteen gates evaluated at the assembled and transferred boundaries, all passed, no phase blocked. The committed acceptance receipt re-cuts to it and gains a spine_battery block — report digest, blocked phase, status census — and the binder test now refuses a receipt without green battery evidence, so the certified candidate can never again be one whose build was not gated. Every evidence layer was re-measured on the candidate itself rather than carried over: the e4-e8 identity ladder green, strict parity signed_parity with zero unsigned differences at the contract band, and the twin note now records the three-vintage chain — spine-d before the evidence layer, spine-e before the battery, spine-g under it, pairwise payload-identical — which is both the twin-determinism receipt and the proof that receipts and gates never moved a byte of the artifact. Co-Authored-By: Claude Fable 5 --- .../build/uk/spine_candidate_acceptance.json | 21 ++++++++++++------- .../tests/test_uk_spine_acceptance_receipt.py | 4 ++++ 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/packages/microcosm-build/src/microcosm/build/uk/spine_candidate_acceptance.json b/packages/microcosm-build/src/microcosm/build/uk/spine_candidate_acceptance.json index ad2aaca64..3c91f9359 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/spine_candidate_acceptance.json +++ b/packages/microcosm-build/src/microcosm/build/uk/spine_candidate_acceptance.json @@ -6,9 +6,9 @@ "household": 52846, "person": 113649 }, - "name": "spine-e", - "sha256": "3c8799970851c409e4cb8578d33a180acb30ae600f4bd99ca3a190f9c5eb870a", - "sidecar_sha256": "5fa5f27065140a3a7d2fd56923150a9f89c5da52504648aebfb70e18da100d32", + "name": "spine-g", + "sha256": "02900abeec29d52596f16c9e29b2a8991e0acd5f5118c3be28a50ca77403787b", + "sidecar_sha256": "d5f4dc645c130484b05d8d7ef00f3f81ffb9c3633eff305bc9e6754c52b1ec0e", "stage_count": 25, "stage_roster": [ "frs_spine", @@ -64,8 +64,15 @@ }, "measured_on": "2026-08-26", "schema_version": 1, + "spine_battery": { + "blocked_at_phase": null, + "report_sha256": "57c4a44d9627b1e47b7a94438a71bc0d96de06073931e37ff096f1f988c3c58b", + "statuses": { + "passed": 14 + } + }, "strict_parity": { - "receipt_sha256": "b9e5118ef5785b4fb16f4aa13db9f302d73429e118d51d7bff6f60e7afe2513f", + "receipt_sha256": "8fff9bf6952f6985eb71b8e81d6adad8dfb862810a7c8e4c5da7d2c278d13bdd", "share_band": { "contract": 0.02, "effective": 0.02 @@ -75,9 +82,9 @@ "verdict": "signed_parity" }, "twin": { - "name": "spine-d", - "note": "Cross-commit twin: spine-d built pre-Run-1 evidence layer, spine-e post; payload_identical across all tables, keys and root attrs, so the pair is simultaneously the twin-determinism receipt and the Run-1 payload-inertness receipt.", + "name": "spine-e (and spine-d before it)", + "note": "Three code vintages, one payload: spine-d (pre-evidence-layer), spine-e (pre-battery), spine-g (full battery armed) are pairwise payload_identical across all tables, keys and root attrs \u2014 the twin-determinism receipt and the proof that receipts and gates never moved a byte of the artifact.", "payload_identical": true, - "sha256": "f3f96805102d2e147408ee03dc3a5702a55b8232df417d7dc13109ac3caf75d1" + "sha256": "3c8799970851c409e4cb8578d33a180acb30ae600f4bd99ca3a190f9c5eb870a" } } diff --git a/packages/microcosm-build/tests/test_uk_spine_acceptance_receipt.py b/packages/microcosm-build/tests/test_uk_spine_acceptance_receipt.py index 361b5346a..214e8f436 100644 --- a/packages/microcosm-build/tests/test_uk_spine_acceptance_receipt.py +++ b/packages/microcosm-build/tests/test_uk_spine_acceptance_receipt.py @@ -58,3 +58,7 @@ def test_receipt_identity_and_verdicts_are_the_accepted_ones(): assert parity["unsigned_differences"] == 0 assert parity["strict_failure"] is False assert parity["share_band"]["effective"] == parity["share_band"]["contract"] + battery = receipt["spine_battery"] + assert battery["blocked_at_phase"] is None + assert battery["statuses"] == {"passed": 14} + assert len(battery["report_sha256"]) == 64 From 8ff3ff06d622b5b3b0ee10ed709db4a6c77ea55e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Wed, 26 Aug 2026 18:51:38 +0200 Subject: [PATCH 13/14] Cite the incumbent issue without naming the retired package MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The live-tree guard (test_us_plan) rightly refused the boundary constant's comment: it named the retired data package where the standing convention — set at the E6 owned_land adjudication — cites uk-data#NNN without the package name, reserving the literal name for the allowlisted frozen references. The comment cites the issue the same way every other live-tree reference does. The UK lanes were green; this was the sole failure across both us-p and fast/rest, which carry the guard. Co-Authored-By: Claude Fable 5 --- tools/build_uk_frs_spine.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/build_uk_frs_spine.py b/tools/build_uk_frs_spine.py index 27b2c9922..b739c57e2 100644 --- a/tools/build_uk_frs_spine.py +++ b/tools/build_uk_frs_spine.py @@ -103,7 +103,7 @@ #: The last stage of the assembled checkpoint: everything through the base #: FRS mapping and the stochastic draws. A name, not an index — a position #: standing in for a key is correct only while two independently-maintained -#: orderings happen to agree (the policyengine-uk-data#468 class). +#: orderings happen to agree (the uk-data#468 class). UK_SPINE_ASSEMBLED_FINAL_STAGE = "frs_brma" _STAGE_NAMES = ( "frs_spine", From c5b08f29cb483c1310670405fb45ac68e1bfe03a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Wed, 26 Aug 2026 18:54:23 +0200 Subject: [PATCH 14/14] Retrigger CI: the push event for the incumbent-name fix produced no run MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit GitHub dropped the workflow event for 8ff3ff06 while two superseded-tip runs occupied the queue (both now cancelled — their tips are no longer the PR head). Empty commit; no tree change. Co-Authored-By: Claude Fable 5