From 5ad473f73241c2385168248964bf48b0a58b3a90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Thu, 27 Aug 2026 17:16:58 +0200 Subject: [PATCH 1/5] Derive band edges from the compiled register, never the pruned roster MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Banded measures took each band's upper edge from the lower edges of sibling specs in the live registry, so excluding a band silently widened its lower neighbour to the next surviving edge (or to infinity at the top) — the #792 movers: 12 of 361 shared measures shifted by up to +858% when the exclusion register grew 5->47, each by exactly the mass of the absorbed excluded siblings. materialize_target_bindings and resolve_target_measures now accept a band_edge_registry (default: the materialized registry, bit-identical for existing callers), and _band_bounds refuses a spec whose own lower edge is absent from the supplied edge set rather than guessing. Regression fence: survivors must be bit-identical under sibling exclusion, for numeric edges and published range labels. Co-Authored-By: Claude Fable 5 --- .../792-band-edges-compiled-register.fixed.md | 1 + .../microcosm/build/target_materialization.py | 23 ++- .../tests/test_target_materialization.py | 188 ++++++++++++++++++ 3 files changed, 208 insertions(+), 4 deletions(-) create mode 100644 changelog.d/792-band-edges-compiled-register.fixed.md diff --git a/changelog.d/792-band-edges-compiled-register.fixed.md b/changelog.d/792-band-edges-compiled-register.fixed.md new file mode 100644 index 00000000..afe982e6 --- /dev/null +++ b/changelog.d/792-band-edges-compiled-register.fixed.md @@ -0,0 +1 @@ +Banded calibration targets now derive upper edges from the compiled pre-exclusion register, fixing microcosm#792 so excluding a banded target no longer widens its surviving lower neighbour. The materialization seam now refuses a pruned register unless callers provide its compiled band-edge source. diff --git a/packages/microcosm-build/src/microcosm/build/target_materialization.py b/packages/microcosm-build/src/microcosm/build/target_materialization.py index 79fdfb22..41468008 100644 --- a/packages/microcosm-build/src/microcosm/build/target_materialization.py +++ b/packages/microcosm-build/src/microcosm/build/target_materialization.py @@ -21,9 +21,11 @@ # encodings are in use — a numeric lower bound (HMRC SPI income bands) and a # published range label (DWP award bands, in monthly units, hence # ``band_period_factor``) — and both reduce to one lower edge, because no -# reference anywhere declares an upper bound. A band's upper edge is -# therefore its sibling's lower edge within the same contract target, and the -# top band runs to infinity. +# reference anywhere declares an upper bound. A band's upper edge is its +# sibling's lower edge within the same compiled contract target. Edges must +# never derive from an exclusion-pruned roster, or excluding a band silently +# widens its lower neighbour; only the compiled register's top band runs to +# infinity. # Entity-count indicators: a value of one per record of the owning entity. _COUNT_VALUE_VARIABLES = frozenset( {"household_count", "person_count", "benunit_count"} @@ -126,6 +128,7 @@ def resolve_target_measures( period: int | str, max_rounds: int = 8, contract_targets: Mapping[str, Mapping[str, Any]] | None = None, + band_edge_registry: TargetRegistry | None = None, ) -> MeasureResolution: """Resolve provider-computed inputs until target materialization binds. @@ -159,6 +162,7 @@ def resolve_target_measures( registry, contract, period=period, + band_edge_registry=band_edge_registry, ) skipped = tuple(result.skipped) round_receipt = { @@ -414,6 +418,13 @@ def _band_bounds( f"groupby_variable {binding['groupby_variable']!r} but its spec " "carries no readable band edge" ) + if lower not in band_edges: + raise ValueError( + f"banded measure {getattr(spec, 'measure', '?')!r} carries band lower edge " + f"{lower!r} that is absent from its contract target's band-edge set " + f"{list(band_edges)!r}; the band-edge register does not cover this spec's " + "band — pass the compiled (pre-exclusion) register this spec was pruned from." + ) upper = math.inf for edge in band_edges: if edge > lower: @@ -429,11 +440,15 @@ def materialize_target_bindings( *, period: int | str, providers: Mapping[str, Provider] | None = None, + band_edge_registry: TargetRegistry | None = None, ) -> TargetMaterializationResult: """Prepare measure columns declared by compiled Ledger target specs.""" provider_registry = {**default_provider_registry(), **(providers or {})} - band_edges = _band_edges_by_group(registry, contract_targets) + band_edges = _band_edges_by_group( + registry if band_edge_registry is None else band_edge_registry, + contract_targets, + ) skipped: list[MaterializationSkip] = [] for spec in registry.specs: if hasattr(adapter, "has_column") and adapter.has_column( diff --git a/packages/microcosm-build/tests/test_target_materialization.py b/packages/microcosm-build/tests/test_target_materialization.py index b4af50fb..1d726f3c 100644 --- a/packages/microcosm-build/tests/test_target_materialization.py +++ b/packages/microcosm-build/tests/test_target_materialization.py @@ -512,6 +512,63 @@ def test_bands_slice_the_population_and_partition_it(): assert list(total) == [1.0, 1.0, 1.0] +def test_band_measures_are_roster_invariant_under_sibling_exclusion(): + registry = _banded_registry() + full_adapter = StubAdapter() + + result = materialize_target_bindings( + full_adapter, registry, _BANDED_CONTRACT, period=2025 + ) + + assert result.skipped == () + snapshots = { + label: full_adapter.tables["person"][f"income_band_{label}"].copy() + for label in ("0", "20", "40") + } + + middle_pruned = TargetRegistry( + [spec for spec in registry.specs if spec.name != "band_20"], + country="uk", + ) + middle_adapter = StubAdapter() + + result = materialize_target_bindings( + middle_adapter, + middle_pruned, + _BANDED_CONTRACT, + period=2025, + band_edge_registry=registry, + ) + + assert result.skipped == () + for label in ("0", "40"): + assert np.array_equal( + middle_adapter.tables["person"][f"income_band_{label}"], + snapshots[label], + ) + + top_pruned = TargetRegistry( + [spec for spec in registry.specs if spec.name != "band_40"], + country="uk", + ) + top_adapter = StubAdapter() + + result = materialize_target_bindings( + top_adapter, + top_pruned, + _BANDED_CONTRACT, + period=2025, + band_edge_registry=registry, + ) + + assert result.skipped == () + for label in ("0", "20"): + assert np.array_equal( + top_adapter.tables["person"][f"income_band_{label}"], + snapshots[label], + ) + + def test_adjacent_bands_are_not_identical(): # The regression that would have caught the unsliced-measure defect: # before banding was implemented every band returned the same unsliced @@ -582,6 +639,137 @@ def test_published_range_labels_band_in_model_units(): assert list(adapter.tables["person"]["award_high"]) == [0.0, 0.0, 1.0] +def test_published_range_label_edges_survive_sibling_exclusion(): + registry = TargetRegistry( + [ + TargetSpec( + name="award_low", + entity="person", + measure="award_low", + value=1.0, + source="test", + family="dwp_universal_credit", + metadata={ + "contract_target_id": "uc.award_bands", + "ledger_filter_family_type": "Single, no children", + "ledger_filter_monthly_award_bands": "£1.01 to £2.00", + }, + ), + TargetSpec( + name="award_high", + entity="person", + measure="award_high", + value=1.0, + source="test", + family="dwp_universal_credit", + metadata={ + "contract_target_id": "uc.award_bands", + "ledger_filter_monthly_award_bands": "£2.01 to £3.00", + }, + ), + ], + country="uk", + ) + contract = { + "uc.award_bands": { + "bindings": { + "policyengine": { + "value_variable": "person_count", + "groupby_variable": "income", + "from_entity": "person", + "band_period_factor": 12, + } + } + } + } + pruned = TargetRegistry( + [spec for spec in registry.specs if spec.name != "award_high"], + country="uk", + ) + adapter = StubAdapter() + + result = materialize_target_bindings( + adapter, + pruned, + contract, + period=2025, + band_edge_registry=registry, + ) + + assert result.skipped == () + assert list(adapter.tables["person"]["award_low"]) == [0.0, 1.0, 0.0] + + +def test_band_bounds_refuse_a_spec_absent_from_the_band_edge_register(): + registry = TargetRegistry([_banded_registry().specs[0]], country="uk") + adapter = StubAdapter() + + result = materialize_target_bindings( + adapter, + registry, + _BANDED_CONTRACT, + period=2025, + band_edge_registry=TargetRegistry([], country="uk"), + ) + + assert len(result.skipped) == 1 + assert result.skipped[0].name == "band_0" + assert "absent from its contract target's band-edge set" in result.skipped[0].reason + assert "income_band_0" not in adapter.tables["person"] + + +def test_resolve_target_measures_threads_the_band_edge_registry(): + class BandedInputProvider(StubMeasureProvider): + def compute(self, entity, variable): + assert (entity, variable) == ("person", "input_a") + return np.array([1.0, 2.0, 3.0]), "stub:person.input_a" + + source = { + "person": pd.DataFrame( + { + "person_id": [1, 2, 3], + "income": [10.0, 20.0, 30.0], + } + ) + } + registry = _banded_registry() + pruned = TargetRegistry( + [spec for spec in registry.specs if spec.name != "band_20"], + country="uk", + ) + probes = [] + + def adapter_factory(): + adapter = ResolutionAdapter(source) + probes.append(adapter) + return adapter + + resolution = resolve_target_measures( + adapter_factory, + pruned, + BandedInputProvider(), + period=2025, + contract_targets={ + "spi.income_by_band": { + "bindings": { + "policyengine": { + "value_variable": "input_a", + "groupby_variable": "income", + "from_entity": "person", + } + } + } + }, + band_edge_registry=registry, + ) + + assert resolution.receipt["attached"] == { + "person.input_a": "stub:person.input_a" + } + assert list(probes[-1].tables["person"]["income_band_0"]) == [1.0, 0.0, 0.0] + assert list(probes[-1].tables["person"]["income_band_40"]) == [0.0, 0.0, 0.0] + + def test_unreadable_band_is_skipped_not_silently_unsliced(): adapter = StubAdapter() registry = TargetRegistry( From 4429487e1693cc39aeb55d588911aec8c16ab7c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Thu, 27 Aug 2026 17:17:10 +0200 Subject: [PATCH 2/5] Thread the compiled band-edge register through the UK calibration seam MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The seam prunes measure exclusions before materialization, so the stage only ever saw the pruned registry. run_uk_calibration now takes the compiled pre-exclusion register as band_edge_registry and refuses, before any artifact is written, a non-empty exclusion receipt without it — and a register whose extra names do not exactly reconstitute the receipt (the applier already fails exclusions that match zero specs, so pruned + receipt == compiled is exact). The registry content hash is recorded as run_config.band_edge_register_sha256, flowing into the build record and identity digest. The seam tool passes compilation.registry; census counts are unchanged — only survivors' measured values return to published band widths. The stale prose in test_uk_measure_simulation that read the widening artifact as the resolution mechanism is corrected. Co-Authored-By: Claude Fable 5 --- .../build/uk_runtime/calibration_run.py | 49 +++++++ .../build/uk_runtime/ledger_targets.py | 2 + .../build/uk_runtime/national_calibration.py | 7 + .../tests/test_uk_calibration_run.py | 132 ++++++++++++++++++ .../tests/test_uk_calibration_seam_driver.py | 6 +- .../tests/test_uk_measure_simulation.py | 6 +- .../tests/test_uk_national_calibration.py | 38 +++++ tools/calibrate_uk_national_dataset.py | 1 + 8 files changed, 237 insertions(+), 4 deletions(-) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/calibration_run.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/calibration_run.py index 5bb27c48..28fc29ed 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/calibration_run.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/calibration_run.py @@ -197,6 +197,7 @@ def run_uk_calibration( input_sha256: str, ledger_artifact: Any, register_registry: TargetRegistry, + band_edge_registry: TargetRegistry | None = None, calibration_year: int, exclusion_receipt: Mapping[str, Mapping[str, str]], doctrine: Any, @@ -211,6 +212,15 @@ def run_uk_calibration( started_at = time.perf_counter() started_ts = datetime.now(UTC) + # Pure-argument validation precedes every environment probe: a pruned + # register without its compiled band-edge source must refuse identically + # whether or not a git checkout or Logbook chain is reachable. + _validate_band_edge_registry( + register_registry=register_registry, + band_edge_registry=band_edge_registry, + exclusion_receipt=exclusion_receipt, + ) + edge_registry = band_edge_registry or register_registry code_pin = git_code_pin(_REPOSITORY) # Predecessor configuration is validated before anything is written: a # disagreeing chain must refuse with no artifact on disk, not after a @@ -230,6 +240,7 @@ def run_uk_calibration( "ledger": _ledger_provenance(ledger_artifact), **dict(run_config_extra), } + run_config["band_edge_register_sha256"] = edge_registry.version state = AttemptState( # Attempts are distinct rows even when they re-run one release: both # the local chain and the store refuse a repeated build id. @@ -246,6 +257,7 @@ def run_uk_calibration( input_sha256=input_sha256, ledger_artifact=ledger_artifact, register_registry=register_registry, + band_edge_registry=edge_registry, calibration_year=calibration_year, exclusion_receipt=exclusion_receipt, doctrine=doctrine, @@ -287,6 +299,41 @@ def _new_calibration_attempt_id(*, timestamp: datetime) -> str: ) +def _validate_band_edge_registry( + *, + register_registry: TargetRegistry, + band_edge_registry: TargetRegistry | None, + exclusion_receipt: Mapping[str, Mapping[str, str]], +) -> None: + if exclusion_receipt and band_edge_registry is None: + raise ValueError( + "a pruned register cannot derive published band edges from itself; " + "pass the compiled pre-exclusion register." + ) + if band_edge_registry is None: + return + register_names = _registry_spec_names(register_registry) + edge_names = _registry_spec_names(band_edge_registry) + excluded_names = {str(name) for name in exclusion_receipt} + if not register_names <= edge_names: + missing = sorted(register_names - edge_names) + raise ValueError( + "band-edge register must include every materialized registry spec; " + f"missing={missing}." + ) + extra = edge_names - register_names + if extra != excluded_names: + raise ValueError( + "band-edge register extra spec names must match the measure " + f"exclusion receipt; extra={sorted(extra)}, " + f"receipt={sorted(excluded_names)}." + ) + + +def _registry_spec_names(registry: TargetRegistry) -> set[str]: + return {str(spec.name) for spec in registry.specs} + + def _record_failed_attempt( *, error: BaseException, @@ -336,6 +383,7 @@ def _run_uk_calibration_attempt( input_sha256: str, ledger_artifact: Any, register_registry: TargetRegistry, + band_edge_registry: TargetRegistry, calibration_year: int, exclusion_receipt: Mapping[str, Mapping[str, str]], doctrine: Any, @@ -374,6 +422,7 @@ def _run_uk_calibration_attempt( period=calibration_year, doctrine=doctrine, measure_resolver=measure_resolver, + band_edge_registry=band_edge_registry, ) calibrated = stage(frame) append_phase(state, "national_calibration_solved") diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/ledger_targets.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/ledger_targets.py index 7e3608fa..0a1f0493 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/ledger_targets.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/ledger_targets.py @@ -88,6 +88,7 @@ def materialize_uk_ledger_targets( registry: TargetRegistry, *, period: int | str, + band_edge_registry: TargetRegistry | None = None, ) -> TargetMaterializationResult: """Materialize compiled UK Ledger target bindings on an adapter.""" @@ -102,6 +103,7 @@ def materialize_uk_ledger_targets( "baseline_flag_crosstab": _uk_baseline_flag_crosstab, "input_substitution_counterfactual": _uk_input_substitution, }, + band_edge_registry=band_edge_registry, ) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/national_calibration.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/national_calibration.py index 4ff938bb..df294ee2 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/national_calibration.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/national_calibration.py @@ -48,6 +48,7 @@ def __init__( period: int, doctrine: UKNationalSolveDoctrine = UK_NATIONAL_SOLVE_DOCTRINE, measure_resolver: object | None = None, + band_edge_registry: TargetRegistry | None = None, ) -> None: self.compilation = ( registry @@ -55,6 +56,9 @@ def __init__( else UKLedgerTargetCompilation(registry=registry, unsupported=()) ) self.registry = self.compilation.registry + self.band_edge_registry = ( + band_edge_registry if band_edge_registry is not None else self.registry + ) # The materialization period is the declared calibration year the # registry was compiled at — never the input frame's base-year # time_period, which lags it (survey 2024, calibration 2025). @@ -90,6 +94,7 @@ def __call__(self, frame: Frame) -> Frame: adapter, self.registry, period=self.period, + band_edge_registry=self.band_edge_registry, ) if materialized.skipped: skipped = [skip.__dict__ for skip in materialized.skipped] @@ -209,12 +214,14 @@ def _resolve_measures(self, frame: Frame) -> MeasureResolution | None: lambda: _CalibrationFrameAdapter(frame), self.registry, period=self.period, + band_edge_registry=self.band_edge_registry, ) return resolve_target_measures( lambda: _CalibrationFrameAdapter(frame), self.registry, self.measure_resolver, period=self.period, + band_edge_registry=self.band_edge_registry, ) def checkpoint_metadata(self) -> Mapping[str, object]: diff --git a/packages/microcosm-build/tests/test_uk_calibration_run.py b/packages/microcosm-build/tests/test_uk_calibration_run.py index 42420d26..96865c3d 100644 --- a/packages/microcosm-build/tests/test_uk_calibration_run.py +++ b/packages/microcosm-build/tests/test_uk_calibration_run.py @@ -87,6 +87,16 @@ def _registry(): ) +def _paths(tmp_path: Path) -> UKCalibrationRunPaths: + return UKCalibrationRunPaths( + input_h5=tmp_path / "input.h5", + staging_h5=tmp_path / "staged.h5", + diagnostics_json=tmp_path / "diagnostics.json", + build_record_json=tmp_path / "build_record.json", + terminal_gate_json=tmp_path / "terminal_gates.json", + ) + + def _sha(path: Path) -> str: return hashlib.sha256(path.read_bytes()).hexdigest() @@ -265,6 +275,128 @@ def test_run_uk_calibration_writes_cross_pinned_outputs(monkeypatch, tmp_path: P assert result.logbook_spool.exists() +def test_run_uk_calibration_refuses_pruned_register_without_band_edge_register( + tmp_path: Path, +): + paths = _paths(tmp_path) + + with pytest.raises(ValueError, match="compiled pre-exclusion register"): + run_uk_calibration( + paths=paths, + input_sha256="a" * 64, + ledger_artifact=object(), + register_registry=_registry(), + calibration_year=2025, + exclusion_receipt={"excluded.target": {"reason": "reviewed"}}, + doctrine=UKNationalSolveDoctrine(epochs=1), + doctrine_overrides={}, + measure_resolver=None, + source_pins={}, + run_config_extra={}, + release_id="pruned-without-edge-register", + ) + + assert not paths.staging_h5.exists() + assert not paths.diagnostics_json.exists() + assert not paths.build_record_json.exists() + + +def test_run_uk_calibration_refuses_incoherent_band_edge_register(tmp_path: Path): + paths = _paths(tmp_path) + edge_registry = TargetRegistry( + [ + *_registry().specs, + TargetSpec( + name="different.excluded", + entity="benunit", + measure="different/excluded", + value=1.0, + source="test", + metadata={"contract_target_id": "different.excluded"}, + ), + ], + country="uk", + ) + + with pytest.raises(ValueError, match="exclusion receipt"): + run_uk_calibration( + paths=paths, + input_sha256="a" * 64, + ledger_artifact=object(), + register_registry=_registry(), + band_edge_registry=edge_registry, + calibration_year=2025, + exclusion_receipt={"other.excluded": {"reason": "reviewed"}}, + doctrine=UKNationalSolveDoctrine(epochs=1), + doctrine_overrides={}, + measure_resolver=None, + source_pins={}, + run_config_extra={}, + release_id="incoherent-edge-register", + ) + + assert not paths.staging_h5.exists() + assert not paths.diagnostics_json.exists() + assert not paths.build_record_json.exists() + + +def test_run_uk_calibration_records_band_edge_register_sha256( + monkeypatch, tmp_path: Path +): + pytest.importorskip("tables") # pandas HDF backend + monkeypatch.setattr( + calibration_run, + "uk_aggregate_admin_totals", + lambda frame, manifest: (_admin_anchor_values(), []), + ) + input_h5 = tmp_path / "input.h5" + frame = _frame() + write_uk_national_frame(frame, input_h5) + _write_spine_sidecar(input_h5, frame) + paths = _paths(tmp_path) + register = _registry() + edge_registry = TargetRegistry( + [ + TargetSpec( + name="dwp.uc.households", + entity="benunit", + measure="dwp/uc/households", + value=99.0, + source="test", + family="dwp_universal_credit", + metadata={"contract_target_id": "dwp.uc.households"}, + ) + ], + country="uk", + ) + + result = run_uk_calibration( + paths=paths, + input_sha256=_sha(input_h5), + ledger_artifact=object(), + register_registry=register, + band_edge_registry=edge_registry, + calibration_year=2025, + exclusion_receipt={}, + doctrine=UKNationalSolveDoctrine(epochs=5), + doctrine_overrides={}, + measure_resolver=None, + source_pins={ + "input_h5": { + "sha256": _sha(input_h5), + "size_bytes": input_h5.stat().st_size, + } + }, + run_config_extra={}, + release_id="band-edge-provenance", + ) + + assert ( + result.build_record["run_config"]["band_edge_register_sha256"] + == edge_registry.version + ) + + def test_run_uk_calibration_refuses_input_sha_before_outputs(tmp_path: Path): pytest.importorskip("tables") # pandas HDF backend input_h5 = tmp_path / "input.h5" diff --git a/packages/microcosm-build/tests/test_uk_calibration_seam_driver.py b/packages/microcosm-build/tests/test_uk_calibration_seam_driver.py index 8ed5850e..f8cf5b3a 100644 --- a/packages/microcosm-build/tests/test_uk_calibration_seam_driver.py +++ b/packages/microcosm-build/tests/test_uk_calibration_seam_driver.py @@ -129,6 +129,7 @@ def test_driver_threads_registry_exclusions_resolver_and_overrides(monkeypatch, driver = _load_driver_module() calls = [] registry = _registry() + pruned_registry = TargetRegistry([], country="uk") artifact = SimpleNamespace( path=tmp_path / "ledger", facts=({"fact": 1},), @@ -152,7 +153,7 @@ def test_driver_threads_registry_exclusions_resolver_and_overrides(monkeypatch, monkeypatch.setattr( driver, "apply_uk_calibration_measure_exclusions", - lambda reg, exclusions: (reg, {"excluded": "reviewed"}), + lambda reg, exclusions: (pruned_registry, {"excluded": "reviewed"}), ) class FakeResolver: @@ -183,7 +184,8 @@ def fake_run(**kwargs): assert result == 0 call = calls[0] - assert call["register_registry"] is registry + assert call["register_registry"] is pruned_registry + assert call["band_edge_registry"] is registry assert call["calibration_year"] == 2025 assert call["exclusion_receipt"] == {"excluded": "reviewed"} assert call["doctrine"].epochs == 128 diff --git a/packages/microcosm-build/tests/test_uk_measure_simulation.py b/packages/microcosm-build/tests/test_uk_measure_simulation.py index 4e380007..59ba0648 100644 --- a/packages/microcosm-build/tests/test_uk_measure_simulation.py +++ b/packages/microcosm-build/tests/test_uk_measure_simulation.py @@ -289,8 +289,10 @@ def test_packaged_exclusions_load(): assert entry["tracking"] == "microcosm#791", entry["name"] # The lever targets are deliberately NOT excluded: the six UC - # caseload / two-child-limit cells ride the would_claim_uc lever run, - # and the two expected-to-resolve cells ride the exclusion re-run. + # caseload / two-child-limit cells ride the would_claim_uc lever run. + # Exclusions cannot move surviving cells because band edges are pinned + # to the compiled register (#792); the non-excluded cells are expected + # to pass at published band widths on the next seam run. excluded = set(names) for riding in ( "dwp.uc.households", diff --git a/packages/microcosm-build/tests/test_uk_national_calibration.py b/packages/microcosm-build/tests/test_uk_national_calibration.py index 1886242e..099c577b 100644 --- a/packages/microcosm-build/tests/test_uk_national_calibration.py +++ b/packages/microcosm-build/tests/test_uk_national_calibration.py @@ -21,6 +21,7 @@ UK_NATIONAL_TARGET_LOSS_CAP, UK_NATIONAL_TARGET_WEIGHT_RULE, UKNationalSolveDoctrine, + national_calibration, uk_doctrine_with_overrides, uk_national_target_loss_weights, ) @@ -372,6 +373,43 @@ def test_stage_manifest_omits_measure_resolution_without_resolver() -> None: assert "measure_resolution" not in stage.manifest +def test_stage_threads_band_edge_registry_to_materialization(monkeypatch) -> None: + captured = [] + real_materialize = national_calibration.materialize_uk_ledger_targets + + def capture_materialize(*args, **kwargs): + captured.append(kwargs) + return real_materialize(*args, **kwargs) + + monkeypatch.setattr( + national_calibration, + "materialize_uk_ledger_targets", + capture_materialize, + ) + sentinel = TargetRegistry([], country="uk") + stage = UKNationalCalibrationStage( + _registry(), + period=2025, + doctrine=UKNationalSolveDoctrine(epochs=1), + band_edge_registry=sentinel, + ) + + stage(_frame()) + + assert captured[-1]["band_edge_registry"] is sentinel + + captured.clear() + default_stage = UKNationalCalibrationStage( + _registry(), + period=2025, + doctrine=UKNationalSolveDoctrine(epochs=1), + ) + + default_stage(_frame()) + + assert captured[-1]["band_edge_registry"] is default_stage.registry + + def test_activated_unresolvable_compiled_reference_aborts_loudly() -> None: stage = UKNationalCalibrationStage( UKLedgerTargetCompilation( diff --git a/tools/calibrate_uk_national_dataset.py b/tools/calibrate_uk_national_dataset.py index 2f969784..c6e2118a 100644 --- a/tools/calibrate_uk_national_dataset.py +++ b/tools/calibrate_uk_national_dataset.py @@ -98,6 +98,7 @@ def main(argv: list[str] | None = None) -> int: input_sha256=args.input_sha256, ledger_artifact=artifact, register_registry=registry, + band_edge_registry=compilation.registry, calibration_year=calibration_year, exclusion_receipt=exclusion_receipt, doctrine=doctrine, From 7ccd09946f7efe4195107299d8b56b895f4a6568 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Thu, 27 Aug 2026 17:19:26 +0200 Subject: [PATCH 3/5] Require policyengine-uk 2.92 for the uk extra 2.92.0 carries private_pension_wealth (policyengine-uk#1824): the explicit capital disregard, the total_wealth re-sum, and the corporate_sector_wealth allocation key. The #750 split emits that column from the WAS stage, and an older engine's loader would silently drop it, narrowing total_wealth and the exposure keys. Locked upgrade moves policyengine-uk alone (2.89.0 -> 2.92.0, no transitive churn). Co-Authored-By: Claude Fable 5 --- packages/microcosm-build/pyproject.toml | 2 +- packages/microcosm-data/pyproject.toml | 2 +- packages/microcosm-frame/pyproject.toml | 2 +- uv.lock | 12 ++++++------ 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/packages/microcosm-build/pyproject.toml b/packages/microcosm-build/pyproject.toml index c7e75e36..dec94c00 100644 --- a/packages/microcosm-build/pyproject.toml +++ b/packages/microcosm-build/pyproject.toml @@ -38,7 +38,7 @@ us = [ # The UK extra adds the rules engine for local metric generation from a # Microcosm UK H5. Target tables remain explicit inputs, and the base package # still does not import policyengine-uk at import time. -uk = ["policyengine-uk>=2.88", "h5py>=3", "tables>=3"] +uk = ["policyengine-uk>=2.92", "h5py>=3", "tables>=3"] [project.scripts] microcosm-export-us-l0-refit-h5 = "microcosm.build.us_runtime.l0_refit_export:main" diff --git a/packages/microcosm-data/pyproject.toml b/packages/microcosm-data/pyproject.toml index 6ef7d481..c221c2a5 100644 --- a/packages/microcosm-data/pyproject.toml +++ b/packages/microcosm-data/pyproject.toml @@ -28,7 +28,7 @@ us = ["policyengine-us>=1.745.0,<2"] # The UK artifact lives in a PRIVATE repo (UK Data Service licence): loads # require an authenticated HF token with access. The loader surfaces the # 401 with that explanation rather than retrying. -uk = ["policyengine-uk>=2.88"] +uk = ["policyengine-uk>=2.92"] [project.urls] Homepage = "https://microcosm.institute" diff --git a/packages/microcosm-frame/pyproject.toml b/packages/microcosm-frame/pyproject.toml index 94d93027..6bc0c05b 100644 --- a/packages/microcosm-frame/pyproject.toml +++ b/packages/microcosm-frame/pyproject.toml @@ -12,7 +12,7 @@ dependencies = [ [project.optional-dependencies] us = ["microunit>=0.1.0"] policyengine = ["policyengine-us>=1.745.0,<2", "microunit>=0.1.0"] -uk = ["policyengine-uk>=2.88"] +uk = ["policyengine-uk>=2.92"] # The Axiom adapter's PyPI-resolvable dependencies. The engine itself # (axiom-rules-engine + its dense native extension) is not on PyPI yet and # installs from a checkout; see microcosm/frame/adapters/axiom.py. diff --git a/uv.lock b/uv.lock index 0e1eae87..66193d3d 100644 --- a/uv.lock +++ b/uv.lock @@ -712,7 +712,7 @@ requires-dist = [ { name = "microunit", marker = "extra == 'us'", specifier = ">=0.1.0" }, { name = "numpy", specifier = ">=1.26" }, { name = "pandas", specifier = ">=2" }, - { name = "policyengine-uk", marker = "extra == 'uk'", specifier = ">=2.88" }, + { name = "policyengine-uk", marker = "extra == 'uk'", specifier = ">=2.92" }, { name = "policyengine-us", marker = "extra == 'us'", specifier = ">=1.745.0,<2" }, { name = "pyarrow", specifier = ">=15" }, { name = "pyyaml", specifier = ">=6" }, @@ -783,7 +783,7 @@ requires-dist = [ { name = "h5py", specifier = ">=3" }, { name = "huggingface-hub", specifier = ">=0.20" }, { name = "packaging", specifier = ">=24" }, - { name = "policyengine-uk", marker = "extra == 'uk'", specifier = ">=2.88" }, + { name = "policyengine-uk", marker = "extra == 'uk'", specifier = ">=2.92" }, { name = "policyengine-us", marker = "extra == 'us'", specifier = ">=1.745.0,<2" }, ] provides-extras = ["us", "uk"] @@ -857,7 +857,7 @@ requires-dist = [ { name = "microunit", marker = "extra == 'us'", specifier = ">=0.1.0" }, { name = "numpy", specifier = ">=2" }, { name = "pandas", specifier = ">=2.3" }, - { name = "policyengine-uk", marker = "extra == 'uk'", specifier = ">=2.88" }, + { name = "policyengine-uk", marker = "extra == 'uk'", specifier = ">=2.92" }, { name = "policyengine-us", marker = "extra == 'policyengine'", specifier = ">=1.745.0,<2" }, { name = "tables", marker = "extra == 'axiom'", specifier = ">=3" }, ] @@ -1391,7 +1391,7 @@ wheels = [ [[package]] name = "policyengine-uk" -version = "2.89.0" +version = "2.92.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "microdf-python" }, @@ -1399,9 +1399,9 @@ dependencies = [ { name = "pydantic" }, { name = "tables" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/e4/49/75c80cd9b1145d8cfd5efbdb534aca2c9d5c5722fef3fc8f4887831505ea/policyengine_uk-2.89.0.tar.gz", hash = "sha256:aea49956a350ec35c1e0def7cd28db324b055711ec5ab7f331395dca8d103341", size = 1216776, upload-time = "2026-06-11T13:06:04.826Z" } +sdist = { url = "https://files.pythonhosted.org/packages/9f/3a/316968411a749f9efa3ea88b8ec5eb11420a1ec6df8dca34acdb06abf38f/policyengine_uk-2.92.0.tar.gz", hash = "sha256:c6d4a0be0f2ca3a2fe2c75e278ab5899c5d9e53630dc58da605f62328a34a2ce", size = 1242165, upload-time = "2026-08-26T05:35:34.152Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/bb/5f/fd32a3f7314b02b3b6a8362d6b2869af5b51569301945c00ee1172e01914/policyengine_uk-2.89.0-py3-none-any.whl", hash = "sha256:72acae45a35a59cf7429bba69363601e624bfd643d50cd74b8d20d325497e9fd", size = 1999087, upload-time = "2026-06-11T13:06:02.819Z" }, + { url = "https://files.pythonhosted.org/packages/a4/22/37a8e7ed43ae8322d408e97753c8608959ac1dc1ae08bf2f881e4aa9196c/policyengine_uk-2.92.0-py3-none-any.whl", hash = "sha256:af0c29baa7ee0fd555d1f92cbfc506382f40cbc3387cc86c7c20a9e9d830de1d", size = 2041047, upload-time = "2026-08-26T05:35:32.363Z" }, ] [[package]] From 6dad6e0e47c7e1d4c4b7d34691837926d39be9cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Thu, 27 Aug 2026 21:21:46 +0200 Subject: [PATCH 4/5] Split private_pension_wealth out of corporate_wealth on the WAS stage The stage folded WAS private pension wealth less current-employment DB (totalpenr8_aggr - dvvaldbt_scaper8_aggr) into corporate_wealth, which policyengine-uk counts as capital in every means test - the largest mechanism behind the UC caseload shortfall (uk-data#452 M2). The stage now emits private_pension_wealth as its own household output, drawn first in the second chain segment with the share-like holdings conditional on it and the third segment conditional on both; corporate_wealth keeps employee shares/options, UK shares, unit and investment trusts and the stocks-and-shares-ISA fold. Row identity old corporate_wealth == corporate_wealth + private_pension_wealth holds on the donor by construction and is pinned in tests, alongside a segment-1 containment test documenting why the first segment's draws cannot move. Lockstep spec pair, support bounds (operator-measured on the pinned tab: min 0.00 over 15,128 rows, outward bound [0, 8000000], byte-verified via the tool's --check), export allow-list, stage-support gate columns and zero allowances, coverage manifest, and the moved digests (battery policy/manifest/fingerprint, spine and release_cut certification part mirrors, UK spec bundle sha) re-cut in the same change. The QRF household-tail note records the 13th unarmed household column; arming stays the microcosm#796 follow-up. Closes microcosm#750 machinery; the spine rebuild, parity re-mints and seam measurement ride the follow-up commits. Co-Authored-By: Claude Fable 5 --- .../750-pension-wealth-split.changed.md | 1 + .../src/microcosm/build/uk/gates.json | 6 +- .../uk/release_input_coverage_manifest.json | 69 +++++++------- .../src/microcosm/build/uk/source_stages.json | 8 +- .../src/microcosm/build/uk/spec/sources.yaml | 8 +- .../build/uk/was_wealth_support_bounds.json | 4 + .../build/uk_runtime/terminal_gates.py | 1 + .../microcosm/build/uk_runtime/was_wealth.py | 30 ++++-- .../tests/test_spec_engine_country_bundles.py | 2 +- .../tests/test_uk_was_wealth.py | 95 ++++++++++++++++++- .../src/microcosm/data/contract.py | 14 +-- .../microcosm-data/tests/test_contract.py | 6 +- 12 files changed, 184 insertions(+), 60 deletions(-) create mode 100644 changelog.d/750-pension-wealth-split.changed.md diff --git a/changelog.d/750-pension-wealth-split.changed.md b/changelog.d/750-pension-wealth-split.changed.md new file mode 100644 index 00000000..aec06f1a --- /dev/null +++ b/changelog.d/750-pension-wealth-split.changed.md @@ -0,0 +1 @@ +The UK WAS stage now emits `private_pension_wealth` (WAS private pension wealth less current-employment DB, uk-data#452 M2) as its own household output, drawn first in the second chain segment; `corporate_wealth` keeps the share-like holdings and the stocks-and-shares-ISA fold, row identity old corporate wealth == new corporate wealth + pension wealth holds on the donor, and the `uk` extra now requires policyengine-uk 2.92, whose means tests disregard the new column. diff --git a/packages/microcosm-build/src/microcosm/build/uk/gates.json b/packages/microcosm-build/src/microcosm/build/uk/gates.json index ed3447da..fc0984e4 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/gates.json +++ b/packages/microcosm-build/src/microcosm/build/uk/gates.json @@ -70,6 +70,7 @@ "num_vehicles", "other_residential_property_value", "owned_land", + "private_pension_wealth", "property_wealth", "savings", "stocks_and_shares_isa", @@ -85,6 +86,7 @@ "num_vehicles": 0, "other_residential_property_value": 0, "owned_land": 0, + "private_pension_wealth": 0, "property_wealth": 0, "savings": 0, "stocks_and_shares_isa": 0, @@ -100,6 +102,7 @@ "num_vehicles": 0, "other_residential_property_value": 0, "owned_land": 0, + "private_pension_wealth": 0, "property_wealth": 0, "savings": 0, "stocks_and_shares_isa": 0, @@ -526,6 +529,7 @@ "household.msoa_code", "household.num_vehicles", "household.oa_code", + "household.private_pension_wealth", "household.property_purchased", "household.rail_usage", "household.region_code_oa", @@ -662,7 +666,7 @@ "max_top_share": 0.9994670564654868, "min_nonzero_records": 104 }, - "notes": "Weighted tail-concentration audit of the QRF-imputed output columns derived from the HMRC source manifest. Armed from the #686 L3 baselines per #757 B4, each threshold recording the run that measured it: the 47-column qrf_tail grids of uk_weighted_integrity_baselines_686.json (sha256 8b3f6c4e9c522445bf3e05d3451ee8557f4e14d25524ceac3af8bfe05401a146, licensed acceptance dir 686-spine-swap), measured on spine-a.h5 (sha256 a65f2132736d1dcecb91709a513a0d54a5887635ea03760629cc888d188ee306, 113,649 person rows, design weights) for the #609/#578 threshold adjudication. top_k 100 is the measurement grid anchor [10, 100, 500, 1000]; max_top_share is the exact measured maximum over the checked surface (hmrc_spi_other_social_security_income, 104 carriers), no headroom; min_nonzero_records is the thinnest measured column above the grid anchor (the same column) - the policy domain requires min_nonzero_records > top_k, so the three saturated sub-anchor columns (hmrc_spi_taxable_termination_pay 12, charitable_investment_gifts 22, sda_reported 24 carriers, each top-100 share 1.0) sit below it and go thin visibly in the signed details on every run, never a silent pass. The baselines are design-weight measurements and the terminal gate runs on the calibrated release frame; a calibrated-weight breach names its column and is a finding, not noise. The 12 household_qrf_tail grids (was_wealth surface) are measured in the same baselines file and not yet armed - a household-surface gate entry is a declared follow-up. Reviewed exclusions live in the named register resource of this package." + "notes": "Weighted tail-concentration audit of the QRF-imputed output columns derived from the HMRC source manifest. Armed from the #686 L3 baselines per #757 B4, each threshold recording the run that measured it: the 47-column qrf_tail grids of uk_weighted_integrity_baselines_686.json (sha256 8b3f6c4e9c522445bf3e05d3451ee8557f4e14d25524ceac3af8bfe05401a146, licensed acceptance dir 686-spine-swap), measured on spine-a.h5 (sha256 a65f2132736d1dcecb91709a513a0d54a5887635ea03760629cc888d188ee306, 113,649 person rows, design weights) for the #609/#578 threshold adjudication. top_k 100 is the measurement grid anchor [10, 100, 500, 1000]; max_top_share is the exact measured maximum over the checked surface (hmrc_spi_other_social_security_income, 104 carriers), no headroom; min_nonzero_records is the thinnest measured column above the grid anchor (the same column) - the policy domain requires min_nonzero_records > top_k, so the three saturated sub-anchor columns (hmrc_spi_taxable_termination_pay 12, charitable_investment_gifts 22, sda_reported 24 carriers, each top-100 share 1.0) sit below it and go thin visibly in the signed details on every run, never a silent pass. The baselines are design-weight measurements and the terminal gate runs on the calibrated release frame; a calibrated-weight breach names its column and is a finding, not noise. The 12 household_qrf_tail grids (was_wealth surface) are measured in the same baselines file and not yet armed - a household-surface gate entry is a declared follow-up. The #750 split adds a 13th household QRF column (private_pension_wealth) not yet in the measured baselines; the household-surface arming follow-up on microcosm#796 re-measures. Reviewed exclusions live in the named register resource of this package." } ] } diff --git a/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json b/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json index 92b35167..416e547a 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json +++ b/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json @@ -473,7 +473,7 @@ "capital_gains" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "84d1c1c85f0172a9a396ef69d445ee33cf1340f62ff73c8653c399af40087bf4", + "source_manifest_sha256": "2dc160b3c1f3067d34757a5f566a76545bf3f4f92035b4327f4662658a793adb", "source_vintages": { "source": "HMRC Capital Gains Tax statistics, July 2025, Table 2.1a", "survey": "HMRC Capital Gains Tax statistics Table 2.1a and Advani-Summers capital-gains incidence" @@ -496,7 +496,7 @@ "capital_gains" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "84d1c1c85f0172a9a396ef69d445ee33cf1340f62ff73c8653c399af40087bf4", + "source_manifest_sha256": "2dc160b3c1f3067d34757a5f566a76545bf3f4f92035b4327f4662658a793adb", "source_vintages": { "source": "Advani and Summers (2020), Capital Gains and UK Inequality, CAGE Working Paper 465", "survey": "Family Resources Survey 2024-25, SPI synthetic support, and Advani-Summers capital-gains incidence" @@ -525,7 +525,7 @@ "required_mass_change_reason": "E5 source-stage transform preserves household rows and typed household weights; total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "84d1c1c85f0172a9a396ef69d445ee33cf1340f62ff73c8653c399af40087bf4", + "source_manifest_sha256": "2dc160b3c1f3067d34757a5f566a76545bf3f4f92035b4327f4662658a793adb", "source_vintages": { "source": "UK Data Service SN 8856 Effects of Taxes and Benefits household tab, DfT rail fare index, and public NHS activity/cost table.", "survey": "Effects of Taxes and Benefits 1977-2024 and NHS age-gender public table" @@ -545,7 +545,7 @@ "required_mass_change_reason": "E5 source-stage transform preserves household rows and typed household weights; total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "84d1c1c85f0172a9a396ef69d445ee33cf1340f62ff73c8653c399af40087bf4", + "source_manifest_sha256": "2dc160b3c1f3067d34757a5f566a76545bf3f4f92035b4327f4662658a793adb", "source_vintages": { "source": "UK Data Service SN 8856 Effects of Taxes and Benefits household tab and cited VAT anchor resource.", "survey": "Effects of Taxes and Benefits 1977-2024" @@ -565,20 +565,20 @@ "capital_gains" ], "required_mass_change_reason": "Amounts-only capital gains redraw: household weights pass through unchanged and total household mass is conserved.", - "source_manifest": "cgt_source_stages.json", - "source_manifest_sha256": "71104111b4b9f2da00ce49ad5abec54a35d300196032c9742d62e02aa1730774", - "superseded_by": { - "reason": "The FRS spine build executes hmrc_cgt_gains_spine, which applies the same HMRC Table 3 amounts redraw directly in source_stages.json before calibration.", - "source_manifest": "source_stages.json", - "source_manifest_sha256": "84d1c1c85f0172a9a396ef69d445ee33cf1340f62ff73c8653c399af40087bf4", - "stage": "hmrc_cgt_gains_spine" - }, - "source_vintages": { - "hmrc_surface": "2023-24", - "mapped_build_period": "2024" + "source_manifest": "cgt_source_stages.json", + "source_manifest_sha256": "71104111b4b9f2da00ce49ad5abec54a35d300196032c9742d62e02aa1730774", + "source_vintages": { + "hmrc_surface": "2023-24", + "mapped_build_period": "2024" }, "stage": "hmrc_cgt_gains", - "status": "required_at_build" + "status": "required_at_build", + "superseded_by": { + "reason": "The FRS spine build executes hmrc_cgt_gains_spine, which applies the same HMRC Table 3 amounts redraw directly in source_stages.json before calibration.", + "source_manifest": "source_stages.json", + "source_manifest_sha256": "2dc160b3c1f3067d34757a5f566a76545bf3f4f92035b4327f4662658a793adb", + "stage": "hmrc_cgt_gains_spine" + } }, "hmrc_cgt_gains_spine": { "base_candidate_sha256": "f17306ccb2aad7ff0130be3589b560afb2e2a12a943570911cd0c77f07934833", @@ -596,7 +596,7 @@ "capital_gains" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "84d1c1c85f0172a9a396ef69d445ee33cf1340f62ff73c8653c399af40087bf4", + "source_manifest_sha256": "2dc160b3c1f3067d34757a5f566a76545bf3f4f92035b4327f4662658a793adb", "source_vintages": { "hmrc_surface": "2023-24", "mapped_build_period": "2024" @@ -610,7 +610,7 @@ "base_candidate_tier": "frs", "calibration_permitted": false, "canonical_source_manifest": "source_stages.json", - "canonical_source_manifest_sha256": "84d1c1c85f0172a9a396ef69d445ee33cf1340f62ff73c8653c399af40087bf4", + "canonical_source_manifest_sha256": "2dc160b3c1f3067d34757a5f566a76545bf3f4f92035b4327f4662658a793adb", "effective_mass_requirements": { "charitable_investment_gifts": { "mass_share_denominator": "all_person_effective_mass", @@ -669,7 +669,7 @@ "OTHERINC" ] }, - "reviewed_fence_ids": [ + "reviewed_fence_ids": [ "frs_epb_source_absent", "frs_exps_source_absent", "frs_taxterm_source_absent", @@ -679,15 +679,9 @@ "frs_srp_regular_code5_subset", "full_frs_tei_band_unavailable" ], - "source_manifest": "hmrc_income_source_stages.json", - "source_manifest_sha256": "c0341af7166ae3a85a3c1164e7d9e880c4b4aec122f1a8fa90c73b46c596e1ea", - "superseded_by": { - "reason": "The FRS spine build executes hmrc_spi_income_spine, which supersedes the June retained-leaves/hmrc_spi_income pair inside source_stages.json.", - "source_manifest": "source_stages.json", - "source_manifest_sha256": "84d1c1c85f0172a9a396ef69d445ee33cf1340f62ff73c8653c399af40087bf4", - "stage": "hmrc_spi_income_spine" - }, - "source_vintages": { + "source_manifest": "hmrc_income_source_stages.json", + "source_manifest_sha256": "c0341af7166ae3a85a3c1164e7d9e880c4b4aec122f1a8fa90c73b46c596e1ea", + "source_vintages": { "hmrc_surface": "2023-24", "mapped_build_period": "2024", "period_mapping": "latest_published_tax_year", @@ -695,7 +689,13 @@ }, "spi_prior_national_household_mass_share": 0.5, "stage": "hmrc_spi_income", - "status": "required_at_build" + "status": "required_at_build", + "superseded_by": { + "reason": "The FRS spine build executes hmrc_spi_income_spine, which supersedes the June retained-leaves/hmrc_spi_income pair inside source_stages.json.", + "source_manifest": "source_stages.json", + "source_manifest_sha256": "2dc160b3c1f3067d34757a5f566a76545bf3f4f92035b4327f4662658a793adb", + "stage": "hmrc_spi_income_spine" + } }, "lcfs_consumption": { "base_candidate_sha256": "f17306ccb2aad7ff0130be3589b560afb2e2a12a943570911cd0c77f07934833", @@ -727,7 +727,7 @@ "required_mass_change_reason": "E5 source-stage transform preserves household rows and typed household weights; total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "84d1c1c85f0172a9a396ef69d445ee33cf1340f62ff73c8653c399af40087bf4", + "source_manifest_sha256": "2dc160b3c1f3067d34757a5f566a76545bf3f4f92035b4327f4662658a793adb", "source_vintages": { "source": "UK Data Service SN 9468 Living Costs and Food Survey 2023-24 household/person tabs, NEED 2023 headline energy tables, Ofgem Q2 2026 unit rates, and WAS round-8 bridge donor.", "survey": "Living Costs and Food Survey 2023-24" @@ -748,7 +748,7 @@ "property_wealth" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "84d1c1c85f0172a9a396ef69d445ee33cf1340f62ff73c8653c399af40087bf4", + "source_manifest_sha256": "2dc160b3c1f3067d34757a5f566a76545bf3f4f92035b4327f4662658a793adb", "source_vintages": { "source": "MHCLG dwellings and ONS UK House Price Index December 2025 regional average prices.", "survey": "Public regional property reference" @@ -772,7 +772,7 @@ "employee_pension_contributions" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "84d1c1c85f0172a9a396ef69d445ee33cf1340f62ff73c8653c399af40087bf4", + "source_manifest_sha256": "2dc160b3c1f3067d34757a5f566a76545bf3f4f92035b4327f4662658a793adb", "source_vintages": { "source": "HMRC, Salary sacrifice reform for pension contributions effective from 6 April 2029", "survey": "Family Resources Survey 2024-25 salary-sacrifice respondents and HMRC salary-sacrifice reform analysis" @@ -794,7 +794,7 @@ "student_loan_plan" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "84d1c1c85f0172a9a396ef69d445ee33cf1340f62ff73c8653c399af40087bf4", + "source_manifest_sha256": "2dc160b3c1f3067d34757a5f566a76545bf3f4f92035b4327f4662658a793adb", "source_vintages": { "source": "Explore Education Statistics Table 6a, Higher education total", "survey": "Family Resources Survey 2024-25 and Student Loans Company borrower forecasts for England" @@ -811,6 +811,7 @@ "outputs": [ "owned_land", "property_wealth", + "private_pension_wealth", "corporate_wealth", "gross_financial_wealth", "net_financial_wealth", @@ -826,7 +827,7 @@ "required_mass_change_reason": "E5 source-stage transform preserves household rows and typed household weights; total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "84d1c1c85f0172a9a396ef69d445ee33cf1340f62ff73c8653c399af40087bf4", + "source_manifest_sha256": "2dc160b3c1f3067d34757a5f566a76545bf3f4f92035b4327f4662658a793adb", "source_vintages": { "source": "Office for National Statistics Wealth and Assets Survey, UK Data Service SN 7215, DOI 10.5255/UKDA-SN-7215-20; local licensed 2006-22 household tab.", "survey": "Wealth and Assets Survey round 8" diff --git a/packages/microcosm-build/src/microcosm/build/uk/source_stages.json b/packages/microcosm-build/src/microcosm/build/uk/source_stages.json index 27417362..c8d749a0 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/source_stages.json +++ b/packages/microcosm-build/src/microcosm/build/uk/source_stages.json @@ -907,7 +907,8 @@ "weight": "R8xshhwgt" }, "derived": { - "corporate_wealth_excl_isa": "(totalpenr8_aggr - dvvaldbt_scaper8_aggr) + DVFESHARESR8_aggr + DVFShUKVR8_aggr + DVFCollVR8_aggr", + "private_pension_wealth": "totalpenr8_aggr - dvvaldbt_scaper8_aggr", + "corporate_wealth_excl_isa": "DVFESHARESR8_aggr + DVFShUKVR8_aggr + DVFCollVR8_aggr", "stocks_and_shares_isa": "DVIISAVR8_aggr", "cash_isa": "DVCISAVR8_aggr", "student_loan_balance": "Tot_LosR8_aggr - Tot_los_exc_SLCR8_aggr", @@ -960,6 +961,7 @@ "chain_order": [ "owned_land", "property_wealth", + "private_pension_wealth", "corporate_wealth_excl_isa", "stocks_and_shares_isa", "corporate_wealth", @@ -1007,6 +1009,7 @@ "outputs": [ "owned_land", "property_wealth", + "private_pension_wealth", "corporate_wealth", "gross_financial_wealth", "net_financial_wealth", @@ -1022,6 +1025,7 @@ "nonnegative_outputs": [ "owned_land", "property_wealth", + "private_pension_wealth", "corporate_wealth", "gross_financial_wealth", "main_residence_value", @@ -1033,7 +1037,7 @@ "stocks_and_shares_isa", "student_loan_balance" ], - "notes": "Ports incumbent WAS round-8 wealth imputation with signed E5 differences: exact lower-case column matching replaces the fuzzy r/w fallback; cash ISA uses DVCISAVR8_aggr and stocks-and-shares ISA uses DVIISAVR8_aggr; corporate_wealth folds stocks-and-shares ISA after drawing corporate_wealth_excl_isa; recipient Northern Ireland regions are mapped to Wales for prediction only; student_loan_balance is allocated by household id rather than the incumbent positional off-by-one. Engine predictors materialize at their native entity and person/benunit values are summed to household, reproducing the incumbent map_to=household semantics; region is one-hot encoded jointly across donor and recipient (the incumbent's dummy encoding), with unmapped donor GOR codes becoming all-zero dummy rows. The WAS and FRS predictor definitions are not fully like-for-like and are ported as-is; raw WAS missing values are blanket-filled with zero; UKDS negative sentinel codes (-9/-8/-7/-6) are recoded to zero for the nonnegative-domain columns the licensed audit found carrying them (vcarnr8: 2 rows; HBedRmR8: 95.8 percent - the bedrooms question is effectively unasked in the WAS household file, predictor-quality revisit registered on microcosm#145) - a signed difference vs the incumbent, which trains on raw sentinels; DVPriRntR8's -9 is structural not-applicable so the is_renting mapping is unchanged; genuinely negative domains are never recoded." + "notes": "Ports incumbent WAS round-8 wealth imputation with signed E5 differences: exact lower-case column matching replaces the fuzzy r/w fallback; cash ISA uses DVCISAVR8_aggr and stocks-and-shares ISA uses DVIISAVR8_aggr; corporate_wealth folds stocks-and-shares ISA after drawing corporate_wealth_excl_isa; under uk-data#452 M2 / microcosm#750, WAS private pension wealth less current-employment DB is drawn first in the second chain segment as private_pension_wealth, and row identity old corporate_wealth == corporate_wealth + private_pension_wealth holds on the donor; recipient Northern Ireland regions are mapped to Wales for prediction only; student_loan_balance is allocated by household id rather than the incumbent positional off-by-one. Engine predictors materialize at their native entity and person/benunit values are summed to household, reproducing the incumbent map_to=household semantics; region is one-hot encoded jointly across donor and recipient (the incumbent's dummy encoding), with unmapped donor GOR codes becoming all-zero dummy rows. The WAS and FRS predictor definitions are not fully like-for-like and are ported as-is; raw WAS missing values are blanket-filled with zero; UKDS negative sentinel codes (-9/-8/-7/-6) are recoded to zero for the nonnegative-domain columns the licensed audit found carrying them (vcarnr8: 2 rows; HBedRmR8: 95.8 percent - the bedrooms question is effectively unasked in the WAS household file, predictor-quality revisit registered on microcosm#145) - a signed difference vs the incumbent, which trains on raw sentinels; DVPriRntR8's -9 is structural not-applicable so the is_renting mapping is unchanged; genuinely negative domains are never recoded." }, { "stage": "regional_property_uprating", diff --git a/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml b/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml index d30a9419..86c2af67 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml +++ b/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml @@ -724,7 +724,8 @@ stages: num_vehicles: vcarnr8 weight: R8xshhwgt derived: - corporate_wealth_excl_isa: (totalpenr8_aggr - dvvaldbt_scaper8_aggr) + DVFESHARESR8_aggr + DVFShUKVR8_aggr + DVFCollVR8_aggr + private_pension_wealth: totalpenr8_aggr - dvvaldbt_scaper8_aggr + corporate_wealth_excl_isa: DVFESHARESR8_aggr + DVFShUKVR8_aggr + DVFCollVR8_aggr stocks_and_shares_isa: DVIISAVR8_aggr cash_isa: DVCISAVR8_aggr student_loan_balance: Tot_LosR8_aggr - Tot_los_exc_SLCR8_aggr @@ -767,6 +768,7 @@ stages: chain_order: - owned_land - property_wealth + - private_pension_wealth - corporate_wealth_excl_isa - stocks_and_shares_isa - corporate_wealth @@ -802,6 +804,7 @@ stages: outputs: - owned_land - property_wealth + - private_pension_wealth - corporate_wealth - gross_financial_wealth - net_financial_wealth @@ -816,6 +819,7 @@ stages: nonnegative_outputs: - owned_land - property_wealth + - private_pension_wealth - corporate_wealth - gross_financial_wealth - main_residence_value @@ -826,7 +830,7 @@ stages: - cash_isa - stocks_and_shares_isa - student_loan_balance - notes: 'Ports incumbent WAS round-8 wealth imputation with signed E5 differences: exact lower-case column matching replaces the fuzzy r/w fallback; cash ISA uses DVCISAVR8_aggr and stocks-and-shares ISA uses DVIISAVR8_aggr; corporate_wealth folds stocks-and-shares ISA after drawing corporate_wealth_excl_isa; recipient Northern Ireland regions are mapped to Wales for prediction only; student_loan_balance is allocated by household id rather than the incumbent positional off-by-one. Engine predictors materialize at their native entity and person/benunit values are summed to household, reproducing the incumbent map_to=household semantics; region is one-hot encoded jointly across donor and recipient (the incumbent''s dummy encoding), with unmapped donor GOR codes becoming all-zero dummy rows. The WAS and FRS predictor definitions are not fully like-for-like and are ported as-is; raw WAS missing values are blanket-filled with zero; UKDS negative sentinel codes (-9/-8/-7/-6) are recoded to zero + notes: 'Ports incumbent WAS round-8 wealth imputation with signed E5 differences: exact lower-case column matching replaces the fuzzy r/w fallback; cash ISA uses DVCISAVR8_aggr and stocks-and-shares ISA uses DVIISAVR8_aggr; corporate_wealth folds stocks-and-shares ISA after drawing corporate_wealth_excl_isa; under uk-data#452 M2 / microcosm#750, WAS private pension wealth less current-employment DB is drawn first in the second chain segment as private_pension_wealth, and row identity old corporate_wealth == corporate_wealth + private_pension_wealth holds on the donor; recipient Northern Ireland regions are mapped to Wales for prediction only; student_loan_balance is allocated by household id rather than the incumbent positional off-by-one. Engine predictors materialize at their native entity and person/benunit values are summed to household, reproducing the incumbent map_to=household semantics; region is one-hot encoded jointly across donor and recipient (the incumbent''s dummy encoding), with unmapped donor GOR codes becoming all-zero dummy rows. The WAS and FRS predictor definitions are not fully like-for-like and are ported as-is; raw WAS missing values are blanket-filled with zero; UKDS negative sentinel codes (-9/-8/-7/-6) are recoded to zero for the nonnegative-domain columns the licensed audit found carrying them (vcarnr8: 2 rows; HBedRmR8: 95.8 percent - the bedrooms question is effectively unasked in the WAS household file, predictor-quality revisit registered on microcosm#145) - a signed difference vs the incumbent, which trains on raw sentinels; DVPriRntR8''s -9 is structural not-applicable so the is_renting mapping is unchanged; genuinely negative domains are never recoded.' - stage: regional_property_uprating survey: Public regional property reference diff --git a/packages/microcosm-build/src/microcosm/build/uk/was_wealth_support_bounds.json b/packages/microcosm-build/src/microcosm/build/uk/was_wealth_support_bounds.json index 0d9bce9b..a9fc037d 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/was_wealth_support_bounds.json +++ b/packages/microcosm-build/src/microcosm/build/uk/was_wealth_support_bounds.json @@ -18,6 +18,10 @@ 0.0, 20000000 ], + "private_pension_wealth": [ + 0.0, + 8000000 + ], "corporate_wealth": [ 0.0, 30000000 diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/terminal_gates.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/terminal_gates.py index 625d9602..1e4660ca 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/terminal_gates.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/terminal_gates.py @@ -178,6 +178,7 @@ def __post_init__(self) -> None: "household.msoa_code", "household.num_vehicles", "household.oa_code", + "household.private_pension_wealth", "household.property_purchased", "household.rail_usage", "household.region_code_oa", diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/was_wealth.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/was_wealth.py index 27a1b8d0..53a1cad4 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/was_wealth.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/was_wealth.py @@ -1,4 +1,8 @@ -"""UK WAS wealth imputation stage.""" +"""UK WAS wealth imputation stage. + +The WAS pension component is emitted separately from corporate wealth under +uk-data#452 M2 / microcosm#750. +""" from __future__ import annotations @@ -58,6 +62,7 @@ UK_WAS_WEALTH_OUTPUT_COLUMNS = ( "owned_land", "property_wealth", + "private_pension_wealth", "corporate_wealth", "gross_financial_wealth", "net_financial_wealth", @@ -283,10 +288,9 @@ def clean_was_household_table(raw: pd.DataFrame) -> pd.DataFrame: values = cleaned[column] cleaned[column] = values.where(~values.isin(_SENTINEL_CODES), 0) cleaned["is_renting"] = cleaned["private_rent_code"] == 1 + cleaned["private_pension_wealth"] = cleaned["pensions"] - cleaned["db_pensions"] cleaned["corporate_wealth_excl_isa"] = ( - cleaned["pensions"] - - cleaned["db_pensions"] - + cleaned["emp_shares_options"] + cleaned["emp_shares_options"] + cleaned["uk_shares"] + cleaned["unit_investment_trusts"] ) @@ -427,19 +431,33 @@ def run_segment(base_predictors: Sequence[str], targets: Sequence[str]) -> None: base = encoded_predictors run_segment(base, ("owned_land", "property_wealth")) + donor_encoded["private_pension_wealth"] = donor_encoded[ + "private_pension_wealth" + ].astype(float) donor_encoded["corporate_wealth"] = donor_encoded["corporate_wealth"].astype(float) recipient_encoded["owned_land"] = raw["owned_land"] recipient_encoded["property_wealth"] = raw["property_wealth"] run_segment( (*base, "owned_land", "property_wealth"), - ("corporate_wealth_excl_isa", "stocks_and_shares_isa"), + ( + "private_pension_wealth", + "corporate_wealth_excl_isa", + "stocks_and_shares_isa", + ), ) raw["corporate_wealth"] = ( raw["corporate_wealth_excl_isa"] + raw["stocks_and_shares_isa"] ) + recipient_encoded["private_pension_wealth"] = raw["private_pension_wealth"] recipient_encoded["corporate_wealth"] = raw["corporate_wealth"] run_segment( - (*base, "owned_land", "property_wealth", "corporate_wealth"), + ( + *base, + "owned_land", + "property_wealth", + "private_pension_wealth", + "corporate_wealth", + ), ( "gross_financial_wealth", "net_financial_wealth", diff --git a/packages/microcosm-build/tests/test_spec_engine_country_bundles.py b/packages/microcosm-build/tests/test_spec_engine_country_bundles.py index 9d028071..1cdd82c1 100644 --- a/packages/microcosm-build/tests/test_spec_engine_country_bundles.py +++ b/packages/microcosm-build/tests/test_spec_engine_country_bundles.py @@ -42,7 +42,7 @@ ), ( "uk", - "4793baa59a90930956faf8f5b422ece57da3445c7ed0030622ef62ff35dfb142", + "d56a4d0ecb32325921efb38a366b674891eb5d97ae18a74428532e405c4f144b", { "benunit.benunit_id", "household.household_id", diff --git a/packages/microcosm-build/tests/test_uk_was_wealth.py b/packages/microcosm-build/tests/test_uk_was_wealth.py index 872ae69f..1f34586c 100644 --- a/packages/microcosm-build/tests/test_uk_was_wealth.py +++ b/packages/microcosm-build/tests/test_uk_was_wealth.py @@ -152,14 +152,39 @@ def test_was_donor_cleaning_arithmetic_and_exact_case_insensitive_columns() -> N assert donor["stocks_and_shares_isa"].tolist() == [5.0, 6.0] assert donor["cash_isa"].tolist() == [7.0, 8.0] - assert donor["corporate_wealth_excl_isa"].tolist() == [73.0, 166.0] - assert donor["corporate_wealth"].tolist() == [78.0, 172.0] + assert "private_pension_wealth" in donor.columns + assert donor["private_pension_wealth"].tolist() == [60.0, 150.0] + assert donor["corporate_wealth_excl_isa"].tolist() == [13.0, 16.0] + assert donor["corporate_wealth"].tolist() == [18.0, 22.0] assert donor["student_loan_balance"].tolist() == [5000.0, 2000.0] assert donor["region"].tolist() == ["LONDON", "SCOTLAND"] assert donor["is_renting"].tolist() == [True, False] assert 3 not in REGIONS +def test_was_pension_split_preserves_old_corporate_wealth_identity() -> None: + donor = clean_was_household_table(_raw_was()) + raw = _raw_was() + + pd.testing.assert_series_equal( + donor["corporate_wealth"] + donor["private_pension_wealth"], + pd.Series([78.0, 172.0]), + check_names=False, + ) + pd.testing.assert_series_equal( + donor["corporate_wealth"] + donor["private_pension_wealth"], + ( + raw["totalpenr8_aggr"] + - raw["dvvaldbt_scaper8_aggr"] + + raw["DVFESHARESR8_aggr"] + + raw["DVFShUKVR8_aggr"] + + raw["DVFCollVR8_aggr"] + + raw["DVIISAVR8_aggR"] + ), + check_names=False, + ) + + def test_was_donor_sentinel_codes_recode_to_zero_for_nonnegative_domains() -> None: raw = _raw_was() raw.loc[0, "vcarnr8"] = -8 @@ -343,6 +368,7 @@ def test_stage_transform_is_deterministic_with_fast_synthetic_imputer( pd.testing.assert_frame_equal(a.table("household"), b.table("household")) pd.testing.assert_frame_equal(a.table("person"), b.table("person")) + assert "private_pension_wealth" in a.table("household").columns def test_stage_transform_requires_a_tab_path_or_donor() -> None: @@ -387,6 +413,60 @@ def test_stage_transform_refuses_sha_mismatched_tab(tmp_path) -> None: transform(_frame()) +def test_was_imputer_keeps_first_segment_before_pension_split( + monkeypatch: pytest.MonkeyPatch, +) -> None: + import microcosm.build.uk_runtime.was_wealth as module + import microcosm.fit + + calls = [] + + class FakeQRF: + def __init__(self, *, n_estimators, seed): + assert n_estimators == 7 + assert seed == 0 + + def start_chain(self, donor, predictors, targets, *, weights): + assert weights == "weight" + calls.append((tuple(predictors), tuple(targets))) + return SimpleNamespace(targets=tuple(targets), position=0) + + def fit_draw_next( + self, + donor, + recipient_predictors, + raw_prior_draws, + *, + state, + weights, + ): + assert weights == "weight" + target = state.targets[state.position] + return SimpleNamespace( + target=target, + raw_draw=np.full(len(recipient_predictors), float(state.position + 1)), + weight_kind="explicit", + state=SimpleNamespace( + targets=state.targets, + position=state.position + 1, + ), + ) + + monkeypatch.setattr(microcosm.fit, "RegimeGatedQRF", FakeQRF) + donor = clean_was_household_table(_raw_was()) + recipient = recipient_predictors(_frame(), _FakeEngine()) + expected_base = tuple( + predictor for predictor in module.UK_WAS_WEALTH_PREDICTORS if predictor != "region" + ) + ("region_LONDON", "region_SCOTLAND", "region_WALES") + + module.impute_was_wealth(donor, recipient, seed=0, n_estimators=7) + + assert calls[0] == ( + expected_base, + ("owned_land", "property_wealth"), + ) + + def test_was_imputer_uses_checkpointed_chain_segments( monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -437,11 +517,18 @@ def fit_draw_next( assert result.draws.columns.tolist() == list(UK_WAS_WEALTH_OUTPUT_COLUMNS) assert calls[0][1] == ("owned_land", "property_wealth") - assert calls[1][1] == ("corporate_wealth_excl_isa", "stocks_and_shares_isa") + assert calls[1][1] == ( + "private_pension_wealth", + "corporate_wealth_excl_isa", + "stocks_and_shares_isa", + ) + assert "private_pension_wealth" in calls[2][0] assert "corporate_wealth" in calls[2][0] assert calls[2][1][-1] == "cash_isa" fitted_targets = [name for _, targets in calls for name in targets] - assert [record.fit_name for record in result.fit_weight_records] == [ + fit_names = [record.fit_name for record in result.fit_weight_records] + assert "uk_was_2018_20_wealth:private_pension_wealth" in fit_names + assert fit_names == [ f"uk_was_2018_20_wealth:{target}" for target in fitted_targets ] assert {record.weight_kind for record in result.fit_weight_records} == {"explicit"} diff --git a/packages/microcosm-data/src/microcosm/data/contract.py b/packages/microcosm-data/src/microcosm/data/contract.py index c423461c..0362dde1 100644 --- a/packages/microcosm-data/src/microcosm/data/contract.py +++ b/packages/microcosm-data/src/microcosm/data/contract.py @@ -375,13 +375,13 @@ # fingerprint derives from the manifest digest. Editing the spec moves all # three here in the same reviewed change. _UK_GATE_BATTERY_POLICY_SHA256 = ( - "5459347c9077b2acd5970a62d818e3ddd063d86d6c3dbce4d32dcacec3bdc414" + "8584de8e792e03ab18a7c7c12855393bb0a4358c529566aac1e3cef89e6c6fe6" ) _UK_GATE_BATTERY_GATES_MANIFEST_SHA256 = ( - "f68e10d8ff6654b7fc707da0508ea063b0f3c96b8a813b7098298727d43b9d6d" + "23b3adb8618d7edd905d8c32620914fdc26b2c8feb68e2e48805ed992e58686e" ) _UK_GATE_BATTERY_SPEC_FINGERPRINT = ( - "2a4a8b18d024f80782b375539c87d57006592d64470553a4da5a378791254faa" + "bfed261f7af3077d1c3fc0351adf6a9dc717e5ea1e3b50e980637608ac99223f" ) #: Spec entry id -> the legacy gate name whose observable detail checks #: apply unchanged (the battery re-keys the report by entry id; the gate @@ -591,10 +591,10 @@ _UK_CERTIFICATION_PART_DIGESTS: Mapping[str, Mapping[str, str]] = { "spine": { "gates_manifest_sha256": ( - "1605cf3fe1be4983cfb4ed806a34d69375cdc3e4e0c8883cc49481ac5870399a" + "af18105ef46720574c4a3a347656d689d2ccc239a2897fda8b694bef7144e8dd" ), "policy_sha256": ( - "3d14ad24eff7f5afd343164560db24095d27fafb36c619ddf725c32e00b35a69" + "7f9ba299c1e756f6ba217d35431334678815f66258540e1eea4fcdd143b183f1" ), }, "calibration_seam": { @@ -607,10 +607,10 @@ }, "release_cut": { "gates_manifest_sha256": ( - "18f07f40eead198a4436de7a43d4c0b13b9f2a9335bc84d4ab6b6a2ed75e597e" + "70c4c938767fdbae4c07d915a2e1cdc036cdb2f4a3e1af2cee7437cdf94ab7d7" ), "policy_sha256": ( - "b2a6446da0ffc53c0a894618795d918163368a266ef01a2a3609587085162115" + "e73dcbcaa07e34d23c8cf40b251f19fd00d9d2b85202647ca0ea13cb1021769a" ), }, } diff --git a/packages/microcosm-data/tests/test_contract.py b/packages/microcosm-data/tests/test_contract.py index 4f4910eb..79950ad7 100644 --- a/packages/microcosm-data/tests/test_contract.py +++ b/packages/microcosm-data/tests/test_contract.py @@ -135,13 +135,13 @@ def _trusted_terminal_gate_signing_key(monkeypatch) -> None: UK_GATE_BATTERY_PRODUCER = "microcosm.build.gate_battery" UK_GATE_BATTERY_SIGNING_KEY_ENV = "MICROCOSM_UK_TERMINAL_GATE_SIGNING_KEY" UK_GATE_BATTERY_POLICY_SHA256 = ( - "5459347c9077b2acd5970a62d818e3ddd063d86d6c3dbce4d32dcacec3bdc414" + "8584de8e792e03ab18a7c7c12855393bb0a4358c529566aac1e3cef89e6c6fe6" ) UK_GATE_BATTERY_GATES_MANIFEST_SHA256 = ( - "f68e10d8ff6654b7fc707da0508ea063b0f3c96b8a813b7098298727d43b9d6d" + "23b3adb8618d7edd905d8c32620914fdc26b2c8feb68e2e48805ed992e58686e" ) UK_GATE_BATTERY_SPEC_FINGERPRINT = ( - "2a4a8b18d024f80782b375539c87d57006592d64470553a4da5a378791254faa" + "bfed261f7af3077d1c3fc0351adf6a9dc717e5ea1e3b50e980637608ac99223f" ) UK_GATE_BATTERY_DEGENERATE_EVIDENCE_SHA256 = ( "d0d024043132fa07c378c393dbe2b24fe99bf19e876bcc39997d2c80cc9bd4f6" From ae9457a312d85c26add0639c439187609b97e616 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Thu, 27 Aug 2026 23:01:59 +0200 Subject: [PATCH 5/5] Re-record the licensed extraction evidence under policyengine-uk 2.92 The 2.92 engine newly recognizes three loader inputs the pinned artifacts already carry and populate - bus_fare_spending (household), employment_sector and sic_industry_division (person), previously in the reference's unknown-export-columns list - so the committed extraction evidence recorded under 2.89 no longer matches its own regeneration. The parity reference, the known-gaps candidate evidence and the coverage manifest are regenerated with their own tools from the same pinned artifacts: identities unchanged (filename/revision/sha256 identical), every pre-existing share value byte-identical, surface 145 -> 148 required, count pins and version strings move, nothing else. The June candidate and the spine both carry all three columns, so the widened contract is satisfiable on both sides. This is not the efrs-post-calibration input-mass reference re-pin - that stays deferred behind the uk-data mirror per the #750 ordering, and the input-mass totals evidence is untouched. Co-Authored-By: Claude Fable 5 --- .../build/uk/efrs_parity_known_gaps.json | 20 ++++++++++++---- .../build/uk/efrs_parity_reference.json | 23 +++++++++++-------- .../uk/release_input_coverage_manifest.json | 15 +++++++++--- .../tests/test_uk_parity_reference.py | 4 ++-- ...test_uk_release_input_coverage_manifest.py | 10 ++++---- 5 files changed, 48 insertions(+), 24 deletions(-) diff --git a/packages/microcosm-build/src/microcosm/build/uk/efrs_parity_known_gaps.json b/packages/microcosm-build/src/microcosm/build/uk/efrs_parity_known_gaps.json index 9fe5e1d5..ceb30652 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/efrs_parity_known_gaps.json +++ b/packages/microcosm-build/src/microcosm/build/uk/efrs_parity_known_gaps.json @@ -14,6 +14,7 @@ "attends_private_school_random_draw": "person", "brma": "household", "bsp_reported": "person", + "bus_fare_spending": "household", "bus_subsidy_spending": "household", "capital_gains": "person", "carers_allowance_reported": "person", @@ -43,6 +44,7 @@ "employee_pension_contributions": "person", "employer_pension_contributions": "person", "employment_income": "person", + "employment_sector": "person", "employment_status": "person", "esa_contrib_reported": "person", "esa_income_reported": "person", @@ -120,6 +122,7 @@ "savings_interest_income": "person", "sda_reported": "person", "self_employment_income": "person", + "sic_industry_division": "person", "ssmg_reported": "person", "state_pension_reported": "person", "statutory_maternity_pay": "person", @@ -168,6 +171,7 @@ "attends_private_school_random_draw": 1.0, "brma": 0.998307955714, "bsp_reported": 0.002269750231, + "bus_fare_spending": 0.176972057506, "bus_subsidy_spending": 0.181612426148, "capital_gains": 0.028136660276, "carers_allowance_reported": 0.013745759756, @@ -197,6 +201,7 @@ "employee_pension_contributions": 0.213260351368, "employer_pension_contributions": 0.273574493593, "employment_income": 0.387291454309, + "employment_sector": 0.488599258708, "employment_status": 0.985687923006, "esa_contrib_reported": 0.004342270757, "esa_income_reported": 0.008391805365, @@ -274,6 +279,7 @@ "savings_interest_income": 0.357580855206, "sda_reported": 0.000107122652, "self_employment_income": 0.066184105292, + "sic_industry_division": 0.713824072671, "ssmg_reported": 0.000359030478, "state_pension_reported": 0.181567916228, "statutory_maternity_pay": 0.002379481691, @@ -301,7 +307,7 @@ "would_claim_universal_childcare": 0.429571577769, "would_evade_tv_licence_fee": 0.106472398355 }, - "effective_signal_columns": 143, + "effective_signal_columns": 146, "engine": { "h5_input_aliases": { "capital_gains": "capital_gains_before_response", @@ -309,7 +315,7 @@ "employment_income": "employment_income_before_lsr" }, "package": "policyengine-uk", - "version": "2.89.0" + "version": "2.92.0" }, "entity_records": { "benunit": 618980, @@ -335,6 +341,7 @@ "attends_private_school_random_draw": 1.0, "brma": 0.997869, "bsp_reported": 0.005877, + "bus_fare_spending": 0.204941, "bus_subsidy_spending": 0.208829, "capital_gains": 0.231216, "carers_allowance_reported": 0.011287, @@ -364,6 +371,7 @@ "employee_pension_contributions": 0.233878, "employer_pension_contributions": 0.295463, "employment_income": 0.478282, + "employment_sector": 0.458405, "employment_status": 0.989284, "esa_contrib_reported": 0.005514, "esa_income_reported": 0.009507, @@ -441,6 +449,7 @@ "savings_interest_income": 0.335736, "sda_reported": 0.000104, "self_employment_income": 0.056123, + "sic_industry_division": 0.733091, "ssmg_reported": 0.000415, "state_pension_reported": 0.235745, "statutory_maternity_pay": 0.002506, @@ -482,6 +491,7 @@ "attends_private_school_random_draw": 1.0, "brma": 1.0, "bsp_reported": 0.005877, + "bus_fare_spending": 0.204941, "bus_subsidy_spending": 0.208829, "capital_gains": 0.231216, "carers_allowance_reported": 0.011287, @@ -511,6 +521,7 @@ "employee_pension_contributions": 0.233878, "employer_pension_contributions": 0.295463, "employment_income": 0.478282, + "employment_sector": 1.0, "employment_status": 1.0, "esa_contrib_reported": 0.005514, "esa_income_reported": 0.009507, @@ -588,6 +599,7 @@ "savings_interest_income": 0.335736, "sda_reported": 0.000104, "self_employment_income": 0.056123, + "sic_industry_division": 0.733091, "ssmg_reported": 0.000415, "state_pension_reported": 0.235745, "statutory_maternity_pay": 0.002506, @@ -615,8 +627,8 @@ "would_claim_universal_childcare": 0.560923, "would_evade_tv_licence_fee": 0.105666 }, - "reference_columns_evaluated": 145, - "signal_columns": 145, + "reference_columns_evaluated": 148, + "signal_columns": 148, "source": { "filename": "populace_uk_2023.h5", "hf_commit": "a75a9a831d6b07aaffbd09713f2a1124f5c0f08f", diff --git a/packages/microcosm-build/src/microcosm/build/uk/efrs_parity_reference.json b/packages/microcosm-build/src/microcosm/build/uk/efrs_parity_reference.json index 2637e42e..e00ccbad 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/efrs_parity_reference.json +++ b/packages/microcosm-build/src/microcosm/build/uk/efrs_parity_reference.json @@ -1,7 +1,7 @@ { "description": "Frozen enhanced-FRS parity reference for the UK release input-coverage contract. Every PolicyEngine-UK loader variable the pinned artifact populates is recorded as its unweighted owning-entity nonzero share. This includes formula-owned persisted overrides because the UK Simulation loader passes every engine-known H5 column to set_input; pipeline scratch columns, structural IDs, and all-zero loader layers are not requirements.", "engine": { - "engine_known_persisted_variable_count": 866, + "engine_known_persisted_variable_count": 881, "formula_owned_persisted_overrides_included": [ "current_education", "is_benunit_head", @@ -22,7 +22,7 @@ "employee_pension_contributions": "employee_pension_contributions_reported", "employment_income": "employment_income_before_lsr" }, - "input_variable_count": 223, + "input_variable_count": 227, "input_variable_scope": "Every CountryTaxBenefitSystem variable persisted in the H5: Simulation.build_from_multi_year_dataset calls set_input for all engine-known columns, including formula-owned overrides", "package": "policyengine-uk", "structural_columns_excluded": [ @@ -33,10 +33,8 @@ "household_id" ], "unknown_export_columns_excluded": [ - "bus_fare_spending", "clone_index", "constituency_code_oa", - "employment_sector", "esa_health_condition_proxy", "esa_support_group_proxy", "free_school_breakfasts", @@ -53,11 +51,10 @@ "region_code_oa", "salary_sacrifice_asked", "salary_sacrifice_reported", - "sic_industry_division", "source_household_id", "source_year" ], - "version": "2.89.0", + "version": "2.92.0", "zero_share_input_columns_excluded": [ "disabled_students_allowance_eligible_expenses", "incapacity_benefit_reported", @@ -73,14 +70,14 @@ }, "household": { "export_columns": 66, - "input_columns": 51, - "populated_input_columns": 51, + "input_columns": 52, + "populated_input_columns": 52, "records": 52846 }, "person": { "export_columns": 99, - "input_columns": 87, - "populated_input_columns": 84, + "input_columns": 89, + "populated_input_columns": 86, "records": 113617 } }, @@ -98,6 +95,7 @@ "attends_private_school_random_draw": "person", "brma": "household", "bsp_reported": "person", + "bus_fare_spending": "household", "bus_subsidy_spending": "household", "capital_gains": "person", "carers_allowance_reported": "person", @@ -127,6 +125,7 @@ "employee_pension_contributions": "person", "employer_pension_contributions": "person", "employment_income": "person", + "employment_sector": "person", "employment_status": "person", "esa_contrib_reported": "person", "esa_income_reported": "person", @@ -204,6 +203,7 @@ "savings_interest_income": "person", "sda_reported": "person", "self_employment_income": "person", + "sic_industry_division": "person", "ssmg_reported": "person", "state_pension_reported": "person", "statutory_maternity_pay": "person", @@ -245,6 +245,7 @@ "attends_private_school_random_draw": 1.0, "brma": 1.0, "bsp_reported": 0.004929, + "bus_fare_spending": 0.202872, "bus_subsidy_spending": 0.316675, "capital_gains": 0.23375, "carers_allowance_reported": 0.010535, @@ -274,6 +275,7 @@ "employee_pension_contributions": 0.273489, "employer_pension_contributions": 0.314865, "employment_income": 0.492928, + "employment_sector": 1.0, "employment_status": 1.0, "esa_contrib_reported": 0.008053, "esa_income_reported": 0.009558, @@ -351,6 +353,7 @@ "savings_interest_income": 0.424963, "sda_reported": 8.8e-05, "self_employment_income": 0.054939, + "sic_industry_division": 0.734371, "ssmg_reported": 0.000405, "state_pension_reported": 0.245694, "statutory_maternity_pay": 0.002139, diff --git a/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json b/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json index 416e547a..da606690 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json +++ b/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json @@ -47,6 +47,9 @@ "bsp_reported": { "status": "required" }, + "bus_fare_spending": { + "status": "required" + }, "bus_subsidy_spending": { "status": "required" }, @@ -134,6 +137,9 @@ "employment_income": { "status": "required" }, + "employment_sector": { + "status": "required" + }, "employment_status": { "status": "required" }, @@ -365,6 +371,9 @@ "self_employment_income": { "status": "required" }, + "sic_industry_division": { + "status": "required" + }, "ssmg_reported": { "status": "required" }, @@ -445,9 +454,9 @@ } }, "counts": { - "required": 145, + "required": 148, "reviewed_exclusion": 0, - "total": 145 + "total": 148 }, "derivation": "Surface = efrs_parity_reference.json populated effective loader inputs. status='required' when the sha-pinned candidate evidence records non-default signal on at least the reviewed owning-entity effective-mass share OR the column is pinned in restored_required_columns after a source-family restoration; all remaining surface columns are reviewed_exclusion with reason 'not yet ported from enhanced FRS pipeline — pending review' and a UK_COVERAGE_PROGRESS.md tracking note. The final release gate applies the same effective-mass floor, and distributional restorations must pass it on their required source channel.", "description": "Declared full-coverage contract for a UK release: every populated effective loader input in the pinned enhanced FRS must be persisted with non-default signal, or carry a reviewed exclusion with the campaign reason and tracking note.", @@ -840,7 +849,7 @@ "derived_from": "efrs_parity_reference.json", "filename": "enhanced_frs_2024_25.h5", "period": "2024", - "populated_input_columns": 145, + "populated_input_columns": 148, "revision": "a9e52499b6a6cca100a5ce4f36ca27b2e8a213df", "sha256": "e433e532b17bd8ce76030156285816e33d44e93edabd2204adbef71d19a68712", "vintage": "2024_25" diff --git a/packages/microcosm-build/tests/test_uk_parity_reference.py b/packages/microcosm-build/tests/test_uk_parity_reference.py index ac7199a3..c517be17 100644 --- a/packages/microcosm-build/tests/test_uk_parity_reference.py +++ b/packages/microcosm-build/tests/test_uk_parity_reference.py @@ -70,8 +70,8 @@ class TestEfrsParityReference: def test_reference_loads_with_populated_layers(self) -> None: reference = load_efrs_parity_reference() assert reference.schema_version == 3 - assert len(reference.nonzero_shares) == 145 - assert len(reference.populated_layers) == 145 + assert len(reference.nonzero_shares) == 148 + assert len(reference.populated_layers) == 148 assert all(share > 0.0 for share in reference.nonzero_shares.values()) assert set(reference.input_entities) == set(reference.nonzero_shares) assert set(reference.input_entities.values()) == { diff --git a/packages/microcosm-build/tests/test_uk_release_input_coverage_manifest.py b/packages/microcosm-build/tests/test_uk_release_input_coverage_manifest.py index 2dd872ef..81c041f7 100644 --- a/packages/microcosm-build/tests/test_uk_release_input_coverage_manifest.py +++ b/packages/microcosm-build/tests/test_uk_release_input_coverage_manifest.py @@ -55,7 +55,7 @@ def test_candidate_evidence_is_sha_pinned_and_covers_reference() -> None: assert evidence["nonzero_shares"]["household_weight"] == 0.626224 assert evidence["nondefault_shares"]["household_weight"] == 1.0 assert evidence["nondefault_shares"]["employment_income"] == 0.478282 - assert evidence["effective_signal_columns"] == 143 + assert evidence["effective_signal_columns"] == 146 assert evidence["insufficient_effective_mass_columns"] == [ "charitable_investment_gifts", "gift_aid", @@ -80,8 +80,8 @@ def test_known_gap_register_records_post_candidate_restoration_separately() -> N } assert gaps["candidate_evidence"]["missing_columns"] == [] assert gaps["candidate_evidence"]["default_only_columns"] == [] - assert gaps["candidate_evidence"]["signal_columns"] == 145 - assert gaps["candidate_evidence"]["effective_signal_columns"] == 143 + assert gaps["candidate_evidence"]["signal_columns"] == 148 + assert gaps["candidate_evidence"]["effective_signal_columns"] == 146 assert gaps["exclusion_policy"]["reason"] == ( "not yet ported from enhanced FRS pipeline — pending review" ) @@ -175,9 +175,9 @@ def test_promoted_manifest_requires_the_full_reference_surface() -> None: reference = _resource("efrs_parity_reference.json") manifest = _resource("release_input_coverage_manifest.json") assert manifest["counts"] == { - "required": 145, + "required": 148, "reviewed_exclusion": 0, - "total": 145, + "total": 148, } assert set(manifest["columns"]) == set(reference["nonzero_shares"]) assert all(