diff --git a/changelog.d/750-pension-wealth-split.changed.md b/changelog.d/750-pension-wealth-split.changed.md new file mode 100644 index 00000000..aec06f1a --- /dev/null +++ b/changelog.d/750-pension-wealth-split.changed.md @@ -0,0 +1 @@ +The UK WAS stage now emits `private_pension_wealth` (WAS private pension wealth less current-employment DB, uk-data#452 M2) as its own household output, drawn first in the second chain segment; `corporate_wealth` keeps the share-like holdings and the stocks-and-shares-ISA fold, row identity old corporate wealth == new corporate wealth + pension wealth holds on the donor, and the `uk` extra now requires policyengine-uk 2.92, whose means tests disregard the new column. diff --git a/changelog.d/792-band-edges-compiled-register.fixed.md b/changelog.d/792-band-edges-compiled-register.fixed.md new file mode 100644 index 00000000..afe982e6 --- /dev/null +++ b/changelog.d/792-band-edges-compiled-register.fixed.md @@ -0,0 +1 @@ +Banded calibration targets now derive upper edges from the compiled pre-exclusion register, fixing microcosm#792 so excluding a banded target no longer widens its surviving lower neighbour. The materialization seam now refuses a pruned register unless callers provide its compiled band-edge source. diff --git a/packages/microcosm-build/pyproject.toml b/packages/microcosm-build/pyproject.toml index c7e75e36..dec94c00 100644 --- a/packages/microcosm-build/pyproject.toml +++ b/packages/microcosm-build/pyproject.toml @@ -38,7 +38,7 @@ us = [ # The UK extra adds the rules engine for local metric generation from a # Microcosm UK H5. Target tables remain explicit inputs, and the base package # still does not import policyengine-uk at import time. -uk = ["policyengine-uk>=2.88", "h5py>=3", "tables>=3"] +uk = ["policyengine-uk>=2.92", "h5py>=3", "tables>=3"] [project.scripts] microcosm-export-us-l0-refit-h5 = "microcosm.build.us_runtime.l0_refit_export:main" diff --git a/packages/microcosm-build/src/microcosm/build/target_materialization.py b/packages/microcosm-build/src/microcosm/build/target_materialization.py index 79fdfb22..41468008 100644 --- a/packages/microcosm-build/src/microcosm/build/target_materialization.py +++ b/packages/microcosm-build/src/microcosm/build/target_materialization.py @@ -21,9 +21,11 @@ # encodings are in use — a numeric lower bound (HMRC SPI income bands) and a # published range label (DWP award bands, in monthly units, hence # ``band_period_factor``) — and both reduce to one lower edge, because no -# reference anywhere declares an upper bound. A band's upper edge is -# therefore its sibling's lower edge within the same contract target, and the -# top band runs to infinity. +# reference anywhere declares an upper bound. A band's upper edge is its +# sibling's lower edge within the same compiled contract target. Edges must +# never derive from an exclusion-pruned roster, or excluding a band silently +# widens its lower neighbour; only the compiled register's top band runs to +# infinity. # Entity-count indicators: a value of one per record of the owning entity. _COUNT_VALUE_VARIABLES = frozenset( {"household_count", "person_count", "benunit_count"} @@ -126,6 +128,7 @@ def resolve_target_measures( period: int | str, max_rounds: int = 8, contract_targets: Mapping[str, Mapping[str, Any]] | None = None, + band_edge_registry: TargetRegistry | None = None, ) -> MeasureResolution: """Resolve provider-computed inputs until target materialization binds. @@ -159,6 +162,7 @@ def resolve_target_measures( registry, contract, period=period, + band_edge_registry=band_edge_registry, ) skipped = tuple(result.skipped) round_receipt = { @@ -414,6 +418,13 @@ def _band_bounds( f"groupby_variable {binding['groupby_variable']!r} but its spec " "carries no readable band edge" ) + if lower not in band_edges: + raise ValueError( + f"banded measure {getattr(spec, 'measure', '?')!r} carries band lower edge " + f"{lower!r} that is absent from its contract target's band-edge set " + f"{list(band_edges)!r}; the band-edge register does not cover this spec's " + "band — pass the compiled (pre-exclusion) register this spec was pruned from." + ) upper = math.inf for edge in band_edges: if edge > lower: @@ -429,11 +440,15 @@ def materialize_target_bindings( *, period: int | str, providers: Mapping[str, Provider] | None = None, + band_edge_registry: TargetRegistry | None = None, ) -> TargetMaterializationResult: """Prepare measure columns declared by compiled Ledger target specs.""" provider_registry = {**default_provider_registry(), **(providers or {})} - band_edges = _band_edges_by_group(registry, contract_targets) + band_edges = _band_edges_by_group( + registry if band_edge_registry is None else band_edge_registry, + contract_targets, + ) skipped: list[MaterializationSkip] = [] for spec in registry.specs: if hasattr(adapter, "has_column") and adapter.has_column( diff --git a/packages/microcosm-build/src/microcosm/build/uk/efrs_parity_known_gaps.json b/packages/microcosm-build/src/microcosm/build/uk/efrs_parity_known_gaps.json index 9fe5e1d5..ceb30652 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/efrs_parity_known_gaps.json +++ b/packages/microcosm-build/src/microcosm/build/uk/efrs_parity_known_gaps.json @@ -14,6 +14,7 @@ "attends_private_school_random_draw": "person", "brma": "household", "bsp_reported": "person", + "bus_fare_spending": "household", "bus_subsidy_spending": "household", "capital_gains": "person", "carers_allowance_reported": "person", @@ -43,6 +44,7 @@ "employee_pension_contributions": "person", "employer_pension_contributions": "person", "employment_income": "person", + "employment_sector": "person", "employment_status": "person", "esa_contrib_reported": "person", "esa_income_reported": "person", @@ -120,6 +122,7 @@ "savings_interest_income": "person", "sda_reported": "person", "self_employment_income": "person", + "sic_industry_division": "person", "ssmg_reported": "person", "state_pension_reported": "person", "statutory_maternity_pay": "person", @@ -168,6 +171,7 @@ "attends_private_school_random_draw": 1.0, "brma": 0.998307955714, "bsp_reported": 0.002269750231, + "bus_fare_spending": 0.176972057506, "bus_subsidy_spending": 0.181612426148, "capital_gains": 0.028136660276, "carers_allowance_reported": 0.013745759756, @@ -197,6 +201,7 @@ "employee_pension_contributions": 0.213260351368, "employer_pension_contributions": 0.273574493593, "employment_income": 0.387291454309, + "employment_sector": 0.488599258708, "employment_status": 0.985687923006, "esa_contrib_reported": 0.004342270757, "esa_income_reported": 0.008391805365, @@ -274,6 +279,7 @@ "savings_interest_income": 0.357580855206, "sda_reported": 0.000107122652, "self_employment_income": 0.066184105292, + "sic_industry_division": 0.713824072671, "ssmg_reported": 0.000359030478, "state_pension_reported": 0.181567916228, "statutory_maternity_pay": 0.002379481691, @@ -301,7 +307,7 @@ "would_claim_universal_childcare": 0.429571577769, "would_evade_tv_licence_fee": 0.106472398355 }, - "effective_signal_columns": 143, + "effective_signal_columns": 146, "engine": { "h5_input_aliases": { "capital_gains": "capital_gains_before_response", @@ -309,7 +315,7 @@ "employment_income": "employment_income_before_lsr" }, "package": "policyengine-uk", - "version": "2.89.0" + "version": "2.92.0" }, "entity_records": { "benunit": 618980, @@ -335,6 +341,7 @@ "attends_private_school_random_draw": 1.0, "brma": 0.997869, "bsp_reported": 0.005877, + "bus_fare_spending": 0.204941, "bus_subsidy_spending": 0.208829, "capital_gains": 0.231216, "carers_allowance_reported": 0.011287, @@ -364,6 +371,7 @@ "employee_pension_contributions": 0.233878, "employer_pension_contributions": 0.295463, "employment_income": 0.478282, + "employment_sector": 0.458405, "employment_status": 0.989284, "esa_contrib_reported": 0.005514, "esa_income_reported": 0.009507, @@ -441,6 +449,7 @@ "savings_interest_income": 0.335736, "sda_reported": 0.000104, "self_employment_income": 0.056123, + "sic_industry_division": 0.733091, "ssmg_reported": 0.000415, "state_pension_reported": 0.235745, "statutory_maternity_pay": 0.002506, @@ -482,6 +491,7 @@ "attends_private_school_random_draw": 1.0, "brma": 1.0, "bsp_reported": 0.005877, + "bus_fare_spending": 0.204941, "bus_subsidy_spending": 0.208829, "capital_gains": 0.231216, "carers_allowance_reported": 0.011287, @@ -511,6 +521,7 @@ "employee_pension_contributions": 0.233878, "employer_pension_contributions": 0.295463, "employment_income": 0.478282, + "employment_sector": 1.0, "employment_status": 1.0, "esa_contrib_reported": 0.005514, "esa_income_reported": 0.009507, @@ -588,6 +599,7 @@ "savings_interest_income": 0.335736, "sda_reported": 0.000104, "self_employment_income": 0.056123, + "sic_industry_division": 0.733091, "ssmg_reported": 0.000415, "state_pension_reported": 0.235745, "statutory_maternity_pay": 0.002506, @@ -615,8 +627,8 @@ "would_claim_universal_childcare": 0.560923, "would_evade_tv_licence_fee": 0.105666 }, - "reference_columns_evaluated": 145, - "signal_columns": 145, + "reference_columns_evaluated": 148, + "signal_columns": 148, "source": { "filename": "populace_uk_2023.h5", "hf_commit": "a75a9a831d6b07aaffbd09713f2a1124f5c0f08f", diff --git a/packages/microcosm-build/src/microcosm/build/uk/efrs_parity_reference.json b/packages/microcosm-build/src/microcosm/build/uk/efrs_parity_reference.json index 2637e42e..e00ccbad 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/efrs_parity_reference.json +++ b/packages/microcosm-build/src/microcosm/build/uk/efrs_parity_reference.json @@ -1,7 +1,7 @@ { "description": "Frozen enhanced-FRS parity reference for the UK release input-coverage contract. Every PolicyEngine-UK loader variable the pinned artifact populates is recorded as its unweighted owning-entity nonzero share. This includes formula-owned persisted overrides because the UK Simulation loader passes every engine-known H5 column to set_input; pipeline scratch columns, structural IDs, and all-zero loader layers are not requirements.", "engine": { - "engine_known_persisted_variable_count": 866, + "engine_known_persisted_variable_count": 881, "formula_owned_persisted_overrides_included": [ "current_education", "is_benunit_head", @@ -22,7 +22,7 @@ "employee_pension_contributions": "employee_pension_contributions_reported", "employment_income": "employment_income_before_lsr" }, - "input_variable_count": 223, + "input_variable_count": 227, "input_variable_scope": "Every CountryTaxBenefitSystem variable persisted in the H5: Simulation.build_from_multi_year_dataset calls set_input for all engine-known columns, including formula-owned overrides", "package": "policyengine-uk", "structural_columns_excluded": [ @@ -33,10 +33,8 @@ "household_id" ], "unknown_export_columns_excluded": [ - "bus_fare_spending", "clone_index", "constituency_code_oa", - "employment_sector", "esa_health_condition_proxy", "esa_support_group_proxy", "free_school_breakfasts", @@ -53,11 +51,10 @@ "region_code_oa", "salary_sacrifice_asked", "salary_sacrifice_reported", - "sic_industry_division", "source_household_id", "source_year" ], - "version": "2.89.0", + "version": "2.92.0", "zero_share_input_columns_excluded": [ "disabled_students_allowance_eligible_expenses", "incapacity_benefit_reported", @@ -73,14 +70,14 @@ }, "household": { "export_columns": 66, - "input_columns": 51, - "populated_input_columns": 51, + "input_columns": 52, + "populated_input_columns": 52, "records": 52846 }, "person": { "export_columns": 99, - "input_columns": 87, - "populated_input_columns": 84, + "input_columns": 89, + "populated_input_columns": 86, "records": 113617 } }, @@ -98,6 +95,7 @@ "attends_private_school_random_draw": "person", "brma": "household", "bsp_reported": "person", + "bus_fare_spending": "household", "bus_subsidy_spending": "household", "capital_gains": "person", "carers_allowance_reported": "person", @@ -127,6 +125,7 @@ "employee_pension_contributions": "person", "employer_pension_contributions": "person", "employment_income": "person", + "employment_sector": "person", "employment_status": "person", "esa_contrib_reported": "person", "esa_income_reported": "person", @@ -204,6 +203,7 @@ "savings_interest_income": "person", "sda_reported": "person", "self_employment_income": "person", + "sic_industry_division": "person", "ssmg_reported": "person", "state_pension_reported": "person", "statutory_maternity_pay": "person", @@ -245,6 +245,7 @@ "attends_private_school_random_draw": 1.0, "brma": 1.0, "bsp_reported": 0.004929, + "bus_fare_spending": 0.202872, "bus_subsidy_spending": 0.316675, "capital_gains": 0.23375, "carers_allowance_reported": 0.010535, @@ -274,6 +275,7 @@ "employee_pension_contributions": 0.273489, "employer_pension_contributions": 0.314865, "employment_income": 0.492928, + "employment_sector": 1.0, "employment_status": 1.0, "esa_contrib_reported": 0.008053, "esa_income_reported": 0.009558, @@ -351,6 +353,7 @@ "savings_interest_income": 0.424963, "sda_reported": 8.8e-05, "self_employment_income": 0.054939, + "sic_industry_division": 0.734371, "ssmg_reported": 0.000405, "state_pension_reported": 0.245694, "statutory_maternity_pay": 0.002139, diff --git a/packages/microcosm-build/src/microcosm/build/uk/gates.json b/packages/microcosm-build/src/microcosm/build/uk/gates.json index ed3447da..fc0984e4 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/gates.json +++ b/packages/microcosm-build/src/microcosm/build/uk/gates.json @@ -70,6 +70,7 @@ "num_vehicles", "other_residential_property_value", "owned_land", + "private_pension_wealth", "property_wealth", "savings", "stocks_and_shares_isa", @@ -85,6 +86,7 @@ "num_vehicles": 0, "other_residential_property_value": 0, "owned_land": 0, + "private_pension_wealth": 0, "property_wealth": 0, "savings": 0, "stocks_and_shares_isa": 0, @@ -100,6 +102,7 @@ "num_vehicles": 0, "other_residential_property_value": 0, "owned_land": 0, + "private_pension_wealth": 0, "property_wealth": 0, "savings": 0, "stocks_and_shares_isa": 0, @@ -526,6 +529,7 @@ "household.msoa_code", "household.num_vehicles", "household.oa_code", + "household.private_pension_wealth", "household.property_purchased", "household.rail_usage", "household.region_code_oa", @@ -662,7 +666,7 @@ "max_top_share": 0.9994670564654868, "min_nonzero_records": 104 }, - "notes": "Weighted tail-concentration audit of the QRF-imputed output columns derived from the HMRC source manifest. Armed from the #686 L3 baselines per #757 B4, each threshold recording the run that measured it: the 47-column qrf_tail grids of uk_weighted_integrity_baselines_686.json (sha256 8b3f6c4e9c522445bf3e05d3451ee8557f4e14d25524ceac3af8bfe05401a146, licensed acceptance dir 686-spine-swap), measured on spine-a.h5 (sha256 a65f2132736d1dcecb91709a513a0d54a5887635ea03760629cc888d188ee306, 113,649 person rows, design weights) for the #609/#578 threshold adjudication. top_k 100 is the measurement grid anchor [10, 100, 500, 1000]; max_top_share is the exact measured maximum over the checked surface (hmrc_spi_other_social_security_income, 104 carriers), no headroom; min_nonzero_records is the thinnest measured column above the grid anchor (the same column) - the policy domain requires min_nonzero_records > top_k, so the three saturated sub-anchor columns (hmrc_spi_taxable_termination_pay 12, charitable_investment_gifts 22, sda_reported 24 carriers, each top-100 share 1.0) sit below it and go thin visibly in the signed details on every run, never a silent pass. The baselines are design-weight measurements and the terminal gate runs on the calibrated release frame; a calibrated-weight breach names its column and is a finding, not noise. The 12 household_qrf_tail grids (was_wealth surface) are measured in the same baselines file and not yet armed - a household-surface gate entry is a declared follow-up. Reviewed exclusions live in the named register resource of this package." + "notes": "Weighted tail-concentration audit of the QRF-imputed output columns derived from the HMRC source manifest. Armed from the #686 L3 baselines per #757 B4, each threshold recording the run that measured it: the 47-column qrf_tail grids of uk_weighted_integrity_baselines_686.json (sha256 8b3f6c4e9c522445bf3e05d3451ee8557f4e14d25524ceac3af8bfe05401a146, licensed acceptance dir 686-spine-swap), measured on spine-a.h5 (sha256 a65f2132736d1dcecb91709a513a0d54a5887635ea03760629cc888d188ee306, 113,649 person rows, design weights) for the #609/#578 threshold adjudication. top_k 100 is the measurement grid anchor [10, 100, 500, 1000]; max_top_share is the exact measured maximum over the checked surface (hmrc_spi_other_social_security_income, 104 carriers), no headroom; min_nonzero_records is the thinnest measured column above the grid anchor (the same column) - the policy domain requires min_nonzero_records > top_k, so the three saturated sub-anchor columns (hmrc_spi_taxable_termination_pay 12, charitable_investment_gifts 22, sda_reported 24 carriers, each top-100 share 1.0) sit below it and go thin visibly in the signed details on every run, never a silent pass. The baselines are design-weight measurements and the terminal gate runs on the calibrated release frame; a calibrated-weight breach names its column and is a finding, not noise. The 12 household_qrf_tail grids (was_wealth surface) are measured in the same baselines file and not yet armed - a household-surface gate entry is a declared follow-up. The #750 split adds a 13th household QRF column (private_pension_wealth) not yet in the measured baselines; the household-surface arming follow-up on microcosm#796 re-measures. Reviewed exclusions live in the named register resource of this package." } ] } diff --git a/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json b/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json index 92b35167..da606690 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json +++ b/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json @@ -47,6 +47,9 @@ "bsp_reported": { "status": "required" }, + "bus_fare_spending": { + "status": "required" + }, "bus_subsidy_spending": { "status": "required" }, @@ -134,6 +137,9 @@ "employment_income": { "status": "required" }, + "employment_sector": { + "status": "required" + }, "employment_status": { "status": "required" }, @@ -365,6 +371,9 @@ "self_employment_income": { "status": "required" }, + "sic_industry_division": { + "status": "required" + }, "ssmg_reported": { "status": "required" }, @@ -445,9 +454,9 @@ } }, "counts": { - "required": 145, + "required": 148, "reviewed_exclusion": 0, - "total": 145 + "total": 148 }, "derivation": "Surface = efrs_parity_reference.json populated effective loader inputs. status='required' when the sha-pinned candidate evidence records non-default signal on at least the reviewed owning-entity effective-mass share OR the column is pinned in restored_required_columns after a source-family restoration; all remaining surface columns are reviewed_exclusion with reason 'not yet ported from enhanced FRS pipeline — pending review' and a UK_COVERAGE_PROGRESS.md tracking note. The final release gate applies the same effective-mass floor, and distributional restorations must pass it on their required source channel.", "description": "Declared full-coverage contract for a UK release: every populated effective loader input in the pinned enhanced FRS must be persisted with non-default signal, or carry a reviewed exclusion with the campaign reason and tracking note.", @@ -473,7 +482,7 @@ "capital_gains" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "84d1c1c85f0172a9a396ef69d445ee33cf1340f62ff73c8653c399af40087bf4", + "source_manifest_sha256": "2dc160b3c1f3067d34757a5f566a76545bf3f4f92035b4327f4662658a793adb", "source_vintages": { "source": "HMRC Capital Gains Tax statistics, July 2025, Table 2.1a", "survey": "HMRC Capital Gains Tax statistics Table 2.1a and Advani-Summers capital-gains incidence" @@ -496,7 +505,7 @@ "capital_gains" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "84d1c1c85f0172a9a396ef69d445ee33cf1340f62ff73c8653c399af40087bf4", + "source_manifest_sha256": "2dc160b3c1f3067d34757a5f566a76545bf3f4f92035b4327f4662658a793adb", "source_vintages": { "source": "Advani and Summers (2020), Capital Gains and UK Inequality, CAGE Working Paper 465", "survey": "Family Resources Survey 2024-25, SPI synthetic support, and Advani-Summers capital-gains incidence" @@ -525,7 +534,7 @@ "required_mass_change_reason": "E5 source-stage transform preserves household rows and typed household weights; total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "84d1c1c85f0172a9a396ef69d445ee33cf1340f62ff73c8653c399af40087bf4", + "source_manifest_sha256": "2dc160b3c1f3067d34757a5f566a76545bf3f4f92035b4327f4662658a793adb", "source_vintages": { "source": "UK Data Service SN 8856 Effects of Taxes and Benefits household tab, DfT rail fare index, and public NHS activity/cost table.", "survey": "Effects of Taxes and Benefits 1977-2024 and NHS age-gender public table" @@ -545,7 +554,7 @@ "required_mass_change_reason": "E5 source-stage transform preserves household rows and typed household weights; total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "84d1c1c85f0172a9a396ef69d445ee33cf1340f62ff73c8653c399af40087bf4", + "source_manifest_sha256": "2dc160b3c1f3067d34757a5f566a76545bf3f4f92035b4327f4662658a793adb", "source_vintages": { "source": "UK Data Service SN 8856 Effects of Taxes and Benefits household tab and cited VAT anchor resource.", "survey": "Effects of Taxes and Benefits 1977-2024" @@ -565,20 +574,20 @@ "capital_gains" ], "required_mass_change_reason": "Amounts-only capital gains redraw: household weights pass through unchanged and total household mass is conserved.", - "source_manifest": "cgt_source_stages.json", - "source_manifest_sha256": "71104111b4b9f2da00ce49ad5abec54a35d300196032c9742d62e02aa1730774", - "superseded_by": { - "reason": "The FRS spine build executes hmrc_cgt_gains_spine, which applies the same HMRC Table 3 amounts redraw directly in source_stages.json before calibration.", - "source_manifest": "source_stages.json", - "source_manifest_sha256": "84d1c1c85f0172a9a396ef69d445ee33cf1340f62ff73c8653c399af40087bf4", - "stage": "hmrc_cgt_gains_spine" - }, - "source_vintages": { - "hmrc_surface": "2023-24", - "mapped_build_period": "2024" + "source_manifest": "cgt_source_stages.json", + "source_manifest_sha256": "71104111b4b9f2da00ce49ad5abec54a35d300196032c9742d62e02aa1730774", + "source_vintages": { + "hmrc_surface": "2023-24", + "mapped_build_period": "2024" }, "stage": "hmrc_cgt_gains", - "status": "required_at_build" + "status": "required_at_build", + "superseded_by": { + "reason": "The FRS spine build executes hmrc_cgt_gains_spine, which applies the same HMRC Table 3 amounts redraw directly in source_stages.json before calibration.", + "source_manifest": "source_stages.json", + "source_manifest_sha256": "2dc160b3c1f3067d34757a5f566a76545bf3f4f92035b4327f4662658a793adb", + "stage": "hmrc_cgt_gains_spine" + } }, "hmrc_cgt_gains_spine": { "base_candidate_sha256": "f17306ccb2aad7ff0130be3589b560afb2e2a12a943570911cd0c77f07934833", @@ -596,7 +605,7 @@ "capital_gains" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "84d1c1c85f0172a9a396ef69d445ee33cf1340f62ff73c8653c399af40087bf4", + "source_manifest_sha256": "2dc160b3c1f3067d34757a5f566a76545bf3f4f92035b4327f4662658a793adb", "source_vintages": { "hmrc_surface": "2023-24", "mapped_build_period": "2024" @@ -610,7 +619,7 @@ "base_candidate_tier": "frs", "calibration_permitted": false, "canonical_source_manifest": "source_stages.json", - "canonical_source_manifest_sha256": "84d1c1c85f0172a9a396ef69d445ee33cf1340f62ff73c8653c399af40087bf4", + "canonical_source_manifest_sha256": "2dc160b3c1f3067d34757a5f566a76545bf3f4f92035b4327f4662658a793adb", "effective_mass_requirements": { "charitable_investment_gifts": { "mass_share_denominator": "all_person_effective_mass", @@ -669,7 +678,7 @@ "OTHERINC" ] }, - "reviewed_fence_ids": [ + "reviewed_fence_ids": [ "frs_epb_source_absent", "frs_exps_source_absent", "frs_taxterm_source_absent", @@ -679,15 +688,9 @@ "frs_srp_regular_code5_subset", "full_frs_tei_band_unavailable" ], - "source_manifest": "hmrc_income_source_stages.json", - "source_manifest_sha256": "c0341af7166ae3a85a3c1164e7d9e880c4b4aec122f1a8fa90c73b46c596e1ea", - "superseded_by": { - "reason": "The FRS spine build executes hmrc_spi_income_spine, which supersedes the June retained-leaves/hmrc_spi_income pair inside source_stages.json.", - "source_manifest": "source_stages.json", - "source_manifest_sha256": "84d1c1c85f0172a9a396ef69d445ee33cf1340f62ff73c8653c399af40087bf4", - "stage": "hmrc_spi_income_spine" - }, - "source_vintages": { + "source_manifest": "hmrc_income_source_stages.json", + "source_manifest_sha256": "c0341af7166ae3a85a3c1164e7d9e880c4b4aec122f1a8fa90c73b46c596e1ea", + "source_vintages": { "hmrc_surface": "2023-24", "mapped_build_period": "2024", "period_mapping": "latest_published_tax_year", @@ -695,7 +698,13 @@ }, "spi_prior_national_household_mass_share": 0.5, "stage": "hmrc_spi_income", - "status": "required_at_build" + "status": "required_at_build", + "superseded_by": { + "reason": "The FRS spine build executes hmrc_spi_income_spine, which supersedes the June retained-leaves/hmrc_spi_income pair inside source_stages.json.", + "source_manifest": "source_stages.json", + "source_manifest_sha256": "2dc160b3c1f3067d34757a5f566a76545bf3f4f92035b4327f4662658a793adb", + "stage": "hmrc_spi_income_spine" + } }, "lcfs_consumption": { "base_candidate_sha256": "f17306ccb2aad7ff0130be3589b560afb2e2a12a943570911cd0c77f07934833", @@ -727,7 +736,7 @@ "required_mass_change_reason": "E5 source-stage transform preserves household rows and typed household weights; total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "84d1c1c85f0172a9a396ef69d445ee33cf1340f62ff73c8653c399af40087bf4", + "source_manifest_sha256": "2dc160b3c1f3067d34757a5f566a76545bf3f4f92035b4327f4662658a793adb", "source_vintages": { "source": "UK Data Service SN 9468 Living Costs and Food Survey 2023-24 household/person tabs, NEED 2023 headline energy tables, Ofgem Q2 2026 unit rates, and WAS round-8 bridge donor.", "survey": "Living Costs and Food Survey 2023-24" @@ -748,7 +757,7 @@ "property_wealth" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "84d1c1c85f0172a9a396ef69d445ee33cf1340f62ff73c8653c399af40087bf4", + "source_manifest_sha256": "2dc160b3c1f3067d34757a5f566a76545bf3f4f92035b4327f4662658a793adb", "source_vintages": { "source": "MHCLG dwellings and ONS UK House Price Index December 2025 regional average prices.", "survey": "Public regional property reference" @@ -772,7 +781,7 @@ "employee_pension_contributions" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "84d1c1c85f0172a9a396ef69d445ee33cf1340f62ff73c8653c399af40087bf4", + "source_manifest_sha256": "2dc160b3c1f3067d34757a5f566a76545bf3f4f92035b4327f4662658a793adb", "source_vintages": { "source": "HMRC, Salary sacrifice reform for pension contributions effective from 6 April 2029", "survey": "Family Resources Survey 2024-25 salary-sacrifice respondents and HMRC salary-sacrifice reform analysis" @@ -794,7 +803,7 @@ "student_loan_plan" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "84d1c1c85f0172a9a396ef69d445ee33cf1340f62ff73c8653c399af40087bf4", + "source_manifest_sha256": "2dc160b3c1f3067d34757a5f566a76545bf3f4f92035b4327f4662658a793adb", "source_vintages": { "source": "Explore Education Statistics Table 6a, Higher education total", "survey": "Family Resources Survey 2024-25 and Student Loans Company borrower forecasts for England" @@ -811,6 +820,7 @@ "outputs": [ "owned_land", "property_wealth", + "private_pension_wealth", "corporate_wealth", "gross_financial_wealth", "net_financial_wealth", @@ -826,7 +836,7 @@ "required_mass_change_reason": "E5 source-stage transform preserves household rows and typed household weights; total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "84d1c1c85f0172a9a396ef69d445ee33cf1340f62ff73c8653c399af40087bf4", + "source_manifest_sha256": "2dc160b3c1f3067d34757a5f566a76545bf3f4f92035b4327f4662658a793adb", "source_vintages": { "source": "Office for National Statistics Wealth and Assets Survey, UK Data Service SN 7215, DOI 10.5255/UKDA-SN-7215-20; local licensed 2006-22 household tab.", "survey": "Wealth and Assets Survey round 8" @@ -839,7 +849,7 @@ "derived_from": "efrs_parity_reference.json", "filename": "enhanced_frs_2024_25.h5", "period": "2024", - "populated_input_columns": 145, + "populated_input_columns": 148, "revision": "a9e52499b6a6cca100a5ce4f36ca27b2e8a213df", "sha256": "e433e532b17bd8ce76030156285816e33d44e93edabd2204adbef71d19a68712", "vintage": "2024_25" diff --git a/packages/microcosm-build/src/microcosm/build/uk/source_stages.json b/packages/microcosm-build/src/microcosm/build/uk/source_stages.json index 27417362..c8d749a0 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/source_stages.json +++ b/packages/microcosm-build/src/microcosm/build/uk/source_stages.json @@ -907,7 +907,8 @@ "weight": "R8xshhwgt" }, "derived": { - "corporate_wealth_excl_isa": "(totalpenr8_aggr - dvvaldbt_scaper8_aggr) + DVFESHARESR8_aggr + DVFShUKVR8_aggr + DVFCollVR8_aggr", + "private_pension_wealth": "totalpenr8_aggr - dvvaldbt_scaper8_aggr", + "corporate_wealth_excl_isa": "DVFESHARESR8_aggr + DVFShUKVR8_aggr + DVFCollVR8_aggr", "stocks_and_shares_isa": "DVIISAVR8_aggr", "cash_isa": "DVCISAVR8_aggr", "student_loan_balance": "Tot_LosR8_aggr - Tot_los_exc_SLCR8_aggr", @@ -960,6 +961,7 @@ "chain_order": [ "owned_land", "property_wealth", + "private_pension_wealth", "corporate_wealth_excl_isa", "stocks_and_shares_isa", "corporate_wealth", @@ -1007,6 +1009,7 @@ "outputs": [ "owned_land", "property_wealth", + "private_pension_wealth", "corporate_wealth", "gross_financial_wealth", "net_financial_wealth", @@ -1022,6 +1025,7 @@ "nonnegative_outputs": [ "owned_land", "property_wealth", + "private_pension_wealth", "corporate_wealth", "gross_financial_wealth", "main_residence_value", @@ -1033,7 +1037,7 @@ "stocks_and_shares_isa", "student_loan_balance" ], - "notes": "Ports incumbent WAS round-8 wealth imputation with signed E5 differences: exact lower-case column matching replaces the fuzzy r/w fallback; cash ISA uses DVCISAVR8_aggr and stocks-and-shares ISA uses DVIISAVR8_aggr; corporate_wealth folds stocks-and-shares ISA after drawing corporate_wealth_excl_isa; recipient Northern Ireland regions are mapped to Wales for prediction only; student_loan_balance is allocated by household id rather than the incumbent positional off-by-one. Engine predictors materialize at their native entity and person/benunit values are summed to household, reproducing the incumbent map_to=household semantics; region is one-hot encoded jointly across donor and recipient (the incumbent's dummy encoding), with unmapped donor GOR codes becoming all-zero dummy rows. The WAS and FRS predictor definitions are not fully like-for-like and are ported as-is; raw WAS missing values are blanket-filled with zero; UKDS negative sentinel codes (-9/-8/-7/-6) are recoded to zero for the nonnegative-domain columns the licensed audit found carrying them (vcarnr8: 2 rows; HBedRmR8: 95.8 percent - the bedrooms question is effectively unasked in the WAS household file, predictor-quality revisit registered on microcosm#145) - a signed difference vs the incumbent, which trains on raw sentinels; DVPriRntR8's -9 is structural not-applicable so the is_renting mapping is unchanged; genuinely negative domains are never recoded." + "notes": "Ports incumbent WAS round-8 wealth imputation with signed E5 differences: exact lower-case column matching replaces the fuzzy r/w fallback; cash ISA uses DVCISAVR8_aggr and stocks-and-shares ISA uses DVIISAVR8_aggr; corporate_wealth folds stocks-and-shares ISA after drawing corporate_wealth_excl_isa; under uk-data#452 M2 / microcosm#750, WAS private pension wealth less current-employment DB is drawn first in the second chain segment as private_pension_wealth, and row identity old corporate_wealth == corporate_wealth + private_pension_wealth holds on the donor; recipient Northern Ireland regions are mapped to Wales for prediction only; student_loan_balance is allocated by household id rather than the incumbent positional off-by-one. Engine predictors materialize at their native entity and person/benunit values are summed to household, reproducing the incumbent map_to=household semantics; region is one-hot encoded jointly across donor and recipient (the incumbent's dummy encoding), with unmapped donor GOR codes becoming all-zero dummy rows. The WAS and FRS predictor definitions are not fully like-for-like and are ported as-is; raw WAS missing values are blanket-filled with zero; UKDS negative sentinel codes (-9/-8/-7/-6) are recoded to zero for the nonnegative-domain columns the licensed audit found carrying them (vcarnr8: 2 rows; HBedRmR8: 95.8 percent - the bedrooms question is effectively unasked in the WAS household file, predictor-quality revisit registered on microcosm#145) - a signed difference vs the incumbent, which trains on raw sentinels; DVPriRntR8's -9 is structural not-applicable so the is_renting mapping is unchanged; genuinely negative domains are never recoded." }, { "stage": "regional_property_uprating", diff --git a/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml b/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml index d30a9419..86c2af67 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml +++ b/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml @@ -724,7 +724,8 @@ stages: num_vehicles: vcarnr8 weight: R8xshhwgt derived: - corporate_wealth_excl_isa: (totalpenr8_aggr - dvvaldbt_scaper8_aggr) + DVFESHARESR8_aggr + DVFShUKVR8_aggr + DVFCollVR8_aggr + private_pension_wealth: totalpenr8_aggr - dvvaldbt_scaper8_aggr + corporate_wealth_excl_isa: DVFESHARESR8_aggr + DVFShUKVR8_aggr + DVFCollVR8_aggr stocks_and_shares_isa: DVIISAVR8_aggr cash_isa: DVCISAVR8_aggr student_loan_balance: Tot_LosR8_aggr - Tot_los_exc_SLCR8_aggr @@ -767,6 +768,7 @@ stages: chain_order: - owned_land - property_wealth + - private_pension_wealth - corporate_wealth_excl_isa - stocks_and_shares_isa - corporate_wealth @@ -802,6 +804,7 @@ stages: outputs: - owned_land - property_wealth + - private_pension_wealth - corporate_wealth - gross_financial_wealth - net_financial_wealth @@ -816,6 +819,7 @@ stages: nonnegative_outputs: - owned_land - property_wealth + - private_pension_wealth - corporate_wealth - gross_financial_wealth - main_residence_value @@ -826,7 +830,7 @@ stages: - cash_isa - stocks_and_shares_isa - student_loan_balance - notes: 'Ports incumbent WAS round-8 wealth imputation with signed E5 differences: exact lower-case column matching replaces the fuzzy r/w fallback; cash ISA uses DVCISAVR8_aggr and stocks-and-shares ISA uses DVIISAVR8_aggr; corporate_wealth folds stocks-and-shares ISA after drawing corporate_wealth_excl_isa; recipient Northern Ireland regions are mapped to Wales for prediction only; student_loan_balance is allocated by household id rather than the incumbent positional off-by-one. Engine predictors materialize at their native entity and person/benunit values are summed to household, reproducing the incumbent map_to=household semantics; region is one-hot encoded jointly across donor and recipient (the incumbent''s dummy encoding), with unmapped donor GOR codes becoming all-zero dummy rows. The WAS and FRS predictor definitions are not fully like-for-like and are ported as-is; raw WAS missing values are blanket-filled with zero; UKDS negative sentinel codes (-9/-8/-7/-6) are recoded to zero + notes: 'Ports incumbent WAS round-8 wealth imputation with signed E5 differences: exact lower-case column matching replaces the fuzzy r/w fallback; cash ISA uses DVCISAVR8_aggr and stocks-and-shares ISA uses DVIISAVR8_aggr; corporate_wealth folds stocks-and-shares ISA after drawing corporate_wealth_excl_isa; under uk-data#452 M2 / microcosm#750, WAS private pension wealth less current-employment DB is drawn first in the second chain segment as private_pension_wealth, and row identity old corporate_wealth == corporate_wealth + private_pension_wealth holds on the donor; recipient Northern Ireland regions are mapped to Wales for prediction only; student_loan_balance is allocated by household id rather than the incumbent positional off-by-one. Engine predictors materialize at their native entity and person/benunit values are summed to household, reproducing the incumbent map_to=household semantics; region is one-hot encoded jointly across donor and recipient (the incumbent''s dummy encoding), with unmapped donor GOR codes becoming all-zero dummy rows. The WAS and FRS predictor definitions are not fully like-for-like and are ported as-is; raw WAS missing values are blanket-filled with zero; UKDS negative sentinel codes (-9/-8/-7/-6) are recoded to zero for the nonnegative-domain columns the licensed audit found carrying them (vcarnr8: 2 rows; HBedRmR8: 95.8 percent - the bedrooms question is effectively unasked in the WAS household file, predictor-quality revisit registered on microcosm#145) - a signed difference vs the incumbent, which trains on raw sentinels; DVPriRntR8''s -9 is structural not-applicable so the is_renting mapping is unchanged; genuinely negative domains are never recoded.' - stage: regional_property_uprating survey: Public regional property reference diff --git a/packages/microcosm-build/src/microcosm/build/uk/was_wealth_support_bounds.json b/packages/microcosm-build/src/microcosm/build/uk/was_wealth_support_bounds.json index 0d9bce9b..a9fc037d 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/was_wealth_support_bounds.json +++ b/packages/microcosm-build/src/microcosm/build/uk/was_wealth_support_bounds.json @@ -18,6 +18,10 @@ 0.0, 20000000 ], + "private_pension_wealth": [ + 0.0, + 8000000 + ], "corporate_wealth": [ 0.0, 30000000 diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/calibration_run.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/calibration_run.py index 5bb27c48..28fc29ed 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/calibration_run.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/calibration_run.py @@ -197,6 +197,7 @@ def run_uk_calibration( input_sha256: str, ledger_artifact: Any, register_registry: TargetRegistry, + band_edge_registry: TargetRegistry | None = None, calibration_year: int, exclusion_receipt: Mapping[str, Mapping[str, str]], doctrine: Any, @@ -211,6 +212,15 @@ def run_uk_calibration( started_at = time.perf_counter() started_ts = datetime.now(UTC) + # Pure-argument validation precedes every environment probe: a pruned + # register without its compiled band-edge source must refuse identically + # whether or not a git checkout or Logbook chain is reachable. + _validate_band_edge_registry( + register_registry=register_registry, + band_edge_registry=band_edge_registry, + exclusion_receipt=exclusion_receipt, + ) + edge_registry = band_edge_registry or register_registry code_pin = git_code_pin(_REPOSITORY) # Predecessor configuration is validated before anything is written: a # disagreeing chain must refuse with no artifact on disk, not after a @@ -230,6 +240,7 @@ def run_uk_calibration( "ledger": _ledger_provenance(ledger_artifact), **dict(run_config_extra), } + run_config["band_edge_register_sha256"] = edge_registry.version state = AttemptState( # Attempts are distinct rows even when they re-run one release: both # the local chain and the store refuse a repeated build id. @@ -246,6 +257,7 @@ def run_uk_calibration( input_sha256=input_sha256, ledger_artifact=ledger_artifact, register_registry=register_registry, + band_edge_registry=edge_registry, calibration_year=calibration_year, exclusion_receipt=exclusion_receipt, doctrine=doctrine, @@ -287,6 +299,41 @@ def _new_calibration_attempt_id(*, timestamp: datetime) -> str: ) +def _validate_band_edge_registry( + *, + register_registry: TargetRegistry, + band_edge_registry: TargetRegistry | None, + exclusion_receipt: Mapping[str, Mapping[str, str]], +) -> None: + if exclusion_receipt and band_edge_registry is None: + raise ValueError( + "a pruned register cannot derive published band edges from itself; " + "pass the compiled pre-exclusion register." + ) + if band_edge_registry is None: + return + register_names = _registry_spec_names(register_registry) + edge_names = _registry_spec_names(band_edge_registry) + excluded_names = {str(name) for name in exclusion_receipt} + if not register_names <= edge_names: + missing = sorted(register_names - edge_names) + raise ValueError( + "band-edge register must include every materialized registry spec; " + f"missing={missing}." + ) + extra = edge_names - register_names + if extra != excluded_names: + raise ValueError( + "band-edge register extra spec names must match the measure " + f"exclusion receipt; extra={sorted(extra)}, " + f"receipt={sorted(excluded_names)}." + ) + + +def _registry_spec_names(registry: TargetRegistry) -> set[str]: + return {str(spec.name) for spec in registry.specs} + + def _record_failed_attempt( *, error: BaseException, @@ -336,6 +383,7 @@ def _run_uk_calibration_attempt( input_sha256: str, ledger_artifact: Any, register_registry: TargetRegistry, + band_edge_registry: TargetRegistry, calibration_year: int, exclusion_receipt: Mapping[str, Mapping[str, str]], doctrine: Any, @@ -374,6 +422,7 @@ def _run_uk_calibration_attempt( period=calibration_year, doctrine=doctrine, measure_resolver=measure_resolver, + band_edge_registry=band_edge_registry, ) calibrated = stage(frame) append_phase(state, "national_calibration_solved") diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/ledger_targets.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/ledger_targets.py index 7e3608fa..0a1f0493 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/ledger_targets.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/ledger_targets.py @@ -88,6 +88,7 @@ def materialize_uk_ledger_targets( registry: TargetRegistry, *, period: int | str, + band_edge_registry: TargetRegistry | None = None, ) -> TargetMaterializationResult: """Materialize compiled UK Ledger target bindings on an adapter.""" @@ -102,6 +103,7 @@ def materialize_uk_ledger_targets( "baseline_flag_crosstab": _uk_baseline_flag_crosstab, "input_substitution_counterfactual": _uk_input_substitution, }, + band_edge_registry=band_edge_registry, ) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/national_calibration.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/national_calibration.py index 4ff938bb..df294ee2 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/national_calibration.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/national_calibration.py @@ -48,6 +48,7 @@ def __init__( period: int, doctrine: UKNationalSolveDoctrine = UK_NATIONAL_SOLVE_DOCTRINE, measure_resolver: object | None = None, + band_edge_registry: TargetRegistry | None = None, ) -> None: self.compilation = ( registry @@ -55,6 +56,9 @@ def __init__( else UKLedgerTargetCompilation(registry=registry, unsupported=()) ) self.registry = self.compilation.registry + self.band_edge_registry = ( + band_edge_registry if band_edge_registry is not None else self.registry + ) # The materialization period is the declared calibration year the # registry was compiled at — never the input frame's base-year # time_period, which lags it (survey 2024, calibration 2025). @@ -90,6 +94,7 @@ def __call__(self, frame: Frame) -> Frame: adapter, self.registry, period=self.period, + band_edge_registry=self.band_edge_registry, ) if materialized.skipped: skipped = [skip.__dict__ for skip in materialized.skipped] @@ -209,12 +214,14 @@ def _resolve_measures(self, frame: Frame) -> MeasureResolution | None: lambda: _CalibrationFrameAdapter(frame), self.registry, period=self.period, + band_edge_registry=self.band_edge_registry, ) return resolve_target_measures( lambda: _CalibrationFrameAdapter(frame), self.registry, self.measure_resolver, period=self.period, + band_edge_registry=self.band_edge_registry, ) def checkpoint_metadata(self) -> Mapping[str, object]: diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/terminal_gates.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/terminal_gates.py index 625d9602..1e4660ca 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/terminal_gates.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/terminal_gates.py @@ -178,6 +178,7 @@ def __post_init__(self) -> None: "household.msoa_code", "household.num_vehicles", "household.oa_code", + "household.private_pension_wealth", "household.property_purchased", "household.rail_usage", "household.region_code_oa", diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/was_wealth.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/was_wealth.py index 27a1b8d0..53a1cad4 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/was_wealth.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/was_wealth.py @@ -1,4 +1,8 @@ -"""UK WAS wealth imputation stage.""" +"""UK WAS wealth imputation stage. + +The WAS pension component is emitted separately from corporate wealth under +uk-data#452 M2 / microcosm#750. +""" from __future__ import annotations @@ -58,6 +62,7 @@ UK_WAS_WEALTH_OUTPUT_COLUMNS = ( "owned_land", "property_wealth", + "private_pension_wealth", "corporate_wealth", "gross_financial_wealth", "net_financial_wealth", @@ -283,10 +288,9 @@ def clean_was_household_table(raw: pd.DataFrame) -> pd.DataFrame: values = cleaned[column] cleaned[column] = values.where(~values.isin(_SENTINEL_CODES), 0) cleaned["is_renting"] = cleaned["private_rent_code"] == 1 + cleaned["private_pension_wealth"] = cleaned["pensions"] - cleaned["db_pensions"] cleaned["corporate_wealth_excl_isa"] = ( - cleaned["pensions"] - - cleaned["db_pensions"] - + cleaned["emp_shares_options"] + cleaned["emp_shares_options"] + cleaned["uk_shares"] + cleaned["unit_investment_trusts"] ) @@ -427,19 +431,33 @@ def run_segment(base_predictors: Sequence[str], targets: Sequence[str]) -> None: base = encoded_predictors run_segment(base, ("owned_land", "property_wealth")) + donor_encoded["private_pension_wealth"] = donor_encoded[ + "private_pension_wealth" + ].astype(float) donor_encoded["corporate_wealth"] = donor_encoded["corporate_wealth"].astype(float) recipient_encoded["owned_land"] = raw["owned_land"] recipient_encoded["property_wealth"] = raw["property_wealth"] run_segment( (*base, "owned_land", "property_wealth"), - ("corporate_wealth_excl_isa", "stocks_and_shares_isa"), + ( + "private_pension_wealth", + "corporate_wealth_excl_isa", + "stocks_and_shares_isa", + ), ) raw["corporate_wealth"] = ( raw["corporate_wealth_excl_isa"] + raw["stocks_and_shares_isa"] ) + recipient_encoded["private_pension_wealth"] = raw["private_pension_wealth"] recipient_encoded["corporate_wealth"] = raw["corporate_wealth"] run_segment( - (*base, "owned_land", "property_wealth", "corporate_wealth"), + ( + *base, + "owned_land", + "property_wealth", + "private_pension_wealth", + "corporate_wealth", + ), ( "gross_financial_wealth", "net_financial_wealth", diff --git a/packages/microcosm-build/tests/test_spec_engine_country_bundles.py b/packages/microcosm-build/tests/test_spec_engine_country_bundles.py index 9d028071..1cdd82c1 100644 --- a/packages/microcosm-build/tests/test_spec_engine_country_bundles.py +++ b/packages/microcosm-build/tests/test_spec_engine_country_bundles.py @@ -42,7 +42,7 @@ ), ( "uk", - "4793baa59a90930956faf8f5b422ece57da3445c7ed0030622ef62ff35dfb142", + "d56a4d0ecb32325921efb38a366b674891eb5d97ae18a74428532e405c4f144b", { "benunit.benunit_id", "household.household_id", diff --git a/packages/microcosm-build/tests/test_target_materialization.py b/packages/microcosm-build/tests/test_target_materialization.py index b4af50fb..1d726f3c 100644 --- a/packages/microcosm-build/tests/test_target_materialization.py +++ b/packages/microcosm-build/tests/test_target_materialization.py @@ -512,6 +512,63 @@ def test_bands_slice_the_population_and_partition_it(): assert list(total) == [1.0, 1.0, 1.0] +def test_band_measures_are_roster_invariant_under_sibling_exclusion(): + registry = _banded_registry() + full_adapter = StubAdapter() + + result = materialize_target_bindings( + full_adapter, registry, _BANDED_CONTRACT, period=2025 + ) + + assert result.skipped == () + snapshots = { + label: full_adapter.tables["person"][f"income_band_{label}"].copy() + for label in ("0", "20", "40") + } + + middle_pruned = TargetRegistry( + [spec for spec in registry.specs if spec.name != "band_20"], + country="uk", + ) + middle_adapter = StubAdapter() + + result = materialize_target_bindings( + middle_adapter, + middle_pruned, + _BANDED_CONTRACT, + period=2025, + band_edge_registry=registry, + ) + + assert result.skipped == () + for label in ("0", "40"): + assert np.array_equal( + middle_adapter.tables["person"][f"income_band_{label}"], + snapshots[label], + ) + + top_pruned = TargetRegistry( + [spec for spec in registry.specs if spec.name != "band_40"], + country="uk", + ) + top_adapter = StubAdapter() + + result = materialize_target_bindings( + top_adapter, + top_pruned, + _BANDED_CONTRACT, + period=2025, + band_edge_registry=registry, + ) + + assert result.skipped == () + for label in ("0", "20"): + assert np.array_equal( + top_adapter.tables["person"][f"income_band_{label}"], + snapshots[label], + ) + + def test_adjacent_bands_are_not_identical(): # The regression that would have caught the unsliced-measure defect: # before banding was implemented every band returned the same unsliced @@ -582,6 +639,137 @@ def test_published_range_labels_band_in_model_units(): assert list(adapter.tables["person"]["award_high"]) == [0.0, 0.0, 1.0] +def test_published_range_label_edges_survive_sibling_exclusion(): + registry = TargetRegistry( + [ + TargetSpec( + name="award_low", + entity="person", + measure="award_low", + value=1.0, + source="test", + family="dwp_universal_credit", + metadata={ + "contract_target_id": "uc.award_bands", + "ledger_filter_family_type": "Single, no children", + "ledger_filter_monthly_award_bands": "£1.01 to £2.00", + }, + ), + TargetSpec( + name="award_high", + entity="person", + measure="award_high", + value=1.0, + source="test", + family="dwp_universal_credit", + metadata={ + "contract_target_id": "uc.award_bands", + "ledger_filter_monthly_award_bands": "£2.01 to £3.00", + }, + ), + ], + country="uk", + ) + contract = { + "uc.award_bands": { + "bindings": { + "policyengine": { + "value_variable": "person_count", + "groupby_variable": "income", + "from_entity": "person", + "band_period_factor": 12, + } + } + } + } + pruned = TargetRegistry( + [spec for spec in registry.specs if spec.name != "award_high"], + country="uk", + ) + adapter = StubAdapter() + + result = materialize_target_bindings( + adapter, + pruned, + contract, + period=2025, + band_edge_registry=registry, + ) + + assert result.skipped == () + assert list(adapter.tables["person"]["award_low"]) == [0.0, 1.0, 0.0] + + +def test_band_bounds_refuse_a_spec_absent_from_the_band_edge_register(): + registry = TargetRegistry([_banded_registry().specs[0]], country="uk") + adapter = StubAdapter() + + result = materialize_target_bindings( + adapter, + registry, + _BANDED_CONTRACT, + period=2025, + band_edge_registry=TargetRegistry([], country="uk"), + ) + + assert len(result.skipped) == 1 + assert result.skipped[0].name == "band_0" + assert "absent from its contract target's band-edge set" in result.skipped[0].reason + assert "income_band_0" not in adapter.tables["person"] + + +def test_resolve_target_measures_threads_the_band_edge_registry(): + class BandedInputProvider(StubMeasureProvider): + def compute(self, entity, variable): + assert (entity, variable) == ("person", "input_a") + return np.array([1.0, 2.0, 3.0]), "stub:person.input_a" + + source = { + "person": pd.DataFrame( + { + "person_id": [1, 2, 3], + "income": [10.0, 20.0, 30.0], + } + ) + } + registry = _banded_registry() + pruned = TargetRegistry( + [spec for spec in registry.specs if spec.name != "band_20"], + country="uk", + ) + probes = [] + + def adapter_factory(): + adapter = ResolutionAdapter(source) + probes.append(adapter) + return adapter + + resolution = resolve_target_measures( + adapter_factory, + pruned, + BandedInputProvider(), + period=2025, + contract_targets={ + "spi.income_by_band": { + "bindings": { + "policyengine": { + "value_variable": "input_a", + "groupby_variable": "income", + "from_entity": "person", + } + } + } + }, + band_edge_registry=registry, + ) + + assert resolution.receipt["attached"] == { + "person.input_a": "stub:person.input_a" + } + assert list(probes[-1].tables["person"]["income_band_0"]) == [1.0, 0.0, 0.0] + assert list(probes[-1].tables["person"]["income_band_40"]) == [0.0, 0.0, 0.0] + + def test_unreadable_band_is_skipped_not_silently_unsliced(): adapter = StubAdapter() registry = TargetRegistry( diff --git a/packages/microcosm-build/tests/test_uk_calibration_run.py b/packages/microcosm-build/tests/test_uk_calibration_run.py index 42420d26..96865c3d 100644 --- a/packages/microcosm-build/tests/test_uk_calibration_run.py +++ b/packages/microcosm-build/tests/test_uk_calibration_run.py @@ -87,6 +87,16 @@ def _registry(): ) +def _paths(tmp_path: Path) -> UKCalibrationRunPaths: + return UKCalibrationRunPaths( + input_h5=tmp_path / "input.h5", + staging_h5=tmp_path / "staged.h5", + diagnostics_json=tmp_path / "diagnostics.json", + build_record_json=tmp_path / "build_record.json", + terminal_gate_json=tmp_path / "terminal_gates.json", + ) + + def _sha(path: Path) -> str: return hashlib.sha256(path.read_bytes()).hexdigest() @@ -265,6 +275,128 @@ def test_run_uk_calibration_writes_cross_pinned_outputs(monkeypatch, tmp_path: P assert result.logbook_spool.exists() +def test_run_uk_calibration_refuses_pruned_register_without_band_edge_register( + tmp_path: Path, +): + paths = _paths(tmp_path) + + with pytest.raises(ValueError, match="compiled pre-exclusion register"): + run_uk_calibration( + paths=paths, + input_sha256="a" * 64, + ledger_artifact=object(), + register_registry=_registry(), + calibration_year=2025, + exclusion_receipt={"excluded.target": {"reason": "reviewed"}}, + doctrine=UKNationalSolveDoctrine(epochs=1), + doctrine_overrides={}, + measure_resolver=None, + source_pins={}, + run_config_extra={}, + release_id="pruned-without-edge-register", + ) + + assert not paths.staging_h5.exists() + assert not paths.diagnostics_json.exists() + assert not paths.build_record_json.exists() + + +def test_run_uk_calibration_refuses_incoherent_band_edge_register(tmp_path: Path): + paths = _paths(tmp_path) + edge_registry = TargetRegistry( + [ + *_registry().specs, + TargetSpec( + name="different.excluded", + entity="benunit", + measure="different/excluded", + value=1.0, + source="test", + metadata={"contract_target_id": "different.excluded"}, + ), + ], + country="uk", + ) + + with pytest.raises(ValueError, match="exclusion receipt"): + run_uk_calibration( + paths=paths, + input_sha256="a" * 64, + ledger_artifact=object(), + register_registry=_registry(), + band_edge_registry=edge_registry, + calibration_year=2025, + exclusion_receipt={"other.excluded": {"reason": "reviewed"}}, + doctrine=UKNationalSolveDoctrine(epochs=1), + doctrine_overrides={}, + measure_resolver=None, + source_pins={}, + run_config_extra={}, + release_id="incoherent-edge-register", + ) + + assert not paths.staging_h5.exists() + assert not paths.diagnostics_json.exists() + assert not paths.build_record_json.exists() + + +def test_run_uk_calibration_records_band_edge_register_sha256( + monkeypatch, tmp_path: Path +): + pytest.importorskip("tables") # pandas HDF backend + monkeypatch.setattr( + calibration_run, + "uk_aggregate_admin_totals", + lambda frame, manifest: (_admin_anchor_values(), []), + ) + input_h5 = tmp_path / "input.h5" + frame = _frame() + write_uk_national_frame(frame, input_h5) + _write_spine_sidecar(input_h5, frame) + paths = _paths(tmp_path) + register = _registry() + edge_registry = TargetRegistry( + [ + TargetSpec( + name="dwp.uc.households", + entity="benunit", + measure="dwp/uc/households", + value=99.0, + source="test", + family="dwp_universal_credit", + metadata={"contract_target_id": "dwp.uc.households"}, + ) + ], + country="uk", + ) + + result = run_uk_calibration( + paths=paths, + input_sha256=_sha(input_h5), + ledger_artifact=object(), + register_registry=register, + band_edge_registry=edge_registry, + calibration_year=2025, + exclusion_receipt={}, + doctrine=UKNationalSolveDoctrine(epochs=5), + doctrine_overrides={}, + measure_resolver=None, + source_pins={ + "input_h5": { + "sha256": _sha(input_h5), + "size_bytes": input_h5.stat().st_size, + } + }, + run_config_extra={}, + release_id="band-edge-provenance", + ) + + assert ( + result.build_record["run_config"]["band_edge_register_sha256"] + == edge_registry.version + ) + + def test_run_uk_calibration_refuses_input_sha_before_outputs(tmp_path: Path): pytest.importorskip("tables") # pandas HDF backend input_h5 = tmp_path / "input.h5" diff --git a/packages/microcosm-build/tests/test_uk_calibration_seam_driver.py b/packages/microcosm-build/tests/test_uk_calibration_seam_driver.py index 8ed5850e..f8cf5b3a 100644 --- a/packages/microcosm-build/tests/test_uk_calibration_seam_driver.py +++ b/packages/microcosm-build/tests/test_uk_calibration_seam_driver.py @@ -129,6 +129,7 @@ def test_driver_threads_registry_exclusions_resolver_and_overrides(monkeypatch, driver = _load_driver_module() calls = [] registry = _registry() + pruned_registry = TargetRegistry([], country="uk") artifact = SimpleNamespace( path=tmp_path / "ledger", facts=({"fact": 1},), @@ -152,7 +153,7 @@ def test_driver_threads_registry_exclusions_resolver_and_overrides(monkeypatch, monkeypatch.setattr( driver, "apply_uk_calibration_measure_exclusions", - lambda reg, exclusions: (reg, {"excluded": "reviewed"}), + lambda reg, exclusions: (pruned_registry, {"excluded": "reviewed"}), ) class FakeResolver: @@ -183,7 +184,8 @@ def fake_run(**kwargs): assert result == 0 call = calls[0] - assert call["register_registry"] is registry + assert call["register_registry"] is pruned_registry + assert call["band_edge_registry"] is registry assert call["calibration_year"] == 2025 assert call["exclusion_receipt"] == {"excluded": "reviewed"} assert call["doctrine"].epochs == 128 diff --git a/packages/microcosm-build/tests/test_uk_measure_simulation.py b/packages/microcosm-build/tests/test_uk_measure_simulation.py index 4e380007..59ba0648 100644 --- a/packages/microcosm-build/tests/test_uk_measure_simulation.py +++ b/packages/microcosm-build/tests/test_uk_measure_simulation.py @@ -289,8 +289,10 @@ def test_packaged_exclusions_load(): assert entry["tracking"] == "microcosm#791", entry["name"] # The lever targets are deliberately NOT excluded: the six UC - # caseload / two-child-limit cells ride the would_claim_uc lever run, - # and the two expected-to-resolve cells ride the exclusion re-run. + # caseload / two-child-limit cells ride the would_claim_uc lever run. + # Exclusions cannot move surviving cells because band edges are pinned + # to the compiled register (#792); the non-excluded cells are expected + # to pass at published band widths on the next seam run. excluded = set(names) for riding in ( "dwp.uc.households", diff --git a/packages/microcosm-build/tests/test_uk_national_calibration.py b/packages/microcosm-build/tests/test_uk_national_calibration.py index 1886242e..099c577b 100644 --- a/packages/microcosm-build/tests/test_uk_national_calibration.py +++ b/packages/microcosm-build/tests/test_uk_national_calibration.py @@ -21,6 +21,7 @@ UK_NATIONAL_TARGET_LOSS_CAP, UK_NATIONAL_TARGET_WEIGHT_RULE, UKNationalSolveDoctrine, + national_calibration, uk_doctrine_with_overrides, uk_national_target_loss_weights, ) @@ -372,6 +373,43 @@ def test_stage_manifest_omits_measure_resolution_without_resolver() -> None: assert "measure_resolution" not in stage.manifest +def test_stage_threads_band_edge_registry_to_materialization(monkeypatch) -> None: + captured = [] + real_materialize = national_calibration.materialize_uk_ledger_targets + + def capture_materialize(*args, **kwargs): + captured.append(kwargs) + return real_materialize(*args, **kwargs) + + monkeypatch.setattr( + national_calibration, + "materialize_uk_ledger_targets", + capture_materialize, + ) + sentinel = TargetRegistry([], country="uk") + stage = UKNationalCalibrationStage( + _registry(), + period=2025, + doctrine=UKNationalSolveDoctrine(epochs=1), + band_edge_registry=sentinel, + ) + + stage(_frame()) + + assert captured[-1]["band_edge_registry"] is sentinel + + captured.clear() + default_stage = UKNationalCalibrationStage( + _registry(), + period=2025, + doctrine=UKNationalSolveDoctrine(epochs=1), + ) + + default_stage(_frame()) + + assert captured[-1]["band_edge_registry"] is default_stage.registry + + def test_activated_unresolvable_compiled_reference_aborts_loudly() -> None: stage = UKNationalCalibrationStage( UKLedgerTargetCompilation( diff --git a/packages/microcosm-build/tests/test_uk_parity_reference.py b/packages/microcosm-build/tests/test_uk_parity_reference.py index ac7199a3..c517be17 100644 --- a/packages/microcosm-build/tests/test_uk_parity_reference.py +++ b/packages/microcosm-build/tests/test_uk_parity_reference.py @@ -70,8 +70,8 @@ class TestEfrsParityReference: def test_reference_loads_with_populated_layers(self) -> None: reference = load_efrs_parity_reference() assert reference.schema_version == 3 - assert len(reference.nonzero_shares) == 145 - assert len(reference.populated_layers) == 145 + assert len(reference.nonzero_shares) == 148 + assert len(reference.populated_layers) == 148 assert all(share > 0.0 for share in reference.nonzero_shares.values()) assert set(reference.input_entities) == set(reference.nonzero_shares) assert set(reference.input_entities.values()) == { diff --git a/packages/microcosm-build/tests/test_uk_release_input_coverage_manifest.py b/packages/microcosm-build/tests/test_uk_release_input_coverage_manifest.py index 2dd872ef..81c041f7 100644 --- a/packages/microcosm-build/tests/test_uk_release_input_coverage_manifest.py +++ b/packages/microcosm-build/tests/test_uk_release_input_coverage_manifest.py @@ -55,7 +55,7 @@ def test_candidate_evidence_is_sha_pinned_and_covers_reference() -> None: assert evidence["nonzero_shares"]["household_weight"] == 0.626224 assert evidence["nondefault_shares"]["household_weight"] == 1.0 assert evidence["nondefault_shares"]["employment_income"] == 0.478282 - assert evidence["effective_signal_columns"] == 143 + assert evidence["effective_signal_columns"] == 146 assert evidence["insufficient_effective_mass_columns"] == [ "charitable_investment_gifts", "gift_aid", @@ -80,8 +80,8 @@ def test_known_gap_register_records_post_candidate_restoration_separately() -> N } assert gaps["candidate_evidence"]["missing_columns"] == [] assert gaps["candidate_evidence"]["default_only_columns"] == [] - assert gaps["candidate_evidence"]["signal_columns"] == 145 - assert gaps["candidate_evidence"]["effective_signal_columns"] == 143 + assert gaps["candidate_evidence"]["signal_columns"] == 148 + assert gaps["candidate_evidence"]["effective_signal_columns"] == 146 assert gaps["exclusion_policy"]["reason"] == ( "not yet ported from enhanced FRS pipeline — pending review" ) @@ -175,9 +175,9 @@ def test_promoted_manifest_requires_the_full_reference_surface() -> None: reference = _resource("efrs_parity_reference.json") manifest = _resource("release_input_coverage_manifest.json") assert manifest["counts"] == { - "required": 145, + "required": 148, "reviewed_exclusion": 0, - "total": 145, + "total": 148, } assert set(manifest["columns"]) == set(reference["nonzero_shares"]) assert all( diff --git a/packages/microcosm-build/tests/test_uk_was_wealth.py b/packages/microcosm-build/tests/test_uk_was_wealth.py index 872ae69f..1f34586c 100644 --- a/packages/microcosm-build/tests/test_uk_was_wealth.py +++ b/packages/microcosm-build/tests/test_uk_was_wealth.py @@ -152,14 +152,39 @@ def test_was_donor_cleaning_arithmetic_and_exact_case_insensitive_columns() -> N assert donor["stocks_and_shares_isa"].tolist() == [5.0, 6.0] assert donor["cash_isa"].tolist() == [7.0, 8.0] - assert donor["corporate_wealth_excl_isa"].tolist() == [73.0, 166.0] - assert donor["corporate_wealth"].tolist() == [78.0, 172.0] + assert "private_pension_wealth" in donor.columns + assert donor["private_pension_wealth"].tolist() == [60.0, 150.0] + assert donor["corporate_wealth_excl_isa"].tolist() == [13.0, 16.0] + assert donor["corporate_wealth"].tolist() == [18.0, 22.0] assert donor["student_loan_balance"].tolist() == [5000.0, 2000.0] assert donor["region"].tolist() == ["LONDON", "SCOTLAND"] assert donor["is_renting"].tolist() == [True, False] assert 3 not in REGIONS +def test_was_pension_split_preserves_old_corporate_wealth_identity() -> None: + donor = clean_was_household_table(_raw_was()) + raw = _raw_was() + + pd.testing.assert_series_equal( + donor["corporate_wealth"] + donor["private_pension_wealth"], + pd.Series([78.0, 172.0]), + check_names=False, + ) + pd.testing.assert_series_equal( + donor["corporate_wealth"] + donor["private_pension_wealth"], + ( + raw["totalpenr8_aggr"] + - raw["dvvaldbt_scaper8_aggr"] + + raw["DVFESHARESR8_aggr"] + + raw["DVFShUKVR8_aggr"] + + raw["DVFCollVR8_aggr"] + + raw["DVIISAVR8_aggR"] + ), + check_names=False, + ) + + def test_was_donor_sentinel_codes_recode_to_zero_for_nonnegative_domains() -> None: raw = _raw_was() raw.loc[0, "vcarnr8"] = -8 @@ -343,6 +368,7 @@ def test_stage_transform_is_deterministic_with_fast_synthetic_imputer( pd.testing.assert_frame_equal(a.table("household"), b.table("household")) pd.testing.assert_frame_equal(a.table("person"), b.table("person")) + assert "private_pension_wealth" in a.table("household").columns def test_stage_transform_requires_a_tab_path_or_donor() -> None: @@ -387,6 +413,60 @@ def test_stage_transform_refuses_sha_mismatched_tab(tmp_path) -> None: transform(_frame()) +def test_was_imputer_keeps_first_segment_before_pension_split( + monkeypatch: pytest.MonkeyPatch, +) -> None: + import microcosm.build.uk_runtime.was_wealth as module + import microcosm.fit + + calls = [] + + class FakeQRF: + def __init__(self, *, n_estimators, seed): + assert n_estimators == 7 + assert seed == 0 + + def start_chain(self, donor, predictors, targets, *, weights): + assert weights == "weight" + calls.append((tuple(predictors), tuple(targets))) + return SimpleNamespace(targets=tuple(targets), position=0) + + def fit_draw_next( + self, + donor, + recipient_predictors, + raw_prior_draws, + *, + state, + weights, + ): + assert weights == "weight" + target = state.targets[state.position] + return SimpleNamespace( + target=target, + raw_draw=np.full(len(recipient_predictors), float(state.position + 1)), + weight_kind="explicit", + state=SimpleNamespace( + targets=state.targets, + position=state.position + 1, + ), + ) + + monkeypatch.setattr(microcosm.fit, "RegimeGatedQRF", FakeQRF) + donor = clean_was_household_table(_raw_was()) + recipient = recipient_predictors(_frame(), _FakeEngine()) + expected_base = tuple( + predictor for predictor in module.UK_WAS_WEALTH_PREDICTORS if predictor != "region" + ) + ("region_LONDON", "region_SCOTLAND", "region_WALES") + + module.impute_was_wealth(donor, recipient, seed=0, n_estimators=7) + + assert calls[0] == ( + expected_base, + ("owned_land", "property_wealth"), + ) + + def test_was_imputer_uses_checkpointed_chain_segments( monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -437,11 +517,18 @@ def fit_draw_next( assert result.draws.columns.tolist() == list(UK_WAS_WEALTH_OUTPUT_COLUMNS) assert calls[0][1] == ("owned_land", "property_wealth") - assert calls[1][1] == ("corporate_wealth_excl_isa", "stocks_and_shares_isa") + assert calls[1][1] == ( + "private_pension_wealth", + "corporate_wealth_excl_isa", + "stocks_and_shares_isa", + ) + assert "private_pension_wealth" in calls[2][0] assert "corporate_wealth" in calls[2][0] assert calls[2][1][-1] == "cash_isa" fitted_targets = [name for _, targets in calls for name in targets] - assert [record.fit_name for record in result.fit_weight_records] == [ + fit_names = [record.fit_name for record in result.fit_weight_records] + assert "uk_was_2018_20_wealth:private_pension_wealth" in fit_names + assert fit_names == [ f"uk_was_2018_20_wealth:{target}" for target in fitted_targets ] assert {record.weight_kind for record in result.fit_weight_records} == {"explicit"} diff --git a/packages/microcosm-data/pyproject.toml b/packages/microcosm-data/pyproject.toml index 6ef7d481..c221c2a5 100644 --- a/packages/microcosm-data/pyproject.toml +++ b/packages/microcosm-data/pyproject.toml @@ -28,7 +28,7 @@ us = ["policyengine-us>=1.745.0,<2"] # The UK artifact lives in a PRIVATE repo (UK Data Service licence): loads # require an authenticated HF token with access. The loader surfaces the # 401 with that explanation rather than retrying. -uk = ["policyengine-uk>=2.88"] +uk = ["policyengine-uk>=2.92"] [project.urls] Homepage = "https://microcosm.institute" diff --git a/packages/microcosm-data/src/microcosm/data/contract.py b/packages/microcosm-data/src/microcosm/data/contract.py index c423461c..0362dde1 100644 --- a/packages/microcosm-data/src/microcosm/data/contract.py +++ b/packages/microcosm-data/src/microcosm/data/contract.py @@ -375,13 +375,13 @@ # fingerprint derives from the manifest digest. Editing the spec moves all # three here in the same reviewed change. _UK_GATE_BATTERY_POLICY_SHA256 = ( - "5459347c9077b2acd5970a62d818e3ddd063d86d6c3dbce4d32dcacec3bdc414" + "8584de8e792e03ab18a7c7c12855393bb0a4358c529566aac1e3cef89e6c6fe6" ) _UK_GATE_BATTERY_GATES_MANIFEST_SHA256 = ( - "f68e10d8ff6654b7fc707da0508ea063b0f3c96b8a813b7098298727d43b9d6d" + "23b3adb8618d7edd905d8c32620914fdc26b2c8feb68e2e48805ed992e58686e" ) _UK_GATE_BATTERY_SPEC_FINGERPRINT = ( - "2a4a8b18d024f80782b375539c87d57006592d64470553a4da5a378791254faa" + "bfed261f7af3077d1c3fc0351adf6a9dc717e5ea1e3b50e980637608ac99223f" ) #: Spec entry id -> the legacy gate name whose observable detail checks #: apply unchanged (the battery re-keys the report by entry id; the gate @@ -591,10 +591,10 @@ _UK_CERTIFICATION_PART_DIGESTS: Mapping[str, Mapping[str, str]] = { "spine": { "gates_manifest_sha256": ( - "1605cf3fe1be4983cfb4ed806a34d69375cdc3e4e0c8883cc49481ac5870399a" + "af18105ef46720574c4a3a347656d689d2ccc239a2897fda8b694bef7144e8dd" ), "policy_sha256": ( - "3d14ad24eff7f5afd343164560db24095d27fafb36c619ddf725c32e00b35a69" + "7f9ba299c1e756f6ba217d35431334678815f66258540e1eea4fcdd143b183f1" ), }, "calibration_seam": { @@ -607,10 +607,10 @@ }, "release_cut": { "gates_manifest_sha256": ( - "18f07f40eead198a4436de7a43d4c0b13b9f2a9335bc84d4ab6b6a2ed75e597e" + "70c4c938767fdbae4c07d915a2e1cdc036cdb2f4a3e1af2cee7437cdf94ab7d7" ), "policy_sha256": ( - "b2a6446da0ffc53c0a894618795d918163368a266ef01a2a3609587085162115" + "e73dcbcaa07e34d23c8cf40b251f19fd00d9d2b85202647ca0ea13cb1021769a" ), }, } diff --git a/packages/microcosm-data/tests/test_contract.py b/packages/microcosm-data/tests/test_contract.py index 4f4910eb..79950ad7 100644 --- a/packages/microcosm-data/tests/test_contract.py +++ b/packages/microcosm-data/tests/test_contract.py @@ -135,13 +135,13 @@ def _trusted_terminal_gate_signing_key(monkeypatch) -> None: UK_GATE_BATTERY_PRODUCER = "microcosm.build.gate_battery" UK_GATE_BATTERY_SIGNING_KEY_ENV = "MICROCOSM_UK_TERMINAL_GATE_SIGNING_KEY" UK_GATE_BATTERY_POLICY_SHA256 = ( - "5459347c9077b2acd5970a62d818e3ddd063d86d6c3dbce4d32dcacec3bdc414" + "8584de8e792e03ab18a7c7c12855393bb0a4358c529566aac1e3cef89e6c6fe6" ) UK_GATE_BATTERY_GATES_MANIFEST_SHA256 = ( - "f68e10d8ff6654b7fc707da0508ea063b0f3c96b8a813b7098298727d43b9d6d" + "23b3adb8618d7edd905d8c32620914fdc26b2c8feb68e2e48805ed992e58686e" ) UK_GATE_BATTERY_SPEC_FINGERPRINT = ( - "2a4a8b18d024f80782b375539c87d57006592d64470553a4da5a378791254faa" + "bfed261f7af3077d1c3fc0351adf6a9dc717e5ea1e3b50e980637608ac99223f" ) UK_GATE_BATTERY_DEGENERATE_EVIDENCE_SHA256 = ( "d0d024043132fa07c378c393dbe2b24fe99bf19e876bcc39997d2c80cc9bd4f6" diff --git a/packages/microcosm-frame/pyproject.toml b/packages/microcosm-frame/pyproject.toml index 94d93027..6bc0c05b 100644 --- a/packages/microcosm-frame/pyproject.toml +++ b/packages/microcosm-frame/pyproject.toml @@ -12,7 +12,7 @@ dependencies = [ [project.optional-dependencies] us = ["microunit>=0.1.0"] policyengine = ["policyengine-us>=1.745.0,<2", "microunit>=0.1.0"] -uk = ["policyengine-uk>=2.88"] +uk = ["policyengine-uk>=2.92"] # The Axiom adapter's PyPI-resolvable dependencies. The engine itself # (axiom-rules-engine + its dense native extension) is not on PyPI yet and # installs from a checkout; see microcosm/frame/adapters/axiom.py. diff --git a/tools/calibrate_uk_national_dataset.py b/tools/calibrate_uk_national_dataset.py index 2f969784..c6e2118a 100644 --- a/tools/calibrate_uk_national_dataset.py +++ b/tools/calibrate_uk_national_dataset.py @@ -98,6 +98,7 @@ def main(argv: list[str] | None = None) -> int: input_sha256=args.input_sha256, ledger_artifact=artifact, register_registry=registry, + band_edge_registry=compilation.registry, calibration_year=calibration_year, exclusion_receipt=exclusion_receipt, doctrine=doctrine, diff --git a/uv.lock b/uv.lock index 0e1eae87..66193d3d 100644 --- a/uv.lock +++ b/uv.lock @@ -712,7 +712,7 @@ requires-dist = [ { name = "microunit", marker = "extra == 'us'", specifier = ">=0.1.0" }, { name = "numpy", specifier = ">=1.26" }, { name = "pandas", specifier = ">=2" }, - { name = "policyengine-uk", marker = "extra == 'uk'", specifier = ">=2.88" }, + { name = "policyengine-uk", marker = "extra == 'uk'", specifier = ">=2.92" }, { name = "policyengine-us", marker = "extra == 'us'", specifier = ">=1.745.0,<2" }, { name = "pyarrow", specifier = ">=15" }, { name = "pyyaml", specifier = ">=6" }, @@ -783,7 +783,7 @@ requires-dist = [ { name = "h5py", specifier = ">=3" }, { name = "huggingface-hub", specifier = ">=0.20" }, { name = "packaging", specifier = ">=24" }, - { name = "policyengine-uk", marker = "extra == 'uk'", specifier = ">=2.88" }, + { name = "policyengine-uk", marker = "extra == 'uk'", specifier = ">=2.92" }, { name = "policyengine-us", marker = "extra == 'us'", specifier = ">=1.745.0,<2" }, ] provides-extras = ["us", "uk"] @@ -857,7 +857,7 @@ requires-dist = [ { name = "microunit", marker = "extra == 'us'", specifier = ">=0.1.0" }, { name = "numpy", specifier = ">=2" }, { name = "pandas", specifier = ">=2.3" }, - { name = "policyengine-uk", marker = "extra == 'uk'", specifier = ">=2.88" }, + { name = "policyengine-uk", marker = "extra == 'uk'", specifier = ">=2.92" }, { name = "policyengine-us", marker = "extra == 'policyengine'", specifier = ">=1.745.0,<2" }, { name = "tables", marker = "extra == 'axiom'", specifier = ">=3" }, ] @@ -1391,7 +1391,7 @@ wheels = [ [[package]] name = "policyengine-uk" -version = "2.89.0" +version = "2.92.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "microdf-python" }, @@ -1399,9 +1399,9 @@ dependencies = [ { name = "pydantic" }, { name = "tables" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/e4/49/75c80cd9b1145d8cfd5efbdb534aca2c9d5c5722fef3fc8f4887831505ea/policyengine_uk-2.89.0.tar.gz", hash = "sha256:aea49956a350ec35c1e0def7cd28db324b055711ec5ab7f331395dca8d103341", size = 1216776, upload-time = "2026-06-11T13:06:04.826Z" } +sdist = { url = "https://files.pythonhosted.org/packages/9f/3a/316968411a749f9efa3ea88b8ec5eb11420a1ec6df8dca34acdb06abf38f/policyengine_uk-2.92.0.tar.gz", hash = "sha256:c6d4a0be0f2ca3a2fe2c75e278ab5899c5d9e53630dc58da605f62328a34a2ce", size = 1242165, upload-time = "2026-08-26T05:35:34.152Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/bb/5f/fd32a3f7314b02b3b6a8362d6b2869af5b51569301945c00ee1172e01914/policyengine_uk-2.89.0-py3-none-any.whl", hash = "sha256:72acae45a35a59cf7429bba69363601e624bfd643d50cd74b8d20d325497e9fd", size = 1999087, upload-time = "2026-06-11T13:06:02.819Z" }, + { url = "https://files.pythonhosted.org/packages/a4/22/37a8e7ed43ae8322d408e97753c8608959ac1dc1ae08bf2f881e4aa9196c/policyengine_uk-2.92.0-py3-none-any.whl", hash = "sha256:af0c29baa7ee0fd555d1f92cbfc506382f40cbc3387cc86c7c20a9e9d830de1d", size = 2041047, upload-time = "2026-08-26T05:35:32.363Z" }, ] [[package]]