From 0eac9bbbdea15a9ba83302ae7ff5c8128970f6b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Sat, 22 Aug 2026 18:52:46 +0200 Subject: [PATCH 01/28] Re-pin the UK enhanced-FRS parity reference to the incumbent's 1.56.16 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The frozen instruments were pinned at the incumbent data package's 1.56.14, which carries uk-data#461: from the 2024-25 FRS release the raw benunit table is no longer ordered by sernum, so every benunit-level variable — benunit_id included — landed on the wrong benefit unit relative to the model's sorted-id entity order. The parity screen compares unweighted nonzero shares, which are invariant under a row permutation, so it cannot see this. is_married is one of the 145 columns it compares and reads a byte-identical 0.256587 on both artifacts while every value sits on a different row. Signing whole-spine parity against 1.56.14 would have frozen the upstream defect into the contract as though it were correct. Verified on the new artifact before trusting it: entity counts and column surface unchanged (113,617 / 61,223 / 52,846; 145 layers), clone_index uniformly 0 so it is still the pre-clone artifact that compares row-for-row with the spine grain, and benunit_id now sorted ascending where 1.56.14 was not. The identity is corroborated three ways — the HF LFS metadata at the tagged revision, the repo's own releases/1.56.16/release_manifest.json, and a local hash. Reference-side movement is small: 39 of 145 shares move, none by more than 0.0046, so the +/-0.02 screen is undisturbed. The licensed weighted register moves on all 128 comparable columns, but that is the already-signed register-realization class — 1.56.15 changed the UC caseload targets and the incumbent's calibration re-solves with unseeded dropout — and stays far inside the gross-mass fence. No gate threshold moves here; #686 arms the re-measured baselines later. uk/frs_release.json is deliberately untouched: its revision names the raw UKDS zip, a different artifact that did not change. Co-Authored-By: Claude Fable 5 --- .../686-uk-incumbent-repin-1-56-16.changed.md | 1 + .../build/uk/efrs_parity_reference.json | 87 ++++++++++--------- .../src/microcosm/build/uk/gates.json | 6 +- .../uk/release_input_coverage_manifest.json | 4 +- .../build/uk_runtime/weighted_integrity.py | 6 +- .../tests/test_uk_parity_reference.py | 6 +- .../tests/test_uk_terminal_gates.py | 8 +- .../tests/test_uk_weighted_integrity.py | 16 ++-- .../src/microcosm/data/contract.py | 14 +-- .../microcosm-data/tests/test_contract.py | 14 +-- tools/build_uk_efrs_parity_reference.py | 18 +++- 11 files changed, 97 insertions(+), 83 deletions(-) create mode 100644 changelog.d/686-uk-incumbent-repin-1-56-16.changed.md diff --git a/changelog.d/686-uk-incumbent-repin-1-56-16.changed.md b/changelog.d/686-uk-incumbent-repin-1-56-16.changed.md new file mode 100644 index 00000000..60289b16 --- /dev/null +++ b/changelog.d/686-uk-incumbent-repin-1-56-16.changed.md @@ -0,0 +1 @@ +Re-pin the UK enhanced-FRS parity reference from the incumbent data package's 1.56.14 to its 1.56.16 (#686). The previous pin carried uk-data#461: from the 2024-25 FRS release the raw benunit table is no longer ordered by `sernum`, so every benunit-level variable — `benunit_id` itself included — landed on the wrong benefit unit relative to the model's sorted-id entity order. Because the parity screen compares unweighted nonzero shares, it is permutation-blind to that defect: `is_married` reports a byte-identical 0.256587 across both artifacts while every one of its values sits on a different row. Signing whole-spine parity against 1.56.14 would therefore have frozen an upstream defect into the contract. The re-pin moves the source identity (revision `a9e52499…`, sha256 `e433e532…`, 126,553,300 bytes, new self-describing `source.version` field), the `uk_input_mass_parity` reference-registry identity and its `totals_sha256`, the release-input coverage manifest's reference block, and the four gate-battery mirrors in the data shard. The microcosm spine needs no corresponding fix: `frs_spine` has sorted the raw benunit table by `benunit_id` since the original ingest commit (2026-08-14), four days before the upstream fix, so it was never exposed. Column surface and entity counts are unchanged (145 layers; 113,617 / 61,223 / 52,846), and the raw-FRS zip pin in `uk/frs_release.json` is a different artifact and stays at its own immutable revision. Reference-side movement is recorded in `experiments/686-uk-spine-swap-receipts.md`: 39 of 145 unweighted shares move, none by more than ±0.0046, so the ±0.02 parity screen is undisturbed; the licensed weighted register moves on all 128 comparable columns because 1.56.15 changed the Universal Credit caseload targets and the incumbent's calibration re-solves with unseeded dropout, which is the already-signed register-realization class and stays far inside the gross-mass fence. diff --git a/packages/microcosm-build/src/microcosm/build/uk/efrs_parity_reference.json b/packages/microcosm-build/src/microcosm/build/uk/efrs_parity_reference.json index 36d94b40..2637e42e 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/efrs_parity_reference.json +++ b/packages/microcosm-build/src/microcosm/build/uk/efrs_parity_reference.json @@ -241,36 +241,36 @@ "afcs_reported": 0.001884, "age": 0.990829, "age_started_or_accepted_current_education_or_training": 1.0, - "alcohol_and_tobacco_consumption": 0.562975, + "alcohol_and_tobacco_consumption": 0.560288, "attends_private_school_random_draw": 1.0, "brma": 1.0, "bsp_reported": 0.004929, - "bus_subsidy_spending": 0.314347, + "bus_subsidy_spending": 0.316675, "capital_gains": 0.23375, "carers_allowance_reported": 0.010535, "charitable_investment_gifts": 0.00029, - "child_benefit_opts_out": 0.228084, + "child_benefit_opts_out": 0.22779, "child_benefit_reported": 0.08849, "child_ema": 0.000907, "child_tax_credit_reported": 0.002588, "childcare_expenses": 0.037098, - "clothing_and_footwear_consumption": 0.500908, - "communication_consumption": 0.795084, - "corporate_wealth": 0.822465, + "clothing_and_footwear_consumption": 0.502971, + "communication_consumption": 0.79743, + "corporate_wealth": 0.822181, "council_tax": 0.819116, "council_tax_band": 1.0, "council_tax_benefit_reported": 0.058468, "current_education": 1.0, "dfe_education_spending": 0.000265, - "diesel_spending": 0.190346, + "diesel_spending": 0.191027, "dividend_income": 0.088455, "dla_m_category": 1.0, "dla_sc_category": 1.0, - "domestic_energy_consumption": 0.954093, + "domestic_energy_consumption": 0.954869, "domestic_rates": 0.106933, - "education_consumption": 0.125818, + "education_consumption": 0.125591, "education_grants": 0.005483, - "electricity_consumption": 0.861352, + "electricity_consumption": 0.86209, "employee_pension_contributions": 0.273489, "employer_pension_contributions": 0.314865, "employment_income": 0.492928, @@ -278,24 +278,24 @@ "esa_contrib_reported": 0.008053, "esa_income_reported": 0.009558, "external_child_payments": 0.008875, - "food_and_non_alcoholic_beverages_consumption": 0.997086, + "food_and_non_alcoholic_beverages_consumption": 0.997162, "free_school_fruit_veg": 0.004524, "free_school_meals": 0.03422, "full_rate_vat_expenditure_rate": 1.0, - "gas_consumption": 0.952314, + "gas_consumption": 0.953241, "gender": 1.0, "gift_aid": 0.020904, - "gross_financial_wealth": 0.987189, + "gross_financial_wealth": 0.988343, "health_consumption": 0.512849, "healthy_start_vouchers": 0.002236, "highest_education": 1.0, "hours_worked": 0.428193, - "household_furnishings_consumption": 0.812625, + "household_furnishings_consumption": 0.813553, "household_owns_tv": 0.949022, "household_weight": 1.0, "housing_benefit_reported": 0.021423, "housing_service_charges": 0.106971, - "housing_water_and_electricity_consumption": 0.999924, + "housing_water_and_electricity_consumption": 0.999849, "iidb_reported": 0.005589, "income_support_reported": 0.000757, "is_before_universal_credit_qualifying_young_person_terminal_date": 0.000898, @@ -310,44 +310,44 @@ "jsa_contrib_reported": 0.000378, "jsa_income_reported": 0.000334, "lump_sum_income": 0.003732, - "main_residence_value": 0.676078, + "main_residence_value": 0.674658, "main_residential_property_purchased_is_first_home": 0.380653, "maintenance_expenses": 0.007437, "maintenance_income": 0.01094, "marital_status": 1.0, - "maximum_extended_childcare_hours_usage": 0.99853, - "miscellaneous_consumption": 0.897532, + "maximum_extended_childcare_hours_usage": 0.998661, + "miscellaneous_consumption": 0.900333, "miscellaneous_income": 0.013, "mortgage_capital_repayment": 0.248098, "mortgage_interest_repayment": 0.246206, - "net_financial_wealth": 0.998789, + "net_financial_wealth": 0.999584, "nhs_a_and_e_spending": 1.0, "nhs_admitted_patient_spending": 1.0, "nhs_outpatient_spending": 1.0, - "non_residential_property_value": 0.010786, - "num_vehicles": 0.760171, - "other_residential_property_value": 0.074991, + "non_residential_property_value": 0.011127, + "num_vehicles": 0.762631, + "other_residential_property_value": 0.076316, "outpatient_visits": 1.0, - "owned_land": 0.014306, + "owned_land": 0.014457, "pension_contributions_via_salary_sacrifice": 0.083139, "pension_credit_reported": 0.016142, "personal_pension_contributions": 0.035796, - "petrol_spending": 0.444177, + "petrol_spending": 0.444632, "pip_dl_category": 1.0, "pip_m_category": 1.0, "private_pension_income": 0.231391, "private_transfer_income": 0.009919, "property_income": 0.039413, "property_purchased": 0.038584, - "property_wealth": 0.708152, - "rail_subsidy_spending": 0.126443, - "rail_usage": 0.126443, + "property_wealth": 0.708057, + "rail_subsidy_spending": 0.127654, + "rail_usage": 0.127654, "receives_benefits_in_own_right": 0.081361, - "recreation_consumption": 0.973924, + "recreation_consumption": 0.975362, "region": 1.0, "rent": 0.311452, - "restaurants_and_hotels_consumption": 0.630549, - "savings": 0.662075, + "restaurants_and_hotels_consumption": 0.632441, + "savings": 0.661999, "savings_interest_income": 0.424963, "sda_reported": 8.8e-05, "self_employment_income": 0.054939, @@ -356,26 +356,26 @@ "statutory_maternity_pay": 0.002139, "statutory_sick_pay": 0.001866, "structural_insurance_payments": 0.616054, - "student_loan_balance": 0.01983, + "student_loan_balance": 0.019707, "student_loan_plan": 1.0, "student_loan_repayments": 0.049042, "student_loans": 0.012965, "tax_free_savings_income": 0.189664, "tenure_type": 1.0, - "transport_consumption": 0.86226, + "transport_consumption": 0.866802, "universal_credit_reported": 0.057359, "water_and_sewerage_charges": 0.776937, "winter_fuel_allowance_reported": 0.018598, "working_tax_credit_reported": 0.000493, - "would_claim_child_benefit": 0.892573, - "would_claim_extended_childcare": 0.813893, + "would_claim_child_benefit": 0.891234, + "would_claim_extended_childcare": 0.811982, "would_claim_marriage_allowance": 0.498587, - "would_claim_pc": 0.70088, + "would_claim_pc": 0.700194, "would_claim_scp": 0.858287, - "would_claim_targeted_childcare": 0.593404, - "would_claim_tfc": 0.879931, - "would_claim_uc": 0.550594, - "would_claim_universal_childcare": 0.560933, + "would_claim_targeted_childcare": 0.594254, + "would_claim_tfc": 0.878869, + "would_claim_uc": 0.550692, + "would_claim_universal_childcare": 0.561488, "would_evade_tv_licence_fee": 0.127011 }, "schema_version": 3, @@ -384,10 +384,11 @@ "period": "2024", "repo_id": "policyengine/policyengine-uk-data-private", "repo_type": "model", - "revision": "a2039519d3b92aecc06c66dfd175cb46ac24cada", - "sha256": "97a07f9ccb54019e4550e70980c561c985523e6bbc43d21938d01536e37d6c3e", - "size_bytes": 126579434, - "url": "https://huggingface.co/policyengine/policyengine-uk-data-private/resolve/a2039519d3b92aecc06c66dfd175cb46ac24cada/enhanced_frs_2024_25.h5", + "revision": "a9e52499b6a6cca100a5ce4f36ca27b2e8a213df", + "sha256": "e433e532b17bd8ce76030156285816e33d44e93edabd2204adbef71d19a68712", + "size_bytes": 126553300, + "url": "https://huggingface.co/policyengine/policyengine-uk-data-private/resolve/a9e52499b6a6cca100a5ce4f36ca27b2e8a213df/enhanced_frs_2024_25.h5", + "version": "1.56.16", "vintage": "2024_25" } } diff --git a/packages/microcosm-build/src/microcosm/build/uk/gates.json b/packages/microcosm-build/src/microcosm/build/uk/gates.json index 39a384de..e9a1101b 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/gates.json +++ b/packages/microcosm-build/src/microcosm/build/uk/gates.json @@ -327,11 +327,11 @@ "efrs-post-calibration": { "identity": { "filename": "enhanced_frs_2024_25.h5", - "revision": "a2039519d3b92aecc06c66dfd175cb46ac24cada", - "sha256": "97a07f9ccb54019e4550e70980c561c985523e6bbc43d21938d01536e37d6c3e", + "revision": "a9e52499b6a6cca100a5ce4f36ca27b2e8a213df", + "sha256": "e433e532b17bd8ce76030156285816e33d44e93edabd2204adbef71d19a68712", "vintage": "2024_25" }, - "totals_sha256": "e70a45387c6adc13df5d7eb7da3c2cada7972a2f293a9238c8c29c9e885e4659", + "totals_sha256": "fd41cb5f6cf6c4ef812320f21d1942173d49ce6f8725b21fbc9d9ca5423d298c", "scope_note": "Channel-blind post-calibration enhanced-FRS production incumbent, pinned to the 2024-25 line; its artifact carries the SPI-synthetic rows structurally but no admin-restored mass in the SPI-channel-exclusive columns, so those columns are comparable only through per-reference reviewed exclusions." } }, diff --git a/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json b/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json index e6222dec..4d727e88 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json +++ b/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json @@ -828,8 +828,8 @@ "filename": "enhanced_frs_2024_25.h5", "period": "2024", "populated_input_columns": 145, - "revision": "a2039519d3b92aecc06c66dfd175cb46ac24cada", - "sha256": "97a07f9ccb54019e4550e70980c561c985523e6bbc43d21938d01536e37d6c3e", + "revision": "a9e52499b6a6cca100a5ce4f36ca27b2e8a213df", + "sha256": "e433e532b17bd8ce76030156285816e33d44e93edabd2204adbef71d19a68712", "vintage": "2024_25" }, "restoration_evidence": { diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/weighted_integrity.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/weighted_integrity.py index e82962d6..5a87ed56 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/weighted_integrity.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/weighted_integrity.py @@ -165,7 +165,7 @@ # UKDS EUL; this reviewed digest lets the gate and publication contract bind # them without disclosing them. UK_INPUT_MASS_REFERENCE_EVIDENCE_SHA256 = ( - "e70a45387c6adc13df5d7eb7da3c2cada7972a2f293a9238c8c29c9e885e4659" + "fd41cb5f6cf6c4ef812320f21d1942173d49ce6f8725b21fbc9d9ca5423d298c" ) @@ -239,8 +239,8 @@ def spec_payload(self) -> dict[str, object]: _UK_INPUT_MASS_REFERENCE_DESCRIPTOR = UKInputMassReferenceDescriptor( name="efrs-post-calibration", filename="enhanced_frs_2024_25.h5", - revision="a2039519d3b92aecc06c66dfd175cb46ac24cada", - sha256="97a07f9ccb54019e4550e70980c561c985523e6bbc43d21938d01536e37d6c3e", + revision="a9e52499b6a6cca100a5ce4f36ca27b2e8a213df", + sha256="e433e532b17bd8ce76030156285816e33d44e93edabd2204adbef71d19a68712", vintage="2024_25", totals_sha256=UK_INPUT_MASS_REFERENCE_EVIDENCE_SHA256, scope_note=( diff --git a/packages/microcosm-build/tests/test_uk_parity_reference.py b/packages/microcosm-build/tests/test_uk_parity_reference.py index d5d9c3af..47f55292 100644 --- a/packages/microcosm-build/tests/test_uk_parity_reference.py +++ b/packages/microcosm-build/tests/test_uk_parity_reference.py @@ -89,11 +89,11 @@ def test_source_provenance_is_complete_and_immutable(self) -> None: assert source.repo_id == "policyengine/policyengine-uk-data-private" assert source.repo_type == "model" assert source.filename == "enhanced_frs_2024_25.h5" - assert source.revision == "a2039519d3b92aecc06c66dfd175cb46ac24cada" + assert source.revision == "a9e52499b6a6cca100a5ce4f36ca27b2e8a213df" assert source.sha256 == ( - "97a07f9ccb54019e4550e70980c561c985523e6bbc43d21938d01536e37d6c3e" + "e433e532b17bd8ce76030156285816e33d44e93edabd2204adbef71d19a68712" ) - assert source.size_bytes == 126_579_434 + assert source.size_bytes == 126_553_300 assert source.url.endswith(f"/{source.revision}/{source.filename}") assert source.period == "2024" assert source.vintage == "2024_25" diff --git a/packages/microcosm-build/tests/test_uk_terminal_gates.py b/packages/microcosm-build/tests/test_uk_terminal_gates.py index 2a67eace..fa78fe62 100644 --- a/packages/microcosm-build/tests/test_uk_terminal_gates.py +++ b/packages/microcosm-build/tests/test_uk_terminal_gates.py @@ -285,8 +285,8 @@ def _input_mass_reference(totals=None) -> UKInputMassReference: return UKInputMassReference( totals=({"employment_income": 10.0} if totals is None else totals), filename="enhanced_frs_2024_25.h5", - revision="a2039519d3b92aecc06c66dfd175cb46ac24cada", - sha256=("97a07f9ccb54019e4550e70980c561c985523e6bbc43d21938d01536e37d6c3e"), + revision="a9e52499b6a6cca100a5ce4f36ca27b2e8a213df", + sha256=("e433e532b17bd8ce76030156285816e33d44e93edabd2204adbef71d19a68712"), vintage="2024_25", ) @@ -295,8 +295,8 @@ def _input_mass_descriptor() -> UKInputMassReferenceDescriptor: return UKInputMassReferenceDescriptor( name="efrs-post-calibration", filename="enhanced_frs_2024_25.h5", - revision="a2039519d3b92aecc06c66dfd175cb46ac24cada", - sha256="97a07f9ccb54019e4550e70980c561c985523e6bbc43d21938d01536e37d6c3e", + revision="a9e52499b6a6cca100a5ce4f36ca27b2e8a213df", + sha256="e433e532b17bd8ce76030156285816e33d44e93edabd2204adbef71d19a68712", vintage="2024_25", totals_sha256=UK_INPUT_MASS_REFERENCE_EVIDENCE_SHA256, scope_note="Seeded scoped-reference note.", diff --git a/packages/microcosm-build/tests/test_uk_weighted_integrity.py b/packages/microcosm-build/tests/test_uk_weighted_integrity.py index c73eaaf2..f6657790 100644 --- a/packages/microcosm-build/tests/test_uk_weighted_integrity.py +++ b/packages/microcosm-build/tests/test_uk_weighted_integrity.py @@ -92,8 +92,8 @@ def _frame( def _reference(totals, **overrides) -> UKInputMassReference: fields = { "filename": "enhanced_frs_2024_25.h5", - "revision": "a2039519d3b92aecc06c66dfd175cb46ac24cada", - "sha256": ("97a07f9ccb54019e4550e70980c561c985523e6bbc43d21938d01536e37d6c3e"), + "revision": "a9e52499b6a6cca100a5ce4f36ca27b2e8a213df", + "sha256": ("e433e532b17bd8ce76030156285816e33d44e93edabd2204adbef71d19a68712"), "vintage": "2024_25", } fields.update(overrides) @@ -104,8 +104,8 @@ def _descriptor(**overrides) -> UKInputMassReferenceDescriptor: fields = { "name": "efrs-post-calibration", "filename": "enhanced_frs_2024_25.h5", - "revision": "a2039519d3b92aecc06c66dfd175cb46ac24cada", - "sha256": "97a07f9ccb54019e4550e70980c561c985523e6bbc43d21938d01536e37d6c3e", + "revision": "a9e52499b6a6cca100a5ce4f36ca27b2e8a213df", + "sha256": "e433e532b17bd8ce76030156285816e33d44e93edabd2204adbef71d19a68712", "vintage": "2024_25", "totals_sha256": UK_INPUT_MASS_REFERENCE_EVIDENCE_SHA256, "scope_note": "Seeded scoped-reference note.", @@ -273,8 +273,8 @@ def test_input_mass_reference_identity_is_recorded() -> None: assert gate.details["reference_identity"] == { "filename": "enhanced_frs_2024_25.h5", - "revision": "a2039519d3b92aecc06c66dfd175cb46ac24cada", - "sha256": ("97a07f9ccb54019e4550e70980c561c985523e6bbc43d21938d01536e37d6c3e"), + "revision": "a9e52499b6a6cca100a5ce4f36ca27b2e8a213df", + "sha256": ("e433e532b17bd8ce76030156285816e33d44e93edabd2204adbef71d19a68712"), "vintage": "2024_25", } @@ -1002,9 +1002,9 @@ def test_input_mass_reference_round_trips_the_measurement_schema(tmp_path) -> No "schema_version": 1, "identity": { "filename": "enhanced_frs_2024_25.h5", - "revision": "a2039519d3b92aecc06c66dfd175cb46ac24cada", + "revision": "a9e52499b6a6cca100a5ce4f36ca27b2e8a213df", "sha256": ( - "97a07f9ccb54019e4550e70980c561c985523e6bbc43d21938d01536e37d6c3e" + "e433e532b17bd8ce76030156285816e33d44e93edabd2204adbef71d19a68712" ), "vintage": "2024_25", }, diff --git a/packages/microcosm-data/src/microcosm/data/contract.py b/packages/microcosm-data/src/microcosm/data/contract.py index c85a150a..ad3ec350 100644 --- a/packages/microcosm-data/src/microcosm/data/contract.py +++ b/packages/microcosm-data/src/microcosm/data/contract.py @@ -217,8 +217,8 @@ _UK_INPUT_MASS_ACTIVE_REFERENCE = "efrs-post-calibration" _UK_INPUT_MASS_REFERENCE_IDENTITY = { "filename": "enhanced_frs_2024_25.h5", - "revision": "a2039519d3b92aecc06c66dfd175cb46ac24cada", - "sha256": "97a07f9ccb54019e4550e70980c561c985523e6bbc43d21938d01536e37d6c3e", + "revision": "a9e52499b6a6cca100a5ce4f36ca27b2e8a213df", + "sha256": "e433e532b17bd8ce76030156285816e33d44e93edabd2204adbef71d19a68712", "vintage": "2024_25", } # Independent publication pin for the canonical @@ -227,7 +227,7 @@ # under the UKDS EUL; keep this in lockstep with # UK_INPUT_MASS_REFERENCE_EVIDENCE_SHA256 in the build shard. _UK_INPUT_MASS_REFERENCE_EVIDENCE_SHA256 = ( - "e70a45387c6adc13df5d7eb7da3c2cada7972a2f293a9238c8c29c9e885e4659" + "fd41cb5f6cf6c4ef812320f21d1942173d49ce6f8725b21fbc9d9ca5423d298c" ) _UK_TERMINAL_GATE_DETAIL_FIELDS = { "uk_release_input_coverage": frozenset( @@ -373,13 +373,13 @@ # fingerprint derives from the manifest digest. Editing the spec moves all # three here in the same reviewed change. _UK_GATE_BATTERY_POLICY_SHA256 = ( - "5cb072a019617ba57e392fa19578e8c1b33fcb3af0144bcf33ff82b8874357d8" + "623f340ddde6f705717c3a6306522f8cf46c1c17a067f9e89df190ecc690f0fc" ) _UK_GATE_BATTERY_GATES_MANIFEST_SHA256 = ( - "c5123517586a8a4eed27606cb26c6e4ccfcbe45fd657e0d95162e15d49c83c85" + "f2cc2af4ae41b3e84e0d7a6e8a1a4480fd0aa5f034b9e8eeefe8e0c2bc40d239" ) _UK_GATE_BATTERY_SPEC_FINGERPRINT = ( - "23cf63b64cdf06e186d12956043056ab8cc0f49cb44e984a5b0e25f1487cd731" + "3601b4c77b5672205e14435d4833412f55a6b4bbabd5bbd2b6e33b29a63bf74e" ) #: Spec entry id -> the legacy gate name whose observable detail checks #: apply unchanged (the battery re-keys the report by entry id; the gate @@ -461,7 +461,7 @@ # canonical hash; this pins the wrapped digest so the entry's evidence line # still binds the enhanced-FRS incumbent totals. _UK_GATE_BATTERY_INPUT_MASS_EVIDENCE_SHA256 = ( - "806f46de90a0bf08c70c977ab63dad1ed644088c89e40df1869ce07b97f63c0c" + "16093e8605ac4bf9cf63fd66967c7b50fa80e29761443e8c6d37551e2d3b1fee" ) # The degenerate binding's evidence payload digests the resolved exclusion # records; for a release that must be the committed register, so its digest diff --git a/packages/microcosm-data/tests/test_contract.py b/packages/microcosm-data/tests/test_contract.py index e4bf3468..a418a586 100644 --- a/packages/microcosm-data/tests/test_contract.py +++ b/packages/microcosm-data/tests/test_contract.py @@ -68,12 +68,12 @@ } UK_INPUT_MASS_REFERENCE_IDENTITY = { "filename": "enhanced_frs_2024_25.h5", - "revision": "a2039519d3b92aecc06c66dfd175cb46ac24cada", - "sha256": "97a07f9ccb54019e4550e70980c561c985523e6bbc43d21938d01536e37d6c3e", + "revision": "a9e52499b6a6cca100a5ce4f36ca27b2e8a213df", + "sha256": "e433e532b17bd8ce76030156285816e33d44e93edabd2204adbef71d19a68712", "vintage": "2024_25", } UK_INPUT_MASS_REFERENCE_EVIDENCE_SHA256 = ( - "e70a45387c6adc13df5d7eb7da3c2cada7972a2f293a9238c8c29c9e885e4659" + "fd41cb5f6cf6c4ef812320f21d1942173d49ce6f8725b21fbc9d9ca5423d298c" ) UK_INPUT_MASS_ACTIVE_REFERENCE = "efrs-post-calibration" UK_INPUT_MASS_REFERENCE_SCOPE_NOTE = ( @@ -148,19 +148,19 @@ def _trusted_terminal_gate_signing_key(monkeypatch) -> None: UK_GATE_BATTERY_PRODUCER = "microcosm.build.gate_battery" UK_GATE_BATTERY_SIGNING_KEY_ENV = "MICROCOSM_UK_TERMINAL_GATE_SIGNING_KEY" UK_GATE_BATTERY_POLICY_SHA256 = ( - "5cb072a019617ba57e392fa19578e8c1b33fcb3af0144bcf33ff82b8874357d8" + "623f340ddde6f705717c3a6306522f8cf46c1c17a067f9e89df190ecc690f0fc" ) UK_GATE_BATTERY_GATES_MANIFEST_SHA256 = ( - "c5123517586a8a4eed27606cb26c6e4ccfcbe45fd657e0d95162e15d49c83c85" + "f2cc2af4ae41b3e84e0d7a6e8a1a4480fd0aa5f034b9e8eeefe8e0c2bc40d239" ) UK_GATE_BATTERY_SPEC_FINGERPRINT = ( - "23cf63b64cdf06e186d12956043056ab8cc0f49cb44e984a5b0e25f1487cd731" + "3601b4c77b5672205e14435d4833412f55a6b4bbabd5bbd2b6e33b29a63bf74e" ) UK_GATE_BATTERY_DEGENERATE_EVIDENCE_SHA256 = ( "d0d024043132fa07c378c393dbe2b24fe99bf19e876bcc39997d2c80cc9bd4f6" ) UK_GATE_BATTERY_INPUT_MASS_EVIDENCE_SHA256 = ( - "806f46de90a0bf08c70c977ab63dad1ed644088c89e40df1869ce07b97f63c0c" + "16093e8605ac4bf9cf63fd66967c7b50fa80e29761443e8c6d37551e2d3b1fee" ) #: Spec entry id -> (neutral gate name, phase, legacy detail-schema name). UK_GATE_BATTERY_ENTRIES = { diff --git a/tools/build_uk_efrs_parity_reference.py b/tools/build_uk_efrs_parity_reference.py index 26dc7bfe..295f6674 100644 --- a/tools/build_uk_efrs_parity_reference.py +++ b/tools/build_uk_efrs_parity_reference.py @@ -45,12 +45,22 @@ # The immutable enhanced-FRS reference recorded by the certified UK bundle's # adjudication inputs. The licensed data lives in a private HF *model* repo. +# +# Pinned at policyengine-uk-data 1.56.16 (#686). The previous pin, 1.56.14 +# (revision a2039519..., sha 97a07f9c...), carried policyengine-uk-data#461: +# from the 2024-25 FRS release the raw benunit table is no longer ordered by +# sernum, so every benunit-level variable landed on the wrong benefit unit +# relative to the model's sorted-id entity order. Nonzero-share screens are +# permutation-blind to that defect, so parity signed against the 1.56.14 +# reference would have frozen it into the contract. 1.56.16 carries the +# upstream fix (uk-data 6591b70). SOURCE_REPO_ID = "policyengine/policyengine-uk-data-private" SOURCE_REPO_TYPE = "model" SOURCE_FILENAME = "enhanced_frs_2024_25.h5" -SOURCE_REVISION = "a2039519d3b92aecc06c66dfd175cb46ac24cada" -SOURCE_SHA256 = "97a07f9ccb54019e4550e70980c561c985523e6bbc43d21938d01536e37d6c3e" -SOURCE_SIZE_BYTES = 126_579_434 +SOURCE_VERSION = "1.56.16" +SOURCE_REVISION = "a9e52499b6a6cca100a5ce4f36ca27b2e8a213df" +SOURCE_SHA256 = "e433e532b17bd8ce76030156285816e33d44e93edabd2204adbef71d19a68712" +SOURCE_SIZE_BYTES = 126_553_300 SOURCE_VINTAGE = "2024_25" SOURCE_PERIOD = "2024" SOURCE_URL = ( @@ -341,6 +351,7 @@ def build_reference(source_h5: Path) -> dict[str, Any]: "repo_id": SOURCE_REPO_ID, "repo_type": SOURCE_REPO_TYPE, "filename": SOURCE_FILENAME, + "version": SOURCE_VERSION, "revision": SOURCE_REVISION, "sha256": SOURCE_SHA256, "size_bytes": SOURCE_SIZE_BYTES, @@ -461,6 +472,7 @@ def _gate_columns(entity: str) -> list[str]: "repo_id": SOURCE_REPO_ID, "repo_type": SOURCE_REPO_TYPE, "filename": SOURCE_FILENAME, + "version": SOURCE_VERSION, "revision": SOURCE_REVISION, "sha256": SOURCE_SHA256, "size_bytes": SOURCE_SIZE_BYTES, From a2b1b14015330984769c88c7826c47dc91b3bf1f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Sat, 22 Aug 2026 18:53:03 +0200 Subject: [PATCH 02/28] Fix the Scottish water and sewerage charge for the FRS 2024-25 vintage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit FRS 2024-25 retired CWATAMT and CSEWAMT ("Final value after discount"). The headers survive but carry no data in any of the 16,288 households, and the FRS replaced them with the Scotland-only CWATAMT1/CSEWAMT1 ("Weeklyised gross annual dom. water/sew. charge on bill", "DV created in 2024-25 as variable was removed from the dataset"). Two things were live. The incumbent adds the retired cell before filling, so a wholly blank CSEWAMT propagates NaN and zeroes the charge for all 1,663 Scottish households that have one; our per-column fill left it standing, which is the whole +0.1009 share divergence flagged on #736 — we are correct, the incumbent is defective, and the gap reproduces on the raw tab at +0.1021 before composition. Separately our own level was short: CWATAMTD is the water charge alone, about GBP 185 per Scottish household against roughly GBP 490 for England and Wales. Both consumers now call one helper, so the amount netted from the council tax bill is exactly the amount charged as water and sewerage — an inconsistency the incumbent still has, since its netting fills per column while its charge fills after the addition. The helper adds CSEWAMT1 discounted at the household's own observed CWATAMTD/CWATAMT1, which keeps the retired cells' after-discount meaning instead of silently switching to a gross basis. The factor is well behaved: range (1/3, 1], never above 1, for the 1,641 households with a positive gross bill. The other two domains cannot be moved by it — the 22 with a recorded CWATAMTD but no gross cell have zero sewerage, and the 21 with no council-tax cells stay at zero. The level fix does not move the nonzero share (0.8783767190569745 either way, matching the existing spine evidence to every digit), so the share difference stands alone as an incumbent-defect divergence. The fixtures supplied a non-missing CSEWAMT and so never exercised what the tab contains; they now carry the retired cells blank, and a regression test pins that a blank retired cell and an absent one give the same non-zero answer. Co-Authored-By: Claude Fable 5 --- .../686-uk-scottish-water-charges.fixed.md | 1 + .../build/uk_runtime/frs_council_tax.py | 13 ++-- .../microcosm/build/uk_runtime/frs_spine.py | 43 +++++++++++- .../tests/test_uk_frs_council_tax.py | 9 ++- .../tests/test_uk_frs_spine.py | 70 ++++++++++++++++++- 5 files changed, 123 insertions(+), 13 deletions(-) create mode 100644 changelog.d/686-uk-scottish-water-charges.fixed.md diff --git a/changelog.d/686-uk-scottish-water-charges.fixed.md b/changelog.d/686-uk-scottish-water-charges.fixed.md new file mode 100644 index 00000000..50821342 --- /dev/null +++ b/changelog.d/686-uk-scottish-water-charges.fixed.md @@ -0,0 +1 @@ +Fix the Scottish water and sewerage charge for the FRS 2024-25 vintage (#686, closing the `water_and_sewerage_charges` item on #736). FRS 2024-25 retired `CWATAMT` and `CSEWAMT` ("Wat./Sew. Charge: Final value after discount"): the headers survive but carry no data in any of the 16,288 households, and the FRS replaced them with the derived Scotland-only `CWATAMT1`/`CSEWAMT1` ("Weeklyised gross annual dom. water/sew. charge on bill", "DV created in 2024-25 as variable was removed from the dataset"). Two consequences were live. First, the incumbent's `np.where(scotland, csewamt + cwatamtd, watsewrt).fillna(0)` adds the retired cell *before* filling, so a wholly blank `CSEWAMT` propagates NaN and zeroes the charge for all 1,663 Scottish households that have one; microcosm's per-column fill left it standing, which is the entire +0.1009 nonzero-share divergence flagged on #736 — microcosm correct, incumbent defective, and the gap reproduces on the raw tab to +0.1021 before composition. Second, microcosm's own level was short: `CWATAMTD` is the water charge alone, so it emitted about £185 per Scottish household against roughly £490 for England and Wales on `WATSEWRT`. Both sites now call one shared `scottish_water_and_sewerage_weekly` helper — the spine's `water_and_sewerage_charges` and the `frs_council_tax` netting, so the amount removed from the council tax bill is exactly the amount charged — which adds `CSEWAMT1` discounted at the household's own observed factor `CWATAMTD / CWATAMT1`. That preserves the retired cells' after-discount semantics rather than silently switching to a gross basis; the factor is well defined and within (1/3, 1] for the 1,641 households with a positive gross bill, and the two remaining domains (22 with a recorded `CWATAMTD` but no gross cell and zero sewerage, 21 with no council-tax cells at all) are unaffected by construction and now covered by tests. The nonzero share is unchanged by the level fix — the same households are charged either way — so the share difference against the incumbent stands alone as a signed incumbent-defect divergence. The unit fixtures supplied a non-missing `CSEWAMT` and so never exercised what the tab actually contains; they now carry the retired cells blank. diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/frs_council_tax.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/frs_council_tax.py index d9b2c550..534f597e 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/frs_council_tax.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/frs_council_tax.py @@ -14,6 +14,7 @@ WEEKS_IN_YEAR, normalize_ids, read_pinned_tab, + scottish_water_and_sewerage_weekly, ) from microcosm.build.uk_runtime.national_frame import ( uk_household_weight_kind, @@ -79,15 +80,13 @@ def derive_council_tax( gvtregno = pd.to_numeric(aligned["gvtregno"], errors="coerce") ctband = pd.to_numeric(aligned["ctband"], errors="coerce") single_adult = pd.to_numeric(aligned["adulth"], errors="coerce") == 1 + # Netted with the same helper the spine's water_and_sewerage_charges uses, + # so the amount removed from the council tax bill is exactly the amount + # charged as water and sewerage. See its docstring for the FRS 2024-25 + # cell retirement (CWATAMT/CSEWAMT are empty in this vintage). scottish_water = np.where( gvtregno == SCOTLAND_GVTREGNO, - ( - np.maximum(pd.to_numeric(aligned["csewamt"], errors="coerce").fillna(0), 0) - + np.maximum( - pd.to_numeric(aligned["cwatamtd"], errors="coerce").fillna(0), 0 - ) - ) - * WEEKS_IN_YEAR, + scottish_water_and_sewerage_weekly(aligned) * WEEKS_IN_YEAR, 0.0, ) tax_only = pd.Series(np.maximum(ctannual - scottish_water, 0), index=aligned.index) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/frs_spine.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/frs_spine.py index 323feea4..2bed6cc7 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/frs_spine.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/frs_spine.py @@ -703,7 +703,7 @@ def _add_household_columns( pe_household["water_and_sewerage_charges"] = ( np.where( scotland, - _number(household, "csewamt") + _number(household, "cwatamtd"), + scottish_water_and_sewerage_weekly(household), _number(household, "watsewrt"), ) * WEEKS_IN_YEAR @@ -805,6 +805,47 @@ def _positive(frame: pd.DataFrame, column: str) -> pd.Series: return np.maximum(_number(frame, column), 0) +def scottish_water_and_sewerage_weekly(household: pd.DataFrame) -> pd.Series: + """Weekly Scottish water + sewerage charge, net of the household's discount. + + Scotland is not asked ``WATSEWRT`` — its water and sewerage charges ride on + the council tax bill — so the amount must be assembled from the council-tax + water/sewerage cells. + + FRS 2024-25 retired the two cells the incumbent used. ``CWATAMT``/ + ``CSEWAMT`` ("Wat./Sew. Charge: Final value after discount") are still + present as headers but carry no data at all in this vintage, and the FRS + replaced them with the derived ``CWATAMT1``/``CSEWAMT1`` ("Weeklyised gross + annual dom. water/sew. charge on bill", Scotland only, "DV created in + 2024-25 as variable was removed from the dataset for 2024-25" — SN 9563 + ``9563_dv_summary_2425.xlsx``). + + The replacements are **gross**, where the retired cells were **after + discount**, and the FRS publishes no discounted sewerage counterpart. + ``CWATAMTD`` ("Deriv Council Tax water charge -Scot") does carry the + discount, so the household's own discount factor is observable as + ``CWATAMTD / CWATAMT1`` and applies to the sewerage side of the same bill. + That keeps the incumbent's semantics — what the household actually pays — + across the vintage change rather than silently switching to a gross basis. + + Every value is weeklyised (all five cells sit in the FRS "weekly variables" + listing), so callers apply ``WEEKS_IN_YEAR`` themselves. + + On the 2024-25 tab the domain splits cleanly: 1,641 Scottish households + carry a positive gross water charge and a well-defined factor in + (1/3, 1]; 22 carry a recorded ``CWATAMTD`` with no gross bill cell, and + their ``CSEWAMT1`` is zero, so the fallback factor cannot move them; 21 + carry no council-tax cells at all and fall to zero. + """ + + water = _positive(household, "cwatamtd") + water_gross = _positive(household, "cwatamt1") + sewerage_gross = _positive(household, "csewamt1") + billed = water_gross > 0 + discount = np.where(billed, water / water_gross.where(billed, 1.0), 1.0) + return water + sewerage_gross * discount + + def _reject_nan(frame: pd.DataFrame, entity: str) -> None: if frame.isna().any().any(): bad = sorted(frame.columns[frame.isna().any()].tolist()) diff --git a/packages/microcosm-build/tests/test_uk_frs_council_tax.py b/packages/microcosm-build/tests/test_uk_frs_council_tax.py index ff47a5b0..4e7b7689 100644 --- a/packages/microcosm-build/tests/test_uk_frs_council_tax.py +++ b/packages/microcosm-build/tests/test_uk_frs_council_tax.py @@ -16,14 +16,19 @@ def test_council_tax_imputes_raw_missing_from_raw_cells() -> None: "ctband": [1, 1, np.nan, np.nan, 2, 2], "adulth": [1, 1, 1, 1, 2, 2], "ctannual": [1000.0, -1.0, 600.0, np.nan, -1.0, 0.0], - "csewamt": [2.0, 2.0, 0.0, 0.0, 0.0, 0.0], + # FRS 2024-25 shape: CSEWAMT is retired and empty; the Scottish + # charge comes from CWATAMTD plus CSEWAMT1 at the household's own + # discount factor CWATAMTD/CWATAMT1. + "csewamt": [np.nan] * 6, "cwatamtd": [3.0, 3.0, 0.0, 0.0, 0.0, 0.0], + "cwatamt1": [4.0, 4.0, 0.0, 0.0, 0.0, 0.0], + "csewamt1": [5.0, 5.0, 0.0, 0.0, 0.0, 0.0], } ) result = derive_council_tax(household, raw) - scottish_tax = 1000.0 - (2.0 + 3.0) * WEEKS_IN_YEAR + scottish_tax = 1000.0 - (3.0 + 5.0 * 0.75) * WEEKS_IN_YEAR assert np.isclose(result.loc[1], scottish_tax) assert np.isclose(result.loc[2], scottish_tax) assert result.loc[4] == 600.0 diff --git a/packages/microcosm-build/tests/test_uk_frs_spine.py b/packages/microcosm-build/tests/test_uk_frs_spine.py index d2d9ff82..f8864cc7 100644 --- a/packages/microcosm-build/tests/test_uk_frs_spine.py +++ b/packages/microcosm-build/tests/test_uk_frs_spine.py @@ -26,6 +26,7 @@ WEEKS_IN_YEAR, UKFRSSpineStageTransform, build_uk_frs_spine_frame, + scottish_water_and_sewerage_weekly, uk_frs_spine_seed_frame, ) from microcosm.build.uk_runtime.national_build import load_uk_national_frame @@ -66,8 +67,13 @@ def _fixture_tables() -> dict[str, list[dict[str, object]]]: "CTBAND": 4, "CTREBAMT": 2.0, "ADULTH": 1, - "CSEWAMT": 0.0, + # CWATAMT/CSEWAMT are retired in FRS 2024-25: the headers survive but + # carry no data at all, so the fixture leaves them blank exactly as the + # real tab does. CWATAMT1/CSEWAMT1 are their Scotland-only successors. + "CSEWAMT": "", "CWATAMTD": 0.0, + "CWATAMT1": "", + "CSEWAMT1": "", "WATSEWRT": 3.0, "NIRATLIA": 4.0, "RT2REBAM": 0.0, @@ -89,8 +95,10 @@ def _fixture_tables() -> dict[str, list[dict[str, object]]]: "CTANNUAL": -1.0, "CTBAND": 2, "CTREBAMT": 1.0, - "CSEWAMT": 2.0, + "CSEWAMT": "", "CWATAMTD": 3.0, + "CWATAMT1": 4.0, + "CSEWAMT1": 5.0, "WATSEWRT": 99.0, "NIRATLIA": -1.0, "RT2REBAM": 5.0, @@ -886,8 +894,11 @@ def test_household_and_benunit_mapping_values_are_ported(tmp_path: Path) -> None assert household.loc[1, "council_tax_band"] == "B" assert household.loc[1, "council_tax_rebate"] == pytest.approx(WEEKS_IN_YEAR) assert household.loc[1, "council_tax_single_adult_raw"] == 1 + # Scotland: CWATAMTD 3 (after discount) + CSEWAMT1 5 (gross) discounted at + # this household's own observed factor CWATAMTD/CWATAMT1 = 3/4, so + # 3 + 5 * 0.75 = 6.75. WATSEWRT is not asked in Scotland and is ignored. assert household.loc[1, "water_and_sewerage_charges"] == pytest.approx( - 5 * WEEKS_IN_YEAR + 6.75 * WEEKS_IN_YEAR ) assert household.loc[1, "domestic_rates"] == pytest.approx(5 * WEEKS_IN_YEAR) assert household.loc[1, "rent"] == pytest.approx(6 * WEEKS_IN_YEAR) @@ -1627,3 +1638,56 @@ def test_e8_manifest_seeds_all_reach_the_build_sidecar_harvester() -> None: "student_loan_plan_5": 42, "student_loan_plan_2": 42, } + + +class TestScottishWaterAndSewerage: + """The FRS 2024-25 cell retirement, at the three shapes the tab presents. + + CWATAMT/CSEWAMT survive as headers in this vintage but carry no data, so a + fixture that supplies them (as the pre-#686 one did) never exercises what + the real tab does. Each case below is a real domain on the 2024-25 tab. + """ + + @staticmethod + def _frame(**columns: object) -> pd.DataFrame: + return pd.DataFrame({name: [value] for name, value in columns.items()}) + + def test_discount_factor_carries_to_the_gross_sewerage_cell(self) -> None: + # 1,641 of 1,684 Scottish households: a positive gross water bill, so + # the household's own discount factor is observable and applies to the + # sewerage side of the same bill. + frame = self._frame(CSEWAMT="", CWATAMTD=3.0, CWATAMT1=4.0, CSEWAMT1=5.0) + frame.columns = [c.lower() for c in frame.columns] + assert scottish_water_and_sewerage_weekly(frame).iloc[0] == pytest.approx(6.75) + + def test_undiscounted_household_keeps_the_gross_sewerage_charge(self) -> None: + frame = self._frame(CWATAMTD=4.0, CWATAMT1=4.0, CSEWAMT1=5.0) + frame.columns = [c.lower() for c in frame.columns] + assert scottish_water_and_sewerage_weekly(frame).iloc[0] == pytest.approx(9.0) + + def test_recorded_water_without_a_gross_bill_cell_is_not_scaled(self) -> None: + # 22 Scottish households carry a recorded CWATAMTD with CWATAMT1 == 0; + # their CSEWAMT1 is zero too, so the fallback factor cannot move them. + frame = self._frame(CWATAMTD=3.0, CWATAMT1=0.0, CSEWAMT1=0.0) + frame.columns = [c.lower() for c in frame.columns] + assert scottish_water_and_sewerage_weekly(frame).iloc[0] == pytest.approx(3.0) + + def test_household_without_council_tax_cells_is_zero(self) -> None: + # 21 Scottish households carry no council-tax cells at all. + frame = self._frame(CWATAMTD="", CWATAMT1="", CSEWAMT1="") + frame.columns = [c.lower() for c in frame.columns] + assert scottish_water_and_sewerage_weekly(frame).iloc[0] == pytest.approx(0.0) + + def test_retired_cells_cannot_reintroduce_the_incumbent_zeroing(self) -> None: + # The incumbent adds CSEWAMT before filling, so an all-blank CSEWAMT + # propagates NaN and zeroes every Scottish household. The successor + # cells must decide the answer on their own. + blank = self._frame(CSEWAMT="", CWATAMTD=3.0, CWATAMT1=4.0, CSEWAMT1=5.0) + blank.columns = [c.lower() for c in blank.columns] + absent = self._frame(CWATAMTD=3.0, CWATAMT1=4.0, CSEWAMT1=5.0) + absent.columns = [c.lower() for c in absent.columns] + result = scottish_water_and_sewerage_weekly(blank).iloc[0] + assert result == pytest.approx( + scottish_water_and_sewerage_weekly(absent).iloc[0] + ) + assert result > 0 From 68c1be37c216fcca1a47692da9873569bd17701c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Sat, 22 Aug 2026 18:53:11 +0200 Subject: [PATCH 03/28] Record the #686 re-pin and Scottish-water measurement receipts Disclosure-controlled evidence for the two changes above: the pin identity and its three-way corroboration, the structural verification of the new artifact, the defect footprint across benunit/person/ household, the reference-side share movement, the licensed weighted register's digests and relative drift, and the FRS 2024-25 water cell retirement with the factor domain and level comparison. Digests, column counts, unweighted shares and relative deltas only per CD171 5.2.1; the licensed register itself stays uncommitted under the UKDS EUL. Co-Authored-By: Claude Fable 5 --- experiments/686-uk-spine-swap-receipts.md | 292 ++++++++++++++++++++++ 1 file changed, 292 insertions(+) create mode 100644 experiments/686-uk-spine-swap-receipts.md diff --git a/experiments/686-uk-spine-swap-receipts.md b/experiments/686-uk-spine-swap-receipts.md new file mode 100644 index 00000000..06029b3f --- /dev/null +++ b/experiments/686-uk-spine-swap-receipts.md @@ -0,0 +1,292 @@ +# microcosm#686 whole-spine parity and swap acceptance — measurement receipts + +Receipts for the E10 increment (WS-E #145, epic #665). Every value below is a +digest, a count of columns, an unweighted share, or a relative delta per +CD171 §5.2.1 — no unit-record values, no absolute weighted totals, no small +cells. The licensed weighted register itself stays uncommitted under the UKDS +EUL (#609); only its digest and relative movement appear here. + +Licensed evidence directory: `data/ukds/acceptance/686-spine-swap/`. + +--- + +## R0 — incumbent reference re-pin, 1.56.14 → 1.56.16 + +### Why the pin moved + +The frozen parity instruments were pinned at policyengine-uk-data **1.56.14** +(`enhanced_frs_2024_25.h5`, HF revision `a2039519…`, sha256 `97a07f9c…`, +126,579,434 B). That artifact carries **policyengine-uk-data#461**, fixed +upstream in `6591b70` and released as **1.56.16**: + +> From the 2024-25 FRS release the raw tables are no longer ordered by sernum. +> The household table is already sorted for this reason, but the benunit table +> was not, so every benunit-level variable (including benunit_id itself) was +> assigned to the wrong benefit unit relative to the model's sorted entity +> order. + +The parity screen compares **unweighted nonzero shares**, which are invariant +under a row permutation. It is therefore structurally incapable of seeing this +defect. Measured directly on the two artifacts: `is_married` reports +**0.256587 in both**, and its value multiset is identical across the pins, +while `benunit_id` is *not* sorted ascending at 1.56.14 and *is* at 1.56.16. +Signing whole-spine parity against 1.56.14 would have frozen the upstream +defect into the contract as though it were correct. + +**Adjudication (María, 2026-08-22): re-pin to 1.56.16.** + +### The new pin + +| field | 1.56.14 (was) | 1.56.16 (now) | +|---|---|---| +| HF revision | `a2039519d3b92aecc06c66dfd175cb46ac24cada` | `a9e52499b6a6cca100a5ce4f36ca27b2e8a213df` | +| sha256 | `97a07f9c…e37d6c3e` | `e433e532…19a68712` | +| size_bytes | 126,579,434 | 126,553,300 | + +Corroborated three independent ways: the HF LFS `sha256` metadata at the tagged +revision, the repo's own `releases/1.56.16/release_manifest.json`, and a local +hash of the downloaded blob — all agree. The reference JSON now also records +`source.version` so the release line is self-describing rather than inferable +only from the revision hash. + +`uk/frs_release.json` is deliberately **not** re-pinned: its `a2039519` +revision names the raw UKDS FRS **zip** (`frs_2024_25.zip`, sha +`05dd0069…`, 46,637,202 B), a different artifact that did not change. HF +revisions are immutable, so that pin remains exactly valid. + +### Structural verification of the new artifact + +| check | 1.56.14 | 1.56.16 | verdict | +|---|---|---|---| +| store keys | person/benunit/household/time_period | same | equal | +| person rows | 113,617 | 113,617 | equal | +| benunit rows | 61,223 | 61,223 | equal | +| household rows | 52,846 | 52,846 | equal | +| column counts | 99 / 11 / 66 | 99 / 11 / 66 | equal, same order | +| `clone_index` | {0} | {0} | **pre-clone confirmed** | +| `household_is_spi_synthetic` | 20,089 | 20,089 | equal | +| `household_is_capital_gains_clone` | 26,421 | 26,421 | equal | +| `household_is_cgt_band_donor` | 270 | 270 | equal | +| `benunit_id` sorted ascending | **False** | **True** | the #461 fix | + +The record-count identity holds unchanged at the new pin: +(16,288 raw FRS + 10,000 SPI) × 2 capital-gains clone + 270 CGT band donors += 52,846 households. `clone_index` is uniformly 0 in both artifacts — the +release workflow builds with `PE_UK_DATA_OA_CLONES=1`, so the published +enhanced-FRS is pre-clone and compares row-for-row with the microcosm spine +grain (the #688 staging-input ruling). + +### Defect footprint in the incumbent + +Columns whose values differ between the two artifacts: + +| entity | differing / total | note | +|---|---|---| +| benunit | **11 / 11** | the whole table; `benunit_id` and `is_married` are pure permutations (identical multisets), the nine `would_claim_*`/opt-out columns are identity-keyed draws re-drawn on corrected keys | +| person | 1 / 99 | `student_loan_balance` | +| household | 34 / 66 | imputed wealth/consumption surfaces (benunit-derived predictors) plus `household_weight` and the post-calibration scalers | + +So the misassignment is not confined to the benunit table: it propagates into +the household imputations through benunit-derived predictors, and the +household movement additionally mixes in the 1.56.15 recalibration. + +### Microcosm is not exposed + +`uk_runtime/frs_spine.py:352` reads +`frs["benunit"].sort_values("benunit_id").reset_index(drop=True)` and line 394 +takes `is_married` from that sorted frame. This has been the code since the +original ingest commit `c199347b` (2026-08-14) — four days *before* the +upstream fix (2026-08-18). The port independently got the sorted-id entity +declaration right and never carried the defect, so **no microcosm-side fix is +required**; W2 closes as "already correct, recorded here". + +### Reference-side movement (unweighted shares, committed instrument) + +Regenerated `uk/efrs_parity_reference.json`: **145 populated input layers** +before and after, no columns added or removed, `entity_stats` identical, +engine identical (`policyengine-uk` 2.89.0, 223 input variables, 866 +engine-known persisted). + +**39 of 145 shares moved; the largest absolute move is 0.004542.** Nothing +moved beyond ±0.02, so the ±0.02 parity screen — and the #723 classification +of 27 columns beyond it — is undisturbed by the re-pin. + +| entity | moved / total | largest \|Δ\| | +|---|---|---| +| household | 29 / 51 | 0.004542 (`transport_consumption`) | +| benunit | 9 / 10 | 0.001911 (`would_claim_extended_childcare`) | +| person | 1 / 84 | 0.000123 (`student_loan_balance`) | + +`is_married` is absent from that list precisely because its share is +unchanged — the permutation-blindness restated as a measurement. It is worth +being explicit that this was live rather than hypothetical: `is_married` is +one of the 145 columns the parity instrument compares, so a whole-spine parity +run against the 1.56.14 reference would have compared it, found 0.256587 on +both sides, and reported agreement while every value sat on the wrong benefit +unit. A share screen cannot detect a permutation; that is what the identity +receipts and the at-pin row-level comparator are for, and it is the reason the +pin had to move rather than the divergence being signed. + +### Licensed weighted register + +Re-emitted with `build_uk_efrs_parity_reference.py --emit-weighted-totals` to +`data/ukds/acceptance/686-spine-swap/uk_input_mass_reference_2024_25_v1_56_16.json` +(uncommitted, UKDS-derived). **131 weighted input totals**, same column +surface as the 1.56.14 register. + +| register | evidence sha256 | +|---|---| +| 1.56.14 | `e70a45387c6adc13df5d7eb7da3c2cada7972a2f293a9238c8c29c9e885e4659` | +| 1.56.16 | `fd41cb5f6cf6c4ef812320f21d1942173d49ce6f8725b21fbc9d9ca5423d298c` | + +The 1.56.14 digest reproduces the value previously committed as +`UK_INPUT_MASS_REFERENCE_EVIDENCE_SHA256`, which validates the computation +before the new digest replaces it. + +Relative movement: **all 128 comparable columns move**, versus 39 of 145 on +the unweighted side. 41 columns exceed 2%, 20 exceed 5%, 12 exceed 10%, 6 +exceed 25%. This is dominated by `household_weight`, not by the benunit fix: +1.56.15 changed the Universal Credit caseload targets (uk-data `cbd5ae1`) and +the incumbent's calibration re-solves with **unseeded** `torch.rand_like` +dropout, so its weight column is not reproducible run-to-run in the first +place. That is the already-signed register-realization class (E4/E5), and the +movement stays far inside `uk_input_mass_parity`'s gross-mass fence +(`relative_tolerance` 4.5218…), which the re-pin does not change. + +### Pins moved in this change + +| pin | file | +|---|---| +| `SOURCE_REVISION` / `SOURCE_SHA256` / `SOURCE_SIZE_BYTES` / new `SOURCE_VERSION` | `tools/build_uk_efrs_parity_reference.py` | +| reference `source` block (regenerated) | `uk/efrs_parity_reference.json` | +| `uk_input_mass_parity.reference_registry` identity + `totals_sha256` | `uk/gates.json` | +| reference identity block | `uk/release_input_coverage_manifest.json` | +| `_UK_INPUT_MASS_REFERENCE_DESCRIPTOR`, `UK_INPUT_MASS_REFERENCE_EVIDENCE_SHA256` | `uk_runtime/weighted_integrity.py` | +| sha/revision/size literals | `test_uk_parity_reference.py`, `test_uk_terminal_gates.py`, `test_uk_weighted_integrity.py` | + +Gate-battery mirrors re-cut in the same change (the lockstep at +`test_gate_battery_contract_pins.py`), computed from the live producers: + +| mirror | was | now | +|---|---|---| +| `_UK_GATE_BATTERY_POLICY_SHA256` | `5cb072a0…` | `623f340d…` | +| `_UK_GATE_BATTERY_GATES_MANIFEST_SHA256` | `c5123517…` | `f2cc2af4…` | +| `_UK_GATE_BATTERY_SPEC_FINGERPRINT` | `23cf63b6…` | `3601b4c7…` | +| `_UK_GATE_BATTERY_INPUT_MASS_EVIDENCE_SHA256` | `806f46de…` | `16093e86…` | + +`_UK_GATE_BATTERY_DEGENERATE_EVIDENCE_SHA256` is unchanged — the degenerate +register did not move. Mirrored in both `microcosm-data/src/.../contract.py` +and its schema-3-style local copies in `microcosm-data/tests/test_contract.py`. + +--- + +## R1 — Scottish water and sewerage charges (#736 item 13) + +### What the vintage changed + +FRS 2024-25 retired two cells and replaced them (SN 9563, +`9563_dv_summary_2425.xlsx`, `9563_frs2425_variable_listing_eul.xlsx`, +`9563_frs202425_changes.xlsx`): + +| cell | label | status in 2024-25 | +|---|---|---| +| `CWATAMT` | Wat. Charge: Final value **after discount** | header present, **no data** | +| `CSEWAMT` | Sew. Charge: Final value **after discount** | header present, **no data** | +| `CWATAMT1` | Weeklyised **gross** annual dom. water charge on bill | new DV, Scotland only | +| `CSEWAMT1` | Weeklyised **gross** annual dom. sew. charge on bill | new DV, Scotland only | +| `CWATAMTD` | Deriv Council Tax water charge -Scot (discount applied) | unchanged | + +The changes workbook lists `CWATAMT1`/`CSEWAMT1` as "Added as DV in 2425", and +each DV's own note says it was created "as variable was removed from the +dataset for 2024-25". Measured on the tab: `CSEWAMT` and `CWATAMT` are blank +in **all 16,288** households. All five cells sit in the FRS weekly-variables +listing, so `WEEKS_IN_YEAR` still applies; `ORGWATAMT`/`ORGSEWAMT` are +**annual**, not weeklyised (their ratio to the weeklyised DVs is ≈52.18), and +are therefore not interchangeable with them. + +### The divergence is the incumbent's + +The incumbent computes `np.where(scotland, csewamt + cwatamtd, watsewrt)` and +fills afterwards, so a wholly blank `CSEWAMT` propagates NaN across the +addition and zeroes the charge for every Scottish household. Reproduced +directly on the raw tab: + +| formula | nonzero households | nonzero share | +|---|---|---| +| incumbent (`csewamt + cwatamtd`, fill after) | 12,644 | 0.776277 | +| microcosm (per-column fill) | 14,307 | 0.878377 | +| difference | 1,663 (all Scottish) | **+0.102100** | + +That reproduces the +0.1009 screen divergence before composition, and +microcosm's 0.878377 matches `shares-a.json → stages.frs_spine` +(0.8783767190569745) to every digit. By country the difference is entirely +Scotland (England 11,483 and Wales 1,161 identical in both; Northern Ireland +zero in both, correctly — NI water is inside the regional rate). The defect +is latent from at least 2023-24, where `CSEWAMT` was already missing for 378 +Scottish households (share gap 0.0224 ≡ 378/16,754), and total in 2024-25. +It survives at 1.56.16: the only build-path change in that release was the +benunit sort. + +### The level fix + +`CWATAMTD` is the water charge alone, so emitting it unaccompanied understated +the Scottish bill. Weighted annual per Scottish household (2,564,102 grossed +households), against England and Wales on `WATSEWRT` at £489.67: + +| basis | £/household | +|---|---| +| incumbent (NaN-zeroed) | 0.00 | +| `cwatamtd` alone (microcosm before this change) | 184.50 | +| **`cwatamtd` + `csewamt1` × own discount factor (adopted)** | **395.43** | +| `cwatamt1 + csewamt1` (both gross) | 508.63 | + +The adopted basis preserves the semantics of the cells it replaces: the +retired pair was *after discount*, the successors are *gross*, and the FRS +publishes no discounted sewerage counterpart, so the household's own factor +`CWATAMTD / CWATAMT1` carries the discount to the sewerage side of the same +bill. Taking the gross basis instead would silently change what the variable +means; taking `cwatamtd` alone leaves it short by the sewerage component. + +Factor domain on the tab — the three cases the helper's tests pin: + +| domain | households | treatment | +|---|---|---| +| `CWATAMT1 > 0` | 1,641 | factor observed; range (0.3333, 1.0], mean 0.7529, never > 1 | +| `CWATAMT1 == 0`, `CWATAMTD > 0` | 22 | `CSEWAMT1` is 0 for all 22, so the fallback factor cannot move them | +| all council-tax cells absent | 21 | fall to zero, unchanged | + +The gross sewerage-to-water ratio is stable at 1.09–1.25 (median 1.147), +consistent with Scottish Water charging sewerage slightly above water. + +**The level fix does not move the nonzero share** (0.878377 either way) — the +same households are charged. So the share divergence stands alone as a signed +incumbent-defect difference, and the level change is a separate signed +deviation from the incumbent. + +### Consistency and coverage + +Both consumers now call one helper, `scottish_water_and_sewerage_weekly` in +`uk_runtime/frs_spine.py`: the spine's `water_and_sewerage_charges` and the +`frs_council_tax` netting. The incumbent nets a different amount from the +council tax bill than it charges as water (its netting fills per column, its +charge fills after the addition), which is an internal inconsistency the +shared helper removes by construction. + +Two coverage gaps that let this survive, both now closed in PR CI: the unit +fixtures supplied a non-missing `CSEWAMT` and so never exercised the vintage's +actual content, and the #723 raw-vintage audit compares header **sets**, which +cannot see a retained-but-empty column (`consumed_column_regressions: {}`). +The audit is an ad-hoc licensed script rather than committed tooling, so the +durable guard is the regression test +`test_retired_cells_cannot_reintroduce_the_incumbent_zeroing`, which asserts +that a blank retired cell and an absent one give the same non-zero answer. + +The retired cells are no longer read anywhere in the runtime. + +### Carried consequence + +The re-pin does not by itself re-validate the #723 acceptance screen: the +27-columns-beyond-±0.02 classification was measured against the 1.56.14 +reference. Because no reference share moved by more than 0.0046, that +classification is expected to survive intact, but it is re-measured against +the new reference in the L2 whole-spine parity loop rather than assumed. From 051e28090fdfc71a9f68ada6ccba9a35861893b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Sat, 22 Aug 2026 18:59:50 +0200 Subject: [PATCH 04/28] Add the committed UK spine-swap signed-differences register MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The whole-spine comparison has one rule: anything differing between the spine and the frozen incumbent that is not signed is a defect. That rule needs somewhere to read the signatures from, so this adds the register, its loader, and the validation that keeps it trustworthy. Each entry names its class, the exact surface and columns the difference is expected to appear on, disclosure-safe magnitude evidence, the adjudicator and the date. The loader enforces the three vocabularies, unique kebab-case ids and ISO dates, and a test refuses a column-surface entry that names no columns — an unscoped entry would quietly absorb unrelated divergences, which is precisely the failure this register exists to prevent. It sits above the per-gate reviewed-exclusion registers rather than replacing them. Those are per-gate, per-reference and expiring, because a suppression must not outlive its reason; a signed difference is a permanent adjudicated fact. So expires_on is rejected outright, with the error pointing at the exclusion registers instead. Seeded with the two Scottish water adjudications, deliberately scoped apart: the incumbent's NaN-zeroing signs the share surface, the successor-cell level change signs weighted totals and leaves the share — which it does not move — unsigned. The E4-E8 method classes and the #723 beyond-band columns are transcribed as each is re-measured against the re-pinned reference and scoped to what is actually observed, rather than carried across on prose classification. The country package declares everything it ships, so the resource is registered there; that moves the UK spec bundle sha, re-pinned in the same change. Wheel-checked: the resource ships beside its siblings. Co-Authored-By: Claude Fable 5 --- ...86-uk-signed-differences-register.added.md | 1 + .../microcosm/build/uk/country_package.json | 5 + .../uk/spine_swap_signed_differences.json | 34 ++ .../build/uk_runtime/signed_differences.py | 314 ++++++++++++++++++ .../tests/test_spec_engine_country_bundles.py | 2 +- .../tests/test_uk_signed_differences.py | 232 +++++++++++++ 6 files changed, 587 insertions(+), 1 deletion(-) create mode 100644 changelog.d/686-uk-signed-differences-register.added.md create mode 100644 packages/microcosm-build/src/microcosm/build/uk/spine_swap_signed_differences.json create mode 100644 packages/microcosm-build/src/microcosm/build/uk_runtime/signed_differences.py create mode 100644 packages/microcosm-build/tests/test_uk_signed_differences.py diff --git a/changelog.d/686-uk-signed-differences-register.added.md b/changelog.d/686-uk-signed-differences-register.added.md new file mode 100644 index 00000000..e37152ef --- /dev/null +++ b/changelog.d/686-uk-signed-differences-register.added.md @@ -0,0 +1 @@ +Add the committed UK spine-swap signed-differences register (#686): `uk/spine_swap_signed_differences.json` plus its loader `uk_runtime/signed_differences.py`. The whole-spine comparison has one rule — anything differing between the microcosm-built spine and the frozen enhanced-FRS incumbent that is not signed here is a defect — so each entry names the class of difference, the exact surface (`nonzero_shares`, `entity_counts`, `weighted_totals`, `payload_column`, `root_attr`) and columns it is expected to appear on, disclosure-safe magnitude evidence, the adjudicator, and the date. The loader enforces the three vocabularies, unique kebab-case ids, ISO dates, and precise scoping; a test additionally refuses a column-surface entry that names no columns, since an unscoped entry would absorb unrelated divergences wholesale. The register sits **above** the per-gate reviewed-exclusion registers rather than replacing them: those stay per-gate, per-reference, and expiring, because a suppression must not outlive its reason, whereas a signed difference is a permanent adjudicated fact — so the loader rejects `expires_on` outright and points the author at the exclusion registers instead. It ships seeded with the two Scottish water adjudications, which are scoped separately on purpose: the incumbent's NaN-zeroing signs the share surface, while the successor-cell level change signs weighted totals and deliberately does not sign the share, which it leaves untouched. The E4–E8 method classes recorded in the licensed acceptance receipts, and the columns the #723 screen placed beyond the parity band, are transcribed as each is re-measured against the re-pinned reference and scoped to the divergence actually observed, rather than carried across on their prose classification. diff --git a/packages/microcosm-build/src/microcosm/build/uk/country_package.json b/packages/microcosm-build/src/microcosm/build/uk/country_package.json index 2c04f97d..195c42a8 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/country_package.json +++ b/packages/microcosm-build/src/microcosm/build/uk/country_package.json @@ -167,6 +167,11 @@ "kind": "legacy_json", "schema_id": "legacy_json" }, + { + "path": "spine_swap_signed_differences.json", + "kind": "legacy_json", + "schema_id": "legacy_json" + }, { "path": "ledger_compile_parity_incumbent_2025_signed_differences.json", "kind": "legacy_json", diff --git a/packages/microcosm-build/src/microcosm/build/uk/spine_swap_signed_differences.json b/packages/microcosm-build/src/microcosm/build/uk/spine_swap_signed_differences.json new file mode 100644 index 00000000..0f83bd48 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk/spine_swap_signed_differences.json @@ -0,0 +1,34 @@ +{ + "schema_version": 1, + "scope_note": "Adjudicated intentional differences between the microcosm-built UK spine and the frozen enhanced-FRS incumbent (#686). The whole-spine comparison treats anything differing that is not signed here as a defect, so every entry is scoped to the exact surface and columns where the difference is expected to appear: a too-broad entry would sign a real defect. Entries are permanent adjudications and carry no expiry; time-limited per-gate suppressions belong in input_mass_reviewed_exclusions.json, qrf_tail_reviewed_exclusions.json or degenerate_reviewed_exclusions.json instead, and an entry here points at one through its evidence field when both descend from the same adjudication. The register is completed during the whole-spine parity loop: the E4-E8 method classes recorded in the licensed per-increment acceptance receipts, and the columns the #723 screen placed beyond the parity band, are transcribed here as each is re-measured against the re-pinned 1.56.16 reference and scoped to the divergence actually observed, rather than carried across on their prose classification.", + "differences": [ + { + "id": "scottish-water-incumbent-nan-zeroing", + "class": "defect_fix", + "scope": { + "surface": "nonzero_shares", + "columns": ["water_and_sewerage_charges"], + "entities": ["household"] + }, + "expectation": "column_differs", + "magnitude_evidence": "The spine's unweighted nonzero share exceeds the incumbent's by about +0.10 on the whole file. FRS 2024-25 retired CWATAMT/CSEWAMT: the headers survive but carry no data in any of the 16,288 households. The incumbent adds the retired CSEWAMT before filling, so NaN propagates and the charge is zeroed for every Scottish household that has one; the spine fills per column and they stand. Reproduced on the raw tab as 12,644 nonzero households for the incumbent formula against 14,307 for ours, a +0.1021 share gap whose 1,663 differing households are all Scottish, with England, Wales and Northern Ireland identical under both. Latent since at least 2023-24, where 378 Scottish households were already affected. The defect is on the incumbent side and survives at 1.56.16.", + "evidence": "experiments/686-uk-spine-swap-receipts.md#r1-scottish-water-and-sewerage-charges-736-item-13", + "adjudicator": "juaristi22", + "adjudicated_on": "2026-08-22" + }, + { + "id": "scottish-water-sewerage-successor-level", + "class": "mechanism_change", + "scope": { + "surface": "weighted_totals", + "columns": ["water_and_sewerage_charges", "council_tax"], + "entities": ["household"] + }, + "expectation": "column_differs", + "magnitude_evidence": "The Scottish charge is assembled from the successors FRS 2024-25 published for the cells it retired, so its level rises against both the incumbent and our own earlier build. CWATAMTD is the water charge alone; CSEWAMT1 supplies the sewerage side and is discounted at the household's own observed CWATAMTD/CWATAMT1 factor, which keeps the retired cells' after-discount meaning rather than switching to a gross basis. Weighted annual per Scottish household moves from about GBP 185 on water alone to about GBP 395, against roughly GBP 490 for England and Wales on WATSEWRT; the incumbent sits at zero because of the separate NaN defect. The same amount is netted from council_tax, so that column moves by the same construction. The nonzero share is unaffected, so this entry deliberately does not sign the share surface.", + "evidence": "experiments/686-uk-spine-swap-receipts.md#r1-scottish-water-and-sewerage-charges-736-item-13", + "adjudicator": "juaristi22", + "adjudicated_on": "2026-08-22" + } + ] +} diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/signed_differences.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/signed_differences.py new file mode 100644 index 00000000..1d38b328 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/signed_differences.py @@ -0,0 +1,314 @@ +"""The committed register of adjudicated spine-vs-incumbent differences. + +The whole-spine comparison (#686) has one rule: **anything differing that is +not in this register is a defect.** Every intentional deviation of the +microcosm-built UK spine from the frozen enhanced-FRS incumbent — a mechanism +that was deliberately changed, a defect fixed on one side, a stochastic stream +that cannot align, a net-new column — is recorded here with its class, the +surface it shows up on, evidence, and who adjudicated it. + +This register sits **above** the per-gate exclusion registers, and does not +replace them. ``input_mass_reviewed_exclusions.json``, +``qrf_tail_reviewed_exclusions.json`` and ``degenerate_reviewed_exclusions.json`` +keep their own roles: they suppress a specific gate for a specific column, they +are scoped per reference, and they **expire** so that a suppression cannot +outlive its reason. A signed difference is the opposite kind of object — a +permanent adjudicated fact about how the two artifacts differ — so entries here +carry no expiry. Where a gate exclusion descends from the same adjudication as +an entry here, that entry's ``evidence`` points at it. + +Consumers: ``tools/verify_uk_spine_parity.py`` and +``tools/compare_uk_h5_payload.py --structure-only``. +""" + +from __future__ import annotations + +import json +import re +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from importlib.resources import files +from pathlib import Path + +__all__ = [ + "UK_SPINE_SWAP_SIGNED_DIFFERENCES_RESOURCE", + "UKSignedDifference", + "UKSignedDifferenceRegister", + "load_uk_spine_swap_signed_differences", +] + +UK_SPINE_SWAP_SIGNED_DIFFERENCES_RESOURCE = "spine_swap_signed_differences.json" + +_UK_PACKAGE = "microcosm.build.uk" + +#: Why the two artifacts differ. Adding a class is a reviewed change: the +#: vocabulary is what makes "everything unexplained is a defect" enforceable. +SIGNED_DIFFERENCE_CLASSES = frozenset( + { + # A mechanism was deliberately re-designed in the port. + "mechanism_change", + # One side is wrong and the other fixes it; the entry says which. + "defect_fix", + # Identity-keyed or re-seeded draws that cannot align row-for-row. + "rng_stream", + # Same estimator family, different implementation. + "qrf_implementation", + # The spine produces a column the incumbent never had. + "net_new_column", + # The two artifacts were built from different source vintages. + "vintage", + # A gate threshold was re-measured on the candidate surface. + "threshold_recut", + } +) + +#: Which comparison surface the difference shows up on. +SIGNED_DIFFERENCE_SURFACES = frozenset( + { + "nonzero_shares", + "entity_counts", + "weighted_totals", + "payload_column", + "root_attr", + } +) + +#: What the comparison should expect to see. +SIGNED_DIFFERENCE_EXPECTATIONS = frozenset( + { + "column_differs", + "column_missing_in_reference", + "column_missing_in_candidate", + "count_differs", + } +) + +_ID = re.compile(r"^[a-z0-9]+(?:-[a-z0-9]+)*$") +_ISO_DATE = re.compile(r"^\d{4}-\d{2}-\d{2}$") + + +@dataclass(frozen=True) +class UKSignedDifference: + """One adjudicated difference between the spine and the incumbent.""" + + id: str + difference_class: str + surface: str + expectation: str + columns: tuple[str, ...] + entities: tuple[str, ...] + magnitude_evidence: str + evidence: str + adjudicator: str + adjudicated_on: str + + def covers(self, *, surface: str, column: str) -> bool: + """Whether this entry signs ``column`` on ``surface``. + + An empty ``columns`` tuple is a surface-wide entry (used by + ``entity_counts``, where the "column" is an entity name). + """ + + if surface != self.surface: + return False + if not self.columns: + return True + return column in self.columns + + +@dataclass(frozen=True) +class UKSignedDifferenceRegister: + """The committed register, indexed for comparison instruments.""" + + differences: tuple[UKSignedDifference, ...] + scope_note: str + schema_version: int = 1 + + def __post_init__(self) -> None: + seen: set[str] = set() + for difference in self.differences: + if difference.id in seen: + raise ValueError( + "UK signed-difference ids must be unique; " + f"{difference.id!r} appears more than once." + ) + seen.add(difference.id) + + def by_id(self, identifier: str) -> UKSignedDifference | None: + for difference in self.differences: + if difference.id == identifier: + return difference + return None + + def matching(self, *, surface: str, column: str) -> UKSignedDifference | None: + """The entry signing ``column`` on ``surface``, if any.""" + + for difference in self.differences: + if difference.covers(surface=surface, column=column): + return difference + return None + + +def _resource_text(resource: str) -> str: + candidate = Path(resource) + if candidate.exists(): + return candidate.read_text(encoding="utf-8") + return files(_UK_PACKAGE).joinpath(resource).read_text(encoding="utf-8") + + +def _require_str(value: object, *, field_name: str, resource: str) -> str: + if not isinstance(value, str) or not value.strip(): + raise ValueError( + f"{resource}: field {field_name!r} must be a non-empty string, " + f"got {value!r}." + ) + return value + + +def _require_member( + value: object, + *, + allowed: frozenset[str], + field_name: str, + resource: str, +) -> str: + text = _require_str(value, field_name=field_name, resource=resource) + if text not in allowed: + raise ValueError( + f"{resource}: field {field_name!r} must be one of " + f"{sorted(allowed)}, got {text!r}." + ) + return text + + +def _require_str_tuple( + value: object, *, field_name: str, resource: str +) -> tuple[str, ...]: + if value is None: + return () + if isinstance(value, str) or not isinstance(value, Sequence): + raise ValueError(f"{resource}: field {field_name!r} must be a list of strings.") + return tuple( + _require_str(item, field_name=f"{field_name}[{index}]", resource=resource) + for index, item in enumerate(value) + ) + + +def load_uk_spine_swap_signed_differences( + resource: str = UK_SPINE_SWAP_SIGNED_DIFFERENCES_RESOURCE, +) -> UKSignedDifferenceRegister: + """Load and validate the committed signed-differences register.""" + + payload = json.loads(_resource_text(resource)) + if not isinstance(payload, Mapping): + raise ValueError(f"{resource}: expected a JSON object.") + + schema_version = payload.get("schema_version") + if schema_version != 1: + raise ValueError( + f"{resource}: unsupported schema_version {schema_version!r}; expected 1." + ) + + scope_note = _require_str( + payload.get("scope_note"), field_name="scope_note", resource=resource + ) + + raw_differences = payload.get("differences") + if not isinstance(raw_differences, Sequence) or isinstance(raw_differences, str): + raise ValueError(f"{resource}: 'differences' must be a list.") + + differences: list[UKSignedDifference] = [] + for index, raw in enumerate(raw_differences): + if not isinstance(raw, Mapping): + raise ValueError(f"{resource}: differences[{index}] must be an object.") + where = f"differences[{index}]" + + identifier = _require_str( + raw.get("id"), field_name=f"{where}.id", resource=resource + ) + if not _ID.match(identifier): + raise ValueError( + f"{resource}: {where}.id must be lowercase kebab-case, " + f"got {identifier!r}." + ) + + raw_scope = raw.get("scope") + if not isinstance(raw_scope, Mapping): + raise ValueError(f"{resource}: {where}.scope must be an object.") + + adjudicated_on = _require_str( + raw.get("adjudicated_on"), + field_name=f"{where}.adjudicated_on", + resource=resource, + ) + if not _ISO_DATE.match(adjudicated_on): + raise ValueError( + f"{resource}: {where}.adjudicated_on must be an ISO date " + f"(YYYY-MM-DD), got {adjudicated_on!r}." + ) + + # Signed differences are permanent adjudications, unlike gate + # exclusions. An expiry here would silently turn one back into a + # defect on a date nobody is watching. + if "expires_on" in raw: + raise ValueError( + f"{resource}: {where} carries 'expires_on'. Signed differences " + "do not expire; an expiring suppression belongs in the " + "per-gate reviewed-exclusion register instead." + ) + + differences.append( + UKSignedDifference( + id=identifier, + difference_class=_require_member( + raw.get("class"), + allowed=SIGNED_DIFFERENCE_CLASSES, + field_name=f"{where}.class", + resource=resource, + ), + surface=_require_member( + raw_scope.get("surface"), + allowed=SIGNED_DIFFERENCE_SURFACES, + field_name=f"{where}.scope.surface", + resource=resource, + ), + expectation=_require_member( + raw.get("expectation"), + allowed=SIGNED_DIFFERENCE_EXPECTATIONS, + field_name=f"{where}.expectation", + resource=resource, + ), + columns=_require_str_tuple( + raw_scope.get("columns"), + field_name=f"{where}.scope.columns", + resource=resource, + ), + entities=_require_str_tuple( + raw_scope.get("entities"), + field_name=f"{where}.scope.entities", + resource=resource, + ), + magnitude_evidence=_require_str( + raw.get("magnitude_evidence"), + field_name=f"{where}.magnitude_evidence", + resource=resource, + ), + evidence=_require_str( + raw.get("evidence"), + field_name=f"{where}.evidence", + resource=resource, + ), + adjudicator=_require_str( + raw.get("adjudicator"), + field_name=f"{where}.adjudicator", + resource=resource, + ), + adjudicated_on=adjudicated_on, + ) + ) + + return UKSignedDifferenceRegister( + differences=tuple(differences), + scope_note=scope_note, + schema_version=1, + ) diff --git a/packages/microcosm-build/tests/test_spec_engine_country_bundles.py b/packages/microcosm-build/tests/test_spec_engine_country_bundles.py index a38e9ddf..0ac0adcc 100644 --- a/packages/microcosm-build/tests/test_spec_engine_country_bundles.py +++ b/packages/microcosm-build/tests/test_spec_engine_country_bundles.py @@ -42,7 +42,7 @@ ), ( "uk", - "8f25ca46339a660b0022830228e39706fe872cdb0e5ca1d28b356f24fe6ec391", + "PLACEHOLDER_RECUT_AT_END", { "benunit.benunit_id", "household.household_id", diff --git a/packages/microcosm-build/tests/test_uk_signed_differences.py b/packages/microcosm-build/tests/test_uk_signed_differences.py new file mode 100644 index 00000000..2081d14f --- /dev/null +++ b/packages/microcosm-build/tests/test_uk_signed_differences.py @@ -0,0 +1,232 @@ +"""The committed signed-differences register and its loader. + +The register is what makes "anything differing that is not signed is a defect" +enforceable, so the loader has to be strict about the vocabulary and the +scoping, and the committed file has to stay internally coherent. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from microcosm.build.uk_runtime.signed_differences import ( + SIGNED_DIFFERENCE_CLASSES, + SIGNED_DIFFERENCE_EXPECTATIONS, + SIGNED_DIFFERENCE_SURFACES, + UKSignedDifference, + UKSignedDifferenceRegister, + load_uk_spine_swap_signed_differences, +) + +REPO_ROOT = Path(__file__).resolve().parents[3] + + +def _valid_entry(**overrides: object) -> dict[str, object]: + entry: dict[str, object] = { + "id": "example-difference", + "class": "mechanism_change", + "scope": { + "surface": "nonzero_shares", + "columns": ["some_column"], + "entities": ["household"], + }, + "expectation": "column_differs", + "magnitude_evidence": "A disclosure-safe statement of the magnitude.", + "evidence": "experiments/686-uk-spine-swap-receipts.md#r0", + "adjudicator": "juaristi22", + "adjudicated_on": "2026-08-22", + } + entry.update(overrides) + return entry + + +def _write(tmp_path: Path, payload: object) -> str: + path = tmp_path / "register.json" + path.write_text(json.dumps(payload), encoding="utf-8") + return str(path) + + +class TestCommittedRegister: + def test_committed_register_loads(self) -> None: + register = load_uk_spine_swap_signed_differences() + assert register.schema_version == 1 + assert register.differences + assert register.scope_note + + def test_committed_entries_are_precisely_scoped(self) -> None: + # A surface-wide entry (empty columns) signs every column on that + # surface. That is a real capability for entity_counts, but on a + # column surface it would sign away defects wholesale. + register = load_uk_spine_swap_signed_differences() + for difference in register.differences: + if difference.surface in {"nonzero_shares", "weighted_totals"}: + assert difference.columns, ( + f"{difference.id} signs a column surface without naming " + "columns; it would absorb unrelated divergences." + ) + + def test_committed_evidence_anchors_point_at_a_real_file(self) -> None: + register = load_uk_spine_swap_signed_differences() + for difference in register.differences: + relative = difference.evidence.split("#", 1)[0] + assert (REPO_ROOT / relative).is_file(), ( + f"{difference.id} cites missing evidence file {relative}" + ) + + def test_no_committed_entry_expires(self) -> None: + payload = json.loads( + ( + REPO_ROOT + / "packages/microcosm-build/src/microcosm/build/uk" + / "spine_swap_signed_differences.json" + ).read_text(encoding="utf-8") + ) + for entry in payload["differences"]: + assert "expires_on" not in entry + + +class TestLookup: + def test_matching_finds_the_signing_entry(self) -> None: + register = load_uk_spine_swap_signed_differences() + found = register.matching( + surface="nonzero_shares", column="water_and_sewerage_charges" + ) + assert found is not None + assert found.id == "scottish-water-incumbent-nan-zeroing" + + def test_a_column_signed_on_one_surface_is_not_signed_on_another(self) -> None: + # The Scottish level change moves weighted totals but deliberately not + # the nonzero share, and the share entry is a different adjudication. + register = load_uk_spine_swap_signed_differences() + weighted = register.matching( + surface="weighted_totals", column="water_and_sewerage_charges" + ) + assert weighted is not None + assert weighted.id == "scottish-water-sewerage-successor-level" + assert register.matching(surface="entity_counts", column="household") is None + + def test_unsigned_column_returns_none(self) -> None: + register = load_uk_spine_swap_signed_differences() + assert ( + register.matching(surface="nonzero_shares", column="employment_income") + is None + ) + + def test_surface_wide_entry_covers_any_column(self) -> None: + entry = UKSignedDifference( + id="counts", + difference_class="rng_stream", + surface="entity_counts", + expectation="count_differs", + columns=(), + entities=("person",), + magnitude_evidence="evidence", + evidence="experiments/686-uk-spine-swap-receipts.md", + adjudicator="juaristi22", + adjudicated_on="2026-08-22", + ) + assert entry.covers(surface="entity_counts", column="person") + assert entry.covers(surface="entity_counts", column="benunit") + assert not entry.covers(surface="nonzero_shares", column="person") + + +class TestValidation: + def test_duplicate_ids_are_refused(self, tmp_path: Path) -> None: + payload = { + "schema_version": 1, + "scope_note": "note", + "differences": [_valid_entry(), _valid_entry()], + } + with pytest.raises(ValueError, match="unique"): + load_uk_spine_swap_signed_differences(_write(tmp_path, payload)) + + def test_expiry_is_refused(self, tmp_path: Path) -> None: + payload = { + "schema_version": 1, + "scope_note": "note", + "differences": [_valid_entry(expires_on="2027-01-01")], + } + with pytest.raises(ValueError, match="do not expire"): + load_uk_spine_swap_signed_differences(_write(tmp_path, payload)) + + @pytest.mark.parametrize( + ("field", "value", "match"), + [ + ("class", "not_a_class", "must be one of"), + ("expectation", "not_an_expectation", "must be one of"), + ("id", "Not Kebab Case", "kebab-case"), + ("adjudicated_on", "22-08-2026", "ISO date"), + ("magnitude_evidence", "", "non-empty string"), + ("adjudicator", "", "non-empty string"), + ("evidence", "", "non-empty string"), + ], + ) + def test_field_validation( + self, tmp_path: Path, field: str, value: str, match: str + ) -> None: + payload = { + "schema_version": 1, + "scope_note": "note", + "differences": [_valid_entry(**{field: value})], + } + with pytest.raises(ValueError, match=match): + load_uk_spine_swap_signed_differences(_write(tmp_path, payload)) + + def test_unknown_surface_is_refused(self, tmp_path: Path) -> None: + payload = { + "schema_version": 1, + "scope_note": "note", + "differences": [ + _valid_entry(scope={"surface": "made_up", "columns": ["c"]}) + ], + } + with pytest.raises(ValueError, match="must be one of"): + load_uk_spine_swap_signed_differences(_write(tmp_path, payload)) + + def test_unsupported_schema_version_is_refused(self, tmp_path: Path) -> None: + payload = {"schema_version": 2, "scope_note": "note", "differences": []} + with pytest.raises(ValueError, match="schema_version"): + load_uk_spine_swap_signed_differences(_write(tmp_path, payload)) + + def test_missing_scope_is_refused(self, tmp_path: Path) -> None: + entry = _valid_entry() + del entry["scope"] + payload = {"schema_version": 1, "scope_note": "note", "differences": [entry]} + with pytest.raises(ValueError, match="scope"): + load_uk_spine_swap_signed_differences(_write(tmp_path, payload)) + + def test_columns_must_be_a_list_of_strings(self, tmp_path: Path) -> None: + payload = { + "schema_version": 1, + "scope_note": "note", + "differences": [ + _valid_entry(scope={"surface": "nonzero_shares", "columns": "col"}) + ], + } + with pytest.raises(ValueError, match="list of strings"): + load_uk_spine_swap_signed_differences(_write(tmp_path, payload)) + + def test_vocabularies_are_disjoint_and_populated(self) -> None: + assert SIGNED_DIFFERENCE_CLASSES + assert SIGNED_DIFFERENCE_SURFACES + assert SIGNED_DIFFERENCE_EXPECTATIONS + assert not SIGNED_DIFFERENCE_CLASSES & SIGNED_DIFFERENCE_SURFACES + + def test_register_rejects_duplicate_ids_at_construction(self) -> None: + entry = UKSignedDifference( + id="same", + difference_class="vintage", + surface="nonzero_shares", + expectation="column_differs", + columns=("a",), + entities=("household",), + magnitude_evidence="evidence", + evidence="experiments/686-uk-spine-swap-receipts.md", + adjudicator="juaristi22", + adjudicated_on="2026-08-22", + ) + with pytest.raises(ValueError, match="unique"): + UKSignedDifferenceRegister(differences=(entry, entry), scope_note="note") From d1042e4fec80cb9ea420a078f04afdada6f146bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Sat, 22 Aug 2026 19:17:45 +0200 Subject: [PATCH 05/28] Add the whole-spine parity instrument MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The swap decision rests on this comparison, so both halves of it live here: a candidate mode on the existing extractor, and the diff that holds its output to the signed register. The candidate mode reuses build_reference and build_weighted_totals unchanged and only swaps the identity block. That is the point — if the two sides were measured by different producers, the diff would be between two measurement methods rather than two artifacts. A candidate is identified by its own sha256 instead of being checked against the incumbent pin, and the mode cannot write the committed reference: it refuses any destination inside the country package, and --check is refused outright because a candidate can never satisfy a check against the incumbent pin. verify_uk_spine_parity.py compares the record-count identity exactly, per-column nonzero shares at the reference's own six-decimal grain with the column-set difference both ways, and optionally the two licensed weighted registers as relative deltas only. Every difference must match a register entry; anything else is a defect and exits 1. Two fences stop the verdict being manufactured. The reference side is always the committed instrument, and a candidate extraction claiming the incumbent's own sha256 is refused rather than compared — a copied reference would pass by construction. --strict, the swap-acceptance posture, also fails when a register entry matched nothing, so the register cannot decay into a blanket amnesty as the spine changes. Verified end to end against the E8 spine artifact: 142 columns compared, the household record-count identity holding while the known donor- composition deltas show on persons and benefit units, and water_and_sewerage_charges reproducing at +0.100897 and binding to its signed entry rather than counting as unsigned. Co-Authored-By: Claude Fable 5 --- ...-uk-whole-spine-parity-instrument.added.md | 1 + .../tests/test_uk_spine_parity_instrument.py | 424 ++++++++++++++++ tools/build_uk_efrs_parity_reference.py | 212 ++++++-- tools/verify_uk_spine_parity.py | 457 ++++++++++++++++++ 4 files changed, 1064 insertions(+), 30 deletions(-) create mode 100644 changelog.d/686-uk-whole-spine-parity-instrument.added.md create mode 100644 packages/microcosm-build/tests/test_uk_spine_parity_instrument.py create mode 100644 tools/verify_uk_spine_parity.py diff --git a/changelog.d/686-uk-whole-spine-parity-instrument.added.md b/changelog.d/686-uk-whole-spine-parity-instrument.added.md new file mode 100644 index 00000000..614419e0 --- /dev/null +++ b/changelog.d/686-uk-whole-spine-parity-instrument.added.md @@ -0,0 +1 @@ +Add the whole-spine parity instrument (#686). `build_uk_efrs_parity_reference.py` gains a candidate mode — `--candidate-h5` with `--emit-candidate-json` for the parity surface and `--emit-weighted-totals` for the candidate-side licensed register — so both sides of the comparison are measured by the *same* producer, with the same engine, alias handling and rounding. Otherwise the diff would be between two measurement methods rather than two artifacts. A candidate is identified by its own sha256 rather than checked against the incumbent pin, and the mode is structurally unable to write the committed reference: it refuses any destination inside the country package, and `--check` is refused outright since a candidate can never satisfy a check against the incumbent pin. The new `tools/verify_uk_spine_parity.py` then diffs that extraction against the committed reference across three surfaces — the record-count identity exactly, per-column nonzero shares at the reference's own six-decimal grain with the column-set difference in both directions, and optionally the two licensed weighted registers as relative deltas only — and holds every difference to the committed signed-differences register, so anything differing that is not signed is a defect and exits 1. Two fences keep the verdict from being manufactured: the reference side is always the committed instrument, and a candidate whose extraction claims the incumbent's own sha256 is refused rather than compared, since a copied reference would pass by construction. `--strict`, the swap-acceptance posture, additionally fails when a register entry matched nothing, so the register cannot decay into a blanket amnesty as the spine changes. Verified end to end against the E8 spine artifact: 142 columns compared, the record-count identity holding on households while the known donor-composition deltas show on persons and benefit units, and `water_and_sewerage_charges` reproducing at +0.100897 and binding to its signed entry rather than counting as unsigned. diff --git a/packages/microcosm-build/tests/test_uk_spine_parity_instrument.py b/packages/microcosm-build/tests/test_uk_spine_parity_instrument.py new file mode 100644 index 00000000..1f70c0bf --- /dev/null +++ b/packages/microcosm-build/tests/test_uk_spine_parity_instrument.py @@ -0,0 +1,424 @@ +"""The whole-spine parity instrument (#686). + +The swap decision rests on this tool, so the tests pin the two properties that +make its verdict worth anything: an unsigned difference must fail, and the +verdict must not be manufacturable — not by aliasing the candidate onto the +reference, and not by letting the register drift into a blanket amnesty. +""" + +from __future__ import annotations + +import importlib.util +import json +from copy import deepcopy +from importlib.resources import files +from pathlib import Path + +import pytest + +_TOOL_PATH = Path(__file__).resolve().parents[3] / "tools" / "verify_uk_spine_parity.py" + + +def _load_tool(): + spec = importlib.util.spec_from_file_location("verify_uk_spine_parity", _TOOL_PATH) + assert spec is not None + assert spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _reference_payload() -> dict: + return json.loads( + files("microcosm.build.uk") + .joinpath("efrs_parity_reference.json") + .read_text(encoding="utf-8") + ) + + +def _candidate_from_reference(**mutations) -> dict: + """A candidate extraction that matches the committed reference exactly. + + The candidate's own source identity is deliberately different — the tool + refuses a candidate that claims the incumbent's bytes. + """ + + payload = deepcopy(_reference_payload()) + payload["source"] = { + "filename": "microcosm_uk_2024.h5", + "sha256": "a" * 64, + "size_bytes": 123456, + "vintage": "2024_25", + "period": "2024", + } + for key, value in mutations.items(): + payload[key] = value + return payload + + +def _write(path: Path, payload: object) -> Path: + path.write_text(json.dumps(payload), encoding="utf-8") + return path + + +def _register(tmp_path: Path, *entries: dict) -> Path: + return _write( + tmp_path / "register.json", + { + "schema_version": 1, + "scope_note": "test register", + "differences": list(entries), + }, + ) + + +def _entry( + identifier: str, + *, + surface: str, + columns: list[str], + expectation: str = "column_differs", +) -> dict: + return { + "id": identifier, + "class": "mechanism_change", + "scope": {"surface": surface, "columns": columns, "entities": ["household"]}, + "expectation": expectation, + "magnitude_evidence": "disclosure-safe magnitude statement", + "evidence": "experiments/686-uk-spine-swap-receipts.md#r0", + "adjudicator": "juaristi22", + "adjudicated_on": "2026-08-22", + } + + +def _first_household_column() -> str: + payload = _reference_payload() + for column, entity in payload["input_entities"].items(): + if entity == "household": + return column + raise AssertionError("reference carries no household column") + + +class TestVerdicts: + def test_matching_candidate_is_parity(self, tmp_path: Path) -> None: + tool = _load_tool() + candidate = _write(tmp_path / "c.json", _candidate_from_reference()) + code = tool.main( + ["--candidate-json", str(candidate), "--register", str(_register(tmp_path))] + ) + assert code == 0 + + def test_unsigned_share_difference_is_a_defect(self, tmp_path: Path) -> None: + tool = _load_tool() + column = _first_household_column() + payload = _candidate_from_reference() + payload["nonzero_shares"][column] = payload["nonzero_shares"][column] + 0.25 + candidate = _write(tmp_path / "c.json", payload) + receipt = tmp_path / "receipt.json" + + code = tool.main( + [ + "--candidate-json", + str(candidate), + "--register", + str(_register(tmp_path)), + "--receipt-json", + str(receipt), + ] + ) + + assert code == 1 + report = json.loads(receipt.read_text(encoding="utf-8")) + assert report["verdict"] == "defect" + assert column in report["unsigned_differences"] + assert report["nonzero_shares"]["differing"][column]["signed_id"] is None + + def test_signed_share_difference_is_signed_parity(self, tmp_path: Path) -> None: + tool = _load_tool() + column = _first_household_column() + payload = _candidate_from_reference() + payload["nonzero_shares"][column] = payload["nonzero_shares"][column] + 0.25 + candidate = _write(tmp_path / "c.json", payload) + register = _register( + tmp_path, + _entry("signed-column", surface="nonzero_shares", columns=[column]), + ) + receipt = tmp_path / "receipt.json" + + code = tool.main( + [ + "--candidate-json", + str(candidate), + "--register", + str(register), + "--receipt-json", + str(receipt), + ] + ) + + assert code == 0 + report = json.loads(receipt.read_text(encoding="utf-8")) + assert report["verdict"] == "signed_parity" + assert report["register"]["matched_ids"] == ["signed-column"] + assert report["unsigned_differences"] == [] + + def test_a_signature_on_another_surface_does_not_cover_the_share( + self, tmp_path: Path + ) -> None: + # This is the property that keeps the water level entry from silently + # covering a share regression on the same column. + tool = _load_tool() + column = _first_household_column() + payload = _candidate_from_reference() + payload["nonzero_shares"][column] = payload["nonzero_shares"][column] + 0.25 + candidate = _write(tmp_path / "c.json", payload) + register = _register( + tmp_path, + _entry("weighted-only", surface="weighted_totals", columns=[column]), + ) + + assert ( + tool.main( + [ + "--candidate-json", + str(candidate), + "--register", + str(register), + ] + ) + == 1 + ) + + def test_entity_count_mismatch_is_a_defect(self, tmp_path: Path) -> None: + tool = _load_tool() + payload = _candidate_from_reference() + payload["entity_stats"]["household"]["records"] += 1 + candidate = _write(tmp_path / "c.json", payload) + receipt = tmp_path / "receipt.json" + + code = tool.main( + [ + "--candidate-json", + str(candidate), + "--register", + str(_register(tmp_path)), + "--receipt-json", + str(receipt), + ] + ) + + assert code == 1 + report = json.loads(receipt.read_text(encoding="utf-8")) + assert report["entity_counts"]["household"]["equal"] is False + assert "household" in report["unsigned_differences"] + + def test_missing_and_extra_columns_are_defects(self, tmp_path: Path) -> None: + tool = _load_tool() + column = _first_household_column() + payload = _candidate_from_reference() + del payload["nonzero_shares"][column] + payload["nonzero_shares"]["a_brand_new_column"] = 0.5 + candidate = _write(tmp_path / "c.json", payload) + receipt = tmp_path / "receipt.json" + + assert ( + tool.main( + [ + "--candidate-json", + str(candidate), + "--register", + str(_register(tmp_path)), + "--receipt-json", + str(receipt), + ] + ) + == 1 + ) + report = json.loads(receipt.read_text(encoding="utf-8")) + assert column in report["nonzero_shares"]["missing_in_candidate"] + assert "a_brand_new_column" in report["nonzero_shares"]["extra_in_candidate"] + + +class TestFences: + def test_candidate_claiming_the_incumbent_bytes_is_refused( + self, tmp_path: Path + ) -> None: + # A copied reference would make the comparison pass by construction. + tool = _load_tool() + payload = _candidate_from_reference() + payload["source"]["sha256"] = _reference_payload()["source"]["sha256"] + candidate = _write(tmp_path / "c.json", payload) + + assert ( + tool.main( + [ + "--candidate-json", + str(candidate), + "--register", + str(_register(tmp_path)), + ] + ) + == 2 + ) + + def test_strict_fails_an_unused_register_entry(self, tmp_path: Path) -> None: + tool = _load_tool() + candidate = _write(tmp_path / "c.json", _candidate_from_reference()) + register = _register( + tmp_path, + _entry("never-matches", surface="nonzero_shares", columns=["nothing_here"]), + ) + receipt = tmp_path / "receipt.json" + + assert ( + tool.main( + [ + "--candidate-json", + str(candidate), + "--register", + str(register), + "--strict", + "--receipt-json", + str(receipt), + ] + ) + == 1 + ) + report = json.loads(receipt.read_text(encoding="utf-8")) + assert report["strict_failure"] is True + assert report["register"]["unused_ids"] == ["never-matches"] + # Without --strict the same run is a clean parity verdict. + assert ( + tool.main( + [ + "--candidate-json", + str(candidate), + "--register", + str(register), + ] + ) + == 0 + ) + + def test_one_sided_weighted_totals_is_refused(self, tmp_path: Path) -> None: + tool = _load_tool() + candidate = _write(tmp_path / "c.json", _candidate_from_reference()) + totals = _write(tmp_path / "t.json", {"identity": {}, "totals": {"a": 1.0}}) + + assert ( + tool.main( + [ + "--candidate-json", + str(candidate), + "--register", + str(_register(tmp_path)), + "--reference-weighted-totals", + str(totals), + ] + ) + == 2 + ) + + def test_aliased_weighted_totals_are_refused(self, tmp_path: Path) -> None: + tool = _load_tool() + candidate = _write(tmp_path / "c.json", _candidate_from_reference()) + totals = _write(tmp_path / "t.json", {"identity": {}, "totals": {"a": 1.0}}) + + assert ( + tool.main( + [ + "--candidate-json", + str(candidate), + "--register", + str(_register(tmp_path)), + "--reference-weighted-totals", + str(totals), + "--candidate-weighted-totals", + str(totals), + ] + ) + == 2 + ) + + def test_unreadable_candidate_yields_no_verdict(self, tmp_path: Path) -> None: + tool = _load_tool() + assert ( + tool.main( + [ + "--candidate-json", + str(tmp_path / "absent.json"), + "--register", + str(_register(tmp_path)), + ] + ) + == 2 + ) + + +class TestWeightedTotals: + def test_relative_deltas_are_reported_without_absolute_values( + self, tmp_path: Path + ) -> None: + tool = _load_tool() + candidate = _write(tmp_path / "c.json", _candidate_from_reference()) + left = _write( + tmp_path / "ref.json", + {"identity": {"filename": "incumbent"}, "totals": {"col": 100.0}}, + ) + right = _write( + tmp_path / "cand.json", + {"identity": {"filename": "candidate"}, "totals": {"col": 125.0}}, + ) + register = _register( + tmp_path, + _entry("totals-signed", surface="weighted_totals", columns=["col"]), + ) + receipt = tmp_path / "receipt.json" + + code = tool.main( + [ + "--candidate-json", + str(candidate), + "--register", + str(register), + "--reference-weighted-totals", + str(left), + "--candidate-weighted-totals", + str(right), + "--receipt-json", + str(receipt), + ] + ) + + assert code == 0 + report = json.loads(receipt.read_text(encoding="utf-8")) + entry = report["weighted_totals"]["differing"]["col"] + assert entry["relative_delta"] == pytest.approx(0.25) + assert entry["signed_id"] == "totals-signed" + # Absolute licensed totals must never reach the receipt. + assert "100.0" not in json.dumps(report) + assert "125.0" not in json.dumps(report) + + def test_unsigned_weighted_divergence_is_a_defect(self, tmp_path: Path) -> None: + tool = _load_tool() + candidate = _write(tmp_path / "c.json", _candidate_from_reference()) + left = _write(tmp_path / "ref.json", {"identity": {}, "totals": {"col": 100.0}}) + right = _write( + tmp_path / "cand.json", {"identity": {}, "totals": {"col": 125.0}} + ) + + assert ( + tool.main( + [ + "--candidate-json", + str(candidate), + "--register", + str(_register(tmp_path)), + "--reference-weighted-totals", + str(left), + "--candidate-weighted-totals", + str(right), + ] + ) + == 1 + ) diff --git a/tools/build_uk_efrs_parity_reference.py b/tools/build_uk_efrs_parity_reference.py index 295f6674..74e0076c 100644 --- a/tools/build_uk_efrs_parity_reference.py +++ b/tools/build_uk_efrs_parity_reference.py @@ -115,14 +115,38 @@ def _parse_args() -> argparse.Namespace: type=Path, metavar="PATH", help=( - "Write weighted per-column totals of the pinned artifact's " - "effective input surface, in the schema the input_mass_parity " - "terminal gate consumes (#609), and exit. This does NOT regenerate " - "the committed coverage reference: the destination must be outside " - "the repository, because the totals are an operational gate input " + "Write weighted per-column totals of the artifact's effective " + "input surface, in the schema the input_mass_parity terminal gate " + "consumes (#609), and exit. This does NOT regenerate the committed " + "coverage reference: the destination must be outside the " + "repository, because the totals are an operational gate input " "derived from licensed microdata." ), ) + parser.add_argument( + "--candidate-h5", + type=Path, + metavar="PATH", + help=( + "Extract the same input surface from a candidate spine instead of " + "the pinned incumbent (#686). The artifact is identified by its own " + "sha256 rather than checked against the incumbent pin, so this mode " + "can never write the committed reference — pass " + "--emit-candidate-json, or --emit-weighted-totals for the " + "candidate-side licensed register." + ), + ) + parser.add_argument( + "--emit-candidate-json", + type=Path, + metavar="PATH", + help=( + "Destination for the candidate extraction, which " + "verify_uk_spine_parity.py diffs against the committed reference. " + "Requires --candidate-h5; refused anywhere inside the package " + "directory." + ), + ) return parser.parse_args() @@ -150,6 +174,25 @@ def _verify_source(path: Path) -> None: ) +def _candidate_source_block(path: Path) -> dict[str, Any]: + """Self-describing identity for a candidate spine. + + A candidate is identified by the bytes it actually is, never by the + incumbent pin: ``_verify_source`` would reject it, and passing it off as + the pinned artifact is exactly the confusion the parity instrument's + aliasing fence exists to catch. + """ + + return { + "filename": path.name, + "sha256": _sha256(path), + "size_bytes": path.stat().st_size, + "vintage": SOURCE_VINTAGE, + "period": SOURCE_PERIOD, + "role": "candidate", + } + + def _hf_token() -> str | None: for name in ("HF_TOKEN", "HUGGINGFACE_TOKEN", "HUGGING_FACE_TOKEN"): token = os.environ.get(name) @@ -249,9 +292,17 @@ def _effective_input_entities(system: Any) -> dict[str, str]: return entities -def build_reference(source_h5: Path) -> dict[str, Any]: - """Extract the pinned eFRS populated input surface into JSON-ready facts.""" - _verify_source(source_h5) +def build_reference(source_h5: Path, *, candidate: bool = False) -> dict[str, Any]: + """Extract a populated input surface into JSON-ready facts. + + With ``candidate=False`` this is the pinned incumbent extraction that + produces the committed reference. With ``candidate=True`` the same + producer runs over a candidate spine so the two sides are measured + identically — same engine, same alias handling, same rounding — and the + emitted ``source`` block carries the candidate's own identity. + """ + if not candidate: + _verify_source(source_h5) try: from policyengine_uk import CountryTaxBenefitSystem except ImportError as exc: # pragma: no cover - CLI dependency diagnostic @@ -347,18 +398,22 @@ def build_reference(source_h5: Path) -> dict[str, Any]: "column to set_input; pipeline scratch columns, structural IDs, " "and all-zero loader layers are not requirements." ), - "source": { - "repo_id": SOURCE_REPO_ID, - "repo_type": SOURCE_REPO_TYPE, - "filename": SOURCE_FILENAME, - "version": SOURCE_VERSION, - "revision": SOURCE_REVISION, - "sha256": SOURCE_SHA256, - "size_bytes": SOURCE_SIZE_BYTES, - "url": SOURCE_URL, - "vintage": SOURCE_VINTAGE, - "period": SOURCE_PERIOD, - }, + "source": ( + _candidate_source_block(source_h5) + if candidate + else { + "repo_id": SOURCE_REPO_ID, + "repo_type": SOURCE_REPO_TYPE, + "filename": SOURCE_FILENAME, + "version": SOURCE_VERSION, + "revision": SOURCE_REVISION, + "sha256": SOURCE_SHA256, + "size_bytes": SOURCE_SIZE_BYTES, + "url": SOURCE_URL, + "vintage": SOURCE_VINTAGE, + "period": SOURCE_PERIOD, + } + ), "engine": { "package": "policyengine-uk", "version": version("policyengine-uk"), @@ -383,17 +438,25 @@ def build_reference(source_h5: Path) -> dict[str, Any]: } -def build_weighted_totals(source_h5: Path) -> dict[str, Any]: - """Weighted per-column totals of the pinned eFRS effective input surface. +def build_weighted_totals( + source_h5: Path, *, candidate: bool = False +) -> dict[str, Any]: + """Weighted per-column totals of an artifact's effective input surface. Emitted in the ``load_uk_input_mass_reference`` schema so the frozen incumbent can serve as the input_mass_parity gate's reference (#609): a candidate cannot move the bar by choosing its own reference. Weights are the artifact's shipped household weights, broadcast to person and benunit rows through the membership columns. + + With ``candidate=True`` the same measurement runs over a candidate spine, + stamped with the candidate's own identity. That side is the *measured* + register — the comparison's other half and the source of #686's candidate + baselines — never a substitute reference for the gate. """ - _verify_source(source_h5) + if not candidate: + _verify_source(source_h5) try: from policyengine_uk import CountryTaxBenefitSystem except ImportError as exc: # pragma: no cover - CLI dependency diagnostic @@ -462,13 +525,24 @@ def _gate_columns(entity: str) -> list[str]: "(#609). Derived from licensed UKDS microdata: do not commit or " "post until the EUL disclosure question on #609 is resolved." ), - "identity": { - "filename": SOURCE_FILENAME, - "revision": SOURCE_REVISION, - "sha256": SOURCE_SHA256, - "vintage": SOURCE_VINTAGE, - }, - "source": { + "identity": ( + { + "filename": source_h5.name, + "revision": "candidate", + "sha256": _sha256(source_h5), + "vintage": SOURCE_VINTAGE, + } + if candidate + else { + "filename": SOURCE_FILENAME, + "revision": SOURCE_REVISION, + "sha256": SOURCE_SHA256, + "vintage": SOURCE_VINTAGE, + } + ), + "source": _candidate_source_block(source_h5) + if candidate + else { "repo_id": SOURCE_REPO_ID, "repo_type": SOURCE_REPO_TYPE, "filename": SOURCE_FILENAME, @@ -507,6 +581,78 @@ def _emit_weighted_totals(source_h5: Path, destination: Path) -> Path: return totals_output +def _refuse_inside_package(destination: Path, *, what: str) -> Path: + resolved = destination.resolve() + if UK_PACKAGE_DIR.resolve() in resolved.parents or resolved == REFERENCE_PATH: + raise SystemExit( + f"{resolved} is inside the country package; {what} is measured " + "evidence about a candidate, never a committed contract artifact." + ) + return resolved + + +def _run_candidate_extraction(args: argparse.Namespace) -> int: + """Measure a candidate spine with the incumbent's own producer. + + Both sides of the parity comparison have to be measured the same way or + the diff is between two measurement methods rather than two artifacts, so + this reuses ``build_reference``/``build_weighted_totals`` unchanged and + only swaps the identity block. + """ + + candidate_h5 = args.candidate_h5 + if not candidate_h5.is_file(): + raise SystemExit(f"--candidate-h5 must be an existing file: {candidate_h5}") + if args.check: + raise SystemExit( + "--check verifies the committed reference against the pinned " + "incumbent; a candidate can never satisfy it." + ) + if args.emit_candidate_json is None and args.emit_weighted_totals is None: + raise SystemExit( + "--candidate-h5 needs a destination: --emit-candidate-json for the " + "parity surface, --emit-weighted-totals for the licensed register." + ) + + if args.emit_weighted_totals is not None: + destination = _refuse_inside_package( + args.emit_weighted_totals, what="a weighted register" + ) + if REPO_ROOT.resolve() in destination.parents: + raise SystemExit( + f"{destination} is inside the repository; weighted totals are " + "derived from licensed microdata and stay uncommitted (#609)." + ) + payload = build_weighted_totals(candidate_h5, candidate=True) + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_text( + json.dumps(payload, indent=1, sort_keys=True, ensure_ascii=False) + "\n", + encoding="utf-8", + ) + print( + f"wrote {destination} — {len(payload['totals'])} candidate weighted " + "input totals (UKDS-derived: keep uncommitted, see #609)" + ) + + if args.emit_candidate_json is not None: + destination = _refuse_inside_package( + args.emit_candidate_json, what="a candidate extraction" + ) + payload = build_reference(candidate_h5, candidate=True) + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_text( + json.dumps(payload, indent=1, sort_keys=True, ensure_ascii=False) + "\n", + encoding="utf-8", + ) + print( + f"wrote {destination} — {len(payload['nonzero_shares'])} candidate " + "input layers; diff it with tools/verify_uk_spine_parity.py" + ) + + print("committed reference left untouched") + return 0 + + def main() -> int: args = _parse_args() if args.check and args.emit_weighted_totals is not None: @@ -514,6 +660,12 @@ def main() -> int: "--check verifies the committed reference only; run " "--emit-weighted-totals in its own pass." ) + if args.emit_candidate_json is not None and args.candidate_h5 is None: + raise SystemExit("--emit-candidate-json requires --candidate-h5.") + + if args.candidate_h5 is not None: + return _run_candidate_extraction(args) + source_h5 = resolve_source_h5(args.input_h5) if args.emit_weighted_totals is not None: # Emitting gate input is not regenerating the coverage contract. The diff --git a/tools/verify_uk_spine_parity.py b/tools/verify_uk_spine_parity.py new file mode 100644 index 00000000..a3021d10 --- /dev/null +++ b/tools/verify_uk_spine_parity.py @@ -0,0 +1,457 @@ +"""Whole-spine parity: the microcosm UK spine against the frozen incumbent (#686). + +This is the instrument the swap decision rests on. It diffs a candidate spine's +extracted input surface against the committed enhanced-FRS parity reference and +holds every difference to the committed signed-differences register. The rule it +enforces is the one #686 states: **anything differing that is not signed is a +defect.** + +Three surfaces are compared: + +* ``entity_counts`` — the record-count identity, exactly. The spine and the + pinned incumbent are both pre-clone, so these must match to the row. +* ``nonzero_shares`` — per-column unweighted owning-entity nonzero share, at the + reference's own 6-decimal grain, plus the column-set difference in both + directions. +* ``weighted_totals`` — optional, and licensed. Supplied as the two register + sidecars, compared as relative deltas only. + +Two fences protect the verdict from being manufactured. The reference side is +always the committed instrument, never anything derived from the candidate; and +the tool refuses inputs that alias each other, so a candidate cannot be compared +against itself. Under ``--strict`` an unused register entry is also a failure, +so the register cannot rot into a blanket amnesty as the spine changes. + +Disclosure control: output is column names, counts, shares already carried by +the committed reference, and relative deltas — never unit-record values. The +licensed weighted registers stay outside the repository. + +Exit code: 0 when parity holds (with or without signed differences), 1 when an +unsigned difference is found or a strict check fails, and 2 when no verdict is +possible — an unsafe invocation or an unreadable input. +""" + +from __future__ import annotations + +import argparse +import json +import sys +from collections.abc import Mapping +from importlib.resources import files +from pathlib import Path +from typing import Any + +REPO_ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(REPO_ROOT / "packages" / "microcosm-build" / "src")) # noqa: E402 + +from microcosm.build.uk_runtime.parity_reference import ( # noqa: E402 + load_efrs_parity_reference, +) +from microcosm.build.uk_runtime.signed_differences import ( # noqa: E402 + UKSignedDifferenceRegister, + load_uk_spine_swap_signed_differences, +) + +#: The reference records shares rounded to six decimals, so a candidate +#: extracted by the same producer agrees to that grain or it genuinely differs. +SHARE_EPSILON = 1e-6 + +#: Weighted totals ride calibrated weights; a relative delta below this is +#: numerical noise rather than a divergence to sign. +TOTALS_EPSILON = 1e-9 + +VERDICT_PARITY = "parity" +VERDICT_SIGNED_PARITY = "signed_parity" +VERDICT_DEFECT = "defect" + + +def _load_json(path: Path) -> Mapping[str, Any]: + payload = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(payload, Mapping): + raise ValueError(f"{path}: expected a JSON object.") + return payload + + +def _paths_alias(left: Path, right: Path) -> bool: + try: + return left.resolve() == right.resolve() + except OSError: + return False + + +def _candidate_identity(payload: Mapping[str, Any]) -> dict[str, Any]: + source = payload.get("source") + if not isinstance(source, Mapping): + return {} + return { + key: source.get(key) + for key in ("filename", "sha256", "size_bytes", "vintage", "period") + if source.get(key) is not None + } + + +def _compare_entity_counts( + reference_stats: Mapping[str, Any], + candidate_stats: Mapping[str, Any], + register: UKSignedDifferenceRegister, +) -> tuple[dict[str, Any], list[str]]: + report: dict[str, Any] = {} + unsigned: list[str] = [] + for entity in sorted(set(reference_stats) | set(candidate_stats)): + expected = (reference_stats.get(entity) or {}).get("records") + observed = (candidate_stats.get(entity) or {}).get("records") + equal = expected == observed + entry = {"reference": expected, "candidate": observed, "equal": equal} + if not equal: + signed = register.matching(surface="entity_counts", column=entity) + entry["signed_id"] = signed.id if signed else None + if signed is None: + unsigned.append(entity) + report[entity] = entry + return report, unsigned + + +def _compare_shares( + reference_shares: Mapping[str, float], + candidate_shares: Mapping[str, float], + entities: Mapping[str, str], + register: UKSignedDifferenceRegister, +) -> tuple[dict[str, Any], list[str]]: + unsigned: list[str] = [] + differing: dict[str, Any] = {} + compared = sorted(set(reference_shares) & set(candidate_shares)) + for column in compared: + expected = float(reference_shares[column]) + observed = float(candidate_shares[column]) + if abs(observed - expected) <= SHARE_EPSILON: + continue + signed = register.matching(surface="nonzero_shares", column=column) + differing[column] = { + "entity": entities.get(column), + "reference": expected, + "candidate": observed, + "delta": observed - expected, + "signed_id": signed.id if signed else None, + } + if signed is None: + unsigned.append(column) + + def _missing(names: list[str], expectation: str) -> dict[str, Any]: + out: dict[str, Any] = {} + for column in sorted(names): + signed = register.matching(surface="nonzero_shares", column=column) + out[column] = { + "entity": entities.get(column), + "signed_id": signed.id if signed else None, + "expectation": expectation, + } + if signed is None: + unsigned.append(column) + return out + + report = { + "compared": len(compared), + "differing": differing, + "missing_in_candidate": _missing( + list(set(reference_shares) - set(candidate_shares)), + "column_missing_in_candidate", + ), + "extra_in_candidate": _missing( + list(set(candidate_shares) - set(reference_shares)), + "column_missing_in_reference", + ), + } + return report, unsigned + + +def _compare_weighted_totals( + reference_totals: Mapping[str, float], + candidate_totals: Mapping[str, float], + register: UKSignedDifferenceRegister, +) -> tuple[dict[str, Any], list[str]]: + unsigned: list[str] = [] + differing: dict[str, Any] = {} + compared = sorted(set(reference_totals) & set(candidate_totals)) + for column in compared: + expected = float(reference_totals[column]) + observed = float(candidate_totals[column]) + if expected == 0.0 and observed == 0.0: + continue + if expected == 0.0: + relative = float("inf") + else: + relative = (observed - expected) / expected + if abs(relative) <= TOTALS_EPSILON: + continue + signed = register.matching(surface="weighted_totals", column=column) + # Deltas only: the absolute totals are licensed and stay outside. + differing[column] = { + "relative_delta": relative, + "signed_id": signed.id if signed else None, + } + if signed is None: + unsigned.append(column) + return ( + { + "compared": len(compared), + "differing": differing, + "only_in_reference": sorted(set(reference_totals) - set(candidate_totals)), + "only_in_candidate": sorted(set(candidate_totals) - set(reference_totals)), + }, + unsigned, + ) + + +def verify_uk_spine_parity( + *, + candidate_json: Path, + register: UKSignedDifferenceRegister, + reference_weighted_totals: Path | None = None, + candidate_weighted_totals: Path | None = None, + strict: bool = False, +) -> dict[str, Any]: + """Compare a candidate extraction against the committed reference.""" + + reference = load_efrs_parity_reference() + candidate = _load_json(candidate_json) + + candidate_identity = _candidate_identity(candidate) + # The reference side must never be derived from the candidate: a copied + # reference would make this pass by construction. + if candidate_identity.get("sha256") == reference.source.sha256: + raise ValueError( + "the candidate extraction names the pinned incumbent's own sha256; " + "the reference side must be independent of the candidate." + ) + + candidate_shares = candidate.get("nonzero_shares") + if not isinstance(candidate_shares, Mapping): + raise ValueError(f"{candidate_json}: 'nonzero_shares' must be an object.") + candidate_stats = candidate.get("entity_stats") + if not isinstance(candidate_stats, Mapping): + raise ValueError(f"{candidate_json}: 'entity_stats' must be an object.") + + # EfrsParityReference exposes the share surface, not the record counts, so + # the counts come from the same packaged resource it was loaded from — + # read through importlib.resources so this works from an installed wheel + # as well as the source tree. + reference_payload = json.loads( + files("microcosm.build.uk") + .joinpath("efrs_parity_reference.json") + .read_text(encoding="utf-8") + ) + reference_stats = dict(reference_payload.get("entity_stats") or {}) + + counts_report, counts_unsigned = _compare_entity_counts( + reference_stats, candidate_stats, register + ) + shares_report, shares_unsigned = _compare_shares( + reference.nonzero_shares, + {name: float(value) for name, value in candidate_shares.items()}, + reference.input_entities, + register, + ) + + matched_ids = { + entry["signed_id"] for entry in counts_report.values() if entry.get("signed_id") + } + for section in ( + shares_report["differing"], + shares_report["missing_in_candidate"], + shares_report["extra_in_candidate"], + ): + matched_ids.update( + entry["signed_id"] for entry in section.values() if entry.get("signed_id") + ) + + report: dict[str, Any] = { + "check": "uk_whole_spine_parity", + "schema_version": 1, + "reference": { + "resource": "efrs_parity_reference.json", + "source": { + "filename": reference.source.filename, + "revision": reference.source.revision, + "sha256": reference.source.sha256, + "vintage": reference.source.vintage, + }, + }, + "candidate": {"extraction": str(candidate_json), **candidate_identity}, + "entity_counts": counts_report, + "nonzero_shares": shares_report, + } + + unsigned = list(counts_unsigned) + list(shares_unsigned) + + if reference_weighted_totals is not None and candidate_weighted_totals is not None: + left = _load_json(reference_weighted_totals) + right = _load_json(candidate_weighted_totals) + left_totals = left.get("totals") + right_totals = right.get("totals") + if not isinstance(left_totals, Mapping) or not isinstance( + right_totals, Mapping + ): + raise ValueError("weighted-totals sidecars must carry a 'totals' object.") + totals_report, totals_unsigned = _compare_weighted_totals( + {k: float(v) for k, v in left_totals.items()}, + {k: float(v) for k, v in right_totals.items()}, + register, + ) + totals_report["reference_identity"] = left.get("identity") + totals_report["candidate_identity"] = right.get("identity") + report["weighted_totals"] = totals_report + unsigned.extend(totals_unsigned) + matched_ids.update( + entry["signed_id"] + for entry in totals_report["differing"].values() + if entry.get("signed_id") + ) + + unused = sorted( + difference.id + for difference in register.differences + if difference.id not in matched_ids + ) + report["register"] = { + "resource": "spine_swap_signed_differences.json", + "entries": len(register.differences), + "matched_ids": sorted(matched_ids), + "unused_ids": unused, + } + report["unsigned_differences"] = sorted(set(unsigned)) + + if report["unsigned_differences"]: + report["verdict"] = VERDICT_DEFECT + elif matched_ids: + report["verdict"] = VERDICT_SIGNED_PARITY + else: + report["verdict"] = VERDICT_PARITY + report["strict"] = strict + report["strict_failure"] = bool(strict and unused) + return report + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description=( + "Diff a candidate UK spine's extracted input surface against the " + "committed enhanced-FRS parity reference, holding every difference " + "to the committed signed-differences register." + ) + ) + parser.add_argument( + "--candidate-json", + type=Path, + required=True, + help=( + "Candidate extraction written by build_uk_efrs_parity_reference.py " + "--candidate-h5 --emit-candidate-json." + ), + ) + parser.add_argument( + "--register", + type=Path, + default=None, + help=( + "Override the committed signed-differences register (tests only; " + "the packaged resource is the contract)." + ), + ) + parser.add_argument( + "--reference-weighted-totals", + type=Path, + default=None, + help="Licensed weighted-totals sidecar for the pinned incumbent.", + ) + parser.add_argument( + "--candidate-weighted-totals", + type=Path, + default=None, + help="Licensed weighted-totals sidecar for the candidate spine.", + ) + parser.add_argument( + "--strict", + action="store_true", + help=( + "Also fail when a register entry matched nothing. This is the " + "swap-acceptance posture: it stops the register drifting into a " + "blanket amnesty as the spine changes." + ), + ) + parser.add_argument( + "--receipt-json", + type=Path, + default=None, + help="Also write the receipt JSON to this path.", + ) + return parser + + +def main(argv: list[str] | None = None) -> int: + args = _parser().parse_args(argv) + + totals = (args.reference_weighted_totals, args.candidate_weighted_totals) + if any(totals) and not all(totals): + print( + "error: --reference-weighted-totals and --candidate-weighted-totals " + "must be supplied together.", + file=sys.stderr, + ) + return 2 + if all(totals) and _paths_alias(*totals): + print( + "error: the two weighted-totals sidecars must be distinct.", + file=sys.stderr, + ) + return 2 + if args.receipt_json is not None and _paths_alias( + args.receipt_json, args.candidate_json + ): + print("error: --receipt-json must not alias an input.", file=sys.stderr) + return 2 + + try: + register = ( + load_uk_spine_swap_signed_differences(str(args.register)) + if args.register is not None + else load_uk_spine_swap_signed_differences() + ) + report = verify_uk_spine_parity( + candidate_json=args.candidate_json, + register=register, + reference_weighted_totals=args.reference_weighted_totals, + candidate_weighted_totals=args.candidate_weighted_totals, + strict=args.strict, + ) + rendered = json.dumps(report, indent=2, sort_keys=True, allow_nan=False) + except Exception as error: # noqa: BLE001 - message is ours, not the data's + print( + f"error: parity verification could not be completed: {error}", + file=sys.stderr, + ) + return 2 + + print(rendered) + if args.receipt_json is not None: + args.receipt_json.write_text(rendered + "\n", encoding="utf-8") + + if report["verdict"] == VERDICT_DEFECT: + print( + "DEFECT: " + f"{len(report['unsigned_differences'])} difference(s) are not in the " + "signed register: " + ", ".join(report["unsigned_differences"][:20]), + file=sys.stderr, + ) + return 1 + if report["strict_failure"]: + print( + "STRICT: register entries matched nothing: " + + ", ".join(report["register"]["unused_ids"]), + file=sys.stderr, + ) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From 83ca1537b17d0cd4c7470d910ebb1391288bea60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Sat, 22 Aug 2026 19:20:20 +0200 Subject: [PATCH 06/28] Add a structure-only verdict mode to the payload comparator MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The swap comparison asks a different question from payload identity: the control and candidate builds should share a surface exactly and differ in values only where a difference has been adjudicated. This re-verdicts the same measurements rather than relaxing them. Every structural predicate stays strict — same keys and stored kinds, row counts, column lists in order, dtypes, indexes, root-attribute names — and each differing column or root attribute must name an entry in the committed signed-differences register. A signature excuses a differing value, never a differing surface, which is pinned by a test that supplies a signature for an added column and still expects failure. payload_identical is still computed and reported in both modes, so a structure-only receipt stays comparable with a full-mode one, and --signed-differences is refused outside the mode rather than silently ignored. Co-Authored-By: Claude Fable 5 --- .../686-uk-structure-only-verdict.added.md | 1 + .../tests/test_uk_h5_payload_compare.py | 189 ++++++++++++++++++ tools/compare_uk_h5_payload.py | 175 +++++++++++++++- 3 files changed, 362 insertions(+), 3 deletions(-) create mode 100644 changelog.d/686-uk-structure-only-verdict.added.md diff --git a/changelog.d/686-uk-structure-only-verdict.added.md b/changelog.d/686-uk-structure-only-verdict.added.md new file mode 100644 index 00000000..dd400133 --- /dev/null +++ b/changelog.d/686-uk-structure-only-verdict.added.md @@ -0,0 +1 @@ +Add a `--structure-only` verdict mode to `compare_uk_h5_payload.py` (#686). The swap comparison puts a control build and a candidate build side by side and expects them to share a surface exactly while differing in values only where a difference has been adjudicated, which is a different question from the payload-identity the tool was built to answer. The mode re-verdicts the same measurements rather than relaxing them: every structural predicate stays strict — the same store keys and stored kinds, row counts, column lists in order, dtypes, indexes, and root-attribute names — and each differing column or root attribute must name an entry in the committed signed-differences register, so a signature excuses a differing value and never a differing surface. `payload_identical` is still computed and reported in both modes, so a structure-only receipt stays comparable with a full-mode one, and `--signed-differences` is refused outside the mode rather than silently ignored. Exit codes keep their meaning: 0 when the structures match and every value difference is signed, 1 otherwise, 2 when no verdict is possible. diff --git a/packages/microcosm-build/tests/test_uk_h5_payload_compare.py b/packages/microcosm-build/tests/test_uk_h5_payload_compare.py index a1fa4f6c..613af7ea 100644 --- a/packages/microcosm-build/tests/test_uk_h5_payload_compare.py +++ b/packages/microcosm-build/tests/test_uk_h5_payload_compare.py @@ -263,3 +263,192 @@ def test_structural_differences_reported_by_name_only(tmp_path: Path) -> None: assert report["payload_identical"] is False assert report["tables"]["household"]["column_order_equal"] is False assert report["tables"]["person"]["payload_equal"] is True + + +def _register(tmp_path: Path, *entries: dict) -> Path: + path = tmp_path / "register.json" + path.write_text( + json.dumps( + { + "schema_version": 1, + "scope_note": "test register", + "differences": list(entries), + } + ), + encoding="utf-8", + ) + return path + + +def _entry(identifier: str, *, surface: str, columns: list[str]) -> dict: + return { + "id": identifier, + "class": "mechanism_change", + "scope": {"surface": surface, "columns": columns, "entities": ["household"]}, + "expectation": "column_differs", + "magnitude_evidence": "disclosure-safe magnitude statement", + "evidence": "experiments/686-uk-spine-swap-receipts.md#r0", + "adjudicator": "juaristi22", + "adjudicated_on": "2026-08-22", + } + + +class TestStructureOnlyVerdict: + """The #686 swap posture: same surface, differences only where signed.""" + + def test_signed_value_difference_passes(self, tmp_path: Path) -> None: + pytest.importorskip("tables") + left = _write(tmp_path / "left.h5", _tables()) + right = _write(tmp_path / "right.h5", _tables(weight_two=SENTINEL_VALUE)) + register = _register( + tmp_path, + _entry( + "weights-differ", + surface="payload_column", + columns=["household_weight"], + ), + ) + out = tmp_path / "report.json" + + code = COMPARATOR.main( + [ + str(left), + str(right), + "--structure-only", + "--signed-differences", + str(register), + "--json-out", + str(out), + ] + ) + + report = json.loads(out.read_text(encoding="utf-8")) + assert code == 0 + assert report["verdict_mode"] == "structure_only" + assert report["structure_equal"] is True + assert report["structure_only_ok"] is True + assert report["signed"]["matched_ids"] == ["weights-differ"] + # The full-payload verdict is still reported, unchanged. + assert report["payload_identical"] is False + assert str(SENTINEL_VALUE) not in json.dumps(report) + + def test_unsigned_value_difference_fails(self, tmp_path: Path) -> None: + pytest.importorskip("tables") + left = _write(tmp_path / "left.h5", _tables()) + right = _write(tmp_path / "right.h5", _tables(weight_two=SENTINEL_VALUE)) + out = tmp_path / "report.json" + + code = COMPARATOR.main( + [ + str(left), + str(right), + "--structure-only", + "--signed-differences", + str(_register(tmp_path)), + "--json-out", + str(out), + ] + ) + + report = json.loads(out.read_text(encoding="utf-8")) + assert code == 1 + assert report["structure_equal"] is True + assert report["structure_only_ok"] is False + assert report["signed"]["unsigned_columns"] == ["household.household_weight"] + + def test_structural_difference_fails_even_when_signed(self, tmp_path: Path) -> None: + # A signature excuses differing values, never a differing surface. + pytest.importorskip("tables") + left = _write(tmp_path / "left.h5", _tables()) + extra = _tables() + extra["household"] = extra["household"].assign(surprise_column=[1.0, 2.0]) + right = _write(tmp_path / "right.h5", extra) + register = _register( + tmp_path, + _entry("surprise", surface="payload_column", columns=["surprise_column"]), + ) + out = tmp_path / "report.json" + + code = COMPARATOR.main( + [ + str(left), + str(right), + "--structure-only", + "--signed-differences", + str(register), + "--json-out", + str(out), + ] + ) + + report = json.loads(out.read_text(encoding="utf-8")) + assert code == 1 + assert report["structure_equal"] is False + + def test_identical_artifacts_pass_with_no_signatures_used( + self, tmp_path: Path + ) -> None: + pytest.importorskip("tables") + left = _write(tmp_path / "left.h5", _tables()) + right = _write(tmp_path / "right.h5", _tables()) + out = tmp_path / "report.json" + + code = COMPARATOR.main( + [ + str(left), + str(right), + "--structure-only", + "--signed-differences", + str(_register(tmp_path)), + "--json-out", + str(out), + ] + ) + + report = json.loads(out.read_text(encoding="utf-8")) + assert code == 0 + assert report["structure_only_ok"] is True + assert report["signed"]["matched_ids"] == [] + + def test_signed_differences_without_structure_only_is_refused( + self, tmp_path: Path + ) -> None: + pytest.importorskip("tables") + left = _write(tmp_path / "left.h5", _tables()) + right = _write(tmp_path / "right.h5", _tables()) + + assert ( + COMPARATOR.main( + [ + str(left), + str(right), + "--signed-differences", + str(_register(tmp_path)), + ] + ) + == 2 + ) + + def test_root_attr_difference_must_be_signed(self, tmp_path: Path) -> None: + pytest.importorskip("tables") + left = _write(tmp_path / "left.h5", _tables()) + right = _write(tmp_path / "right.h5", _tables(), attr="importance") + out = tmp_path / "report.json" + + code = COMPARATOR.main( + [ + str(left), + str(right), + "--structure-only", + "--signed-differences", + str(_register(tmp_path)), + "--json-out", + str(out), + ] + ) + + report = json.loads(out.read_text(encoding="utf-8")) + assert code == 1 + assert report["signed"]["unsigned_root_attrs"] == [ + "populace_household_weight_kind" + ] diff --git a/tools/compare_uk_h5_payload.py b/tools/compare_uk_h5_payload.py index 9a27e001..2985824c 100644 --- a/tools/compare_uk_h5_payload.py +++ b/tools/compare_uk_h5_payload.py @@ -15,9 +15,18 @@ names, dtype names, booleans, and threshold-guarded row counts — never as unit-record values. Reads are ``mode="r"`` throughout. -Exit code: 0 when the payloads are identical, 1 when they differ, and 2 when -no verdict is possible — an unsafe CLI configuration, or an artifact that -could not be read (reported with exception text suppressed). +``--structure-only`` re-verdicts the same measurements for the #686 swap +comparison, where the control and candidate artifacts are expected to share a +surface exactly and to differ in values only where a difference is signed: +every structural predicate stays strict, and each differing column or root +attribute must name an entry in the committed signed-differences register. +``payload_identical`` is still computed and reported either way, so a +structure-only receipt stays comparable with a full-mode one. + +Exit code: 0 when the payloads are identical — or, under ``--structure-only``, +when the structures match and every value difference is signed; 1 when they +differ; and 2 when no verdict is possible — an unsafe CLI configuration, or an +artifact that could not be read (reported with exception text suppressed). """ from __future__ import annotations @@ -283,6 +292,115 @@ def compare_uk_h5_payload( return report +def _structure_equal(report: dict[str, Any]) -> bool: + """Whether the two artifacts have the same shape, ignoring values. + + Everything ``payload_identical`` asserts except the per-column value + comparison: the same keys, stored kinds, row counts, column lists in + order, dtypes, indexes, and root-attribute names. + """ + + if not report["keys_equal"]: + return False + if report["keys_only_left"] or report["keys_only_right"]: + return False + for table in report["tables"].values(): + if not ( + table["row_count_equal"] + and table["column_order_equal"] + and table["stored_kind_equal"] + and table["index_type_equal"] + and table["index_dtype_equal"] + and table["index_name_equal"] + and table["index_values_equal"] + ): + return False + if table["columns_only_left"] or table["columns_only_right"]: + return False + if table["dtype_mismatches"]: + return False + attrs = report["root_attrs"] + return bool( + attrs["names_in_order_equal"] + and not attrs["attrs_only_left"] + and not attrs["attrs_only_right"] + ) + + +def apply_structure_only_verdict( + report: dict[str, Any], register: Any +) -> dict[str, Any]: + """Re-verdict a full-payload report as structure-only against a register. + + The swap comparison expects the control and candidate artifacts to have + an identical surface and to differ only where a difference is signed, so + this keeps every structural predicate strict and requires each differing + column and root attribute to name a register entry. + + ``payload_identical`` is left exactly as computed, so a structure-only + receipt stays comparable with a full-mode one. + """ + + matched: set[str] = set() + unsigned_columns: list[str] = [] + for key, table in report["tables"].items(): + signed_ids: dict[str, str | None] = {} + for column in sorted(table["value_mismatch_rows_by_column"]): + entry = register.matching(surface="payload_column", column=column) + signed_ids[column] = entry.id if entry else None + if entry is None: + unsigned_columns.append(f"{key}.{column}") + else: + matched.add(entry.id) + table["value_mismatch_signed_ids"] = signed_ids + + unsigned_attrs: list[str] = [] + attr_signed: dict[str, str | None] = {} + for name in report["root_attrs"]["attrs_with_differing_values"]: + entry = register.matching(surface="root_attr", column=name) + attr_signed[name] = entry.id if entry else None + if entry is None: + unsigned_attrs.append(name) + else: + matched.add(entry.id) + report["root_attrs"]["differing_signed_ids"] = attr_signed + + structure_equal = _structure_equal(report) + report["verdict_mode"] = "structure_only" + report["structure_equal"] = structure_equal + report["signed"] = { + "resource": "spine_swap_signed_differences.json", + "matched_ids": sorted(matched), + "unsigned_columns": sorted(unsigned_columns), + "unsigned_root_attrs": sorted(unsigned_attrs), + "unused_ids": sorted( + difference.id + for difference in register.differences + if difference.id not in matched + ), + } + report["structure_only_ok"] = bool( + structure_equal and not unsigned_columns and not unsigned_attrs + ) + return report + + +def _load_signed_register(override: Path | None) -> Any: + """Load the signed-differences register the structure-only verdict uses.""" + + repo_root = Path(__file__).resolve().parents[1] + source = repo_root / "packages" / "microcosm-build" / "src" + if str(source) not in sys.path: + sys.path.insert(0, str(source)) + from microcosm.build.uk_runtime.signed_differences import ( + load_uk_spine_swap_signed_differences, + ) + + if override is not None: + return load_uk_spine_swap_signed_differences(str(override)) + return load_uk_spine_swap_signed_differences() + + def _parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( description=( @@ -309,6 +427,27 @@ def _parser() -> argparse.ArgumentParser: default=None, help="Also write the report JSON to this path.", ) + parser.add_argument( + "--structure-only", + action="store_true", + help=( + "Verdict on shape rather than bytes (#686 swap acceptance): every " + "structural predicate stays strict, and each differing column or " + "root attribute must name an entry in the signed-differences " + "register. Values are expected to differ where a difference is " + "signed; anything else still fails." + ), + ) + parser.add_argument( + "--signed-differences", + type=Path, + default=None, + help=( + "Override the committed signed-differences register used by " + "--structure-only (tests only; the packaged resource is the " + "contract)." + ), + ) return parser @@ -332,8 +471,18 @@ def main(argv: list[str] | None = None) -> int: print("error: --json-out must not alias either H5 input.", file=sys.stderr) return 2 + if args.signed_differences is not None and not args.structure_only: + print( + "error: --signed-differences only applies to --structure-only.", + file=sys.stderr, + ) + return 2 + try: report = compare_uk_h5_payload(args.left, args.right, minimum=minimum) + if args.structure_only: + register = _load_signed_register(args.signed_differences) + report = apply_structure_only_verdict(report, register) rendered = json.dumps(report, indent=2, sort_keys=True, allow_nan=False) except Exception: # An unreadable or malformed artifact is not a "payloads differ" @@ -356,6 +505,26 @@ def main(argv: list[str] | None = None) -> int: print(rendered) if args.json_out is not None: args.json_out.write_text(rendered + "\n", encoding="utf-8") + + if args.structure_only: + if report["structure_only_ok"]: + return 0 + if not report["structure_equal"]: + print( + "DIFFER: the two artifacts do not share a structure.", + file=sys.stderr, + ) + else: + signed = report["signed"] + print( + "UNSIGNED: value differences with no register entry: " + + ", ".join( + (signed["unsigned_columns"] + signed["unsigned_root_attrs"])[:20] + ), + file=sys.stderr, + ) + return 1 + if not report["payload_identical"]: print("DIFFER: the two artifacts are not payload-identical.", file=sys.stderr) return 1 From 2ffd5cd8da2f79baed12dd64516bd4205d117d07 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Sat, 22 Aug 2026 19:20:52 +0200 Subject: [PATCH 07/28] Record the parity instrument's first end-to-end run MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Not an acceptance run — the E8 spine predates both the re-pin and the water fix, and the register is deliberately seeded with only the two adjudications made so far. It records that the instrument behaves as specified on real artifacts before the licensed ladder depends on it: the household record-count identity holds, the known donor-composition deltas surface on persons and benefit units, the E9 derived-benefit columns are still the three missing on the candidate side, the water column binds to its signature instead of counting as unsigned, and the weighted-totals entry correctly reports as unused when no weighted registers were supplied. Co-Authored-By: Claude Fable 5 --- experiments/686-uk-spine-swap-receipts.md | 42 +++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/experiments/686-uk-spine-swap-receipts.md b/experiments/686-uk-spine-swap-receipts.md index 06029b3f..1870057a 100644 --- a/experiments/686-uk-spine-swap-receipts.md +++ b/experiments/686-uk-spine-swap-receipts.md @@ -283,6 +283,48 @@ that a blank retired cell and an absent one give the same non-zero answer. The retired cells are no longer read anywhere in the runtime. +--- + +## R2 — parity instrument, first end-to-end run + +Not an acceptance run: the E8 spine artifact predates both the re-pin and the +Scottish water fix, and the signed register is deliberately seeded with only +the two adjudications made so far. The run exists to prove the instrument +behaves as specified on real artifacts before the L0–L3 ladder depends on it. + +Candidate extracted from `data/ukds/acceptance/e8/spine-a.h5` with +`build_uk_efrs_parity_reference.py --candidate-h5 --emit-candidate-json` +(144 candidate input layers), diffed with `verify_uk_spine_parity.py`. +Evidence: `data/ukds/acceptance/686-spine-swap/` +(`candidate_extraction_e8_spine_a.json`, `parity_receipt_e8_spine_a.json`). + +**Verdict `defect`, exit 1** — correct for a deliberately under-seeded +register, and the state the L2 loop starts from. + +| surface | result | +|---|---| +| entity counts | household 52,846 = 52,846 (identity holds); person 113,617 vs 113,649 and benunit 61,223 vs 61,211 differ — the known E8 donor-composition outcome, not yet transcribed into the register | +| shares | 142 compared, 113 differing, 3 missing in candidate, 2 extra | +| unsigned | 119 | + +The three columns missing on the candidate side are `free_school_meals`, +`free_school_fruit_veg` and `healthy_start_vouchers` — the E9 derived-benefit +class the #723 receipt already named, unchanged by the re-pin. + +Two behaviours worth recording, because they are the ones the swap decision +depends on: + +* `water_and_sewerage_charges` measured **+0.100897** (0.776937 → 0.877834) + and bound to `scottish-water-incumbent-nan-zeroing`, so it is reported as a + *signed* difference and excluded from the unsigned list. The share + reproduces the divergence #736 recorded, now against the re-pinned + reference, and this candidate predates the level fix. +* `scottish-water-sewerage-successor-level` correctly appears under + `unused_ids`: it signs `weighted_totals`, and this run supplied no weighted + registers. Under `--strict` that would fail the run, which is the intended + swap-acceptance behaviour — a register entry that matches nothing is either + stale or the run is incomplete, and both deserve to stop the swap. + ### Carried consequence The re-pin does not by itself re-validate the #723 acceptance screen: the From efebed5b78206c9a1dbe72961cd8a8638c591a28 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Sat, 22 Aug 2026 20:35:53 +0200 Subject: [PATCH 08/28] Pin the new country-package resource in the resource-list assertions Both UK resource tuples in test_country_spec.py enumerate the package's resources exactly, so #686's signed-differences register has to appear in them. Caught by the full build-shard run rather than the targeted ones. The first test's name records the #717 question it was written to answer, but what it does now is pin the whole legacy-JSON list; a note says so, so the next reader is not puzzled by a resource arriving in a test that says nothing was added. Co-Authored-By: Claude Fable 5 --- packages/microcosm-build/tests/test_country_spec.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/packages/microcosm-build/tests/test_country_spec.py b/packages/microcosm-build/tests/test_country_spec.py index 53098596..131eea41 100644 --- a/packages/microcosm-build/tests/test_country_spec.py +++ b/packages/microcosm-build/tests/test_country_spec.py @@ -271,6 +271,10 @@ def test_explicit_stage_subset_refuses_empty_or_unknown_names(self) -> None: class TestUKCountryPackage: def test_spi_spine_adds_no_country_package_resources(self) -> None: + # The name records the #717 question this was written to answer; what + # it does now is pin the whole legacy-JSON resource list, so any + # increment that ships a new country-package resource lands here. + # spine_swap_signed_differences.json is #686's deliberate addition. spec = load_country_spec("uk") legacy_rows = tuple( @@ -304,6 +308,7 @@ def test_spi_spine_adds_no_country_package_resources(self) -> None: "source_stages.json", "take_up_contract.json", "input_mass_reviewed_exclusions.json", + "spine_swap_signed_differences.json", "ledger_compile_parity_incumbent_2025_signed_differences.json", "ledger_compile_parity_production_2023_signed_differences.json", "national_staging_build_record.json", @@ -375,6 +380,7 @@ def test_uk_package_loads(self) -> None: "source_stages.json", "take_up_contract.json", "input_mass_reviewed_exclusions.json", + "spine_swap_signed_differences.json", "ledger_compile_parity_incumbent_2025_signed_differences.json", "ledger_compile_parity_production_2023_signed_differences.json", "national_staging_build_record.json", From f3aec8c56e6b4d1a47dfd8b359e9a705a5909f99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Sat, 22 Aug 2026 21:43:55 +0200 Subject: [PATCH 09/28] Record the rebuilt spine and the L2 adjudication queue MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit L0 ladder built from raw licensed tabs on this branch: smoke, dev and full all clean, every attempt landing a Logbook row. The record-count identity closes exactly at full scale — (16,288 + 10,000) x 2 + 270 = 52,846 — and the Scottish water fix reproduces at every rung. Parity against the re-pinned reference finds 26 columns beyond the band, against 27 in the #723 screen, which is what a re-pin that moved no reference share by more than 0.0046 should produce. Includes a correction to how divergences are attributed. The surviving value belongs to the last stage to produce *or rewrite* a column, and attributing by producing stage alone manufactures false findings: savings_interest_income and tax_free_savings_income originate in frs_spine and are rewritten by the SPI channel, so a naive pass reports them as raw-mapping divergences outside every signed class — the exact signature that made the water defect real. With rewrites folded in, every beyond-band divergence lands in an established class and the only raw-mapping one is already signed. The queue itself is left unadjudicated. The verdict is defect by construction because the register holds only the two water entries, which is the correct starting state; each class needs a ruling before it becomes an entry, and the E9 derived-benefit gap is the one item that may argue against swapping rather than for signing. Co-Authored-By: Claude Fable 5 --- experiments/686-uk-spine-swap-receipts.md | 93 +++++++++++++++++++++++ 1 file changed, 93 insertions(+) diff --git a/experiments/686-uk-spine-swap-receipts.md b/experiments/686-uk-spine-swap-receipts.md index 1870057a..b5348bb9 100644 --- a/experiments/686-uk-spine-swap-receipts.md +++ b/experiments/686-uk-spine-swap-receipts.md @@ -325,6 +325,99 @@ depends on: swap-acceptance behaviour — a register entry that matches nothing is either stale or the run is incomplete, and both deserve to stop the swap. +--- + +## R3 — the spine, rebuilt; and the L2 adjudication queue + +### L0 ladder + +Built from the raw licensed tabs on this branch (re-pin + water fix), FRS +2024-25, all 24 stages, every attempt landing a Logbook row with disposition +`iterating`: + +| rung | households (post-stack) | verdict | +|---|---|---| +| smoke `f001` | 794 | built clean | +| dev `f010` | 5,526 | built clean | +| full `1.0` (twin A) | 52,846 | built clean | + +**The record-count identity closes exactly at full scale:** +(16,288 raw FRS + 10,000 SPI) × 2 capital-gains clone + 270 CGT band donors += **52,846** households, matching the pinned incumbent to the row. +Persons 113,649 and benefit units 61,211 against the incumbent's 113,617 and +61,223 — the standing E8 donor-composition outcome, unchanged by this rebuild. + +The Scottish water fix reproduces at every scale: the `frs_spine` share is +`0.8783767190569745` in all three builds (it is measured before sampling, so +the rungs agree by construction), and the weighted Scottish charge lands at +£390.80 / £375.08 / £405.34 across smoke / dev / full against roughly £185 +under the retired mapping. + +### Parity against the re-pinned reference + +`verify_uk_spine_parity.py` over the full twin-A extraction (144 candidate +input layers): 142 columns compared, 113 differing, 3 missing, 2 extra, +**26 beyond ±0.02**. For comparison the #723 screen found 27 beyond the band +against the 1.56.14 reference, which is the predicted outcome of a re-pin +where no reference share moved by more than 0.0046. + +### Attribution — and a correction to how it is done + +Divergences are attributed to **the stage whose values survive**, which is the +last stage to either produce *or rewrite* the column. Attributing by producing +stage alone is wrong and produces false findings: `savings_interest_income` +and `tax_free_savings_income` both originate in `frs_spine` and are then +rewritten by `hmrc_spi_income_spine`, so a naive attribution reports them as +raw-mapping divergences outside every signed class — the same signature that +made the water defect real. They are E7, not new findings. The distinction +matters precisely because the raw-mapping signature is the one that indicates +a genuine defect rather than a method difference. + +With rewrites folded in, **every beyond-band divergence falls in an +established class**, and the only raw-mapping one is already signed: + +| class | count | columns | +|---|---|---| +| E6 consumption | 15 | bus_subsidy_spending, dfe_education_spending, restaurants_and_hotels, petrol, education, household_furnishings, electricity, miscellaneous, communication, alcohol_and_tobacco, gas, domestic_energy, diesel, transport, health | +| E5 wealth QRF | 6 | savings, property_wealth, corporate_wealth, other_residential_property_value, main_residence_value, student_loan_balance | +| E7 SPI channel | 3 | tax_free_savings_income, employer_pension_contributions, savings_interest_income | +| E8 CGT/salsac/loans | 1 | employee_pension_contributions | +| E2 raw FRS mapping | 1 | water_and_sewerage_charges — **signed** | + +Zero divergent columns are unattributable to a stage. + +### The queue — pending María's adjudication + +The parity verdict is `defect` by construction: the register holds only the two +water adjudications, so all 119 remaining differences are unsigned. That is the +correct starting state, not a failure. Each of the following needs a ruling +before it becomes a register entry; none should be transcribed on its prose +classification alone. + +1. **E6 consumption, 15 columns.** The largest deltas in the whole screen + (bus_subsidy +0.238, dfe_education +0.225). The E6 acceptance established + ours is donor-faithful and the incumbent collapses zero-inflated targets; + if that stands, one class entry scoped to these 15 covers them. +2. **E5 wealth QRF, 6 columns.** The standing correlated-rank-draw difference. + `owned_land` is *not* among them — worth noting given its exclusion expires + 2026-09-20. +3. **E7 SPI channel, 3 columns.** The U14 QRF-surface class at ~38% synthetic + composition, carried from #717 and still pending adjudication there. +4. **E8, 1 column** — `employee_pension_contributions`, the signed + conversion-depth difference from #684. +5. **Entity counts.** Persons +32, benefit units −12. Signed at E8 as a + donor-selection RNG outcome; needs an `entity_counts` register entry, which + is the one surface where a surface-wide entry is legitimate. +6. **Missing, 3 columns** — `free_school_meals`, `free_school_fruit_veg`, + `healthy_start_vouchers`: the E9 derived-benefit family. This is a genuine + coverage gap rather than a method difference, and it is the one item here + that may argue against swapping rather than for signing. +7. **Extra, 2 columns** — `num_bedrooms` (`frs_spine`) and + `other_investment_income` (`hmrc_spi_income_spine`). Net-new columns the + incumbent does not populate; `other_investment_income` is declared by the + incumbent's own national restoration, so the spine is ahead of the pinned + artifact rather than behind it. + ### Carried consequence The re-pin does not by itself re-validate the #723 acceptance screen: the From ab674feb1f8c5105f6595a94c3d0653a82345fbf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Sat, 22 Aug 2026 21:49:25 +0200 Subject: [PATCH 10/28] Record twin determinism and a stale scope in the identity ladder MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Twin determinism passes: two independent full builds are payload-identical across every store key, differing only in bytes because HDFStore stamps write times. All four identity receipts pass the property they exist to prove — that recomputation is invariant to row order. e4, e5 and e6 nevertheless report matches_stored_columns false, and the cause is the instrument, not the spine. _frs_only_frame scopes the survey channel by excluding household_is_spi_synthetic alone, which was right when the SPI channel was the only layer stacking rows. E8 added two more: the capital-gains incidence clone and the 270 band donors. Those rows carry values copied from their sources, so recomputing an identity-keyed draw for a clone's own household id disagrees with a stored value that was never drawn for that id. The measurement is unambiguous: excluding all three flags leaves exactly 16,288 households, the raw FRS count, which is the scope the #723 receipts ran at and passed. e8's own receipt passes because it recomputes the clone and donor logic explicitly instead of assuming unstacked rows, and e4's mismatch list is entirely identity-keyed draw columns — precisely those a clone inherits rather than draws. Recorded as unsigned and unfixed: the scope must exclude every stacked layer, the fix belongs with the e7 receipt work since both are ladder maintenance, and until then these three results say nothing about the spine and the L1 leg of the gate is not satisfied. Also noted: e5 and e6 report the failure with an empty mismatch map, which is not actionable evidence and should name what disagreed. Co-Authored-By: Claude Fable 5 --- experiments/686-uk-spine-swap-receipts.md | 69 +++++++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/experiments/686-uk-spine-swap-receipts.md b/experiments/686-uk-spine-swap-receipts.md index b5348bb9..38ab1f9c 100644 --- a/experiments/686-uk-spine-swap-receipts.md +++ b/experiments/686-uk-spine-swap-receipts.md @@ -418,6 +418,75 @@ classification alone. incumbent's own national restoration, so the spine is ahead of the pinned artifact rather than behind it. +--- + +## R4 — determinism, and a stale scope in the identity ladder + +### Twin determinism: PASS + +Two independent full builds compared with `compare_uk_h5_payload.py`: +**payload-identical** across all four store keys, no differing root +attributes, entity rows equal (52,846 / 61,211 / 113,649). The two files +differ in bytes, which is correct — `HDFStore` stamps object-header write +times, which is exactly why payload comparison rather than byte comparison is +the instrument. + +### Identity receipts: the determinism property holds everywhere + +| check | `identical_under_permutation` | `matches_stored_columns` | +|---|---|---| +| e4 | **PASS** | fail | +| e5 | **PASS** | fail | +| e6 | **PASS** | fail | +| e8 | **PASS** | **PASS** | + +The property these receipts exist to prove — that recomputation is invariant +to row order — holds in all four. What fails is the secondary comparison of +the recomputation against the columns stored in the artifact, and the cause is +in the instrument rather than the spine. + +### Why, and the measurement that shows it + +`_frs_only_frame` (`tools/verify_uk_identity_stability.py`) scopes the frame +to the survey channel by excluding `household_is_spi_synthetic`. It was +written for the post-#717 artifact, where the SPI channel was the only layer +stacking rows. **E8 added two more**: the capital-gains incidence clone (×2) +and the 270 CGT band donors. Those rows carry values *copied from their source +rows*, so recomputing an identity-keyed draw for a clone's own household id +legitimately disagrees with the stored value — the stored value was never +drawn for that id. + +Measured on `spine-a.h5`: + +| scope | households | +|---|---| +| all rows | 52,846 | +| `~spi_synthetic` — what the tool currently uses | 32,761 | +| `~spi_synthetic & ~capital_gains_clone` | 16,381 | +| `~spi_synthetic & ~capital_gains_clone & ~cgt_band_donor` | **16,288** | + +16,288 is the raw FRS household count, and it is exactly the scope the #723 +receipts ran at (`entity_row_counts` 16,288 / 18,850 / 34,966 with +`matches_stored_columns: true`). The #723 spine predates E8, so its +`~spi_synthetic` scope *was* the raw-FRS scope; E8's stacking silently widened +it. e8's own receipt passes because it recomputes the clone and donor logic +explicitly rather than assuming unstacked rows. + +The e4 mismatch list is consistent with this reading throughout: it is the +identity-keyed draw columns (`would_claim_*`, `household_owns_tv`, +`would_evade_tv_licence_fee`, `property_purchased`, `brma`, …), which are +precisely the columns whose values a clone inherits rather than draws. e5 and +e6 report the failure with an empty mismatch map, which is its own small +defect in the receipt's reporting — a fail with nothing named is not +actionable evidence, and should name what disagreed. + +**Disposition: fix the instrument, do not sign this.** The scope must exclude +every stacked layer, not just the SPI channel, and the fix belongs with the +e7 receipt work (W4) since both are ladder maintenance. Until then the e4/e5/e6 +`matches_stored_columns` results carry no information about the spine, and the +L1 leg of the gate is not satisfied — the receipts must be re-run after the +fix rather than accepted as-is. + ### Carried consequence The re-pin does not by itself re-validate the #723 acceptance screen: the From 621c5f4765c6046d2ae4f4846a4438a9c3a867fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Sat, 22 Aug 2026 21:59:46 +0200 Subject: [PATCH 11/28] Fix the identity ladder for E8's stacking layers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Building a spine that carries every stage through E8 and running the whole ladder surfaced that e4, e5 and e6 had gone stale. Each was written when the SPI support channel was the only stage stacking rows; E8 added the capital-gains clone and the CGT band donors after them. The failures are in the instrument, not the spine, and the experiment that proves it is the unchanged tool against two artifacts: the pre-E8 spine passes e6, the post-E8 spine fails it. Three mechanisms, which is why one fix did not cover them. e4 recomputes identity-keyed draws and a stacked row carries a value copied from its source, never drawn for its own id. e5's regional uprating scales to a per-region mean over the frame's owner households, so stacked rows move the denominator — confirmed by running it both ways on one artifact. e6 fails scoped as well as unscoped: its NHS allocation normalizes against an absolute budget, so it needs stage-time weights, and it divided out the SPI channel's share while the clone's mass_split went unrestored. Scoping now excludes every stacked layer from one declared flag list, and the weight divisor reads its factors from the declared operations instead of hardcoding them. The divisor is driven by the flags the artifact actually carries rather than by the committed roster. The first implementation used the roster and divided the clone factor out of a spine built before that stage existed, skewing the comparison the other way; the pre-E8 artifact caught it at once. Both vintages now pass. Recorded at the helper: a new mass-redistributing op kind has to be registered there, and the failure mode if it is not is a receipt silently comparing against the wrong grossing scale. Co-Authored-By: Claude Fable 5 --- ...86-uk-identity-ladder-e8-stacking.fixed.md | 1 + experiments/686-uk-spine-swap-receipts.md | 80 ++++++++-- tools/verify_uk_identity_stability.py | 149 ++++++++++++++---- 3 files changed, 182 insertions(+), 48 deletions(-) create mode 100644 changelog.d/686-uk-identity-ladder-e8-stacking.fixed.md diff --git a/changelog.d/686-uk-identity-ladder-e8-stacking.fixed.md b/changelog.d/686-uk-identity-ladder-e8-stacking.fixed.md new file mode 100644 index 00000000..0dcd2f55 --- /dev/null +++ b/changelog.d/686-uk-identity-ladder-e8-stacking.fixed.md @@ -0,0 +1 @@ +Fix the UK identity-stability ladder for E8's stacking layers (#686). Building a spine that carries every stage through E8 and running the whole ladder surfaced that the e4, e5 and e6 receipts had gone stale: each was written when the SPI support channel was the only stage stacking rows, and E8 added the capital-gains incidence clone and the CGT band donors after them. The failures were in the instrument, not the spine — proved by running the unchanged tool against two artifacts, where the pre-E8 spine passes and the post-E8 spine fails. Three distinct mechanisms were involved, which is why one fix did not cover them: e4 recomputes identity-keyed draws, and a stacked row carries a value copied from its source that was never drawn for its own id; e5's regional property uprating scales to a per-region mean over the owner households in the frame, so stacked rows shift the denominator; and e6's NHS allocation normalizes against an absolute budget, so it needs the stage-time *weights* rather than only the stage-time population, and it was dividing out the SPI channel's reserved share while the clone's `mass_split` went unrestored. Scoping now excludes every stacked layer through a single declared flag list, and the weight restoration reads its factors from the declared operations rather than hardcoding them. Crucially the divisor is driven by which stacking flags the artifact actually carries, not by the committed roster: the first implementation used the roster and divided the clone factor out of a spine built before that stage existed, which the pre-E8 artifact caught immediately. Both vintages now pass. The standing hazard is recorded at the helper: a new mass-redistributing operation kind must be registered there, and the failure mode if it is not is a receipt that silently compares against the wrong grossing scale. diff --git a/experiments/686-uk-spine-swap-receipts.md b/experiments/686-uk-spine-swap-receipts.md index 38ab1f9c..965e9fd8 100644 --- a/experiments/686-uk-spine-swap-receipts.md +++ b/experiments/686-uk-spine-swap-receipts.md @@ -433,6 +433,8 @@ the instrument. ### Identity receipts: the determinism property holds everywhere +As first run, before the ladder fix below: + | check | `identical_under_permutation` | `matches_stored_columns` | |---|---|---| | e4 | **PASS** | fail | @@ -440,6 +442,8 @@ the instrument. | e6 | **PASS** | fail | | e8 | **PASS** | **PASS** | +All four pass both columns after the fix, on both the #686 and #723 spines. + The property these receipts exist to prove — that recomputation is invariant to row order — holds in all four. What fails is the secondary comparison of the recomputation against the columns stored in the artifact, and the cause is @@ -472,20 +476,68 @@ receipts ran at (`entity_row_counts` 16,288 / 18,850 / 34,966 with it. e8's own receipt passes because it recomputes the clone and donor logic explicitly rather than assuming unstacked rows. -The e4 mismatch list is consistent with this reading throughout: it is the -identity-keyed draw columns (`would_claim_*`, `household_owns_tv`, -`would_evade_tv_licence_fee`, `property_purchased`, `brma`, …), which are -precisely the columns whose values a clone inherits rather than draws. e5 and -e6 report the failure with an empty mismatch map, which is its own small -defect in the receipt's reporting — a fail with nothing named is not -actionable evidence, and should name what disagreed. - -**Disposition: fix the instrument, do not sign this.** The scope must exclude -every stacked layer, not just the SPI channel, and the fix belongs with the -e7 receipt work (W4) since both are ladder maintenance. Until then the e4/e5/e6 -`matches_stored_columns` results carry no information about the spine, and the -L1 leg of the gate is not satisfied — the receipts must be re-run after the -fix rather than accepted as-is. +The mismatch lists are consistent with this reading throughout: + +| check | columns reported | +|---|---| +| e4 | the identity-keyed draws — `would_claim_*`, `household_owns_tv`, `would_evade_tv_licence_fee`, `property_purchased`, `brma`, … — precisely those a stacked row inherits rather than draws | +| e5 | `main_residence_value`, `property_wealth` | +| e6 | the six NHS person columns (`a_and_e_visits`, `admitted_patient_visits`, `outpatient_visits` and their spending counterparts) | + +e5's and e6's are not identity-keyed draws but **population-dependent +normalizations** — the regional uprating factor divides by a mean over the +scoped rows (the row-order-dependent float mean fixed during E5's licensed +acceptance), and the NHS allocation normalizes against an absolute budget. +Widening the scope to include stacked rows changes the denominator, so those +columns move for a different reason than e4's do but from the same cause. + +(An earlier draft of this receipt recorded e5 and e6 as reporting the failure +with an empty mismatch map, and called that a reporting defect. That was +wrong: those two checks name their columns under `stored_column_mismatches` +while e4 uses `stored_mismatches`, and the first reading looked at the wrong +key. The differing key names across checks are a small inconsistency worth +tidying, but the evidence was there.) + +### Resolution — three mechanisms, not one + +The first diagnosis recorded here said one scoping fix would address all +three. That was wrong, and the controlled experiment that settled it was +running the **unchanged** tool against two spines: the #723 artifact (26,288 +households, only the SPI flag) passes e6; the #686 artifact (52,846, all three +flags) fails it. Same tool, so E8's stacking is the cause — but the mechanism +differs per check. + +| check | mechanism | fix | +|---|---|---| +| e4 | identity-keyed draws; a stacked row carries a value copied from its source, never drawn for its own id | scope to unstacked rows | +| e5 | regional property uprating scales to a per-region **mean over owner households in the frame**; stacked rows shift the denominator | scope to unstacked rows | +| e6 | NHS allocation normalizes against an **absolute budget**, so it needs stage-time *weights*, not just stage-time population | scope, **and** divide out the mass factors applied after E6 | + +e5 was confirmed by running it both ways on the same artifact: fails at +52,846, passes at 16,288. e6 fails *both* ways, which is what disproved the +scoping-only hypothesis: it already divided out `spi_support_channel`'s +`share`, but E8's `cgt_incidence_clone` splits each household's weight by +`mass_split` afterwards and that was never restored. + +The weight divisor now reads both factors from the declared operations rather +than hardcoding them, so a change to either share is picked up automatically. +A genuinely new mass-redistributing op kind still has to be registered in the +helper, and the failure mode if it is not is a receipt silently comparing +against the wrong grossing scale — which is precisely what happened here. + +**One regression caught in the fix itself.** The first implementation derived +the mass factors from the *committed roster*, which divided out the clone's +0.5 even for a spine built before that stage existed — skewing the comparison +the other way. The pre-E8 artifact failed immediately and the divisor is now +driven by which stacking flags the artifact actually carries. Both vintages +pass: e5/e6 green on the #723 spine, e4/e5/e6/e8 green on the #686 spine. + +**Standing lesson.** Each increment that stacks rows or moves mass invalidates +assumptions inside the *earlier* increments' receipts, and nothing surfaces it +until a spine carrying every stage is built and the whole ladder is run. That +is the argument for running the ladder before anything downstream depends on +it, rather than treating per-increment receipts as still-valid once the roster +grows. ### Carried consequence diff --git a/tools/verify_uk_identity_stability.py b/tools/verify_uk_identity_stability.py index 3dfab742..dfd47c1a 100644 --- a/tools/verify_uk_identity_stability.py +++ b/tools/verify_uk_identity_stability.py @@ -390,42 +390,28 @@ def recompute(person_t, benunit_t, household_t) -> dict[str, pd.DataFrame]: benunit = frame.table("benunit") household = frame.table("household").copy() household["household_weight"] = frame.weights_for("household").values - # Scope to the FRS spine rows: the SPI channel stages stack synthetic - # households AFTER the E6 stages ran (clones inherit their donors' - # consumption/services values), so the deterministic-layer identity - # claims apply to the population the consumption stages actually saw. - # The stacked rows are E7's receipt surface, not E6's. The - # spi_support_channel stage also scales the survey channel's weights by - # (1 - share) after the E6 stages ran; the NHS allocation's budget - # normalization is absolute, so restore the stage-time grossing scale - # from the declared share before recomputing. - if "household_is_spi_synthetic" in household.columns: - spine_mask = ~household["household_is_spi_synthetic"].astype(bool) + # Scope to the rows the E6 stages actually saw, and restore the grossing + # scale they saw them at. Every later stacking stage copies its source + # row's consumption/services values onto the new rows, so those rows are + # the later stage's receipt surface, not E6's; and every later stage that + # redistributes mass leaves these rows carrying a fraction of the weight + # E6 normalized against. Three layers stack today (SPI support channel, + # capital-gains clone, CGT band donors) and two of them move mass. + spine_mask = _unstacked_mask(household) + if spine_mask is not None: household = household.loc[spine_mask].reset_index(drop=True) spine_household_ids = set(household["household_id"].tolist()) person = person.loc[ person["person_household_id"].isin(spine_household_ids) ].reset_index(drop=True) - from importlib.resources import files as _files - - spec = json.loads( - _files("microcosm.build.uk") - .joinpath("source_stages.json") - .read_text(encoding="utf-8") - ) - channel = next( - (s for s in spec["stages"] if s["stage"] == "spi_support_channel"), - None, + applied = tuple( + stage + for flag, stage in _MASS_STAGE_BY_FLAG.items() + if flag in frame.table("household").columns ) - if channel is not None: - share = next( - op["share"] - for op in channel["operations"] - if op["kind"] == "allocate_zero_weight_prior_mass" - ) - household["household_weight"] = household["household_weight"].to_numpy( - dtype=float - ) / (1.0 - float(share)) + household["household_weight"] = household["household_weight"].to_numpy( + dtype=float + ) / _stage_time_weight_divisor(after_stages=applied) original = recompute(person, benunit, household) rng = np.random.default_rng(permutation_seed) permuted = recompute( @@ -765,8 +751,13 @@ def main() -> int: receipt["identical_under_permutation"] and receipt["matches_stored_columns"] ) elif args.check == "e5": + # E5's wealth stages ran before every stacking layer, and its regional + # property uprating scales to a per-region mean over the owner + # households in the frame — so a frame carrying stacked rows shifts + # the denominator and the recomputation stops matching what the stage + # stored. Scope to the population the stage saw. receipt = e5_identity_receipt( - frame, + _frs_only_frame(frame), permutation_seed=args.permutation_seed, ) ok = bool( @@ -797,8 +788,96 @@ def main() -> int: return 0 if ok else 1 +#: Every flag that marks a row as stacked onto the survey channel rather than +#: drawn from the raw FRS. A stage that stacks rows copies its source row's +#: already-computed columns onto the new rows, so an identity-keyed +#: recomputation for a stacked row's *own* id legitimately disagrees with what +#: is stored there — that value was drawn for the source id, not this one. +#: A new stacking stage MUST add its flag here, or these receipts silently +#: start comparing inherited values against fresh draws and report a spine +#: defect that is really an instrument defect. +_STACKED_ROW_FLAGS = ( + "household_is_spi_synthetic", # #717 SPI support channel + "household_is_capital_gains_clone", # E8 capital-gains incidence clone + "household_is_cgt_band_donor", # E8 CGT band donors +) + +#: The stage that stacks each flag, for artifacts that carry it. The weight +#: restoration is driven by what the *artifact* actually contains rather than +#: by the committed roster: a spine built before a stacking stage existed +#: never had that stage's mass factor applied, and dividing it out anyway +#: skews the comparison in the opposite direction. +_MASS_STAGE_BY_FLAG = { + "household_is_spi_synthetic": "spi_support_channel", + "household_is_capital_gains_clone": "cgt_incidence_clone", +} + + +def _unstacked_mask(household: pd.DataFrame): + """Rows that were present when the pre-stacking stages ran, or None.""" + + flags = [flag for flag in _STACKED_ROW_FLAGS if flag in household.columns] + if not flags: + return None + return ~household[flags].astype(bool).any(axis=1) + + +def _stage_time_weight_divisor(*, after_stages: Sequence[str]) -> float: + """Product of the declared mass factors applied after a receipted stage. + + Scoping to the unstacked rows restores the stage's *population* but not + its *weights*: every later stage that redistributes household mass leaves + the surviving survey rows carrying a fraction of what the receipted stage + saw. A receipt whose recomputation normalizes against weights — the NHS + allocation's budget normalization is absolute, not relative — must divide + that back out or it compares against a different grossing scale. + + On the E8 roster two stages do this: `spi_support_channel` reserves + ``share`` of prior mass for the synthetic channel, and + `cgt_incidence_clone` splits each household's weight across its copies by + ``mass_split``. Both factors are read from the declared operations rather + than hardcoded, so a change to either is picked up automatically; a *new* + mass-redistributing op kind still has to be added here. + + ``after_stages`` names only the stages that actually ran in the artifact + under receipt — see ``_MASS_STAGE_BY_FLAG``. Deriving it from the + committed roster instead would divide out a factor that a spine built + before that stage never had applied, which is a regression the pre-E8 + artifact catches immediately. + """ + + from importlib.resources import files as _files + + spec = json.loads( + _files("microcosm.build.uk") + .joinpath("source_stages.json") + .read_text(encoding="utf-8") + ) + wanted = set(after_stages) + divisor = 1.0 + for stage in spec["stages"]: + if stage.get("stage") not in wanted: + continue + for operation in stage.get("operations", ()): + kind = operation.get("kind") + if kind == "allocate_zero_weight_prior_mass": + divisor *= 1.0 - float(operation["share"]) + elif kind == "clone_records": + divisor *= float(operation["mass_split"]) + if divisor <= 0.0: + raise ValueError( + "stage-time weight divisor collapsed to zero; the declared mass " + "factors are not usable for a grossing-scale restoration." + ) + return divisor + + def _frs_only_frame(frame): - """Scope a post-#717 artifact to the survey channel (FRS rows only).""" + """Scope the artifact to the unstacked survey rows (raw FRS only). + + On the E8 roster this leaves the 16,288 raw FRS households, which is the + population the pre-stacking stages actually drew for. + """ from microcosm.build.uk_runtime.national_frame import ( uk_household_weight_kind, @@ -806,9 +885,11 @@ def _frs_only_frame(frame): ) household = frame.table("household") - if "household_is_spi_synthetic" not in household.columns: + flags = [flag for flag in _STACKED_ROW_FLAGS if flag in household.columns] + if not flags: return frame - keep = ~household["household_is_spi_synthetic"].astype(bool) + stacked = household[flags].astype(bool).any(axis=1) + keep = ~stacked weights = frame.weights_for("household").values[keep.to_numpy()] household = household.loc[keep].reset_index(drop=True) ids = set(household["household_id"].tolist()) From 76a5822cdc34e2b0034085e3adc1d6a540a15491 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Sun, 23 Aug 2026 00:21:13 +0200 Subject: [PATCH 12/28] Port the four in-kind benefit columns and add the e7 identity receipt MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two gaps the whole-spine parity screen surfaced, both in already-merged increments rather than in new work. The spine never mapped healthy_start_vouchers, free_school_breakfasts, free_school_fruit_veg or free_school_meals, so parity reported three of them as columns the candidate does not produce. The incumbent maps them straight off the person tapes in create_frs, which makes this an E2/E3 omission rather than deferred work — #685 is UC deduction attributes, bus fares and WAS debt, none of which touch these. heartval is on both the adult and child tapes and the three school columns are child-only, so adults read zero rather than propagating NaN, which the test pins. Measured against the reference the three contract columns land at -0.000089, -0.000001 and +0.000008; free_school_breakfasts is not an engine-known variable so it never enters the 145-column surface. The identity ladder had no e7 check at all — the increment that introduced row stacking, and so the one whose interaction with E8 broke e4, e5 and e6, was the only one without a receipt. It now receipts the support-channel layer: each entity's channel and clone index, the composite source key, and the propagation of a household's channel to its persons and benefit units. Bitwise on both surfaces, since these are labels and integer indices. A negative control confirms it is not vacuous: flipping one channel label fails it and names the column. It deliberately excludes the employer_pension_contributions = 3 x employee_pension_contributions derive. That is a real E7 layer, but E8's salary_sacrifice rewrites the multiplicand in place afterwards and the relation survives on only 95.9% of survey-channel persons, so asserting it would fail for the wrong reason — the #721 rewrites-provenance class. The coverage manifest is regenerated for the new source-manifest hash (145 required, 0 exclusions, unchanged) and the spec bundle sha re-pinned. Co-Authored-By: Claude Fable 5 --- .../uk/release_input_coverage_manifest.json | 24 +- .../src/microcosm/build/uk/source_stages.json | 7001 +++++++++-------- .../src/microcosm/build/uk/spec/sources.yaml | 4 + .../microcosm/build/uk_runtime/frs_spine.py | 18 + .../tests/test_spec_engine_country_bundles.py | 2 +- .../tests/test_uk_frs_spine.py | 40 + tools/verify_uk_identity_stability.py | 148 +- 7 files changed, 3746 insertions(+), 3491 deletions(-) diff --git a/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json b/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json index 4d727e88..a52bd9ba 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json +++ b/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json @@ -473,7 +473,7 @@ "capital_gains" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "6f50637a95c340c07382d13af5a43da9d2a1a1167f713aba3605b3ca62efe449", + "source_manifest_sha256": "922d01407a3cb36d47903bab3b30ef25fd35e1d53674aec8f8c96bc7247932ba", "source_vintages": { "source": "HMRC Capital Gains Tax statistics, July 2025, Table 2.1a", "survey": "HMRC Capital Gains Tax statistics Table 2.1a and Advani-Summers capital-gains incidence" @@ -496,7 +496,7 @@ "capital_gains" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "6f50637a95c340c07382d13af5a43da9d2a1a1167f713aba3605b3ca62efe449", + "source_manifest_sha256": "922d01407a3cb36d47903bab3b30ef25fd35e1d53674aec8f8c96bc7247932ba", "source_vintages": { "source": "Advani and Summers (2020), Capital Gains and UK Inequality, CAGE Working Paper 465", "survey": "Family Resources Survey 2024-25, SPI synthetic support, and Advani-Summers capital-gains incidence" @@ -525,7 +525,7 @@ "required_mass_change_reason": "E5 source-stage transform preserves household rows and typed household weights; total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "6f50637a95c340c07382d13af5a43da9d2a1a1167f713aba3605b3ca62efe449", + "source_manifest_sha256": "922d01407a3cb36d47903bab3b30ef25fd35e1d53674aec8f8c96bc7247932ba", "source_vintages": { "source": "UK Data Service SN 8856 Effects of Taxes and Benefits household tab, DfT rail fare index, and public NHS activity/cost table.", "survey": "Effects of Taxes and Benefits 1977-2024 and NHS age-gender public table" @@ -545,7 +545,7 @@ "required_mass_change_reason": "E5 source-stage transform preserves household rows and typed household weights; total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "6f50637a95c340c07382d13af5a43da9d2a1a1167f713aba3605b3ca62efe449", + "source_manifest_sha256": "922d01407a3cb36d47903bab3b30ef25fd35e1d53674aec8f8c96bc7247932ba", "source_vintages": { "source": "UK Data Service SN 8856 Effects of Taxes and Benefits household tab and cited VAT anchor resource.", "survey": "Effects of Taxes and Benefits 1977-2024" @@ -585,12 +585,12 @@ "outputs": [ "capital_gains" ], - "required_mass_change_reason": "Amounts-only capital gains redraw on the source spine: household weights pass through unchanged and total household mass is conserved.", + "required_mass_change_reason": "Amounts-only capital gains redraw: household weights pass through unchanged and total household mass is conserved.", "rewrites": [ "capital_gains" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "6f50637a95c340c07382d13af5a43da9d2a1a1167f713aba3605b3ca62efe449", + "source_manifest_sha256": "922d01407a3cb36d47903bab3b30ef25fd35e1d53674aec8f8c96bc7247932ba", "source_vintages": { "hmrc_surface": "2023-24", "mapped_build_period": "2024" @@ -604,7 +604,7 @@ "base_candidate_tier": "frs", "calibration_permitted": false, "canonical_source_manifest": "source_stages.json", - "canonical_source_manifest_sha256": "6f50637a95c340c07382d13af5a43da9d2a1a1167f713aba3605b3ca62efe449", + "canonical_source_manifest_sha256": "922d01407a3cb36d47903bab3b30ef25fd35e1d53674aec8f8c96bc7247932ba", "effective_mass_requirements": { "charitable_investment_gifts": { "mass_share_denominator": "all_person_effective_mass", @@ -715,7 +715,7 @@ "required_mass_change_reason": "E5 source-stage transform preserves household rows and typed household weights; total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "6f50637a95c340c07382d13af5a43da9d2a1a1167f713aba3605b3ca62efe449", + "source_manifest_sha256": "922d01407a3cb36d47903bab3b30ef25fd35e1d53674aec8f8c96bc7247932ba", "source_vintages": { "source": "UK Data Service SN 9468 Living Costs and Food Survey 2023-24 household/person tabs, NEED 2023 headline energy tables, Ofgem Q2 2026 unit rates, and WAS round-8 bridge donor.", "survey": "Living Costs and Food Survey 2023-24" @@ -736,7 +736,7 @@ "property_wealth" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "6f50637a95c340c07382d13af5a43da9d2a1a1167f713aba3605b3ca62efe449", + "source_manifest_sha256": "922d01407a3cb36d47903bab3b30ef25fd35e1d53674aec8f8c96bc7247932ba", "source_vintages": { "source": "MHCLG dwellings and ONS UK House Price Index December 2025 regional average prices.", "survey": "Public regional property reference" @@ -760,7 +760,7 @@ "employee_pension_contributions" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "6f50637a95c340c07382d13af5a43da9d2a1a1167f713aba3605b3ca62efe449", + "source_manifest_sha256": "922d01407a3cb36d47903bab3b30ef25fd35e1d53674aec8f8c96bc7247932ba", "source_vintages": { "source": "HMRC, Salary sacrifice reform for pension contributions effective from 6 April 2029", "survey": "Family Resources Survey 2024-25 salary-sacrifice respondents and HMRC salary-sacrifice reform analysis" @@ -782,7 +782,7 @@ "student_loan_plan" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "6f50637a95c340c07382d13af5a43da9d2a1a1167f713aba3605b3ca62efe449", + "source_manifest_sha256": "922d01407a3cb36d47903bab3b30ef25fd35e1d53674aec8f8c96bc7247932ba", "source_vintages": { "source": "Explore Education Statistics Table 6a, Higher education total", "survey": "Family Resources Survey 2024-25 and Student Loans Company borrower forecasts for England" @@ -814,7 +814,7 @@ "required_mass_change_reason": "E5 source-stage transform preserves household rows and typed household weights; total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "6f50637a95c340c07382d13af5a43da9d2a1a1167f713aba3605b3ca62efe449", + "source_manifest_sha256": "922d01407a3cb36d47903bab3b30ef25fd35e1d53674aec8f8c96bc7247932ba", "source_vintages": { "source": "Office for National Statistics Wealth and Assets Survey, UK Data Service SN 7215, DOI 10.5255/UKDA-SN-7215-20; local licensed 2006-22 household tab.", "survey": "Wealth and Assets Survey round 8" diff --git a/packages/microcosm-build/src/microcosm/build/uk/source_stages.json b/packages/microcosm-build/src/microcosm/build/uk/source_stages.json index b777f420..3e8dc10a 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/source_stages.json +++ b/packages/microcosm-build/src/microcosm/build/uk/source_stages.json @@ -1,3486 +1,3533 @@ { - "version": 1, - "country": "uk", - "policy": "The UK HMRC/SPI income family is source-manifest-defined. Private donor data must be supplied locally, every artifact must be SHA-256 verified at runtime, retained FRS constituents and published bands fail closed, and the current replay keeps importance-kind weights because all 208 banded facts require an unavailable full FRS total-income measure.", - "stages": [ - { - "stage": "frs_spine", - "survey": "Family Resources Survey 2024-25", - "source": "Department for Work and Pensions Family Resources Survey 2024-25, UK Data Service SN 9563, DOI 10.5255/UKDA-SN-9563-1; local licensed 2024_25 tabs.", - "grain": "household", - "artifacts": [ - { - "role": "frs_table", - "table": "accounts", - "kind": "licensed_microdata", - "format": "tab", - "vintage": "2024_25", - "locator": "accounts.tab", - "sha256": "fa7871eb45cad0db5fd05ede454ced60405d2f9c598651ea5acea5c91a6ff52f", - "size_bytes": 1812923, - "runtime_sha256_required": true, - "tax_year_start": 2024, - "ukds_study_number": 9563, - "doi": "10.5255/UKDA-SN-9563-1" - }, - { - "role": "frs_table", - "table": "adult", - "kind": "licensed_microdata", - "format": "tab", - "vintage": "2024_25", - "locator": "adult.tab", - "sha256": "4eaea0809a7ccca0fddeb98e358771e4a6e5ebb81b21c4fac4070fc9d227658d", - "size_bytes": 34885825, - "runtime_sha256_required": true, - "tax_year_start": 2024, - "ukds_study_number": 9563, - "doi": "10.5255/UKDA-SN-9563-1" - }, - { - "role": "frs_table", - "table": "benefits", - "kind": "licensed_microdata", - "format": "tab", - "vintage": "2024_25", - "locator": "benefits.tab", - "sha256": "f6ad22b408a13e2239c04b0d076a36418dcf5dd89a8c60daa792c4d735b911d3", - "size_bytes": 2362329, - "runtime_sha256_required": true, - "tax_year_start": 2024, - "ukds_study_number": 9563, - "doi": "10.5255/UKDA-SN-9563-1" - }, - { - "role": "frs_table", - "table": "benunit", - "kind": "licensed_microdata", - "format": "tab", - "vintage": "2024_25", - "locator": "benunit.tab", - "sha256": "66b894624498316d19b6259e287a607e98ed3daacc9be3d3e9067d32b8e09a5a", - "size_bytes": 13986782, - "runtime_sha256_required": true, - "tax_year_start": 2024, - "ukds_study_number": 9563, - "doi": "10.5255/UKDA-SN-9563-1" - }, - { - "role": "frs_table", - "table": "child", - "kind": "licensed_microdata", - "format": "tab", - "vintage": "2024_25", - "locator": "child.tab", - "sha256": "88ec53fc52eea4374864bbc74219d551f4b4c5f54a220bf9607c7a6289719aa5", - "size_bytes": 2753961, - "runtime_sha256_required": true, - "tax_year_start": 2024, - "ukds_study_number": 9563, - "doi": "10.5255/UKDA-SN-9563-1" - }, - { - "role": "frs_table", - "table": "chldcare", - "kind": "licensed_microdata", - "format": "tab", - "vintage": "2024_25", - "locator": "chldcare.tab", - "sha256": "7ccd3f92f299a1f49b24063188177cdb8a958d8bcd753fc3d74dadda6ad04023", - "size_bytes": 275878, - "runtime_sha256_required": true, - "tax_year_start": 2024, - "ukds_study_number": 9563, - "doi": "10.5255/UKDA-SN-9563-1" - }, - { - "role": "frs_table", - "table": "extchild", - "kind": "licensed_microdata", - "format": "tab", - "vintage": "2024_25", - "locator": "extchild.tab", - "sha256": "c661379a4aa5079ce482b1f98f0bfb9157ad9b3ba4eb10739b61846f9c9548e4", - "size_bytes": 15150, - "runtime_sha256_required": true, - "tax_year_start": 2024, - "ukds_study_number": 9563, - "doi": "10.5255/UKDA-SN-9563-1" - }, - { - "role": "frs_table", - "table": "househol", - "kind": "licensed_microdata", - "format": "tab", - "vintage": "2024_25", - "locator": "househol.tab", - "sha256": "2b93b6aed49e1591d6f5360736b4aee3a11ef506b276435f4d2c011a8afbb6a5", - "size_bytes": 12108606, - "runtime_sha256_required": true, - "tax_year_start": 2024, - "ukds_study_number": 9563, - "doi": "10.5255/UKDA-SN-9563-1" - }, - { - "role": "frs_table", - "table": "job", - "kind": "licensed_microdata", - "format": "tab", - "vintage": "2024_25", - "locator": "job.tab", - "sha256": "eb7faf7ada3a3851cb2afb83e2983f8907ffeec897cfbe01e56cb0dfefa853e2", - "size_bytes": 10518760, - "runtime_sha256_required": true, - "tax_year_start": 2024, - "ukds_study_number": 9563, - "doi": "10.5255/UKDA-SN-9563-1" - }, - { - "role": "frs_table", - "table": "maint", - "kind": "licensed_microdata", - "format": "tab", - "vintage": "2024_25", - "locator": "maint.tab", - "sha256": "e7a8d6f47cab7bf9db9bfd7b3ad5ebe5830ec75245d065dcf8654c7c20b97a7d", - "size_bytes": 13993, - "runtime_sha256_required": true, - "tax_year_start": 2024, - "ukds_study_number": 9563, - "doi": "10.5255/UKDA-SN-9563-1" - }, - { - "role": "frs_table", - "table": "mortgage", - "kind": "licensed_microdata", - "format": "tab", - "vintage": "2024_25", - "locator": "mortgage.tab", - "sha256": "6a08f6846970dfdc544a7efc8a93fed4f3210d872cd2d160dfb14ca8d92d5ed0", - "size_bytes": 600552, - "runtime_sha256_required": true, - "tax_year_start": 2024, - "ukds_study_number": 9563, - "doi": "10.5255/UKDA-SN-9563-1" - }, - { - "role": "frs_table", - "table": "oddjob", - "kind": "licensed_microdata", - "format": "tab", - "vintage": "2024_25", - "locator": "oddjob.tab", - "sha256": "dfff1baf71a3de05f3a2fcf0c01a3995df5657f242cd7846aa61f6cc27a1cead", - "size_bytes": 5339, - "runtime_sha256_required": true, - "tax_year_start": 2024, - "ukds_study_number": 9563, - "doi": "10.5255/UKDA-SN-9563-1" - }, - { - "role": "frs_table", - "table": "penprov", - "kind": "licensed_microdata", - "format": "tab", - "vintage": "2024_25", - "locator": "penprov.tab", - "sha256": "9e53de0dc969baec000b3cd68387f0f2dfb3f678732e408de175e0a1d6e3fdc1", - "size_bytes": 513614, - "runtime_sha256_required": true, - "tax_year_start": 2024, - "ukds_study_number": 9563, - "doi": "10.5255/UKDA-SN-9563-1" - }, - { - "role": "frs_table", - "table": "pension", - "kind": "licensed_microdata", - "format": "tab", - "vintage": "2024_25", - "locator": "pension.tab", - "sha256": "2b9be1eb6583cc8916fc06294be27e6217f2aea73da24b97b3226293f6a6ec24", - "size_bytes": 1232411, - "runtime_sha256_required": true, - "tax_year_start": 2024, - "ukds_study_number": 9563, - "doi": "10.5255/UKDA-SN-9563-1" - } - ], - "operations": [ - { - "kind": "read_tables", - "format": "tab", - "delimiter": "\t", - "lowercase_columns": true, - "numeric_errors": "coerce", - "runtime_sha256_required": true - }, - { - "kind": "replace_sentinels", - "sentinel_policy": "raw numeric blanks and nonnumeric sentinels are coerced to NaN, then produced E2 columns are filled to the Frame no-NaN contract. Clamping is deliberately per-column, mirroring the incumbent base build: redamt stays unclamped, tuborr clamps at zero, and the property/royalties components clamp only in their aggregate." - }, - { - "kind": "assemble_group_entities", - "household_id": "sernum", - "benunit_id": "sernum * 100 + benunit", - "person_id": "sernum * 1000 + person", - "person_table": "adult union child", - "household_sort": "set_index('household_id').sort_index() before positional household reads" - }, - { - "kind": "map_columns", - "scope": "direct raw-to-column E2 mappings only" - }, - { - "kind": "map_coded_amounts", - "scope": "region, tenure, accommodation, council tax band, benefit-code amount mappings" - }, - { - "kind": "annualize_periodic_amounts", - "weeks_in_year": 52.17857142857143 - } - ], - "outputs": [ - "person_id", - "person_benunit_id", - "person_household_id", - "age", - "gender", - "marital_status", - "hours_worked", - "is_household_head", - "is_benunit_head", - "is_parent", - "employment_income", - "self_employment_income", - "private_pension_income", - "tax_free_savings_income", - "savings_interest_income", - "dividend_income", - "property_income", - "maintenance_income", - "miscellaneous_income", - "private_transfer_income", - "lump_sum_income", - "student_loan_repayments", - "statutory_sick_pay", - "statutory_maternity_pay", - "student_loans", - "access_fund", - "education_grants", - "council_tax_benefit_reported", - "maintenance_expenses", - "childcare_expenses", - "personal_pension_contributions", - "employee_pension_contributions", - "pension_contributions_via_salary_sacrifice", - "salary_sacrifice_reported", - "salary_sacrifice_asked", - "child_benefit_reported", - "income_support_reported", - "housing_benefit_reported", - "attendance_allowance_reported", - "dla_sc_reported", - "dla_m_reported", - "iidb_reported", - "carers_allowance_reported", - "sda_reported", - "afcs_reported", - "ssmg_reported", - "pension_credit_reported", - "child_tax_credit_reported", - "working_tax_credit_reported", - "state_pension_reported", - "winter_fuel_allowance_reported", - "incapacity_benefit_reported", - "universal_credit_reported", - "pip_m_reported", - "pip_dl_reported", - "jsa_contrib_reported", - "jsa_income_reported", - "esa_contrib_reported", - "esa_income_reported", - "bsp_reported", - "benunit_id", - "is_married", - "dependent_children", - "household_id", - "region", - "tenure_type", - "accommodation_type", - "num_bedrooms", - "council_tax_reported", - "council_tax_band", - "council_tax_rebate", - "council_tax_single_adult_raw", - "water_and_sewerage_charges", - "domestic_rates", - "rent", - "subrent", - "mortgage_interest_repayment", - "mortgage_capital_repayment", - "structural_insurance_payments", - "housing_service_charges", - "external_child_payments" - ], - "nonnegative_outputs": [ - "age", - "hours_worked", - "employment_income", - "self_employment_income", - "private_pension_income", - "tax_free_savings_income", - "savings_interest_income", - "dividend_income", - "maintenance_income", - "miscellaneous_income", - "student_loan_repayments", - "statutory_sick_pay", - "statutory_maternity_pay", - "childcare_expenses", - "personal_pension_contributions", - "employee_pension_contributions", - "pension_contributions_via_salary_sacrifice", - "rent", - "mortgage_interest_repayment", - "mortgage_capital_repayment", - "structural_insurance_payments", - "housing_service_charges", - "external_child_payments" - ], - "notes": "Root E2 spine assembly. It carries direct raw mappings only; education-grant aggregate and council-tax reported fields remain raw carriers for E3. Benefit take-up, BRMA/LHA assignment, stochastic flags, and imputations are intentionally absent from this stage. The 2024-25 househol.tab is not sernum-ordered; the declared household sort guards stable identity." - }, - { - "stage": "frs_employment", - "survey": "Family Resources Survey 2024-25", - "source": "Department for Work and Pensions Family Resources Survey 2024-25, UK Data Service SN 9563, DOI 10.5255/UKDA-SN-9563-1; local licensed 2024_25 tabs.", - "grain": "person", - "artifacts": [ - { - "role": "frs_table", - "table": "adult", - "kind": "licensed_microdata", - "format": "tab", - "vintage": "2024_25", - "locator": "adult.tab", - "sha256": "4eaea0809a7ccca0fddeb98e358771e4a6e5ebb81b21c4fac4070fc9d227658d", - "size_bytes": 34885825, - "runtime_sha256_required": true, - "tax_year_start": 2024, - "ukds_study_number": 9563, - "doi": "10.5255/UKDA-SN-9563-1" - } - ], - "operations": [ - { - "kind": "read_tables", - "format": "tab", - "delimiter": "\t", - "lowercase_columns": true, - "numeric_errors": "coerce", - "runtime_sha256_required": true - }, - { - "kind": "map_coded_amounts", - "scope": "empstati employment status, mjobsect sector, and raw SIC division" - } - ], - "outputs": [ - "employment_status", - "employment_sector", - "sic_industry_division" - ], - "nonnegative_outputs": [ - "sic_industry_division" - ], - "notes": "Ports FRS employment derivations. empstati code 11 preserves the incumbent truncated-map artifact as LONG_TERM_DISABLED, so OTHER_INACTIVE is not emitted; mjobsect and sic are direct-indexed and fail loudly if absent." - }, - { - "stage": "frs_council_tax", - "survey": "Family Resources Survey 2024-25", - "source": "Department for Work and Pensions Family Resources Survey 2024-25, UK Data Service SN 9563, DOI 10.5255/UKDA-SN-9563-1; local licensed 2024_25 tabs.", - "grain": "household", - "artifacts": [ - { - "role": "frs_table", - "table": "househol", - "kind": "licensed_microdata", - "format": "tab", - "vintage": "2024_25", - "locator": "househol.tab", - "sha256": "2b93b6aed49e1591d6f5360736b4aee3a11ef506b276435f4d2c011a8afbb6a5", - "size_bytes": 12108606, - "runtime_sha256_required": true, - "tax_year_start": 2024, - "ukds_study_number": 9563, - "doi": "10.5255/UKDA-SN-9563-1" - } - ], - "operations": [ - { - "kind": "read_tables", - "format": "tab", - "delimiter": "\t", - "lowercase_columns": true, - "numeric_errors": "coerce", - "runtime_sha256_required": true - }, - { - "kind": "impute_cell_means", - "cells": [ - "gvtregno", - "ctband", - "adulth == 1" - ], - "donor_filter": "raw ctannual > 0", - "missing": "raw ctannual < 0 or NaN", - "value": "Scottish-water-netted CTANNUAL" - } - ], - "outputs": [ - "council_tax" - ], - "nonnegative_outputs": [ - "council_tax" - ], - "notes": "Re-reads raw househol.tab because spine council_tax_reported clips missing values. Scottish Water charges are netted before cell means; no-donor cells impute zero. The dead ct_mean.replace(-1, ...) branch is intentionally dropped." - }, - { - "stage": "frs_disability", - "survey": "Family Resources Survey 2024-25", - "source": "Department for Work and Pensions Family Resources Survey 2024-25, UK Data Service SN 9563, DOI 10.5255/UKDA-SN-9563-1; local licensed 2024_25 tabs. plus policyengine-uk DWP parameters.", - "grain": "person", - "artifacts": [], - "operations": [ - { - "kind": "derive", - "parameters": "disability category thresholds from baseline.gov.dwp at Jan-1" - }, - { - "kind": "derive", - "parameters": "disability flags from gov.dwp at Jan-1" - } - ], - "outputs": [ - "aa_category", - "dla_sc_category", - "dla_m_category", - "pip_m_category", - "pip_dl_category", - "is_disabled_for_benefits", - "is_enhanced_disabled_for_benefits", - "is_severely_disabled_for_benefits" - ], - "notes": "Consumes E2 reported disability amount carriers. The five internal amount carriers are retained through E7 and stripped at E10; export allowlists stay fail-closed." - }, - { - "stage": "frs_education", - "survey": "Family Resources Survey 2024-25", - "source": "Department for Work and Pensions Family Resources Survey 2024-25, UK Data Service SN 9563, DOI 10.5255/UKDA-SN-9563-1; local licensed 2024_25 tabs.", - "grain": "person", - "artifacts": [ - { - "role": "frs_table", - "table": "adult", - "kind": "licensed_microdata", - "format": "tab", - "vintage": "2024_25", - "locator": "adult.tab", - "sha256": "4eaea0809a7ccca0fddeb98e358771e4a6e5ebb81b21c4fac4070fc9d227658d", - "size_bytes": 34885825, - "runtime_sha256_required": true, - "tax_year_start": 2024, - "ukds_study_number": 9563, - "doi": "10.5255/UKDA-SN-9563-1" - }, - { - "role": "frs_table", - "table": "child", - "kind": "licensed_microdata", - "format": "tab", - "vintage": "2024_25", - "locator": "child.tab", - "sha256": "88ec53fc52eea4374864bbc74219d551f4b4c5f54a220bf9607c7a6289719aa5", - "size_bytes": 2753961, - "runtime_sha256_required": true, - "tax_year_start": 2024, - "ukds_study_number": 9563, - "doi": "10.5255/UKDA-SN-9563-1" - } - ], - "operations": [ - { - "kind": "read_tables", - "format": "tab", - "delimiter": "\t", - "lowercase_columns": true, - "numeric_errors": "coerce", - "runtime_sha256_required": true - }, - { - "kind": "derive", - "scope": "current/highest education, QYP inputs, EMA cell-mean degenerate fills, and benefits-in-own-right flag" - }, - { - "kind": "impute_cell_means", - "cells": [ - "single cell: EMA participants (code == 1; adema/ademaamt pair, eduma/edumaamt when adema is absent; chema/chemaamt for children)" - ], - "donor_filter": "participants with non-negative reported amounts", - "missing": "participants with sentinel negative reported amounts", - "value": "donor-mean fill, floored at zero, annualized with 365.25 / 7" - } - ], - "outputs": [ - "current_education", - "highest_education", - "is_in_non_advanced_education", - "is_in_approved_training", - "age_started_or_accepted_current_education_or_training", - "is_before_universal_credit_qualifying_young_person_terminal_date", - "adult_ema", - "child_ema", - "receives_benefits_in_own_right" - ], - "nonnegative_outputs": [ - "adult_ema", - "child_ema", - "age_started_or_accepted_current_education_or_training" - ], - "notes": "Ports the incumbent education cascade including its unreachable POST_SECONDARY branch order. EDUCQUAL_MAP carries the corrected highest-qualification codeframe (1 = Doctorate, descending) per the FRS 2024-25 data dictionary (UK Data Service SN 9563, DOI 10.5255/UKDA-SN-9563-1, adult table), corroborated against the raw aggregates and adopted as a signed difference (PR #703); an upstream defect report records the incumbent inversion. Codes 1-87 are empirically present at 2024-25; code 87 remains undocumented in the SN 9563 dictionary and falls to the default. EMA uses the shared weeks-in-year constant." - }, - { - "stage": "frs_legacy_proxies", - "survey": "Family Resources Survey 2024-25", - "source": "Department for Work and Pensions Family Resources Survey 2024-25, UK Data Service SN 9563, DOI 10.5255/UKDA-SN-9563-1; local licensed 2024_25 tabs. plus policyengine-uk DWP parameters.", - "grain": "person", - "artifacts": [ - { - "role": "frs_table", - "table": "adult", - "kind": "licensed_microdata", - "format": "tab", - "vintage": "2024_25", - "locator": "adult.tab", - "sha256": "4eaea0809a7ccca0fddeb98e358771e4a6e5ebb81b21c4fac4070fc9d227658d", - "size_bytes": 34885825, - "runtime_sha256_required": true, - "tax_year_start": 2024, - "ukds_study_number": 9563, - "doi": "10.5255/UKDA-SN-9563-1" - } - ], - "operations": [ - { - "kind": "read_tables", - "format": "tab", - "delimiter": "\t", - "lowercase_columns": true, - "numeric_errors": "coerce", - "runtime_sha256_required": true - }, - { - "kind": "materialize_rules_engine_predictors", - "predictors": [ - "state_pension_age" - ], - "consumed_only": true - }, - { - "kind": "derive", - "scope": "legacy JSA and ESA claimant-state proxies" - } - ], - "outputs": [ - "legacy_jobseeker_proxy", - "esa_health_condition_proxy", - "esa_support_group_proxy" - ], - "notes": "The proxies are labels, not entitlement determinations. JSA hours compare against 16 * (365.25 / 7) on the E2 spine hours scale; state_pension_age is consumed but not persisted." - }, - { - "stage": "frs_education_grant_split", - "survey": "Family Resources Survey 2024-25", - "source": "Department for Work and Pensions Family Resources Survey 2024-25, UK Data Service SN 9563, DOI 10.5255/UKDA-SN-9563-1; local licensed 2024_25 tabs. plus policyengine-uk DfE grant parameters.", - "grain": "person", - "artifacts": [], - "operations": [ - { - "kind": "materialize_rules_engine_predictors", - "predictors": [ - "childcare_grant", - "parents_learning_allowance", - "adult_dependants_grant" - ], - "consumed_only": true - }, - { - "kind": "derive", - "scope": "proportional split of aggregate education_grants and DSA residual capacity" - } - ], - "outputs": [ - "disabled_students_allowance_eligible_expenses" - ], - "rewrites": [ - "education_grants" - ], - "nonnegative_outputs": [ - "disabled_students_allowance_eligible_expenses" - ], - "notes": "Runs before BRMA and always runs; both are signed inert differences for 2023-24. Pre-2025 DSA capacity is an aligned zero vector rather than an engine shape-sizing read. EDUCQUAL grant-split compatibility is re-cited to the FRS 2024-25 SN 9563 dictionary; codes 1-87 are empirically present and code 87 remains undocumented/defaulted." - }, - { - "stage": "frs_take_up", - "survey": "Family Resources Survey 2024-25", - "source": "Family Resources Survey 2024-25 reported receipt anchors plus sourced UK take-up contract rates.", - "grain": "benunit", - "artifacts": [], - "operations": [ - { - "kind": "aggregate_person_to_benunit", - "method": "any_positive", - "consumed_only": true, - "aggregates": { - "child_benefit_reported_anchor": "child_benefit_reported", - "pension_credit_reported_anchor": "pension_credit_reported", - "universal_credit_reported_anchor": "universal_credit_reported" - } - }, - { - "kind": "assign_binary_with_anchored_residual", - "output": "would_claim_child_benefit", - "draw": "would_claim_child_benefit", - "rate_key": "child_benefit", - "anchor": "child_benefit_reported_anchor", - "seed": 0 - }, - { - "kind": "assign_binary_from_rate", - "output": "child_benefit_opts_out", - "draw": "child_benefit_opts_out", - "rate_key": "child_benefit_opts_out_rate", - "seed": 0 - }, - { - "kind": "assign_binary_with_anchored_residual", - "output": "would_claim_pc", - "draw": "would_claim_pc", - "rate_key": "pension_credit", - "anchor": "pension_credit_reported_anchor", - "seed": 0 - }, - { - "kind": "assign_binary_with_anchored_residual", - "output": "would_claim_uc", - "draw": "would_claim_uc", - "rate_key": "universal_credit", - "anchor": "universal_credit_reported_anchor", - "seed": 0 - }, - { - "kind": "assign_binary_from_rate", - "output": "would_claim_tfc", - "draw": "would_claim_tfc", - "rate_key": "tax_free_childcare", - "seed": 0 - }, - { - "kind": "assign_binary_from_rate", - "output": "would_claim_extended_childcare", - "draw": "would_claim_extended_childcare", - "rate_key": "extended_childcare", - "seed": 0 - }, - { - "kind": "assign_binary_from_rate", - "output": "would_claim_universal_childcare", - "draw": "would_claim_universal_childcare", - "rate_key": "universal_childcare", - "seed": 0 - }, - { - "kind": "assign_binary_from_rate", - "output": "would_claim_targeted_childcare", - "draw": "would_claim_targeted_childcare", - "rate_key": "targeted_childcare", - "seed": 0 - }, - { - "kind": "assign_clipped_normal", - "output": "maximum_extended_childcare_hours_usage", - "draw": "maximum_extended_childcare_hours_usage", - "distribution_key": "maximum_extended_childcare_hours_usage", - "seed": 0 - } - ], - "outputs": [ - "would_claim_child_benefit", - "child_benefit_opts_out", - "would_claim_pc", - "would_claim_uc", - "would_claim_tfc", - "would_claim_extended_childcare", - "would_claim_universal_childcare", - "would_claim_targeted_childcare", - "maximum_extended_childcare_hours_usage" - ], - "nonnegative_outputs": [ - "maximum_extended_childcare_hours_usage" - ], - "notes": "Identity-keyed seed 0 streams replace the incumbent sequential take-up seed 100; salts are output variable names. Reported positive receipt anchors are transient and consumed only." - }, - { - "stage": "frs_person_draws", - "survey": "Family Resources Survey 2024-25", - "source": "Sourced UK take-up contract rates and identity-keyed deterministic draws.", - "grain": "person", - "artifacts": [], - "operations": [ - { - "kind": "assign_binary_from_rate", - "output": "would_claim_marriage_allowance", - "draw": "would_claim_marriage_allowance", - "rate_key": "marriage_allowance", - "seed": 0 - }, - { - "kind": "assign_binary_from_banded_rates", - "output": "would_claim_scp", - "draw": "would_claim_scp", - "band_column": "age", - "bands": [ - { - "upper_exclusive": 6, - "rate_key": "scp_under_6" - }, - { - "rate_key": "scp_6_plus" - } - ], - "seed": 0 - }, - { - "kind": "assign_uniform_draw", - "output": "attends_private_school_random_draw", - "seed": 0 - } - ], - "outputs": [ - "would_claim_marriage_allowance", - "would_claim_scp", - "attends_private_school_random_draw" - ], - "nonnegative_outputs": [ - "attends_private_school_random_draw" - ], - "notes": "Identity-keyed seed 0 streams replace the incumbent sequential take-up seed 100; higher_earner_tie_break is intentionally not produced." - }, - { - "stage": "frs_household_draws", - "survey": "Family Resources Survey 2024-25", - "source": "Sourced UK stochastic contract rates and identity-keyed deterministic draws.", - "grain": "household", - "artifacts": [], - "operations": [ - { - "kind": "assign_binary_from_rate", - "output": "household_owns_tv", - "draw": "household_owns_tv", - "rate_key": "tv_ownership_rate", - "seed": 0 - }, - { - "kind": "assign_binary_from_rate", - "output": "would_evade_tv_licence_fee", - "draw": "would_evade_tv_licence_fee", - "rate_key": "tv_licence_evasion_rate", - "seed": 0 - }, - { - "kind": "assign_binary_from_rate", - "output": "main_residential_property_purchased_is_first_home", - "draw": "main_residential_property_purchased_is_first_home", - "rate_key": "first_time_buyer_rate", - "seed": 0 - }, - { - "kind": "assign_binary_from_rate", - "output": "property_purchased", - "draw": "property_purchased", - "rate_key": "property_purchase_rate", - "seed": 0 - } - ], - "outputs": [ - "household_owns_tv", - "would_evade_tv_licence_fee", - "main_residential_property_purchased_is_first_home", - "property_purchased" - ], - "notes": "Identity-keyed seed 0 streams replace the incumbent sequential take-up seed 100. TV evasion remains an independent draw, matching the incumbent reference share." - }, - { - "stage": "frs_brma", - "survey": "Family Resources Survey 2024-25", - "source": "Valuation Office Agency LHA list-of-rents count table and policyengine-uk LHA_category predictor.", - "grain": "household", - "artifacts": [ - { - "role": "count_resource", - "resource": "brma_rent_counts.json", - "kind": "public_aggregated_counts", - "format": "json" - } - ], - "operations": [ - { - "kind": "materialize_rules_engine_predictors", - "predictors": [ - "LHA_category" - ], - "consumed_only": true - }, - { - "kind": "sample_categorical_from_count_table", - "output": "brma", - "count_resource": "brma_rent_counts.json", - "cell_columns": [ - "region", - "LHA_category" - ], - "assignment_grain": "benunit", - "collapse_to": "household", - "collapse_method": "uniform_member_pick", - "seed": 0 - } - ], - "outputs": [ - "brma" - ], - "notes": "Identity-keyed seed 0 streams replace the incumbent BRMA seed 0 sequential generator. Household collapse uses salt brma:household_pick." - }, - { - "stage": "was_wealth", - "survey": "Wealth and Assets Survey round 8", - "source": "Office for National Statistics Wealth and Assets Survey, UK Data Service SN 7215, DOI 10.5255/UKDA-SN-7215-20; local licensed 2006-22 household tab.", - "grain": "household", - "base_candidate": { - "filename": "populace_uk_2023.h5", - "revision": "populace-uk-2023-dd68c73-4aa4b14-20260619T023711Z", - "sha256": "f17306ccb2aad7ff0130be3589b560afb2e2a12a943570911cd0c77f07934833", - "tier": "frs" + "version": 1, + "country": "uk", + "policy": "The UK HMRC/SPI income family is source-manifest-defined. Private donor data must be supplied locally, every artifact must be SHA-256 verified at runtime, retained FRS constituents and published bands fail closed, and the current replay keeps importance-kind weights because all 208 banded facts require an unavailable full FRS total-income measure.", + "stages": [ + { + "stage": "frs_spine", + "survey": "Family Resources Survey 2024-25", + "source": "Department for Work and Pensions Family Resources Survey 2024-25, UK Data Service SN 9563, DOI 10.5255/UKDA-SN-9563-1; local licensed 2024_25 tabs.", + "grain": "household", + "artifacts": [ + { + "role": "frs_table", + "table": "accounts", + "kind": "licensed_microdata", + "format": "tab", + "vintage": "2024_25", + "locator": "accounts.tab", + "sha256": "fa7871eb45cad0db5fd05ede454ced60405d2f9c598651ea5acea5c91a6ff52f", + "size_bytes": 1812923, + "runtime_sha256_required": true, + "tax_year_start": 2024, + "ukds_study_number": 9563, + "doi": "10.5255/UKDA-SN-9563-1" + }, + { + "role": "frs_table", + "table": "adult", + "kind": "licensed_microdata", + "format": "tab", + "vintage": "2024_25", + "locator": "adult.tab", + "sha256": "4eaea0809a7ccca0fddeb98e358771e4a6e5ebb81b21c4fac4070fc9d227658d", + "size_bytes": 34885825, + "runtime_sha256_required": true, + "tax_year_start": 2024, + "ukds_study_number": 9563, + "doi": "10.5255/UKDA-SN-9563-1" + }, + { + "role": "frs_table", + "table": "benefits", + "kind": "licensed_microdata", + "format": "tab", + "vintage": "2024_25", + "locator": "benefits.tab", + "sha256": "f6ad22b408a13e2239c04b0d076a36418dcf5dd89a8c60daa792c4d735b911d3", + "size_bytes": 2362329, + "runtime_sha256_required": true, + "tax_year_start": 2024, + "ukds_study_number": 9563, + "doi": "10.5255/UKDA-SN-9563-1" + }, + { + "role": "frs_table", + "table": "benunit", + "kind": "licensed_microdata", + "format": "tab", + "vintage": "2024_25", + "locator": "benunit.tab", + "sha256": "66b894624498316d19b6259e287a607e98ed3daacc9be3d3e9067d32b8e09a5a", + "size_bytes": 13986782, + "runtime_sha256_required": true, + "tax_year_start": 2024, + "ukds_study_number": 9563, + "doi": "10.5255/UKDA-SN-9563-1" + }, + { + "role": "frs_table", + "table": "child", + "kind": "licensed_microdata", + "format": "tab", + "vintage": "2024_25", + "locator": "child.tab", + "sha256": "88ec53fc52eea4374864bbc74219d551f4b4c5f54a220bf9607c7a6289719aa5", + "size_bytes": 2753961, + "runtime_sha256_required": true, + "tax_year_start": 2024, + "ukds_study_number": 9563, + "doi": "10.5255/UKDA-SN-9563-1" + }, + { + "role": "frs_table", + "table": "chldcare", + "kind": "licensed_microdata", + "format": "tab", + "vintage": "2024_25", + "locator": "chldcare.tab", + "sha256": "7ccd3f92f299a1f49b24063188177cdb8a958d8bcd753fc3d74dadda6ad04023", + "size_bytes": 275878, + "runtime_sha256_required": true, + "tax_year_start": 2024, + "ukds_study_number": 9563, + "doi": "10.5255/UKDA-SN-9563-1" + }, + { + "role": "frs_table", + "table": "extchild", + "kind": "licensed_microdata", + "format": "tab", + "vintage": "2024_25", + "locator": "extchild.tab", + "sha256": "c661379a4aa5079ce482b1f98f0bfb9157ad9b3ba4eb10739b61846f9c9548e4", + "size_bytes": 15150, + "runtime_sha256_required": true, + "tax_year_start": 2024, + "ukds_study_number": 9563, + "doi": "10.5255/UKDA-SN-9563-1" + }, + { + "role": "frs_table", + "table": "househol", + "kind": "licensed_microdata", + "format": "tab", + "vintage": "2024_25", + "locator": "househol.tab", + "sha256": "2b93b6aed49e1591d6f5360736b4aee3a11ef506b276435f4d2c011a8afbb6a5", + "size_bytes": 12108606, + "runtime_sha256_required": true, + "tax_year_start": 2024, + "ukds_study_number": 9563, + "doi": "10.5255/UKDA-SN-9563-1" + }, + { + "role": "frs_table", + "table": "job", + "kind": "licensed_microdata", + "format": "tab", + "vintage": "2024_25", + "locator": "job.tab", + "sha256": "eb7faf7ada3a3851cb2afb83e2983f8907ffeec897cfbe01e56cb0dfefa853e2", + "size_bytes": 10518760, + "runtime_sha256_required": true, + "tax_year_start": 2024, + "ukds_study_number": 9563, + "doi": "10.5255/UKDA-SN-9563-1" + }, + { + "role": "frs_table", + "table": "maint", + "kind": "licensed_microdata", + "format": "tab", + "vintage": "2024_25", + "locator": "maint.tab", + "sha256": "e7a8d6f47cab7bf9db9bfd7b3ad5ebe5830ec75245d065dcf8654c7c20b97a7d", + "size_bytes": 13993, + "runtime_sha256_required": true, + "tax_year_start": 2024, + "ukds_study_number": 9563, + "doi": "10.5255/UKDA-SN-9563-1" + }, + { + "role": "frs_table", + "table": "mortgage", + "kind": "licensed_microdata", + "format": "tab", + "vintage": "2024_25", + "locator": "mortgage.tab", + "sha256": "6a08f6846970dfdc544a7efc8a93fed4f3210d872cd2d160dfb14ca8d92d5ed0", + "size_bytes": 600552, + "runtime_sha256_required": true, + "tax_year_start": 2024, + "ukds_study_number": 9563, + "doi": "10.5255/UKDA-SN-9563-1" + }, + { + "role": "frs_table", + "table": "oddjob", + "kind": "licensed_microdata", + "format": "tab", + "vintage": "2024_25", + "locator": "oddjob.tab", + "sha256": "dfff1baf71a3de05f3a2fcf0c01a3995df5657f242cd7846aa61f6cc27a1cead", + "size_bytes": 5339, + "runtime_sha256_required": true, + "tax_year_start": 2024, + "ukds_study_number": 9563, + "doi": "10.5255/UKDA-SN-9563-1" + }, + { + "role": "frs_table", + "table": "penprov", + "kind": "licensed_microdata", + "format": "tab", + "vintage": "2024_25", + "locator": "penprov.tab", + "sha256": "9e53de0dc969baec000b3cd68387f0f2dfb3f678732e408de175e0a1d6e3fdc1", + "size_bytes": 513614, + "runtime_sha256_required": true, + "tax_year_start": 2024, + "ukds_study_number": 9563, + "doi": "10.5255/UKDA-SN-9563-1" + }, + { + "role": "frs_table", + "table": "pension", + "kind": "licensed_microdata", + "format": "tab", + "vintage": "2024_25", + "locator": "pension.tab", + "sha256": "2b9be1eb6583cc8916fc06294be27e6217f2aea73da24b97b3226293f6a6ec24", + "size_bytes": 1232411, + "runtime_sha256_required": true, + "tax_year_start": 2024, + "ukds_study_number": 9563, + "doi": "10.5255/UKDA-SN-9563-1" + } + ], + "operations": [ + { + "kind": "read_tables", + "format": "tab", + "delimiter": "\t", + "lowercase_columns": true, + "numeric_errors": "coerce", + "runtime_sha256_required": true + }, + { + "kind": "replace_sentinels", + "sentinel_policy": "raw numeric blanks and nonnumeric sentinels are coerced to NaN, then produced E2 columns are filled to the Frame no-NaN contract. Clamping is deliberately per-column, mirroring the incumbent base build: redamt stays unclamped, tuborr clamps at zero, and the property/royalties components clamp only in their aggregate." + }, + { + "kind": "assemble_group_entities", + "household_id": "sernum", + "benunit_id": "sernum * 100 + benunit", + "person_id": "sernum * 1000 + person", + "person_table": "adult union child", + "household_sort": "set_index('household_id').sort_index() before positional household reads" + }, + { + "kind": "map_columns", + "scope": "direct raw-to-column E2 mappings only" + }, + { + "kind": "map_coded_amounts", + "scope": "region, tenure, accommodation, council tax band, benefit-code amount mappings" + }, + { + "kind": "annualize_periodic_amounts", + "weeks_in_year": 52.17857142857143 + } + ], + "outputs": [ + "person_id", + "person_benunit_id", + "person_household_id", + "age", + "gender", + "marital_status", + "hours_worked", + "is_household_head", + "is_benunit_head", + "is_parent", + "employment_income", + "self_employment_income", + "private_pension_income", + "tax_free_savings_income", + "savings_interest_income", + "dividend_income", + "property_income", + "maintenance_income", + "miscellaneous_income", + "private_transfer_income", + "lump_sum_income", + "student_loan_repayments", + "statutory_sick_pay", + "statutory_maternity_pay", + "student_loans", + "access_fund", + "education_grants", + "healthy_start_vouchers", + "free_school_breakfasts", + "free_school_fruit_veg", + "free_school_meals", + "council_tax_benefit_reported", + "maintenance_expenses", + "childcare_expenses", + "personal_pension_contributions", + "employee_pension_contributions", + "pension_contributions_via_salary_sacrifice", + "salary_sacrifice_reported", + "salary_sacrifice_asked", + "child_benefit_reported", + "income_support_reported", + "housing_benefit_reported", + "attendance_allowance_reported", + "dla_sc_reported", + "dla_m_reported", + "iidb_reported", + "carers_allowance_reported", + "sda_reported", + "afcs_reported", + "ssmg_reported", + "pension_credit_reported", + "child_tax_credit_reported", + "working_tax_credit_reported", + "state_pension_reported", + "winter_fuel_allowance_reported", + "incapacity_benefit_reported", + "universal_credit_reported", + "pip_m_reported", + "pip_dl_reported", + "jsa_contrib_reported", + "jsa_income_reported", + "esa_contrib_reported", + "esa_income_reported", + "bsp_reported", + "benunit_id", + "is_married", + "dependent_children", + "household_id", + "region", + "tenure_type", + "accommodation_type", + "num_bedrooms", + "council_tax_reported", + "council_tax_band", + "council_tax_rebate", + "council_tax_single_adult_raw", + "water_and_sewerage_charges", + "domestic_rates", + "rent", + "subrent", + "mortgage_interest_repayment", + "mortgage_capital_repayment", + "structural_insurance_payments", + "housing_service_charges", + "external_child_payments" + ], + "nonnegative_outputs": [ + "age", + "hours_worked", + "employment_income", + "self_employment_income", + "private_pension_income", + "tax_free_savings_income", + "savings_interest_income", + "dividend_income", + "maintenance_income", + "miscellaneous_income", + "student_loan_repayments", + "statutory_sick_pay", + "statutory_maternity_pay", + "childcare_expenses", + "personal_pension_contributions", + "employee_pension_contributions", + "pension_contributions_via_salary_sacrifice", + "rent", + "mortgage_interest_repayment", + "mortgage_capital_repayment", + "structural_insurance_payments", + "housing_service_charges", + "external_child_payments" + ], + "notes": "Root E2 spine assembly. It carries direct raw mappings only; education-grant aggregate and council-tax reported fields remain raw carriers for E3. Benefit take-up, BRMA/LHA assignment, stochastic flags, and imputations are intentionally absent from this stage. The 2024-25 househol.tab is not sernum-ordered; the declared household sort guards stable identity." + }, + { + "stage": "frs_employment", + "survey": "Family Resources Survey 2024-25", + "source": "Department for Work and Pensions Family Resources Survey 2024-25, UK Data Service SN 9563, DOI 10.5255/UKDA-SN-9563-1; local licensed 2024_25 tabs.", + "grain": "person", + "artifacts": [ + { + "role": "frs_table", + "table": "adult", + "kind": "licensed_microdata", + "format": "tab", + "vintage": "2024_25", + "locator": "adult.tab", + "sha256": "4eaea0809a7ccca0fddeb98e358771e4a6e5ebb81b21c4fac4070fc9d227658d", + "size_bytes": 34885825, + "runtime_sha256_required": true, + "tax_year_start": 2024, + "ukds_study_number": 9563, + "doi": "10.5255/UKDA-SN-9563-1" + } + ], + "operations": [ + { + "kind": "read_tables", + "format": "tab", + "delimiter": "\t", + "lowercase_columns": true, + "numeric_errors": "coerce", + "runtime_sha256_required": true + }, + { + "kind": "map_coded_amounts", + "scope": "empstati employment status, mjobsect sector, and raw SIC division" + } + ], + "outputs": [ + "employment_status", + "employment_sector", + "sic_industry_division" + ], + "nonnegative_outputs": [ + "sic_industry_division" + ], + "notes": "Ports FRS employment derivations. empstati code 11 preserves the incumbent truncated-map artifact as LONG_TERM_DISABLED, so OTHER_INACTIVE is not emitted; mjobsect and sic are direct-indexed and fail loudly if absent." + }, + { + "stage": "frs_council_tax", + "survey": "Family Resources Survey 2024-25", + "source": "Department for Work and Pensions Family Resources Survey 2024-25, UK Data Service SN 9563, DOI 10.5255/UKDA-SN-9563-1; local licensed 2024_25 tabs.", + "grain": "household", + "artifacts": [ + { + "role": "frs_table", + "table": "househol", + "kind": "licensed_microdata", + "format": "tab", + "vintage": "2024_25", + "locator": "househol.tab", + "sha256": "2b93b6aed49e1591d6f5360736b4aee3a11ef506b276435f4d2c011a8afbb6a5", + "size_bytes": 12108606, + "runtime_sha256_required": true, + "tax_year_start": 2024, + "ukds_study_number": 9563, + "doi": "10.5255/UKDA-SN-9563-1" + } + ], + "operations": [ + { + "kind": "read_tables", + "format": "tab", + "delimiter": "\t", + "lowercase_columns": true, + "numeric_errors": "coerce", + "runtime_sha256_required": true + }, + { + "kind": "impute_cell_means", + "cells": [ + "gvtregno", + "ctband", + "adulth == 1" + ], + "donor_filter": "raw ctannual > 0", + "missing": "raw ctannual < 0 or NaN", + "value": "Scottish-water-netted CTANNUAL" + } + ], + "outputs": [ + "council_tax" + ], + "nonnegative_outputs": [ + "council_tax" + ], + "notes": "Re-reads raw househol.tab because spine council_tax_reported clips missing values. Scottish Water charges are netted before cell means; no-donor cells impute zero. The dead ct_mean.replace(-1, ...) branch is intentionally dropped." + }, + { + "stage": "frs_disability", + "survey": "Family Resources Survey 2024-25", + "source": "Department for Work and Pensions Family Resources Survey 2024-25, UK Data Service SN 9563, DOI 10.5255/UKDA-SN-9563-1; local licensed 2024_25 tabs. plus policyengine-uk DWP parameters.", + "grain": "person", + "artifacts": [], + "operations": [ + { + "kind": "derive", + "parameters": "disability category thresholds from baseline.gov.dwp at Jan-1" + }, + { + "kind": "derive", + "parameters": "disability flags from gov.dwp at Jan-1" + } + ], + "outputs": [ + "aa_category", + "dla_sc_category", + "dla_m_category", + "pip_m_category", + "pip_dl_category", + "is_disabled_for_benefits", + "is_enhanced_disabled_for_benefits", + "is_severely_disabled_for_benefits" + ], + "notes": "Consumes E2 reported disability amount carriers. The five internal amount carriers are retained through E7 and stripped at E10; export allowlists stay fail-closed." + }, + { + "stage": "frs_education", + "survey": "Family Resources Survey 2024-25", + "source": "Department for Work and Pensions Family Resources Survey 2024-25, UK Data Service SN 9563, DOI 10.5255/UKDA-SN-9563-1; local licensed 2024_25 tabs.", + "grain": "person", + "artifacts": [ + { + "role": "frs_table", + "table": "adult", + "kind": "licensed_microdata", + "format": "tab", + "vintage": "2024_25", + "locator": "adult.tab", + "sha256": "4eaea0809a7ccca0fddeb98e358771e4a6e5ebb81b21c4fac4070fc9d227658d", + "size_bytes": 34885825, + "runtime_sha256_required": true, + "tax_year_start": 2024, + "ukds_study_number": 9563, + "doi": "10.5255/UKDA-SN-9563-1" + }, + { + "role": "frs_table", + "table": "child", + "kind": "licensed_microdata", + "format": "tab", + "vintage": "2024_25", + "locator": "child.tab", + "sha256": "88ec53fc52eea4374864bbc74219d551f4b4c5f54a220bf9607c7a6289719aa5", + "size_bytes": 2753961, + "runtime_sha256_required": true, + "tax_year_start": 2024, + "ukds_study_number": 9563, + "doi": "10.5255/UKDA-SN-9563-1" + } + ], + "operations": [ + { + "kind": "read_tables", + "format": "tab", + "delimiter": "\t", + "lowercase_columns": true, + "numeric_errors": "coerce", + "runtime_sha256_required": true + }, + { + "kind": "derive", + "scope": "current/highest education, QYP inputs, EMA cell-mean degenerate fills, and benefits-in-own-right flag" + }, + { + "kind": "impute_cell_means", + "cells": [ + "single cell: EMA participants (code == 1; adema/ademaamt pair, eduma/edumaamt when adema is absent; chema/chemaamt for children)" + ], + "donor_filter": "participants with non-negative reported amounts", + "missing": "participants with sentinel negative reported amounts", + "value": "donor-mean fill, floored at zero, annualized with 365.25 / 7" + } + ], + "outputs": [ + "current_education", + "highest_education", + "is_in_non_advanced_education", + "is_in_approved_training", + "age_started_or_accepted_current_education_or_training", + "is_before_universal_credit_qualifying_young_person_terminal_date", + "adult_ema", + "child_ema", + "receives_benefits_in_own_right" + ], + "nonnegative_outputs": [ + "adult_ema", + "child_ema", + "age_started_or_accepted_current_education_or_training" + ], + "notes": "Ports the incumbent education cascade including its unreachable POST_SECONDARY branch order. EDUCQUAL_MAP carries the corrected highest-qualification codeframe (1 = Doctorate, descending) per the FRS 2024-25 data dictionary (UK Data Service SN 9563, DOI 10.5255/UKDA-SN-9563-1, adult table), corroborated against the raw aggregates and adopted as a signed difference (PR #703); an upstream defect report records the incumbent inversion. Codes 1-87 are empirically present at 2024-25; code 87 remains undocumented in the SN 9563 dictionary and falls to the default. EMA uses the shared weeks-in-year constant." + }, + { + "stage": "frs_legacy_proxies", + "survey": "Family Resources Survey 2024-25", + "source": "Department for Work and Pensions Family Resources Survey 2024-25, UK Data Service SN 9563, DOI 10.5255/UKDA-SN-9563-1; local licensed 2024_25 tabs. plus policyengine-uk DWP parameters.", + "grain": "person", + "artifacts": [ + { + "role": "frs_table", + "table": "adult", + "kind": "licensed_microdata", + "format": "tab", + "vintage": "2024_25", + "locator": "adult.tab", + "sha256": "4eaea0809a7ccca0fddeb98e358771e4a6e5ebb81b21c4fac4070fc9d227658d", + "size_bytes": 34885825, + "runtime_sha256_required": true, + "tax_year_start": 2024, + "ukds_study_number": 9563, + "doi": "10.5255/UKDA-SN-9563-1" + } + ], + "operations": [ + { + "kind": "read_tables", + "format": "tab", + "delimiter": "\t", + "lowercase_columns": true, + "numeric_errors": "coerce", + "runtime_sha256_required": true + }, + { + "kind": "materialize_rules_engine_predictors", + "predictors": [ + "state_pension_age" + ], + "consumed_only": true + }, + { + "kind": "derive", + "scope": "legacy JSA and ESA claimant-state proxies" + } + ], + "outputs": [ + "legacy_jobseeker_proxy", + "esa_health_condition_proxy", + "esa_support_group_proxy" + ], + "notes": "The proxies are labels, not entitlement determinations. JSA hours compare against 16 * (365.25 / 7) on the E2 spine hours scale; state_pension_age is consumed but not persisted." + }, + { + "stage": "frs_education_grant_split", + "survey": "Family Resources Survey 2024-25", + "source": "Department for Work and Pensions Family Resources Survey 2024-25, UK Data Service SN 9563, DOI 10.5255/UKDA-SN-9563-1; local licensed 2024_25 tabs. plus policyengine-uk DfE grant parameters.", + "grain": "person", + "artifacts": [], + "operations": [ + { + "kind": "materialize_rules_engine_predictors", + "predictors": [ + "childcare_grant", + "parents_learning_allowance", + "adult_dependants_grant" + ], + "consumed_only": true + }, + { + "kind": "derive", + "scope": "proportional split of aggregate education_grants and DSA residual capacity" + } + ], + "outputs": [ + "disabled_students_allowance_eligible_expenses" + ], + "rewrites": [ + "education_grants" + ], + "nonnegative_outputs": [ + "disabled_students_allowance_eligible_expenses" + ], + "notes": "Runs before BRMA and always runs; both are signed inert differences for 2023-24. Pre-2025 DSA capacity is an aligned zero vector rather than an engine shape-sizing read. EDUCQUAL grant-split compatibility is re-cited to the FRS 2024-25 SN 9563 dictionary; codes 1-87 are empirically present and code 87 remains undocumented/defaulted." + }, + { + "stage": "frs_take_up", + "survey": "Family Resources Survey 2024-25", + "source": "Family Resources Survey 2024-25 reported receipt anchors plus sourced UK take-up contract rates.", + "grain": "benunit", + "artifacts": [], + "operations": [ + { + "kind": "aggregate_person_to_benunit", + "method": "any_positive", + "consumed_only": true, + "aggregates": { + "child_benefit_reported_anchor": "child_benefit_reported", + "pension_credit_reported_anchor": "pension_credit_reported", + "universal_credit_reported_anchor": "universal_credit_reported" + } + }, + { + "kind": "assign_binary_with_anchored_residual", + "output": "would_claim_child_benefit", + "draw": "would_claim_child_benefit", + "rate_key": "child_benefit", + "anchor": "child_benefit_reported_anchor", + "seed": 0 + }, + { + "kind": "assign_binary_from_rate", + "output": "child_benefit_opts_out", + "draw": "child_benefit_opts_out", + "rate_key": "child_benefit_opts_out_rate", + "seed": 0 + }, + { + "kind": "assign_binary_with_anchored_residual", + "output": "would_claim_pc", + "draw": "would_claim_pc", + "rate_key": "pension_credit", + "anchor": "pension_credit_reported_anchor", + "seed": 0 + }, + { + "kind": "assign_binary_with_anchored_residual", + "output": "would_claim_uc", + "draw": "would_claim_uc", + "rate_key": "universal_credit", + "anchor": "universal_credit_reported_anchor", + "seed": 0 + }, + { + "kind": "assign_binary_from_rate", + "output": "would_claim_tfc", + "draw": "would_claim_tfc", + "rate_key": "tax_free_childcare", + "seed": 0 + }, + { + "kind": "assign_binary_from_rate", + "output": "would_claim_extended_childcare", + "draw": "would_claim_extended_childcare", + "rate_key": "extended_childcare", + "seed": 0 + }, + { + "kind": "assign_binary_from_rate", + "output": "would_claim_universal_childcare", + "draw": "would_claim_universal_childcare", + "rate_key": "universal_childcare", + "seed": 0 + }, + { + "kind": "assign_binary_from_rate", + "output": "would_claim_targeted_childcare", + "draw": "would_claim_targeted_childcare", + "rate_key": "targeted_childcare", + "seed": 0 + }, + { + "kind": "assign_clipped_normal", + "output": "maximum_extended_childcare_hours_usage", + "draw": "maximum_extended_childcare_hours_usage", + "distribution_key": "maximum_extended_childcare_hours_usage", + "seed": 0 + } + ], + "outputs": [ + "would_claim_child_benefit", + "child_benefit_opts_out", + "would_claim_pc", + "would_claim_uc", + "would_claim_tfc", + "would_claim_extended_childcare", + "would_claim_universal_childcare", + "would_claim_targeted_childcare", + "maximum_extended_childcare_hours_usage" + ], + "nonnegative_outputs": [ + "maximum_extended_childcare_hours_usage" + ], + "notes": "Identity-keyed seed 0 streams replace the incumbent sequential take-up seed 100; salts are output variable names. Reported positive receipt anchors are transient and consumed only." + }, + { + "stage": "frs_person_draws", + "survey": "Family Resources Survey 2024-25", + "source": "Sourced UK take-up contract rates and identity-keyed deterministic draws.", + "grain": "person", + "artifacts": [], + "operations": [ + { + "kind": "assign_binary_from_rate", + "output": "would_claim_marriage_allowance", + "draw": "would_claim_marriage_allowance", + "rate_key": "marriage_allowance", + "seed": 0 + }, + { + "kind": "assign_binary_from_banded_rates", + "output": "would_claim_scp", + "draw": "would_claim_scp", + "band_column": "age", + "bands": [ + { + "upper_exclusive": 6, + "rate_key": "scp_under_6" }, - "artifacts": [ - { - "role": "was_qrf_donor", - "kind": "private_microdata", - "ukds_study_number": 7215, - "doi": "10.5255/UKDA-SN-7215-20", - "filename": "was_round_8_hhold_eul_may_2025_230525.tab", - "sha256": "18b3eb980c02c99f3d8a3254af859bee31682b2bdc11703877677292b3ce9374", - "size_bytes": 39073613, - "access": "private_local_input", - "locator": "caller-supplied local input", - "runtime_sha256_required": true - } - ], - "operations": [ - { - "kind": "derive", - "scope": "strict WAS donor cleaning with lower-case exact column matching and no fuzzy r/w fallback", - "fillna": 0, - "direct_maps": { - "owned_land": "DVLUKValR8_sum", - "property_wealth": "DVPropertyR8", - "gross_financial_wealth": "HFINWR8_SUM", - "net_financial_wealth": "HFINWNTR8_Sum", - "main_residence_value": "DVhvalueR8", - "other_residential_property_value": "DVHseValR8_sum", - "non_residential_property_value": "DVBlDValR8_sum", - "savings": "DVSaValR8_aggr", - "num_vehicles": "vcarnr8", - "weight": "R8xshhwgt" - }, - "derived": { - "corporate_wealth_excl_isa": "(totalpenr8_aggr - dvvaldbt_scaper8_aggr) + DVFESHARESR8_aggr + DVFShUKVR8_aggr + DVFCollVR8_aggr", - "stocks_and_shares_isa": "DVIISAVR8_aggr", - "cash_isa": "DVCISAVR8_aggr", - "student_loan_balance": "Tot_LosR8_aggr - Tot_los_exc_SLCR8_aggr", - "is_renting": "DVPriRntR8 == 1", - "region": "GORR8 via incumbent REGIONS map" - } - }, - { - "kind": "materialize_rules_engine_predictors", - "predictors": [ - "household_net_income", - "num_adults", - "num_children", - "private_pension_income", - "employment_income", - "self_employment_income", - "capital_income", - "is_renting" - ], - "consumed_only": true - }, - { - "kind": "fit_weighted_qrf_chain", - "weights": "explicit", - "seed": 0, - "n_estimators": 100, - "predictors": [ - "household_net_income", - "num_adults", - "num_children", - "private_pension_income", - "employment_income", - "self_employment_income", - "capital_income", - "num_bedrooms", - "council_tax", - "is_renting", - "region" - ], - "categorical_predictors": [ - "region", - "is_renting" - ], - "region_remap": { - "NORTHERN_IRELAND": "WALES" - }, - "integer_outputs": [ - "num_vehicles" - ], - "chain_order": [ - "owned_land", - "property_wealth", - "corporate_wealth_excl_isa", - "stocks_and_shares_isa", - "corporate_wealth", - "gross_financial_wealth", - "net_financial_wealth", - "main_residence_value", - "other_residential_property_value", - "non_residential_property_value", - "savings", - "num_vehicles", - "student_loan_balance", - "cash_isa" - ] - }, - { - "kind": "fold_into", - "output": "corporate_wealth", - "inputs": [ - "corporate_wealth_excl_isa", - "stocks_and_shares_isa" - ], - "drop_inputs": [ - "corporate_wealth_excl_isa" - ] - }, - { - "kind": "support_clip", - "range": "donor_realized" - }, - { - "kind": "allocate_within_group_waterfall", - "source": "household.student_loan_balance", - "target": "person.student_loan_balance", - "tiers": [ - "student_loan_repayments > 0", - "student_loans > 0", - "highest_education == TERTIARY", - "current_education == TERTIARY", - "18 <= age <= 55", - "everyone" - ], - "keyed_by": "entity_id" - } - ], - "outputs": [ - "owned_land", - "property_wealth", - "corporate_wealth", - "gross_financial_wealth", - "net_financial_wealth", - "main_residence_value", - "other_residential_property_value", - "non_residential_property_value", - "savings", - "num_vehicles", - "cash_isa", - "stocks_and_shares_isa", - "student_loan_balance" - ], - "nonnegative_outputs": [ - "owned_land", - "property_wealth", - "corporate_wealth", - "gross_financial_wealth", - "main_residence_value", - "other_residential_property_value", - "non_residential_property_value", - "savings", - "num_vehicles", - "cash_isa", - "stocks_and_shares_isa", - "student_loan_balance" - ], - "notes": "Ports incumbent WAS round-8 wealth imputation with signed E5 differences: exact lower-case column matching replaces the fuzzy r/w fallback; cash ISA uses DVCISAVR8_aggr and stocks-and-shares ISA uses DVIISAVR8_aggr; corporate_wealth folds stocks-and-shares ISA after drawing corporate_wealth_excl_isa; recipient Northern Ireland regions are mapped to Wales for prediction only; student_loan_balance is allocated by household id rather than the incumbent positional off-by-one. Engine predictors materialize at their native entity and person/benunit values are summed to household, reproducing the incumbent map_to=household semantics; region is one-hot encoded jointly across donor and recipient (the incumbent's dummy encoding), with unmapped donor GOR codes becoming all-zero dummy rows. The WAS and FRS predictor definitions are not fully like-for-like and are ported as-is; raw WAS missing values are blanket-filled with zero; UKDS negative sentinel codes (-9/-8/-7/-6) are recoded to zero for the nonnegative-domain columns the licensed audit found carrying them (vcarnr8: 2 rows; HBedRmR8: 95.8 percent - the bedrooms question is effectively unasked in the WAS household file, predictor-quality revisit registered on microcosm#145) - a signed difference vs the incumbent, which trains on raw sentinels; DVPriRntR8's -9 is structural not-applicable so the is_renting mapping is unchanged; genuinely negative domains are never recoded." - }, - { - "stage": "regional_property_uprating", - "survey": "Public regional property reference", - "source": "MHCLG dwellings and ONS UK House Price Index December 2025 regional average prices.", - "grain": "household", - "base_candidate": { - "filename": "populace_uk_2023.h5", - "revision": "populace-uk-2023-dd68c73-4aa4b14-20260619T023711Z", - "sha256": "f17306ccb2aad7ff0130be3589b560afb2e2a12a943570911cd0c77f07934833", - "tier": "frs" + { + "rate_key": "scp_6_plus" + } + ], + "seed": 0 + }, + { + "kind": "assign_uniform_draw", + "output": "attends_private_school_random_draw", + "seed": 0 + } + ], + "outputs": [ + "would_claim_marriage_allowance", + "would_claim_scp", + "attends_private_school_random_draw" + ], + "nonnegative_outputs": [ + "attends_private_school_random_draw" + ], + "notes": "Identity-keyed seed 0 streams replace the incumbent sequential take-up seed 100; higher_earner_tie_break is intentionally not produced." + }, + { + "stage": "frs_household_draws", + "survey": "Family Resources Survey 2024-25", + "source": "Sourced UK stochastic contract rates and identity-keyed deterministic draws.", + "grain": "household", + "artifacts": [], + "operations": [ + { + "kind": "assign_binary_from_rate", + "output": "household_owns_tv", + "draw": "household_owns_tv", + "rate_key": "tv_ownership_rate", + "seed": 0 + }, + { + "kind": "assign_binary_from_rate", + "output": "would_evade_tv_licence_fee", + "draw": "would_evade_tv_licence_fee", + "rate_key": "tv_licence_evasion_rate", + "seed": 0 + }, + { + "kind": "assign_binary_from_rate", + "output": "main_residential_property_purchased_is_first_home", + "draw": "main_residential_property_purchased_is_first_home", + "rate_key": "first_time_buyer_rate", + "seed": 0 + }, + { + "kind": "assign_binary_from_rate", + "output": "property_purchased", + "draw": "property_purchased", + "rate_key": "property_purchase_rate", + "seed": 0 + } + ], + "outputs": [ + "household_owns_tv", + "would_evade_tv_licence_fee", + "main_residential_property_purchased_is_first_home", + "property_purchased" + ], + "notes": "Identity-keyed seed 0 streams replace the incumbent sequential take-up seed 100. TV evasion remains an independent draw, matching the incumbent reference share." + }, + { + "stage": "frs_brma", + "survey": "Family Resources Survey 2024-25", + "source": "Valuation Office Agency LHA list-of-rents count table and policyengine-uk LHA_category predictor.", + "grain": "household", + "artifacts": [ + { + "role": "count_resource", + "resource": "brma_rent_counts.json", + "kind": "public_aggregated_counts", + "format": "json" + } + ], + "operations": [ + { + "kind": "materialize_rules_engine_predictors", + "predictors": [ + "LHA_category" + ], + "consumed_only": true + }, + { + "kind": "sample_categorical_from_count_table", + "output": "brma", + "count_resource": "brma_rent_counts.json", + "cell_columns": [ + "region", + "LHA_category" + ], + "assignment_grain": "benunit", + "collapse_to": "household", + "collapse_method": "uniform_member_pick", + "seed": 0 + } + ], + "outputs": [ + "brma" + ], + "notes": "Identity-keyed seed 0 streams replace the incumbent BRMA seed 0 sequential generator. Household collapse uses salt brma:household_pick." + }, + { + "stage": "was_wealth", + "survey": "Wealth and Assets Survey round 8", + "source": "Office for National Statistics Wealth and Assets Survey, UK Data Service SN 7215, DOI 10.5255/UKDA-SN-7215-20; local licensed 2006-22 household tab.", + "grain": "household", + "base_candidate": { + "filename": "populace_uk_2023.h5", + "revision": "populace-uk-2023-dd68c73-4aa4b14-20260619T023711Z", + "sha256": "f17306ccb2aad7ff0130be3589b560afb2e2a12a943570911cd0c77f07934833", + "tier": "frs" + }, + "artifacts": [ + { + "role": "was_qrf_donor", + "kind": "private_microdata", + "ukds_study_number": 7215, + "doi": "10.5255/UKDA-SN-7215-20", + "filename": "was_round_8_hhold_eul_may_2025_230525.tab", + "sha256": "18b3eb980c02c99f3d8a3254af859bee31682b2bdc11703877677292b3ce9374", + "size_bytes": 39073613, + "access": "private_local_input", + "locator": "caller-supplied local input", + "runtime_sha256_required": true + } + ], + "operations": [ + { + "kind": "derive", + "scope": "strict WAS donor cleaning with lower-case exact column matching and no fuzzy r/w fallback", + "fillna": 0, + "direct_maps": { + "owned_land": "DVLUKValR8_sum", + "property_wealth": "DVPropertyR8", + "gross_financial_wealth": "HFINWR8_SUM", + "net_financial_wealth": "HFINWNTR8_Sum", + "main_residence_value": "DVhvalueR8", + "other_residential_property_value": "DVHseValR8_sum", + "non_residential_property_value": "DVBlDValR8_sum", + "savings": "DVSaValR8_aggr", + "num_vehicles": "vcarnr8", + "weight": "R8xshhwgt" + }, + "derived": { + "corporate_wealth_excl_isa": "(totalpenr8_aggr - dvvaldbt_scaper8_aggr) + DVFESHARESR8_aggr + DVFShUKVR8_aggr + DVFCollVR8_aggr", + "stocks_and_shares_isa": "DVIISAVR8_aggr", + "cash_isa": "DVCISAVR8_aggr", + "student_loan_balance": "Tot_LosR8_aggr - Tot_los_exc_SLCR8_aggr", + "is_renting": "DVPriRntR8 == 1", + "region": "GORR8 via incumbent REGIONS map" + } + }, + { + "kind": "materialize_rules_engine_predictors", + "predictors": [ + "household_net_income", + "num_adults", + "num_children", + "private_pension_income", + "employment_income", + "self_employment_income", + "capital_income", + "is_renting" + ], + "consumed_only": true + }, + { + "kind": "fit_weighted_qrf_chain", + "weights": "explicit", + "seed": 0, + "n_estimators": 100, + "predictors": [ + "household_net_income", + "num_adults", + "num_children", + "private_pension_income", + "employment_income", + "self_employment_income", + "capital_income", + "num_bedrooms", + "council_tax", + "is_renting", + "region" + ], + "categorical_predictors": [ + "region", + "is_renting" + ], + "region_remap": { + "NORTHERN_IRELAND": "WALES" + }, + "integer_outputs": [ + "num_vehicles" + ], + "chain_order": [ + "owned_land", + "property_wealth", + "corporate_wealth_excl_isa", + "stocks_and_shares_isa", + "corporate_wealth", + "gross_financial_wealth", + "net_financial_wealth", + "main_residence_value", + "other_residential_property_value", + "non_residential_property_value", + "savings", + "num_vehicles", + "student_loan_balance", + "cash_isa" + ] + }, + { + "kind": "fold_into", + "output": "corporate_wealth", + "inputs": [ + "corporate_wealth_excl_isa", + "stocks_and_shares_isa" + ], + "drop_inputs": [ + "corporate_wealth_excl_isa" + ] + }, + { + "kind": "support_clip", + "range": "donor_realized" + }, + { + "kind": "allocate_within_group_waterfall", + "source": "household.student_loan_balance", + "target": "person.student_loan_balance", + "tiers": [ + "student_loan_repayments > 0", + "student_loans > 0", + "highest_education == TERTIARY", + "current_education == TERTIARY", + "18 <= age <= 55", + "everyone" + ], + "keyed_by": "entity_id" + } + ], + "outputs": [ + "owned_land", + "property_wealth", + "corporate_wealth", + "gross_financial_wealth", + "net_financial_wealth", + "main_residence_value", + "other_residential_property_value", + "non_residential_property_value", + "savings", + "num_vehicles", + "cash_isa", + "stocks_and_shares_isa", + "student_loan_balance" + ], + "nonnegative_outputs": [ + "owned_land", + "property_wealth", + "corporate_wealth", + "gross_financial_wealth", + "main_residence_value", + "other_residential_property_value", + "non_residential_property_value", + "savings", + "num_vehicles", + "cash_isa", + "stocks_and_shares_isa", + "student_loan_balance" + ], + "notes": "Ports incumbent WAS round-8 wealth imputation with signed E5 differences: exact lower-case column matching replaces the fuzzy r/w fallback; cash ISA uses DVCISAVR8_aggr and stocks-and-shares ISA uses DVIISAVR8_aggr; corporate_wealth folds stocks-and-shares ISA after drawing corporate_wealth_excl_isa; recipient Northern Ireland regions are mapped to Wales for prediction only; student_loan_balance is allocated by household id rather than the incumbent positional off-by-one. Engine predictors materialize at their native entity and person/benunit values are summed to household, reproducing the incumbent map_to=household semantics; region is one-hot encoded jointly across donor and recipient (the incumbent's dummy encoding), with unmapped donor GOR codes becoming all-zero dummy rows. The WAS and FRS predictor definitions are not fully like-for-like and are ported as-is; raw WAS missing values are blanket-filled with zero; UKDS negative sentinel codes (-9/-8/-7/-6) are recoded to zero for the nonnegative-domain columns the licensed audit found carrying them (vcarnr8: 2 rows; HBedRmR8: 95.8 percent - the bedrooms question is effectively unasked in the WAS household file, predictor-quality revisit registered on microcosm#145) - a signed difference vs the incumbent, which trains on raw sentinels; DVPriRntR8's -9 is structural not-applicable so the is_renting mapping is unchanged; genuinely negative domains are never recoded." + }, + { + "stage": "regional_property_uprating", + "survey": "Public regional property reference", + "source": "MHCLG dwellings and ONS UK House Price Index December 2025 regional average prices.", + "grain": "household", + "base_candidate": { + "filename": "populace_uk_2023.h5", + "revision": "populace-uk-2023-dd68c73-4aa4b14-20260619T023711Z", + "sha256": "f17306ccb2aad7ff0130be3589b560afb2e2a12a943570911cd0c77f07934833", + "tier": "frs" + }, + "artifacts": [ + { + "role": "regional_reference", + "resource": "regional_land_values.json", + "kind": "public_aggregate_reference", + "format": "json" + } + ], + "operations": [ + { + "kind": "uprate_to_regional_reference", + "resource": "regional_land_values.json", + "factor": "avg_house_price / unweighted_mean(main_residence_value | region, >0)", + "owner_rows": "main_residence_value > 0", + "skip_empty_or_nonpositive_regions": true + } + ], + "outputs": [], + "rewrites": [ + "main_residence_value", + "property_wealth" + ], + "notes": "Deterministically rescales owner rows so regional unweighted owner means match the public house-price reference. Northern Ireland has no reference row and is never scaled; empty and nonpositive regions are skipped. The unweighted mean follows the incumbent behavior." + }, + { + "stage": "lcfs_consumption", + "survey": "Living Costs and Food Survey 2023-24", + "source": "UK Data Service SN 9468 Living Costs and Food Survey 2023-24 household/person tabs, NEED 2023 headline energy tables, Ofgem Q2 2026 unit rates, and WAS round-8 bridge donor.", + "grain": "household", + "base_candidate": { + "filename": "populace_uk_2023.h5", + "revision": "populace-uk-2023-dd68c73-4aa4b14-20260619T023711Z", + "sha256": "f17306ccb2aad7ff0130be3589b560afb2e2a12a943570911cd0c77f07934833", + "tier": "frs" + }, + "artifacts": [ + { + "role": "lcfs_household_tab", + "kind": "private_microdata", + "format": "tab", + "vintage": "2023_24", + "locator": "dvhh_ukanon_v2_2023.tab", + "sha256": "6e78f0914be38e63853165486d641cbd790753cc471086210c6f672bfa18ca72", + "size_bytes": 22812887, + "runtime_sha256_required": true, + "filename": "dvhh_ukanon_v2_2023.tab" + }, + { + "role": "lcfs_person_tab", + "kind": "private_microdata", + "format": "tab", + "vintage": "2023_24", + "locator": "dvper_ukanon_202324_2023.tab", + "sha256": "f32d54d83cdecf023f0ac73530be3a99372099b596e0106a56eae42a64929e50", + "size_bytes": 6545146, + "runtime_sha256_required": true, + "filename": "dvper_ukanon_202324_2023.tab" + }, + { + "role": "was_bridge_donor", + "kind": "private_microdata", + "format": "tab", + "vintage": "2018_20", + "locator": "caller-supplied local input", + "sha256": "18b3eb980c02c99f3d8a3254af859bee31682b2bdc11703877677292b3ce9374", + "size_bytes": 39073613, + "runtime_sha256_required": true, + "filename": "was_round_8_hhold_eul_may_2025_230525.tab" + }, + { + "role": "need_energy_targets", + "resource": "need_energy_targets.json", + "kind": "public_aggregate_reference", + "format": "json" + }, + { + "role": "lcfs_consumption_anchors", + "resource": "lcfs_consumption_anchors.json", + "kind": "public_aggregate_reference", + "format": "json" + } + ], + "operations": [ + { + "kind": "derive", + "lowercase_columns": true, + "annualization_weeks": 52.17857142857143, + "donor_weight": "weighta * 1000", + "lossy_mappings": [ + "LCFS tenure 4 and 8 -> RENT_PRIVATELY", + "LCFS accommodation 4 and 5 -> FLAT" + ], + "logged_dropna_row_count": true + }, + { + "kind": "iterative_proportional_fit", + "columns": [ + "electricity_consumption", + "gas_consumption" + ], + "margins": [ + "gross_income_band" + ], + "iterations": 1, + "weighted": false + }, + { + "kind": "bridge_donor_column_via_qrf", + "source": "was_wealth", + "target": "has_fuel_consumption", + "predictors": [ + "household_net_income", + "num_adults", + "num_children", + "private_pension_income", + "employment_income", + "self_employment_income", + "region" + ], + "weights": "explicit", + "seed": 0, + "n_estimators": 100 + }, + { + "kind": "assign_binary_from_rate", + "target": "has_fuel_consumption", + "rate_key": "nts_ice_share", + "condition": "num_vehicles > 0", + "seed": 0 + }, + { + "kind": "materialize_rules_engine_predictors", + "predictors": [ + "is_adult", + "is_child", + "employment_income", + "self_employment_income", + "private_pension_income", + "hbai_household_net_income" + ] + }, + { + "kind": "fit_weighted_qrf_chain", + "predictors": [ + "is_adult", + "is_child", + "region", + "employment_income", + "self_employment_income", + "private_pension_income", + "hbai_household_net_income", + "tenure_type", + "accommodation_type", + "has_fuel_consumption" + ], + "targets": [ + "food_and_non_alcoholic_beverages_consumption", + "alcohol_and_tobacco_consumption", + "clothing_and_footwear_consumption", + "housing_water_and_electricity_consumption", + "household_furnishings_consumption", + "health_consumption", + "transport_consumption", + "communication_consumption", + "recreation_consumption", + "education_consumption", + "restaurants_and_hotels_consumption", + "miscellaneous_consumption", + "petrol_spending", + "diesel_spending", + "bus_fare_spending", + "domestic_energy_consumption", + "electricity_consumption", + "gas_consumption" + ], + "categorical_predictors": [ + "region", + "tenure_type", + "accommodation_type" + ], + "weights": "explicit", + "seed": 0, + "n_estimators": 100 + }, + { + "kind": "support_clip", + "range": "donor_realized", + "exempt": [ + "electricity_consumption", + "gas_consumption", + "domestic_energy_consumption" + ] + }, + { + "kind": "iterative_proportional_fit", + "columns": [ + "electricity_consumption", + "gas_consumption" + ], + "margins": [ + "income", + "tenure", + "accommodation", + "region" + ], + "iterations": 50, + "weighted": true + }, + { + "kind": "fold_into", + "output": "domestic_energy_consumption", + "inputs": [ + "electricity_consumption", + "gas_consumption" + ], + "drop_inputs": false + }, + { + "kind": "zero_when_false", + "columns": [ + "petrol_spending", + "diesel_spending" + ], + "condition": "has_fuel_consumption == false" + } + ], + "outputs": [ + "food_and_non_alcoholic_beverages_consumption", + "alcohol_and_tobacco_consumption", + "clothing_and_footwear_consumption", + "housing_water_and_electricity_consumption", + "household_furnishings_consumption", + "health_consumption", + "transport_consumption", + "communication_consumption", + "recreation_consumption", + "education_consumption", + "restaurants_and_hotels_consumption", + "miscellaneous_consumption", + "petrol_spending", + "diesel_spending", + "bus_fare_spending", + "domestic_energy_consumption", + "electricity_consumption", + "gas_consumption", + "has_fuel_consumption" + ], + "nonnegative_outputs": [ + "food_and_non_alcoholic_beverages_consumption", + "alcohol_and_tobacco_consumption", + "clothing_and_footwear_consumption", + "housing_water_and_electricity_consumption", + "household_furnishings_consumption", + "health_consumption", + "transport_consumption", + "communication_consumption", + "recreation_consumption", + "education_consumption", + "restaurants_and_hotels_consumption", + "miscellaneous_consumption", + "petrol_spending", + "diesel_spending", + "bus_fare_spending", + "domestic_energy_consumption", + "electricity_consumption", + "gas_consumption", + "has_fuel_consumption" + ], + "notes": "Ports the incumbent LCFS consumption QRF, including NEED energy raking, with adjudicated weighted fits and identity-keyed seed-0 fuel flags. Energy support clipping is exempt because NEED raking governs those columns. Donor uprating is identity at this vintage (LCFS survey year equals the 2023 build year), so the incumbent's CPI and fuel litre-proxy donor uprating is deliberately not declared here; the machinery lands with the FRS 2024-25 refresh (microcosm#687), where a donor/build year gap first exists. The DESNZ pump-price anchors stay committed as the cited litre-proxy denominators for that refresh." + }, + { + "stage": "etb_vat", + "survey": "Effects of Taxes and Benefits 1977-2024", + "source": "UK Data Service SN 8856 Effects of Taxes and Benefits household tab and cited VAT anchor resource.", + "grain": "household", + "base_candidate": { + "filename": "populace_uk_2023.h5", + "revision": "populace-uk-2023-dd68c73-4aa4b14-20260619T023711Z", + "sha256": "f17306ccb2aad7ff0130be3589b560afb2e2a12a943570911cd0c77f07934833", + "tier": "frs" + }, + "artifacts": [ + { + "role": "etb_household_tab", + "kind": "private_microdata", + "format": "tab", + "vintage": "1977_24", + "locator": "householdv2_1977-2024.tab", + "sha256": "d0e94ebc92e85ca1b9fb3a7353dcaf41db2c5110c9f07c7793dc8c0b695250d8", + "size_bytes": 216967663, + "runtime_sha256_required": true, + "filename": "householdv2_1977-2024.tab" + }, + { + "role": "etb_policy_anchors", + "resource": "etb_policy_anchors.json", + "kind": "public_parameter_reference", + "format": "json" + } + ], + "operations": [ + { + "kind": "derive", + "year": 2023, + "annualization_weeks": 52, + "standard_rate": 0.2, + "reduced_rate_share": 0.025, + "fail_loud_on_missing_rate": true + }, + { + "kind": "materialize_rules_engine_predictors", + "predictors": [ + "is_adult", + "is_child", + "is_SP_age", + "household_net_income" + ] + }, + { + "kind": "fit_weighted_qrf", + "predictors": [ + "is_adult", + "is_child", + "is_SP_age", + "household_net_income" + ], + "targets": [ + "full_rate_vat_expenditure_rate" + ], + "weights": "explicit", + "seed": 0, + "n_estimators": 100 + }, + { + "kind": "support_clip", + "range": "donor_realized" + } + ], + "outputs": [ + "full_rate_vat_expenditure_rate" + ], + "nonnegative_outputs": [], + "notes": "Ports ETB VAT imputation using the 2023 donor year and cited VAT anchors; missing or NaN rates fail loud rather than falling back. The donor-realized support includes negative rates (4 of 4,199 cleaned 2023 donor rows, minimum -3.4: totvat can exceed expdis in the raw ETB accounts), so the output is deliberately absent from nonnegative_outputs and the support gate is the guard - the net_financial_wealth precedent from was_wealth." + }, + { + "stage": "etb_services", + "survey": "Effects of Taxes and Benefits 1977-2024 and NHS age-gender public table", + "source": "UK Data Service SN 8856 Effects of Taxes and Benefits household tab, DfT rail fare index, and public NHS activity/cost table.", + "grain": "household+person", + "base_candidate": { + "filename": "populace_uk_2023.h5", + "revision": "populace-uk-2023-dd68c73-4aa4b14-20260619T023711Z", + "sha256": "f17306ccb2aad7ff0130be3589b560afb2e2a12a943570911cd0c77f07934833", + "tier": "frs" + }, + "artifacts": [ + { + "role": "etb_household_tab", + "kind": "private_microdata", + "format": "tab", + "vintage": "1977_24", + "locator": "householdv2_1977-2024.tab", + "sha256": "d0e94ebc92e85ca1b9fb3a7353dcaf41db2c5110c9f07c7793dc8c0b695250d8", + "size_bytes": 216967663, + "runtime_sha256_required": true, + "filename": "householdv2_1977-2024.tab" + }, + { + "role": "nhs_consumption_by_age_gender", + "resource": "nhs_consumption_by_age_gender.json", + "kind": "public_aggregate_reference", + "format": "json" + }, + { + "role": "etb_services_anchors", + "resource": "etb_services_anchors.json", + "kind": "public_parameter_reference", + "format": "json" + } + ], + "operations": [ + { + "kind": "derive", + "year": "max", + "annualization_weeks": 52 + }, + { + "kind": "materialize_rules_engine_predictors", + "predictors": [ + "is_adult", + "is_child", + "is_SP_age", + "dla", + "pip", + "hbai_household_net_income", + "current_education" + ], + "derived_predictors": { + "count_primary_education": "current_education == PRIMARY", + "count_secondary_education": "current_education == LOWER_SECONDARY", + "count_further_education": "current_education in (UPPER_SECONDARY, TERTIARY)" + } + }, + { + "kind": "fit_weighted_qrf_chain", + "predictors": [ + "is_adult", + "is_child", + "is_SP_age", + "count_primary_education", + "count_secondary_education", + "count_further_education", + "dla", + "pip", + "hbai_household_net_income" + ], + "targets": [ + "dfe_education_spending", + "rail_subsidy_spending", + "bus_subsidy_spending" + ], + "weights": "explicit", + "seed": 0, + "n_estimators": 100 + }, + { + "kind": "support_clip", + "range": "donor_realized" + }, + { + "kind": "compute_ratio", + "output": "rail_usage", + "numerator": "rail_subsidy_spending", + "denominator_resource": "etb_services_anchors.json", + "denominator_key": "rail_fare_index_2023" + }, + { + "kind": "allocate_per_capita_from_cell_table", + "resource": "nhs_consumption_by_age_gender.json", + "budget_resource": "etb_services_anchors.json", + "age_bands": "half_open", + "top_band_fold_in": "85+" + } + ], + "outputs": [ + "dfe_education_spending", + "rail_subsidy_spending", + "bus_subsidy_spending", + "rail_usage", + "a_and_e_visits", + "admitted_patient_visits", + "outpatient_visits", + "nhs_a_and_e_spending", + "nhs_admitted_patient_spending", + "nhs_outpatient_spending" + ], + "nonnegative_outputs": [ + "dfe_education_spending", + "rail_subsidy_spending", + "bus_subsidy_spending", + "rail_usage", + "a_and_e_visits", + "admitted_patient_visits", + "outpatient_visits", + "nhs_a_and_e_spending", + "nhs_admitted_patient_spending", + "nhs_outpatient_spending" + ], + "notes": "Ports ETB public-services QRF at household grain, computes rail_usage from the 2023 fare index, and allocates NHS visits/spending to persons using the signed half-open age-band and 85+ fold-in fixes. The year-max donor filter resolves to 2023 on the pinned tab (the file labels financial year ending 2024 as year 2023), so the services training year coincides with the VAT training year and the fare-index year - the incumbent's apparent three-way year mismatch is vacuous on this vintage." + }, + { + "stage": "frs_hmrc_spine_leaves", + "survey": "Family Resources Survey 2024-25", + "source": "Department for Work and Pensions Family Resources Survey 2024-25 raw adult.tab and benefits.tab, caller-supplied local input", + "grain": "person", + "artifacts": [ + { + "role": "frs_table", + "table": "adult", + "kind": "licensed_microdata", + "format": "tab", + "vintage": "2024_25", + "locator": "adult.tab", + "sha256": "4eaea0809a7ccca0fddeb98e358771e4a6e5ebb81b21c4fac4070fc9d227658d", + "size_bytes": 34885825, + "runtime_sha256_required": true, + "tax_year_start": 2024, + "ukds_study_number": 9563, + "doi": "10.5255/UKDA-SN-9563-1" + }, + { + "role": "frs_table", + "table": "benefits", + "kind": "licensed_microdata", + "format": "tab", + "vintage": "2024_25", + "locator": "benefits.tab", + "sha256": "f6ad22b408a13e2239c04b0d076a36418dcf5dd89a8c60daa792c4d735b911d3", + "size_bytes": 2362329, + "runtime_sha256_required": true, + "tax_year_start": 2024, + "ukds_study_number": 9563, + "doi": "10.5255/UKDA-SN-9563-1" + } + ], + "operations": [ + { + "kind": "retain_adjudicated_frs_hmrc_leaves", + "population": "uk_frs_raw_spine", + "source_vintage": "2024-25", + "mapped_build_period": 2024, + "annualization": "weekly raw FRS amounts * (365.25 / 7)", + "status": "adjudicated_partial_replay", + "retained_full_constituents": { + "hmrc_spi_pay": { + "spi_concept": "PAY", + "scope": "full", + "raw_sources": [ + "ADULT.INEARNS" + ], + "formula": "max(0, ADULT.INEARNS) * (365.25 / 7)" }, - "artifacts": [ - { - "role": "regional_reference", - "resource": "regional_land_values.json", - "kind": "public_aggregate_reference", - "format": "json" - } - ], - "operations": [ - { - "kind": "uprate_to_regional_reference", - "resource": "regional_land_values.json", - "factor": "avg_house_price / unweighted_mean(main_residence_value | region, >0)", - "owner_rows": "main_residence_value > 0", - "skip_empty_or_nonpositive_regions": true - } - ], - "outputs": [], - "rewrites": [ - "main_residence_value", - "property_wealth" - ], - "notes": "Deterministically rescales owner rows so regional unweighted owner means match the public house-price reference. Northern Ireland has no reference row and is never scaled; empty and nonpositive regions are skipped. The unweighted mean follows the incumbent behavior." - }, - { - "stage": "lcfs_consumption", - "survey": "Living Costs and Food Survey 2023-24", - "source": "UK Data Service SN 9468 Living Costs and Food Survey 2023-24 household/person tabs, NEED 2023 headline energy tables, Ofgem Q2 2026 unit rates, and WAS round-8 bridge donor.", - "grain": "household", - "base_candidate": { - "filename": "populace_uk_2023.h5", - "revision": "populace-uk-2023-dd68c73-4aa4b14-20260619T023711Z", - "sha256": "f17306ccb2aad7ff0130be3589b560afb2e2a12a943570911cd0c77f07934833", - "tier": "frs" + "hmrc_spi_unemployment_benefit_income": { + "spi_concept": "UBISJA", + "scope": "full", + "raw_sources": [ + "BENEFITS.BENEFIT=14:BENAMT", + "BENEFITS.BENEFIT=19:BENAMT" + ], + "formula": "sum(BENAMT where BENEFIT in {14, 19}) * (365.25 / 7)" }, - "artifacts": [ - { - "role": "lcfs_household_tab", - "kind": "private_microdata", - "format": "tab", - "vintage": "2023_24", - "locator": "dvhh_ukanon_v2_2023.tab", - "sha256": "6e78f0914be38e63853165486d641cbd790753cc471086210c6f672bfa18ca72", - "size_bytes": 22812887, - "runtime_sha256_required": true, - "filename": "dvhh_ukanon_v2_2023.tab" - }, - { - "role": "lcfs_person_tab", - "kind": "private_microdata", - "format": "tab", - "vintage": "2023_24", - "locator": "dvper_ukanon_202324_2023.tab", - "sha256": "f32d54d83cdecf023f0ac73530be3a99372099b596e0106a56eae42a64929e50", - "size_bytes": 6545146, - "runtime_sha256_required": true, - "filename": "dvper_ukanon_202324_2023.tab" - }, - { - "role": "was_bridge_donor", - "kind": "private_microdata", - "format": "tab", - "vintage": "2018_20", - "locator": "caller-supplied local input", - "sha256": "18b3eb980c02c99f3d8a3254af859bee31682b2bdc11703877677292b3ce9374", - "size_bytes": 39073613, - "runtime_sha256_required": true, - "filename": "was_round_8_hhold_eul_may_2025_230525.tab" - }, - { - "role": "need_energy_targets", - "resource": "need_energy_targets.json", - "kind": "public_aggregate_reference", - "format": "json" - }, - { - "role": "lcfs_consumption_anchors", - "resource": "lcfs_consumption_anchors.json", - "kind": "public_aggregate_reference", - "format": "json" - } - ], - "operations": [ - { - "kind": "derive", - "lowercase_columns": true, - "annualization_weeks": 52.17857142857143, - "donor_weight": "weighta * 1000", - "lossy_mappings": [ - "LCFS tenure 4 and 8 -> RENT_PRIVATELY", - "LCFS accommodation 4 and 5 -> FLAT" - ], - "logged_dropna_row_count": true - }, - { - "kind": "iterative_proportional_fit", - "columns": [ - "electricity_consumption", - "gas_consumption" - ], - "margins": [ - "gross_income_band" - ], - "iterations": 1, - "weighted": false - }, - { - "kind": "bridge_donor_column_via_qrf", - "source": "was_wealth", - "target": "has_fuel_consumption", - "predictors": [ - "household_net_income", - "num_adults", - "num_children", - "private_pension_income", - "employment_income", - "self_employment_income", - "region" - ], - "weights": "explicit", - "seed": 0, - "n_estimators": 100 - }, - { - "kind": "assign_binary_from_rate", - "target": "has_fuel_consumption", - "rate_key": "nts_ice_share", - "condition": "num_vehicles > 0", - "seed": 0 - }, - { - "kind": "materialize_rules_engine_predictors", - "predictors": [ - "is_adult", - "is_child", - "employment_income", - "self_employment_income", - "private_pension_income", - "hbai_household_net_income" - ] - }, - { - "kind": "fit_weighted_qrf_chain", - "predictors": [ - "is_adult", - "is_child", - "region", - "employment_income", - "self_employment_income", - "private_pension_income", - "hbai_household_net_income", - "tenure_type", - "accommodation_type", - "has_fuel_consumption" - ], - "targets": [ - "food_and_non_alcoholic_beverages_consumption", - "alcohol_and_tobacco_consumption", - "clothing_and_footwear_consumption", - "housing_water_and_electricity_consumption", - "household_furnishings_consumption", - "health_consumption", - "transport_consumption", - "communication_consumption", - "recreation_consumption", - "education_consumption", - "restaurants_and_hotels_consumption", - "miscellaneous_consumption", - "petrol_spending", - "diesel_spending", - "bus_fare_spending", - "domestic_energy_consumption", - "electricity_consumption", - "gas_consumption" - ], - "categorical_predictors": [ - "region", - "tenure_type", - "accommodation_type" - ], - "weights": "explicit", - "seed": 0, - "n_estimators": 100 - }, - { - "kind": "support_clip", - "range": "donor_realized", - "exempt": [ - "electricity_consumption", - "gas_consumption", - "domestic_energy_consumption" - ] - }, - { - "kind": "iterative_proportional_fit", - "columns": [ - "electricity_consumption", - "gas_consumption" - ], - "margins": [ - "income", - "tenure", - "accommodation", - "region" - ], - "iterations": 50, - "weighted": true - }, - { - "kind": "fold_into", - "output": "domestic_energy_consumption", - "inputs": [ - "electricity_consumption", - "gas_consumption" - ], - "drop_inputs": false - }, - { - "kind": "zero_when_false", - "columns": [ - "petrol_spending", - "diesel_spending" - ], - "condition": "has_fuel_consumption == false" - } - ], - "outputs": [ - "food_and_non_alcoholic_beverages_consumption", - "alcohol_and_tobacco_consumption", - "clothing_and_footwear_consumption", - "housing_water_and_electricity_consumption", - "household_furnishings_consumption", - "health_consumption", - "transport_consumption", - "communication_consumption", - "recreation_consumption", - "education_consumption", - "restaurants_and_hotels_consumption", - "miscellaneous_consumption", - "petrol_spending", - "diesel_spending", - "bus_fare_spending", - "domestic_energy_consumption", - "electricity_consumption", - "gas_consumption", - "has_fuel_consumption" - ], - "nonnegative_outputs": [ - "food_and_non_alcoholic_beverages_consumption", - "alcohol_and_tobacco_consumption", - "clothing_and_footwear_consumption", - "housing_water_and_electricity_consumption", - "household_furnishings_consumption", - "health_consumption", - "transport_consumption", - "communication_consumption", - "recreation_consumption", - "education_consumption", - "restaurants_and_hotels_consumption", - "miscellaneous_consumption", - "petrol_spending", - "diesel_spending", - "bus_fare_spending", - "domestic_energy_consumption", - "electricity_consumption", - "gas_consumption", - "has_fuel_consumption" - ], - "notes": "Ports the incumbent LCFS consumption QRF, including NEED energy raking, with adjudicated weighted fits and identity-keyed seed-0 fuel flags. Energy support clipping is exempt because NEED raking governs those columns. Donor uprating is identity at this vintage (LCFS survey year equals the 2023 build year), so the incumbent's CPI and fuel litre-proxy donor uprating is deliberately not declared here; the machinery lands with the FRS 2024-25 refresh (microcosm#687), where a donor/build year gap first exists. The DESNZ pump-price anchors stay committed as the cited litre-proxy denominators for that refresh." - }, - { - "stage": "etb_vat", - "survey": "Effects of Taxes and Benefits 1977-2024", - "source": "UK Data Service SN 8856 Effects of Taxes and Benefits household tab and cited VAT anchor resource.", - "grain": "household", - "base_candidate": { - "filename": "populace_uk_2023.h5", - "revision": "populace-uk-2023-dd68c73-4aa4b14-20260619T023711Z", - "sha256": "f17306ccb2aad7ff0130be3589b560afb2e2a12a943570911cd0c77f07934833", - "tier": "frs" + "hmrc_spi_incapacity_benefit_income": { + "spi_concept": "INCPBEN", + "scope": "full", + "raw_sources": [ + "BENEFITS.BENEFIT=17:BENAMT" + ], + "formula": "sum(BENAMT where BENEFIT == 17) * (365.25 / 7)", + "observed_support": "structural zero in the audited 2023-24 FRS; retained so future vintages flow" + } + }, + "retained_named_subsets": { + "ossben_identifiable_subset": { + "spi_concept": "OSSBEN", + "raw_sources": [ + "BENEFITS.BENEFIT=13:BENAMT", + "BENEFITS.BENEFIT=16,VAR2 in {1,3}:BENAMT" + ], + "formula": "sum(BENAMT where BENEFIT == 13 or (BENEFIT == 16 and VAR2 in {1, 3})) * (365.25 / 7)", + "scope": "identifiable_subset" }, - "artifacts": [ - { - "role": "etb_household_tab", - "kind": "private_microdata", - "format": "tab", - "vintage": "1977_24", - "locator": "householdv2_1977-2024.tab", - "sha256": "d0e94ebc92e85ca1b9fb3a7353dcaf41db2c5110c9f07c7793dc8c0b695250d8", - "size_bytes": 216967663, - "runtime_sha256_required": true, - "filename": "householdv2_1977-2024.tab" - }, - { - "role": "etb_policy_anchors", - "resource": "etb_policy_anchors.json", - "kind": "public_parameter_reference", - "format": "json" - } - ], - "operations": [ - { - "kind": "derive", - "year": 2023, - "annualization_weeks": 52, - "standard_rate": 0.2, - "reduced_rate_share": 0.025, - "fail_loud_on_missing_rate": true - }, - { - "kind": "materialize_rules_engine_predictors", - "predictors": [ - "is_adult", - "is_child", - "is_SP_age", - "household_net_income" - ] - }, - { - "kind": "fit_weighted_qrf", - "predictors": [ - "is_adult", - "is_child", - "is_SP_age", - "household_net_income" - ], - "targets": [ - "full_rate_vat_expenditure_rate" - ], - "weights": "explicit", - "seed": 0, - "n_estimators": 100 - }, - { - "kind": "support_clip", - "range": "donor_realized" - } - ], - "outputs": [ - "full_rate_vat_expenditure_rate" - ], - "nonnegative_outputs": [], - "notes": "Ports ETB VAT imputation using the 2023 donor year and cited VAT anchors; missing or NaN rates fail loud rather than falling back. The donor-realized support includes negative rates (4 of 4,199 cleaned 2023 donor rows, minimum -3.4: totvat can exceed expdis in the raw ETB accounts), so the output is deliberately absent from nonnegative_outputs and the support gate is the guard - the net_financial_wealth precedent from was_wealth." - }, - { - "stage": "etb_services", - "survey": "Effects of Taxes and Benefits 1977-2024 and NHS age-gender public table", - "source": "UK Data Service SN 8856 Effects of Taxes and Benefits household tab, DfT rail fare index, and public NHS activity/cost table.", - "grain": "household+person", - "base_candidate": { - "filename": "populace_uk_2023.h5", - "revision": "populace-uk-2023-dd68c73-4aa4b14-20260619T023711Z", - "sha256": "f17306ccb2aad7ff0130be3589b560afb2e2a12a943570911cd0c77f07934833", - "tier": "frs" + "srp_regular_code5": { + "spi_concept": "SRP", + "raw_sources": [ + "BENEFITS.BENEFIT=5:BENAMT" + ], + "formula": "sum(BENAMT where BENEFIT == 5) * (365.25 / 7)", + "scope": "regular_code5_subset" + } + }, + "source_absent_full_constituents": [ + "EPB", + "EXPS", + "TAXTERM", + "MOTHINC", + "OTHERINC" + ], + "full_concepts_forbidden_on_frs": [ + "hmrc_spi_employment_benefits", + "hmrc_spi_employment_expenses", + "hmrc_spi_taxable_termination_pay", + "hmrc_spi_miscellaneous_employment_income", + "hmrc_spi_other_income", + "hmrc_spi_other_social_security_income", + "hmrc_spi_state_pension_income" + ], + "forbid_proxy_substitution": [ + "employment_income", + "miscellaneous_income" + ], + "fail_on_missing_retained_constituent": true, + "fail_on_full_concept_alias": true + }, + { + "kind": "derive", + "output": "employer_pension_contributions", + "formula": "3 * employee_pension_contributions", + "source": "incumbent enhanced-FRS frs.py employer-pension contribution estimate" + } + ], + "outputs": [ + "hmrc_spi_pay", + "hmrc_spi_unemployment_benefit_income", + "hmrc_spi_incapacity_benefit_income", + "ossben_identifiable_subset", + "srp_regular_code5", + "employer_pension_contributions" + ], + "nonnegative_outputs": [ + "hmrc_spi_pay", + "hmrc_spi_unemployment_benefit_income", + "hmrc_spi_incapacity_benefit_income", + "ossben_identifiable_subset", + "srp_regular_code5", + "employer_pension_contributions" + ], + "notes": "Retains adjudicated raw-FRS HMRC leaves on the raw FRS spine, where person_id equals the raw sernum*1000+person identity, then ports the incumbent employer-pension-contributions estimate." + }, + { + "stage": "spi_support_channel", + "survey": "Family Resources Survey 2024-25", + "source": "Synthetic SPI support channel sampled uniformly without replacement from the raw FRS spine before cloning.", + "grain": "household", + "artifacts": [], + "operations": [ + { + "kind": "stack_zero_weight_donors", + "count": 10000, + "seed": 42, + "flag_column": "household_is_spi_synthetic", + "channels": { + "base": "frs", + "synthetic": "spi" + }, + "draw": "uniform_without_replacement" + }, + { + "kind": "gate_zero_weight_strata", + "gate": "uk_zero_weight_strata_gate", + "declarations": [ + { + "name": "e7_spi_synthetic_preclone", + "selector": { + "household_is_spi_synthetic": true + }, + "maximum_zero_weight_rows": 10000, + "reason": "E7 stacks exactly 10,000 zero-weight SPI-synthetic households on the raw pre-clone UK FRS spine." + } + ] + }, + { + "kind": "allocate_zero_weight_prior_mass", + "share": 0.5, + "strata": [ + "region" + ], + "weight_kind_out": "importance", + "conservation": "exact_total" + } + ], + "outputs": [ + "household_is_spi_synthetic", + "person_support_channel", + "person_support_clone_index", + "person_source_id", + "benunit_support_channel", + "benunit_support_clone_index", + "benunit_source_id", + "household_support_channel", + "household_support_clone_index", + "household_source_id", + "source_household_id", + "source_year", + "source_household_key" + ], + "nonnegative_outputs": [ + "person_support_clone_index", + "person_source_id", + "benunit_support_clone_index", + "benunit_source_id", + "household_support_clone_index", + "household_source_id", + "source_household_id", + "source_year" + ], + "notes": "Stacks the pre-clone SPI support channel, gates the 10,000 zero-weight synthetic households under a stage-local declaration, and allocates 50 percent of each region stratum prior mass to the SPI channel with exact national conservation." + }, + { + "stage": "hmrc_spi_income_spine", + "survey": "Survey of Personal Incomes Public Use Tape 2022-23 and HMRC Personal Incomes Tables 3.6/3.7 2023-24", + "source": "https://assets.publishing.service.gov.uk/media/69f1f12d2fae53a03709682f/Collated_Tables_3_1_to_3_11_2324.ods", + "grain": "person", + "artifacts": [ + { + "role": "qrf_donor", + "kind": "private_microdata", + "format": "tab_delimited", + "survey": "Survey of Personal Incomes Public Use Tape 2022-23", + "vintage": "2022-23", + "tax_year_start": 2022, + "ukds_study_number": "SN 9422", + "doi": "10.5255/UKDA-SN-9422-1", + "filename": "put2223uk.tab", + "sha256": "5ef829461060c91a2a47be59ad541d9b519fc3976d66ca80d4920f711bb96f66", + "size_bytes": 141323762, + "reviewed_source": "PolicyEngine licensed UKDS mirror (private Hugging Face repository), spi_2022_23.zip", + "access": "private_local_input", + "locator": "caller-supplied local input", + "runtime_sha256_required": true + }, + { + "role": "published_fact_surface", + "kind": "administrative_table", + "format": "ods", + "survey": "HMRC Personal Incomes Tables 3.6 and 3.7", + "publication": "https://www.gov.uk/government/statistics/personal-incomes-statistics-for-the-tax-year-2023-to-2024", + "vintage": "2023-24", + "tax_year_start": 2023, + "locator": "https://assets.publishing.service.gov.uk/media/69f1f12d2fae53a03709682f/Collated_Tables_3_1_to_3_11_2324.ods", + "sha256": "ad063b06b2bdeef8600dbbb09d48153337a4966f8c7eea50df7a2e0304ebd73e", + "size_bytes": 166693, + "mime_type": "application/vnd.oasis.opendocument.spreadsheet", + "sheets": [ + "Table_3_6", + "Table_3_7" + ], + "mapped_build_period": 2024, + "period_mapping": "latest_published_tax_year", + "runtime_sha256_required": true + } + ], + "operations": [ + { + "kind": "verify_pinned_hmrc_source_pair", + "artifact_roles": [ + "qrf_donor", + "published_fact_surface" + ], + "require_before_source_read": true, + "runtime_sha256_required": true, + "fail_on_mismatch": true + }, + { + "kind": "strict_read_private_table", + "artifact_role": "qrf_donor", + "filename": "put2223uk.tab", + "delimiter": "\t", + "weight": "FACT", + "required_columns": [ + "AGERANGE", + "GORCODE", + "SEX", + "FACT", + "PAY", + "EPB", + "EXPS", + "TAXTERM", + "INCPBEN", + "OSSBEN", + "UBISJA", + "MOTHINC", + "OTHERINC", + "PROFITS", + "CAPALL", + "LOSSBF", + "SRP", + "INCBBS", + "DIVIDENDS", + "PENSION", + "INCPROP", + "OTHERINV", + "GIFTAID", + "GIFTINV", + "TEI", + "TII", + "TI" + ], + "runtime_sha256_required": true, + "fail_on_missing_file": true, + "fail_on_missing_columns": true, + "fail_on_invalid_weight": true, + "seed": 42 + }, + { + "kind": "fit_weighted_qrf_stage1", + "training_artifact_role": "qrf_donor", + "predictors": [ + "age", + "gender", + "region" + ], + "categorical_predictors": [ + "gender", + "region" + ], + "source_sampling_weight": "FACT", + "sample_size": 100000, + "sample_with_replacement": true, + "post_sample_fit_weight": "uniform", + "fit_weight_kind": "design", + "double_apply_source_weight": false, + "source_columns": { + "self_employment_income": [ + "PROFITS", + "CAPALL", + "LOSSBF" + ], + "savings_interest_income": [ + "INCBBS" + ], + "dividend_income": [ + "DIVIDENDS" + ], + "private_pension_income": [ + "PENSION" + ], + "property_income": [ + "INCPROP" + ], + "other_investment_income": [ + "OTHERINV" + ], + "gift_aid": [ + "GIFTAID" + ], + "charitable_investment_gifts": [ + "GIFTINV" + ], + "hmrc_spi_pay": [ + "PAY" + ], + "hmrc_spi_employment_benefits": [ + "EPB" + ], + "hmrc_spi_employment_expenses": [ + "EXPS" + ], + "hmrc_spi_incapacity_benefit_income": [ + "INCPBEN" + ], + "hmrc_spi_other_social_security_income": [ + "OSSBEN" + ], + "hmrc_spi_taxable_termination_pay": [ + "TAXTERM" + ], + "hmrc_spi_unemployment_benefit_income": [ + "UBISJA" + ], + "hmrc_spi_miscellaneous_employment_income": [ + "MOTHINC" + ], + "hmrc_spi_other_income": [ + "OTHERINC" + ], + "hmrc_spi_state_pension_income": [ + "SRP" + ] + }, + "derived_policyengine_outputs": { + "employment_income": { + "source_columns": [ + "PAY", + "EPB", + "TAXTERM" + ], + "formula": "hmrc_spi_pay + hmrc_spi_employment_benefits + hmrc_spi_taxable_termination_pay", + "derive_after_draw": true + } + }, + "outputs": [ + "self_employment_income", + "savings_interest_income", + "dividend_income", + "private_pension_income", + "property_income", + "other_investment_income", + "gift_aid", + "charitable_investment_gifts", + "hmrc_spi_pay", + "hmrc_spi_employment_benefits", + "hmrc_spi_employment_expenses", + "hmrc_spi_incapacity_benefit_income", + "hmrc_spi_other_social_security_income", + "hmrc_spi_taxable_termination_pay", + "hmrc_spi_unemployment_benefit_income", + "hmrc_spi_miscellaneous_employment_income", + "hmrc_spi_other_income", + "hmrc_spi_state_pension_income" + ], + "joint_draw": true, + "savings_interest_source_semantics": "INCBBS is taxable bank/building-society interest before reconstruction to the PolicyEngine gross input", + "employment_income_source_semantics": "PolicyEngine input = PAY + EPB + TAXTERM, matching the pinned enhanced-FRS pipeline; it is not the Table 3.6 measure", + "hmrc_employed_income_source_semantics": "Derived after each draw as max(0, PAY + EPB - EXPS) + INCPBEN + OSSBEN + TAXTERM + UBISJA + MOTHINC, using normalized leaves identically on FRS and SPI channels", + "self_employment_income_source_semantics": "max(0, PROFITS - CAPALL - LOSSBF)", + "assessable_income_source_semantics": "QRF draws leaves only; TEI, TII, and TI are deterministic post-draw accounting aggregates and TI equals TEI + TII exactly", + "source_ti_identity_fields": [ + "TI", + "TEI", + "TII" + ], + "source_leaf_reconciliation": { + "documentation_url": "https://doc.ukdataservice.ac.uk/doc/9422/mrdoc/pdf/9422_put_2223_full_documentation.pdf", + "composite_indicator": "AGERANGE == -1", + "formulas": { + "TEI": "max(0, PAY + EPB - EXPS) + INCPBEN + OSSBEN + TAXTERM + UBISJA + MOTHINC + OTHERINC + SRP + PENSION + max(0, PROFITS - CAPALL - LOSSBF)", + "TII": "OTHERINV + DIVIDENDS + INCPROP + INCBBS", + "TI": "TEI + TII" }, - "artifacts": [ - { - "role": "etb_household_tab", - "kind": "private_microdata", - "format": "tab", - "vintage": "1977_24", - "locator": "householdv2_1977-2024.tab", - "sha256": "d0e94ebc92e85ca1b9fb3a7353dcaf41db2c5110c9f07c7793dc8c0b695250d8", - "size_bytes": 216967663, - "runtime_sha256_required": true, - "filename": "householdv2_1977-2024.tab" - }, - { - "role": "nhs_consumption_by_age_gender", - "resource": "nhs_consumption_by_age_gender.json", - "kind": "public_aggregate_reference", - "format": "json" - }, - { - "role": "etb_services_anchors", - "resource": "etb_services_anchors.json", - "kind": "public_parameter_reference", - "format": "json" - } - ], - "operations": [ - { - "kind": "derive", - "year": "max", - "annualization_weeks": 52 - }, - { - "kind": "materialize_rules_engine_predictors", - "predictors": [ - "is_adult", - "is_child", - "is_SP_age", - "dla", - "pip", - "hbai_household_net_income", - "current_education" - ], - "derived_predictors": { - "count_primary_education": "current_education == PRIMARY", - "count_secondary_education": "current_education == LOWER_SECONDARY", - "count_further_education": "current_education in (UPPER_SECONDARY, TERTIARY)" - } - }, - { - "kind": "fit_weighted_qrf_chain", - "predictors": [ - "is_adult", - "is_child", - "is_SP_age", - "count_primary_education", - "count_secondary_education", - "count_further_education", - "dla", - "pip", - "hbai_household_net_income" - ], - "targets": [ - "dfe_education_spending", - "rail_subsidy_spending", - "bus_subsidy_spending" - ], - "weights": "explicit", - "seed": 0, - "n_estimators": 100 - }, - { - "kind": "support_clip", - "range": "donor_realized" - }, - { - "kind": "compute_ratio", - "output": "rail_usage", - "numerator": "rail_subsidy_spending", - "denominator_resource": "etb_services_anchors.json", - "denominator_key": "rail_fare_index_2023" - }, - { - "kind": "allocate_per_capita_from_cell_table", - "resource": "nhs_consumption_by_age_gender.json", - "budget_resource": "etb_services_anchors.json", - "age_bands": "half_open", - "top_band_fold_in": "85+" - } - ], - "outputs": [ - "dfe_education_spending", - "rail_subsidy_spending", - "bus_subsidy_spending", - "rail_usage", - "a_and_e_visits", - "admitted_patient_visits", - "outpatient_visits", - "nhs_a_and_e_spending", - "nhs_admitted_patient_spending", - "nhs_outpatient_spending" - ], - "nonnegative_outputs": [ - "dfe_education_spending", - "rail_subsidy_spending", - "bus_subsidy_spending", - "rail_usage", - "a_and_e_visits", - "admitted_patient_visits", - "outpatient_visits", - "nhs_a_and_e_spending", - "nhs_admitted_patient_spending", - "nhs_outpatient_spending" - ], - "notes": "Ports ETB public-services QRF at household grain, computes rail_usage from the 2023 fare index, and allocates NHS visits/spending to persons using the signed half-open age-band and 85+ fold-in fixes. The year-max donor filter resolves to 2023 on the pinned tab (the file labels financial year ending 2024 as year 2023), so the services training year coincides with the VAT training year and the fare-index year - the incumbent's apparent three-way year mismatch is vacuous on this vintage." - }, - { - "stage": "frs_hmrc_spine_leaves", - "survey": "Family Resources Survey 2024-25", - "source": "Department for Work and Pensions Family Resources Survey 2024-25 raw adult.tab and benefits.tab, caller-supplied local input", - "grain": "person", - "artifacts": [ - { - "role": "frs_table", - "table": "adult", - "kind": "licensed_microdata", - "format": "tab", - "vintage": "2024_25", - "locator": "adult.tab", - "sha256": "4eaea0809a7ccca0fddeb98e358771e4a6e5ebb81b21c4fac4070fc9d227658d", - "size_bytes": 34885825, - "runtime_sha256_required": true, - "tax_year_start": 2024, - "ukds_study_number": 9563, - "doi": "10.5255/UKDA-SN-9563-1" - }, - { - "role": "frs_table", - "table": "benefits", - "kind": "licensed_microdata", - "format": "tab", - "vintage": "2024_25", - "locator": "benefits.tab", - "sha256": "f6ad22b408a13e2239c04b0d076a36418dcf5dd89a8c60daa792c4d735b911d3", - "size_bytes": 2362329, - "runtime_sha256_required": true, - "tax_year_start": 2024, - "ukds_study_number": 9563, - "doi": "10.5255/UKDA-SN-9563-1" - } - ], - "operations": [ - { - "kind": "retain_adjudicated_frs_hmrc_leaves", - "population": "uk_frs_raw_spine", - "source_vintage": "2024-25", - "mapped_build_period": 2024, - "annualization": "weekly raw FRS amounts * (365.25 / 7)", - "status": "adjudicated_partial_replay", - "retained_full_constituents": { - "hmrc_spi_pay": { - "spi_concept": "PAY", - "scope": "full", - "raw_sources": [ - "ADULT.INEARNS" - ], - "formula": "max(0, ADULT.INEARNS) * (365.25 / 7)" - }, - "hmrc_spi_unemployment_benefit_income": { - "spi_concept": "UBISJA", - "scope": "full", - "raw_sources": [ - "BENEFITS.BENEFIT=14:BENAMT", - "BENEFITS.BENEFIT=19:BENAMT" - ], - "formula": "sum(BENAMT where BENEFIT in {14, 19}) * (365.25 / 7)" - }, - "hmrc_spi_incapacity_benefit_income": { - "spi_concept": "INCPBEN", - "scope": "full", - "raw_sources": [ - "BENEFITS.BENEFIT=17:BENAMT" - ], - "formula": "sum(BENAMT where BENEFIT == 17) * (365.25 / 7)", - "observed_support": "structural zero in the audited 2023-24 FRS; retained so future vintages flow" - } - }, - "retained_named_subsets": { - "ossben_identifiable_subset": { - "spi_concept": "OSSBEN", - "raw_sources": [ - "BENEFITS.BENEFIT=13:BENAMT", - "BENEFITS.BENEFIT=16,VAR2 in {1,3}:BENAMT" - ], - "formula": "sum(BENAMT where BENEFIT == 13 or (BENEFIT == 16 and VAR2 in {1, 3})) * (365.25 / 7)", - "scope": "identifiable_subset" - }, - "srp_regular_code5": { - "spi_concept": "SRP", - "raw_sources": [ - "BENEFITS.BENEFIT=5:BENAMT" - ], - "formula": "sum(BENAMT where BENEFIT == 5) * (365.25 / 7)", - "scope": "regular_code5_subset" - } - }, - "source_absent_full_constituents": [ - "EPB", - "EXPS", - "TAXTERM", - "MOTHINC", - "OTHERINC" - ], - "full_concepts_forbidden_on_frs": [ - "hmrc_spi_employment_benefits", - "hmrc_spi_employment_expenses", - "hmrc_spi_taxable_termination_pay", - "hmrc_spi_miscellaneous_employment_income", - "hmrc_spi_other_income", - "hmrc_spi_other_social_security_income", - "hmrc_spi_state_pension_income" - ], - "forbid_proxy_substitution": [ - "employment_income", - "miscellaneous_income" - ], - "fail_on_missing_retained_constituent": true, - "fail_on_full_concept_alias": true - }, - { - "kind": "derive", - "output": "employer_pension_contributions", - "formula": "3 * employee_pension_contributions", - "source": "incumbent enhanced-FRS frs.py employer-pension contribution estimate" - } - ], - "outputs": [ - "hmrc_spi_pay", - "hmrc_spi_unemployment_benefit_income", - "hmrc_spi_incapacity_benefit_income", - "ossben_identifiable_subset", - "srp_regular_code5", - "employer_pension_contributions" - ], - "nonnegative_outputs": [ - "hmrc_spi_pay", - "hmrc_spi_unemployment_benefit_income", - "hmrc_spi_incapacity_benefit_income", - "ossben_identifiable_subset", - "srp_regular_code5", - "employer_pension_contributions" - ], - "notes": "Retains adjudicated raw-FRS HMRC leaves on the raw FRS spine, where person_id equals the raw sernum*1000+person identity, then ports the incumbent employer-pension-contributions estimate." - }, - { - "stage": "spi_support_channel", - "survey": "Family Resources Survey 2024-25", - "source": "Synthetic SPI support channel sampled uniformly without replacement from the raw FRS spine before cloning.", - "grain": "household", - "artifacts": [], - "operations": [ - { - "kind": "stack_zero_weight_donors", - "count": 10000, - "seed": 42, - "flag_column": "household_is_spi_synthetic", - "channels": { - "base": "frs", - "synthetic": "spi" - }, - "draw": "uniform_without_replacement" - }, - { - "kind": "gate_zero_weight_strata", - "gate": "uk_zero_weight_strata_gate", - "declarations": [ - { - "name": "e7_spi_synthetic_preclone", - "selector": { - "household_is_spi_synthetic": true - }, - "maximum_zero_weight_rows": 10000, - "reason": "E7 stacks exactly 10,000 zero-weight SPI-synthetic households on the raw pre-clone UK FRS spine." - } - ] - }, - { - "kind": "allocate_zero_weight_prior_mass", - "share": 0.5, - "strata": [ - "region" - ], - "weight_kind_out": "importance", - "conservation": "exact_total" - } - ], - "outputs": [ - "household_is_spi_synthetic", - "person_support_channel", - "person_support_clone_index", - "person_source_id", - "benunit_support_channel", - "benunit_support_clone_index", - "benunit_source_id", - "household_support_channel", - "household_support_clone_index", - "household_source_id", - "source_household_id", - "source_year", - "source_household_key" - ], - "nonnegative_outputs": [ - "person_support_clone_index", - "person_source_id", - "benunit_support_clone_index", - "benunit_source_id", - "household_support_clone_index", - "household_source_id", - "source_household_id", - "source_year" - ], - "notes": "Stacks the pre-clone SPI support channel, gates the 10,000 zero-weight synthetic households under a stage-local declaration, and allocates 50 percent of each region stratum prior mass to the SPI channel with exact national conservation." - }, - { - "stage": "hmrc_spi_income_spine", - "survey": "Survey of Personal Incomes Public Use Tape 2022-23 and HMRC Personal Incomes Tables 3.6/3.7 2023-24", - "source": "https://assets.publishing.service.gov.uk/media/69f1f12d2fae53a03709682f/Collated_Tables_3_1_to_3_11_2324.ods", - "grain": "person", - "artifacts": [ - { - "role": "qrf_donor", - "kind": "private_microdata", - "format": "tab_delimited", - "survey": "Survey of Personal Incomes Public Use Tape 2022-23", - "vintage": "2022-23", - "tax_year_start": 2022, - "ukds_study_number": "SN 9422", - "doi": "10.5255/UKDA-SN-9422-1", - "filename": "put2223uk.tab", - "sha256": "5ef829461060c91a2a47be59ad541d9b519fc3976d66ca80d4920f711bb96f66", - "size_bytes": 141323762, - "reviewed_source": "PolicyEngine licensed UKDS mirror (private Hugging Face repository), spi_2022_23.zip", - "access": "private_local_input", - "locator": "caller-supplied local input", - "runtime_sha256_required": true - }, - { - "role": "published_fact_surface", - "kind": "administrative_table", - "format": "ods", - "survey": "HMRC Personal Incomes Tables 3.6 and 3.7", - "publication": "https://www.gov.uk/government/statistics/personal-incomes-statistics-for-the-tax-year-2023-to-2024", - "vintage": "2023-24", - "tax_year_start": 2023, - "locator": "https://assets.publishing.service.gov.uk/media/69f1f12d2fae53a03709682f/Collated_Tables_3_1_to_3_11_2324.ods", - "sha256": "ad063b06b2bdeef8600dbbb09d48153337a4966f8c7eea50df7a2e0304ebd73e", - "size_bytes": 166693, - "mime_type": "application/vnd.oasis.opendocument.spreadsheet", - "sheets": [ - "Table_3_6", - "Table_3_7" - ], - "mapped_build_period": 2024, - "period_mapping": "latest_published_tax_year", - "runtime_sha256_required": true - } - ], - "operations": [ - { - "kind": "verify_pinned_hmrc_source_pair", - "artifact_roles": [ - "qrf_donor", - "published_fact_surface" - ], - "require_before_source_read": true, - "runtime_sha256_required": true, - "fail_on_mismatch": true - }, - { - "kind": "strict_read_private_table", - "artifact_role": "qrf_donor", - "filename": "put2223uk.tab", - "delimiter": "\t", - "weight": "FACT", - "required_columns": [ - "AGERANGE", - "GORCODE", - "SEX", - "FACT", - "PAY", - "EPB", - "EXPS", - "TAXTERM", - "INCPBEN", - "OSSBEN", - "UBISJA", - "MOTHINC", - "OTHERINC", - "PROFITS", - "CAPALL", - "LOSSBF", - "SRP", - "INCBBS", - "DIVIDENDS", - "PENSION", - "INCPROP", - "OTHERINV", - "GIFTAID", - "GIFTINV", - "TEI", - "TII", - "TI" - ], - "runtime_sha256_required": true, - "fail_on_missing_file": true, - "fail_on_missing_columns": true, - "fail_on_invalid_weight": true, - "seed": 42 - }, - { - "kind": "fit_weighted_qrf_stage1", - "training_artifact_role": "qrf_donor", - "predictors": [ - "age", - "gender", - "region" - ], - "categorical_predictors": [ - "gender", - "region" - ], - "source_sampling_weight": "FACT", - "sample_size": 100000, - "sample_with_replacement": true, - "post_sample_fit_weight": "uniform", - "fit_weight_kind": "design", - "double_apply_source_weight": false, - "source_columns": { - "self_employment_income": [ - "PROFITS", - "CAPALL", - "LOSSBF" - ], - "savings_interest_income": [ - "INCBBS" - ], - "dividend_income": [ - "DIVIDENDS" - ], - "private_pension_income": [ - "PENSION" - ], - "property_income": [ - "INCPROP" - ], - "other_investment_income": [ - "OTHERINV" - ], - "gift_aid": [ - "GIFTAID" - ], - "charitable_investment_gifts": [ - "GIFTINV" - ], - "hmrc_spi_pay": [ - "PAY" - ], - "hmrc_spi_employment_benefits": [ - "EPB" - ], - "hmrc_spi_employment_expenses": [ - "EXPS" - ], - "hmrc_spi_incapacity_benefit_income": [ - "INCPBEN" - ], - "hmrc_spi_other_social_security_income": [ - "OSSBEN" - ], - "hmrc_spi_taxable_termination_pay": [ - "TAXTERM" - ], - "hmrc_spi_unemployment_benefit_income": [ - "UBISJA" - ], - "hmrc_spi_miscellaneous_employment_income": [ - "MOTHINC" - ], - "hmrc_spi_other_income": [ - "OTHERINC" - ], - "hmrc_spi_state_pension_income": [ - "SRP" - ] - }, - "derived_policyengine_outputs": { - "employment_income": { - "source_columns": [ - "PAY", - "EPB", - "TAXTERM" - ], - "formula": "hmrc_spi_pay + hmrc_spi_employment_benefits + hmrc_spi_taxable_termination_pay", - "derive_after_draw": true - } - }, - "outputs": [ - "self_employment_income", - "savings_interest_income", - "dividend_income", - "private_pension_income", - "property_income", - "other_investment_income", - "gift_aid", - "charitable_investment_gifts", - "hmrc_spi_pay", - "hmrc_spi_employment_benefits", - "hmrc_spi_employment_expenses", - "hmrc_spi_incapacity_benefit_income", - "hmrc_spi_other_social_security_income", - "hmrc_spi_taxable_termination_pay", - "hmrc_spi_unemployment_benefit_income", - "hmrc_spi_miscellaneous_employment_income", - "hmrc_spi_other_income", - "hmrc_spi_state_pension_income" - ], - "joint_draw": true, - "savings_interest_source_semantics": "INCBBS is taxable bank/building-society interest before reconstruction to the PolicyEngine gross input", - "employment_income_source_semantics": "PolicyEngine input = PAY + EPB + TAXTERM, matching the pinned enhanced-FRS pipeline; it is not the Table 3.6 measure", - "hmrc_employed_income_source_semantics": "Derived after each draw as max(0, PAY + EPB - EXPS) + INCPBEN + OSSBEN + TAXTERM + UBISJA + MOTHINC, using normalized leaves identically on FRS and SPI channels", - "self_employment_income_source_semantics": "max(0, PROFITS - CAPALL - LOSSBF)", - "assessable_income_source_semantics": "QRF draws leaves only; TEI, TII, and TI are deterministic post-draw accounting aggregates and TI equals TEI + TII exactly", - "source_ti_identity_fields": [ - "TI", - "TEI", - "TII" - ], - "source_leaf_reconciliation": { - "documentation_url": "https://doc.ukdataservice.ac.uk/doc/9422/mrdoc/pdf/9422_put_2223_full_documentation.pdf", - "composite_indicator": "AGERANGE == -1", - "formulas": { - "TEI": "max(0, PAY + EPB - EXPS) + INCPBEN + OSSBEN + TAXTERM + UBISJA + MOTHINC + OTHERINC + SRP + PENSION + max(0, PROFITS - CAPALL - LOSSBF)", - "TII": "OTHERINV + DIVIDENDS + INCPROP + INCBBS", - "TI": "TEI + TII" - }, - "maximum_absolute_difference_gbp": { - "ordinary": { - "TEI": 15, - "TII": 10, - "TI": 20 - }, - "composite": { - "TEI": 180, - "TII": 10, - "TI": 180 - } - }, - "rationale": "The official PUT rounds source fields, averages documented composite records, then rounds remaining income fields to GBP 5. These are the observed envelopes in the exact sha-pinned donor; post-draw synthetic identities remain exact." - }, - "ti_identity_absolute_tolerance_gbp": 5, - "stochastic_aggregates_forbidden": [ - "hmrc_spi_employed_income", - "hmrc_spi_total_earned_income", - "hmrc_spi_total_investment_income", - "hmrc_spi_assessable_income" - ], - "require_all_predictors": true, - "require_all_outputs": true, - "initialize_frs_channel_columns": { - "gift_aid": 0, - "charitable_investment_gifts": 0 - }, - "n_estimators": 100, - "seed": 42 - }, - { - "kind": "fit_weighted_qrf_stage2", - "training_population": "uk_frs_raw_spine_base_support_channel", - "target_population": "spi_synthetic_support_channel", - "predictors": [ - "age", - "gender", - "region", - "employment_income", - "self_employment_income", - "savings_interest_income", - "dividend_income", - "private_pension_income", - "property_income" - ], - "reviewed_absent_predictors": { - "other_investment_income": "This remains a stage-1 SPI draw and an official HMRC fact component, but it is not an FRS-only stage-2 predictor: the incumbent UK data build's frs_only.py defines exactly six income predictors and the certified Microcosm UK base candidate has no other_investment_income column." - }, - "categorical_predictors": [ - "gender", - "region" - ], - "weight": "household_weight", - "weight_mapping": "household_to_person", - "outputs": [ - "employee_pension_contributions", - "employer_pension_contributions", - "personal_pension_contributions", - "pension_contributions_via_salary_sacrifice", - "tax_free_savings_income", - "universal_credit_reported", - "pension_credit_reported", - "child_benefit_reported", - "housing_benefit_reported", - "income_support_reported", - "working_tax_credit_reported", - "child_tax_credit_reported", - "attendance_allowance_reported", - "state_pension_reported", - "dla_sc_reported", - "dla_m_reported", - "pip_m_reported", - "pip_dl_reported", - "sda_reported", - "carers_allowance_reported", - "iidb_reported", - "afcs_reported", - "bsp_reported", - "winter_fuel_allowance_reported", - "council_tax_benefit_reported", - "jsa_contrib_reported", - "jsa_income_reported", - "esa_contrib_reported", - "esa_income_reported" - ], - "reviewed_absent_outputs": { - "incapacity_benefit_reported": "Absent/all-default on the pinned enhanced-FRS export and certified Microcosm UK base; not a populated loader layer.", - "maternity_allowance_reported": "Absent from the pinned enhanced-FRS export and certified Microcosm UK base; no training source can be materialized for this stage." - }, - "postprocess": { - "gross_savings_interest_income": "stage1 INCBBS draw + stage2 tax_free_savings_income", - "refresh_disability_categories": [ - "aa_category", - "dla_sc_category", - "dla_m_category", - "pip_m_category", - "pip_dl_category" - ], - "refresh_disability_flags": [ - "is_disabled_for_benefits", - "is_enhanced_disabled_for_benefits", - "is_severely_disabled_for_benefits" - ] - }, - "joint_draw": true, - "require_all_predictors": true, - "require_all_materializable_outputs": true, - "require_all_outputs": false, - "n_estimators": 100, - "seed": 43 - }, - { - "kind": "redraw_columns_from_fitted_qrf", - "fit": "stage1", - "columns": [ - "dividend_income" - ], - "rows": "base_support_channel" - }, - { - "kind": "materialize_hmrc_income_bands_fail_closed", - "artifact_role": "published_fact_surface", - "mapped_build_period": 2024, - "period_mapping": "latest_published_tax_year", - "column_index_base": 0, - "data_row_start_index": 5, - "stop_label": "All ranges", - "count_unit_multiplier": 1000, - "amount_unit_multiplier": 1000000, - "component_columns": { - "employment_income": { - "sheet": "Table_3_6", - "count_column_index": 4, - "amount_column_index": 5 - }, - "self_employment_income": { - "sheet": "Table_3_6", - "count_column_index": 1, - "amount_column_index": 2 - }, - "state_pension": { - "sheet": "Table_3_6", - "count_column_index": 7, - "amount_column_index": 8 - }, - "private_pension_income": { - "sheet": "Table_3_6", - "count_column_index": 10, - "amount_column_index": 11 - }, - "property_income": { - "sheet": "Table_3_7", - "count_column_index": 1, - "amount_column_index": 2 - }, - "savings_interest_income": { - "sheet": "Table_3_7", - "count_column_index": 4, - "amount_column_index": 5 - }, - "dividend_income": { - "sheet": "Table_3_7", - "count_column_index": 7, - "amount_column_index": 8 - }, - "other_investment_income": { - "sheet": "Table_3_7", - "count_column_index": 10, - "amount_column_index": 11 - } - }, - "required_band_lower_bounds_gbp": [ - 12570, - 15000, - 20000, - 30000, - 40000, - 50000, - 70000, - 100000, - 150000, - 200000, - 300000, - 500000, - 1000000 - ], - "required_measures": [ - "count", - "amount" - ], - "fail_on_missing_sheet": true, - "fail_on_missing_component": true, - "fail_on_missing_band": true, - "fail_on_non_numeric_value": true - }, - { - "kind": "classify_hmrc_income_facts_with_reviewed_fences", - "target_operation": "materialize_hmrc_income_bands_fail_closed", - "components": [ - "employment_income", - "self_employment_income", - "state_pension", - "private_pension_income", - "property_income", - "savings_interest_income", - "dividend_income", - "other_investment_income" - ], - "breakdown_dependency": "hmrc_spi_assessable_income", - "frs_breakdown_status": "unavailable_full_measure", - "input_weight_kind": "importance", - "output_weight_kind": "importance", - "calibration_permitted": false, - "required_fact_count": 208, - "outcome_counts": { - "exact_pass": 0, - "exact_fail": 0, - "directional_pass": 0, - "directional_fail": 0, - "excluded_with_fence": 208 - }, - "classification_rationale": "Every published fact uses non-overlapping total-income bands. The FRS channel cannot materialize full TEI, and omitted income can move a person between bands, so neither an exact fact nor a per-band directional bound is valid.", - "reviewed_fences": [ - { - "fence_id": "frs_epb_source_absent", - "constituents": [ - "EPB" - ], - "raw_sources_searched": [ - "JOB.EXPBEN01-EXPBEN13", - "JOB.CARVAL", - "JOB.CARAMT", - "JOB.FUELAMT", - "JOB.VCHAMT", - "JOB.CHVAMT" - ], - "finding": "Missing. EXPBEN* are receipt flags, and the amount fields cover only selected benefits; they cannot produce complete taxable expenses payments and benefits.", - "mass_implication": "12.9485464% of certified-candidate FRS effective person mass has at least one receipt flag, but this is not monetary support.", - "rationale": "Receipt flags and selected benefit amounts cannot be promoted to the SPI EPB monetary concept without an imputation or proxy.", - "dependent_fence_ids": [] - }, - { - "fence_id": "frs_exps_source_absent", - "constituents": [ - "EXPS" - ], - "raw_sources_searched": [ - "JOB.EXPBEN04/EXPBEN05", - "JOB.MILEAMT/JOB.MOTAMT", - "JOB.UMILEAMT/JOB.UMOTAMT", - "JOB.DEDUC1-DEDUC9", - "JOB.UDEDUC1-UDEDUC9" - ], - "finding": "Missing. These fields describe reimbursements or payroll deductions, not the complete tax-deductible employment-expense amount required by SPI.", - "mass_implication": "5.1302528% of certified-candidate FRS effective person mass has an adjacent reimbursement flag; the true EXPS mass is not estimable.", - "rationale": "The nearby fields do not measure the required deductible amount, and EXPS enters the employed-income identity with a negative sign.", - "dependent_fence_ids": [] - }, - { - "fence_id": "frs_taxterm_source_absent", - "constituents": [ - "TAXTERM" - ], - "raw_sources_searched": [ - "ADULT.REDAMT", - "ADULT and JOB taxable-termination split search" - ], - "finding": "Missing. REDAMT is gross redundancy pay and has neither the taxable amount nor non-redundancy termination pay.", - "mass_implication": "0.3746084% of certified-candidate FRS effective person mass has positive gross redundancy pay; taxable mass is unknown.", - "rationale": "Gross redundancy pay cannot be relabeled as taxable termination pay.", - "dependent_fence_ids": [] - }, - { - "fence_id": "frs_mothinc_source_absent", - "constituents": [ - "MOTHINC" - ], - "raw_sources_searched": [ - "ODDJOB.OJAMT/ODDJOB.OJNOW", - "ADULT.ALLPAY2", - "ADULT.ROYYR2-ROYYR4", - "JOB.OWNOTHER" - ], - "finding": "Missing. The fields are heterogeneous and belong to distinct income concepts; assigning their union to SPI miscellaneous employment income would be a proxy.", - "mass_implication": "Odd-job-only effective person mass is 0.1724207%; the broader unresolved miscellaneous pool is 1.4650566%.", - "rationale": "The FRS instrument cannot separate the SPI miscellaneous-employment concept source-faithfully.", - "dependent_fence_ids": [] - }, - { - "fence_id": "frs_otherinc_source_absent", - "constituents": [ - "OTHERINC" - ], - "raw_sources_searched": [ - "ADULT, ODDJOB, and JOB miscellaneous fields", - "PENSION", - "ACCOUNTS", - "ASSETS", - "BENEFITS" - ], - "finding": "Missing. No person-level raw FRS variable has SPI OTHERINC semantics, and the miscellaneous pool cannot be split between MOTHINC and OTHERINC from source evidence.", - "mass_implication": "No separable mass estimate exists; the unresolved miscellaneous pool is 1.4650566% of certified-candidate FRS effective person mass.", - "rationale": "A union of heterogeneous residual fields would be a new proxy, not a retained source constituent.", - "dependent_fence_ids": [] - }, - { - "fence_id": "frs_ossben_identifiable_subset", - "constituents": [ - "OSSBEN", - "ossben_identifiable_subset" - ], - "raw_sources_searched": [ - "BENEFITS.BENAMT", - "BENEFITS.BENEFIT", - "BENEFITS.VAR2", - "BENEFITS codes 13, 16, 6, and 30" - ], - "finding": "Incomplete. Carer's Allowance and contribution-based ESA form an identifiable subset, but code 6 mixes tax treatments and code 30 is an undifferentiated catch-all, so the complete taxable family cannot be emitted.", - "mass_implication": "1.8045088% of certified-candidate FRS effective person mass carries the identifiable lower-bound subset; it is not full OSSBEN support.", - "rationale": "The retained column must remain explicitly named as a subset and cannot satisfy the full SPI concept.", - "dependent_fence_ids": [] - }, - { - "fence_id": "frs_srp_regular_code5_subset", - "constituents": [ - "SRP", - "srp_regular_code5" - ], - "raw_sources_searched": [ - "BENEFITS.BENAMT where BENEFIT == 5", - "BENEFITS codes 6 and 9" - ], - "finding": "Incomplete. Code 5 supplies regular State Pension, but the FRS source does not identify the full SPI combination of State Pension lump sums and widow's pension; code 6 mixes benefits and code 9 is tax-free War Widow's Pension.", - "mass_implication": "18.1567916% of certified-candidate FRS effective person mass carries regular code-5 State Pension; it is not complete SRP support.", - "rationale": "The retained column must remain explicitly named as a subset and cannot be reported as the full published state-pension measure.", - "dependent_fence_ids": [] - }, - { - "fence_id": "full_frs_tei_band_unavailable", - "constituents": [ - "EPB", - "EXPS", - "TAXTERM", - "MOTHINC", - "OTHERINC", - "OSSBEN", - "SRP" - ], - "raw_sources_searched": [], - "finding": "The complete FRS TEI measure cannot be materialized from retained source constituents, so exact HMRC total-income band assignment is unavailable on the FRS channel.", - "mass_implication": "Every one of the 208 published facts is banded by total income and therefore depends on this unavailable like-for-like measure.", - "rationale": "A component-level subset does not imply a per-band lower bound: omitted income can move a taxpayer into or out of any non-overlapping published band. Biased partial bands are not emitted as estimates.", - "dependent_fence_ids": [ - "frs_epb_source_absent", - "frs_exps_source_absent", - "frs_taxterm_source_absent", - "frs_mothinc_source_absent", - "frs_otherinc_source_absent", - "frs_ossben_identifiable_subset", - "frs_srp_regular_code5_subset" - ] - } - ], - "fact_fence_id": "full_frs_tei_band_unavailable", - "blocked_dependency": "hmrc_spi_assessable_income", - "fail_on_unfenced_exclusion": true, - "fail_on_fact_count_mismatch": true, - "forbid_biased_estimate_or_delta": true - }, - { - "kind": "gate_distributional_effective_mass", - "columns": [ - "gift_aid", - "charitable_investment_gifts" - ], - "weight": "household_weight", - "weight_mapping": "household_to_person", - "support_channel_column": "person_support_channel", - "required_support_channel": "spi", - "mass_share_denominator": "all_person_effective_mass", - "minimum_nondefault_mass_share": 0.000001, - "fail_below_floor": true - } - ], - "outputs": [ - "other_investment_income", - "gift_aid", - "charitable_investment_gifts", - "hmrc_spi_employment_benefits", - "hmrc_spi_employment_expenses", - "hmrc_spi_other_social_security_income", - "hmrc_spi_taxable_termination_pay", - "hmrc_spi_miscellaneous_employment_income", - "hmrc_spi_other_income", - "hmrc_spi_state_pension_income", - "hmrc_spi_employed_income", - "hmrc_spi_total_earned_income", - "hmrc_spi_total_investment_income", - "hmrc_spi_assessable_income" - ], - "nonnegative_outputs": [ - "other_investment_income", - "gift_aid", - "charitable_investment_gifts", - "hmrc_spi_employment_benefits", - "hmrc_spi_employment_expenses", - "hmrc_spi_other_social_security_income", - "hmrc_spi_taxable_termination_pay", - "hmrc_spi_other_income", - "hmrc_spi_state_pension_income", - "hmrc_spi_employed_income", - "hmrc_spi_total_earned_income", - "hmrc_spi_total_investment_income", - "hmrc_spi_assessable_income" - ], - "rewrites": [ - "employment_income", - "self_employment_income", - "savings_interest_income", - "dividend_income", - "private_pension_income", - "property_income", - "employee_pension_contributions", - "employer_pension_contributions", - "personal_pension_contributions", - "pension_contributions_via_salary_sacrifice", - "tax_free_savings_income", - "universal_credit_reported", - "pension_credit_reported", - "child_benefit_reported", - "housing_benefit_reported", - "income_support_reported", - "working_tax_credit_reported", - "child_tax_credit_reported", - "attendance_allowance_reported", - "state_pension_reported", - "dla_sc_reported", - "dla_m_reported", - "pip_m_reported", - "pip_dl_reported", - "sda_reported", - "carers_allowance_reported", - "iidb_reported", - "afcs_reported", - "bsp_reported", - "winter_fuel_allowance_reported", - "council_tax_benefit_reported", - "jsa_contrib_reported", - "jsa_income_reported", - "esa_contrib_reported", - "esa_income_reported", - "hmrc_spi_pay", - "hmrc_spi_unemployment_benefit_income", - "hmrc_spi_incapacity_benefit_income" - ], - "notes": "Runs the SPI-trained income QRFs on the raw-spine support channel, initializes FRS charity columns to zero, trains FRS-only stage 2 before redrawing base-channel dividends, and emits a sidecar-only 208-fact replay report for the spine path." - }, - { - "stage": "cgt_incidence_clone", - "survey": "Family Resources Survey 2024-25, SPI synthetic support, and Advani-Summers capital-gains incidence", - "source": "Advani and Summers (2020), Capital Gains and UK Inequality, CAGE Working Paper 465", - "grain": "household", - "artifacts": [ - { - "role": "capital_gains_incidence_and_quantiles", - "kind": "public_aggregate_reference", - "resource": "advani_summers_capital_gains_distribution.json", - "format": "json", - "runtime_sha256_required": true - } - ], - "operations": [ - { - "kind": "clone_records", - "entity": "household", - "copies": 2, - "flag_column": "household_is_capital_gains_clone", - "original_flag": false, - "clone_flag": true, - "mass_split": 0.5, - "weight_kind_out": "importance", - "conservation": "exact_total", - "id_remapping": "id_multiplier_for_values", - "declared_factor": 1.0, - "reason": "Capital-gains incidence clone splits every household's mass equally across original and clone records; total household mass is conserved." - }, - { - "kind": "draw_capital_gains_prior_from_banded_quantiles", - "resource": "advani_summers_capital_gains_distribution.json", - "income_proxy_components": [ - "employment_income", - "self_employment_income", - "state_pension_reported", - "private_pension_income", - "property_income", - "savings_interest_income", - "dividend_income", - "miscellaneous_income" - ], - "allowance_subtraction": false, - "carrier": "oldest adult; person_id ascending breaks age ties", - "adult_minimum_age": 16, - "quantile_points": [0.05, 0.1, 0.25, 0.5, 0.75, 0.9, 0.95], - "spline_degree": 1, - "extrapolation": "ext=0", - "keep_negative_draws": true, - "seed": 0, - "salt": "cgt_prior_amount" - } - ], - "outputs": [ - "household_is_capital_gains_clone", - "capital_gains" - ], - "rewrites": ["capital_gains"], - "notes": "Spine-only equal-mass incidence clone. The A&S prior fixes the gainer set and order for the following HMRC Table 3 redraw; negative extrapolated draws remain loss-makers." - }, - { - "stage": "cgt_band_donors", - "survey": "HMRC Capital Gains Tax statistics Table 2.1a and Advani-Summers capital-gains incidence", - "source": "HMRC Capital Gains Tax statistics, July 2025, Table 2.1a", - "grain": "household", - "artifacts": [ - { - "role": "hmrc_cgt_size_bands", - "kind": "public_aggregate_reference", - "resource": "hmrc_cgt_size_bands.json", - "format": "json", - "runtime_sha256_required": true - }, - { - "role": "capital_gains_incidence", - "kind": "public_aggregate_reference", - "resource": "advani_summers_capital_gains_distribution.json", - "format": "json", - "runtime_sha256_required": true - } - ], - "operations": [ - { - "kind": "stack_band_donor_households", - "size_band_resource": "hmrc_cgt_size_bands.json", - "incidence_resource": "advani_summers_capital_gains_distribution.json", - "minimum_band_lower": 12300, - "donors_per_band": 30, - "expected_band_count": 9, - "expected_donor_count": 270, - "candidate_order": "household_id ascending", - "draw": "weighted_without_replacement", - "propensity": "Advani-Summers percent_with_gains at oldest-adult component-sum income", - "seed": 1, - "flag_column": "household_is_cgt_band_donor", - "carrier": "oldest adult; person_id ascending breaks age ties", - "initial_weight": "published band taxpayers / donors_per_band", - "never_zero_weight": true, - "weight_kind_out": "importance", - "reason": "Stack 30 positive-weight HMRC Table 2.1a support households per retained gain band; published donor mass is added explicitly." - } - ], - "outputs": [ - "household_is_cgt_band_donor", - "capital_gains" - ], - "rewrites": ["capital_gains"], - "notes": "Adds 270 positive-weight band donors. Rows below GBP 12,300 are excluded because they mix annual-exempt-amount regimes and the spline body already supplies that support." - }, - { - "stage": "hmrc_cgt_gains_spine", - "survey": "HMRC Capital Gains Tax statistics table 3 (size of gain by taxable income), 2020-21 to 2023-24", - "source": "https://assets.publishing.service.gov.uk/media/6878ac62760bf6cedaf5bd93/Table_3_2025_Size_of_gain_by_income.ods", - "grain": "person", - "artifacts": [ - { - "role": "cgt_published_fact_surface", - "kind": "administrative_table", - "format": "ods", - "survey": "HMRC Capital Gains Tax statistics table 3", - "publication": "https://www.gov.uk/government/statistics/capital-gains-tax-statistics", - "vintage": "2023-24", - "tax_year_start": 2023, - "locator": "https://assets.publishing.service.gov.uk/media/6878ac62760bf6cedaf5bd93/Table_3_2025_Size_of_gain_by_income.ods", - "sha256": "8e75c00bab949348a7238fea6d995f626c85e5d02813b46606dd7fea85e9d0c3", - "size_bytes": 11996, - "mime_type": "application/vnd.oasis.opendocument.spreadsheet", - "sheets": ["3_1_2023-24", "3_2_2022-23", "3_3_2021-22", "3_4_2020-21"], - "mapped_build_period": 2024, - "period_mapping": "latest_published_tax_year", - "runtime_sha256_required": true - }, - { - "role": "policy_parameters", - "kind": "versioned_parameter_tree", - "dependency": "policyengine-uk>=2.88 via microcosm-build[uk]", - "parameters": [ - "gov.hmrc.income_tax.allowances.personal_allowance.amount", - "gov.hmrc.income_tax.allowances.personal_allowance.maximum_ANI", - "gov.hmrc.income_tax.allowances.personal_allowance.reduction_rate", - "gov.hmrc.cgt.annual_exempt_amount" - ], - "instant_rule": "raw dated parameter files evaluated at 1 June of the build period's tax year", - "runtime_sha256_required": false, - "dependency_discipline": "deferred inside uk_cgt_policy_parameters; the base package never imports policyengine-uk at import time" - } - ], - "operations": [ - { - "kind": "verify_pinned_cgt_ods", - "artifact_role": "cgt_published_fact_surface", - "require_before_source_read": true, - "runtime_sha256_required": true, - "fail_on_mismatch": true - }, - { - "kind": "taxable_income_proxy", - "components": [ - "employment_income", - "self_employment_income", - "state_pension_reported", - "private_pension_income", - "property_income", - "savings_interest_income", - "dividend_income", - "miscellaneous_income" - ], - "components_semantics": "Persisted leaves of the model's total_income concept (ITA 2007 s.23); state_pension_reported stands in for social_security_income, whose other taxable benefits are not persisted; reliefs such as pension contributions and Gift Aid are not deducted.", - "allowance": "tapered Personal Allowance from the policy_parameters artifact", - "fail_on_missing_component": true - }, - { - "kind": "rank_preserving_allocation", - "within": "income band", - "ordering": "existing gains descending, person_id ascending on ties", - "band_order": "highest gain band first", - "suppressed_cell_allocation": "count implied by the cell's published gains at the band-total mean", - "column_reconciliation": "every income column rescales onto its published All-row taxpayer total", - "shortfall_policy": "proportional scale-down when the population holds less gainer mass than published taxpayers", - "minimum_allocation_people": 1, - "weights": "household_weight mapped to persons; no person splits across bands" - }, - { - "kind": "within_band_draws", - "bounded_band_family": "truncated exponential matched to the cell's published mean", - "open_band_family": "Pareto with alpha = mean / (mean - lower bound)", - "mean_repair_margin": 0.02, - "mean_repair_reason": "Published counts round to the nearest thousand and amounts to the nearest million; four cells of the 2023-24 table imply a mean outside their own band, and repaired means clamp just inside the violated boundary.", - "bottom_band_floor": "annual exempt amount plus one pound", - "seed_base": 552, - "seed_mixing": "seed combined with the build period; draws ordered by allocation rank", - "deterministic": true - }, - { - "kind": "sub_aea_remainder", - "policy": "gainers beyond the published taxpayer mass keep their existing amounts capped at the annual exempt amount", - "rationale": "Table 3 covers only individuals with a CGT liability; remaining gainers are treated as sub-AEA gainers rather than invented into the liability distribution or deleted." - }, - { - "kind": "record_mass_conservation_receipt", - "entity": "household", - "reason": "Amounts-only capital gains redraw on the source spine: household weights pass through unchanged and total household mass is conserved.", - "declared_factor": 1.0, - "gate_coupling": "The terminal family gate requires a valid mass-conserving MassChangeRecord carrying exactly this spine-specific reason." - }, - { - "kind": "classify_cgt_band_facts_with_reviewed_fence", - "calibration_permitted": false, - "fact_fence_id": "cgt_band_facts_policy_endogenous_proxy_conditioned", - "fenced_fact_count": 76, - "fenced_fact_composition": "60 joint cells, 10 gain-band row totals, 6 income-column totals", - "classification_rationale": "The taxpayer count is endogenous to policy, the income conditioning is an arithmetic proxy, and the published surface needs rounding and suppression reconciliation before any per-band fact is exact.", - "calibrated_facts_unchanged": "The two aggregate facts in UK_CGT_TARGET_SPECS remain the only calibrated CGT facts.", - "promotion_path": "A separately reviewed target profile may lift specific band facts after the reconciliation and proxy adequacy are adjudicated.", - "adjudication": "https://github.com/PolicyEngine/microcosm/issues/552" - } - ], - "outputs": ["capital_gains"], - "rewrites": ["capital_gains"], - "notes": "Spine manifest projection of the merged HMRC Table 3 amounts stage. It deliberately omits base_candidate and verify_certified_candidate, which belong only to the certified-H5 path." - }, - { - "stage": "salary_sacrifice", - "survey": "Family Resources Survey 2024-25 salary-sacrifice respondents and HMRC salary-sacrifice reform analysis", - "source": "HMRC, Salary sacrifice reform for pension contributions effective from 6 April 2029", - "grain": "person", - "artifacts": [ - { - "role": "salary_sacrifice_anchor", - "kind": "public_aggregate_reference", - "resource": "salary_sacrifice_anchor.json", - "format": "json", - "runtime_sha256_required": true - } - ], - "operations": [ - { - "kind": "fit_weighted_qrf", - "training_population": "support_channel == frs and not capital-gains clone and not CGT band donor and salary_sacrifice_asked == 1", - "target_population": "salary_sacrifice_asked != 1 frame-wide", - "predictors": ["age", "employment_income"], - "targets": ["pension_contributions_via_salary_sacrifice"], - "weights": "household_weight", - "weight_mapping": "household_to_person", - "seed": 42, - "n_estimators": 100, - "clamp_minimum": 0, - "preserve_asked_rows": true, - "cache": false - }, - { - "kind": "convert_donors_to_target_stock", - "resource": "salary_sacrifice_anchor.json", - "target": 5400000, - "donor_pool": "employee_pension_contributions > 0 and pension_contributions_via_salary_sacrifice == 0 and employment_income > 0", - "rate_cap": 0.5, - "move": "full employee_pension_contributions to pension_contributions_via_salary_sacrifice; source zeroed", - "seed": 2024, - "salt": "salary_sacrifice_conversion", - "receipt": "weighted_headcount", - "reason": "Salary-sacrifice support stage rewrites pension columns only; household rows and typed household weights pass through and total household mass is conserved." - } - ], - "outputs": [ - "pension_contributions_via_salary_sacrifice", - "employee_pension_contributions" - ], - "nonnegative_outputs": [ - "pension_contributions_via_salary_sacrifice", - "employee_pension_contributions" - ], - "rewrites": [ - "pension_contributions_via_salary_sacrifice", - "employee_pension_contributions" - ], - "notes": "The QRF trains only on the 2024-25 FRS asked subset. The second arm creates support toward the reviewed 5.4m staging target by moving contributors' full pension amounts." - }, - { - "stage": "student_loans", - "survey": "Family Resources Survey 2024-25 and Student Loans Company borrower forecasts for England", - "source": "Explore Education Statistics Table 6a, Higher education total", - "grain": "person", - "artifacts": [ - { - "role": "frs_release", - "kind": "public_aggregate_reference", - "resource": "frs_release.json", - "format": "json", - "runtime_sha256_required": true - }, - { - "role": "slc_liable_stocks", - "kind": "public_aggregate_reference", - "resource": "slc_liable_stocks.json", - "format": "json", - "runtime_sha256_required": true - } - ], - "operations": [ - { - "kind": "assign_student_loan_plan_cohorts", - "year_rule": "calibration_year", - "start_year_formula": "year - age + 18", - "reported_repayment_test": "student_loan_repayments > 0", - "reported_country_gate": false, - "plan_1_before": 2012, - "plan_5_from": 2023, - "enum_domain": ["NONE", "PLAN_1", "PLAN_2", "PLAN_5"], - "plan_4_imputation": false - }, - { - "kind": "top_up_to_stock", - "plan": "PLAN_5", - "priority": 1, - "resource": "slc_liable_stocks.json", - "stock_series": "plan_5.liable", - "year_rule": "calibration_year", - "age_min": 18, - "age_max": 25, - "cohort_start_min": 2023, - "eligible_region_exclusions": ["SCOTLAND", "WALES", "NORTHERN_IRELAND"], - "highest_education": "TERTIARY", - "seed": 42, - "salt": "student_loan_plan_5" - }, - { - "kind": "top_up_to_stock", - "plan": "PLAN_2", - "priority": 2, - "resource": "slc_liable_stocks.json", - "stock_series": "plan_2.liable", - "year_rule": "calibration_year", - "age_min": 21, - "age_max": 55, - "cohort_start_min": 2012, - "cohort_start_max_exclusive": 2023, - "eligible_region_exclusions": ["SCOTLAND", "WALES", "NORTHERN_IRELAND"], - "highest_education": "TERTIARY", - "seed": 42, - "salt": "student_loan_plan_2", - "reason": "Student-loan plan assignment writes an enum column only; household rows and typed household weights pass through and total household mass is conserved." - } - ], - "outputs": ["student_loan_plan"], - "rewrites": ["student_loan_plan"], - "notes": "Reported PAYE repayers are classified without a country gate. England tertiary cohorts are then topped up PLAN_5 first and PLAN_2 second to the pinned liable stocks at the FRS release calibration year; PLAN_4 is never imputed." - }, - { - "stage": "frs_hmrc_retained_leaves", - "survey": "Family Resources Survey 2024-25", - "source": "Department for Work and Pensions Family Resources Survey 2024-25 raw adult.tab and benefits.tab, caller-supplied local input", - "grain": "person", - "artifacts": [], - "operations": [ - { - "kind": "verify_certified_candidate", - "artifact": "base_candidate", - "runtime_sha256_required": true, - "fail_on_mismatch": true - }, - { - "kind": "retain_adjudicated_frs_hmrc_leaves", - "population": "certified_microcosm_uk_candidate_base_channel", - "source_vintage": "2024-25", - "mapped_build_period": 2024, - "annualization": "weekly raw FRS amounts * (365.25 / 7)", - "status": "adjudicated_partial_replay", - "retained_full_constituents": { - "hmrc_spi_pay": { - "spi_concept": "PAY", - "scope": "full", - "raw_sources": [ - "ADULT.INEARNS" - ], - "formula": "max(0, ADULT.INEARNS) * (365.25 / 7)" - }, - "hmrc_spi_unemployment_benefit_income": { - "spi_concept": "UBISJA", - "scope": "full", - "raw_sources": [ - "BENEFITS.BENEFIT=14:BENAMT", - "BENEFITS.BENEFIT=19:BENAMT" - ], - "formula": "sum(BENAMT where BENEFIT in {14, 19}) * (365.25 / 7)" - }, - "hmrc_spi_incapacity_benefit_income": { - "spi_concept": "INCPBEN", - "scope": "full", - "raw_sources": [ - "BENEFITS.BENEFIT=17:BENAMT" - ], - "formula": "sum(BENAMT where BENEFIT == 17) * (365.25 / 7)", - "observed_support": "structural zero in the audited 2023-24 FRS; retained so future vintages flow" - } - }, - "retained_named_subsets": { - "ossben_identifiable_subset": { - "spi_concept": "OSSBEN", - "raw_sources": [ - "BENEFITS.BENEFIT=13:BENAMT", - "BENEFITS.BENEFIT=16,VAR2 in {1,3}:BENAMT" - ], - "formula": "sum(BENAMT where BENEFIT == 13 or (BENEFIT == 16 and VAR2 in {1, 3})) * (365.25 / 7)", - "scope": "identifiable_subset" - }, - "srp_regular_code5": { - "spi_concept": "SRP", - "raw_sources": [ - "BENEFITS.BENEFIT=5:BENAMT" - ], - "formula": "sum(BENAMT where BENEFIT == 5) * (365.25 / 7)", - "scope": "regular_code5_subset" - } - }, - "source_absent_full_constituents": [ - "EPB", - "EXPS", - "TAXTERM", - "MOTHINC", - "OTHERINC" - ], - "full_concepts_forbidden_on_frs": [ - "hmrc_spi_employment_benefits", - "hmrc_spi_employment_expenses", - "hmrc_spi_taxable_termination_pay", - "hmrc_spi_miscellaneous_employment_income", - "hmrc_spi_other_income", - "hmrc_spi_other_social_security_income", - "hmrc_spi_state_pension_income" - ], - "forbid_proxy_substitution": [ - "employment_income", - "miscellaneous_income" - ], - "fail_on_missing_retained_constituent": true, - "fail_on_full_concept_alias": true - } - ], - "outputs": [ - "hmrc_spi_pay", - "hmrc_spi_unemployment_benefit_income", - "hmrc_spi_incapacity_benefit_income", - "ossben_identifiable_subset", + "maximum_absolute_difference_gbp": { + "ordinary": { + "TEI": 15, + "TII": 10, + "TI": 20 + }, + "composite": { + "TEI": 180, + "TII": 10, + "TI": 180 + } + }, + "rationale": "The official PUT rounds source fields, averages documented composite records, then rounds remaining income fields to GBP 5. These are the observed envelopes in the exact sha-pinned donor; post-draw synthetic identities remain exact." + }, + "ti_identity_absolute_tolerance_gbp": 5, + "stochastic_aggregates_forbidden": [ + "hmrc_spi_employed_income", + "hmrc_spi_total_earned_income", + "hmrc_spi_total_investment_income", + "hmrc_spi_assessable_income" + ], + "require_all_predictors": true, + "require_all_outputs": true, + "initialize_frs_channel_columns": { + "gift_aid": 0, + "charitable_investment_gifts": 0 + }, + "n_estimators": 100, + "seed": 42 + }, + { + "kind": "fit_weighted_qrf_stage2", + "training_population": "uk_frs_raw_spine_base_support_channel", + "target_population": "spi_synthetic_support_channel", + "predictors": [ + "age", + "gender", + "region", + "employment_income", + "self_employment_income", + "savings_interest_income", + "dividend_income", + "private_pension_income", + "property_income" + ], + "reviewed_absent_predictors": { + "other_investment_income": "This remains a stage-1 SPI draw and an official HMRC fact component, but it is not an FRS-only stage-2 predictor: the incumbent UK data build's frs_only.py defines exactly six income predictors and the certified Microcosm UK base candidate has no other_investment_income column." + }, + "categorical_predictors": [ + "gender", + "region" + ], + "weight": "household_weight", + "weight_mapping": "household_to_person", + "outputs": [ + "employee_pension_contributions", + "employer_pension_contributions", + "personal_pension_contributions", + "pension_contributions_via_salary_sacrifice", + "tax_free_savings_income", + "universal_credit_reported", + "pension_credit_reported", + "child_benefit_reported", + "housing_benefit_reported", + "income_support_reported", + "working_tax_credit_reported", + "child_tax_credit_reported", + "attendance_allowance_reported", + "state_pension_reported", + "dla_sc_reported", + "dla_m_reported", + "pip_m_reported", + "pip_dl_reported", + "sda_reported", + "carers_allowance_reported", + "iidb_reported", + "afcs_reported", + "bsp_reported", + "winter_fuel_allowance_reported", + "council_tax_benefit_reported", + "jsa_contrib_reported", + "jsa_income_reported", + "esa_contrib_reported", + "esa_income_reported" + ], + "reviewed_absent_outputs": { + "incapacity_benefit_reported": "Absent/all-default on the pinned enhanced-FRS export and certified Microcosm UK base; not a populated loader layer.", + "maternity_allowance_reported": "Absent from the pinned enhanced-FRS export and certified Microcosm UK base; no training source can be materialized for this stage." + }, + "postprocess": { + "gross_savings_interest_income": "stage1 INCBBS draw + stage2 tax_free_savings_income", + "refresh_disability_categories": [ + "aa_category", + "dla_sc_category", + "dla_m_category", + "pip_m_category", + "pip_dl_category" + ], + "refresh_disability_flags": [ + "is_disabled_for_benefits", + "is_enhanced_disabled_for_benefits", + "is_severely_disabled_for_benefits" + ] + }, + "joint_draw": true, + "require_all_predictors": true, + "require_all_materializable_outputs": true, + "require_all_outputs": false, + "n_estimators": 100, + "seed": 43 + }, + { + "kind": "redraw_columns_from_fitted_qrf", + "fit": "stage1", + "columns": [ + "dividend_income" + ], + "rows": "base_support_channel" + }, + { + "kind": "materialize_hmrc_income_bands_fail_closed", + "artifact_role": "published_fact_surface", + "mapped_build_period": 2024, + "period_mapping": "latest_published_tax_year", + "column_index_base": 0, + "data_row_start_index": 5, + "stop_label": "All ranges", + "count_unit_multiplier": 1000, + "amount_unit_multiplier": 1000000, + "component_columns": { + "employment_income": { + "sheet": "Table_3_6", + "count_column_index": 4, + "amount_column_index": 5 + }, + "self_employment_income": { + "sheet": "Table_3_6", + "count_column_index": 1, + "amount_column_index": 2 + }, + "state_pension": { + "sheet": "Table_3_6", + "count_column_index": 7, + "amount_column_index": 8 + }, + "private_pension_income": { + "sheet": "Table_3_6", + "count_column_index": 10, + "amount_column_index": 11 + }, + "property_income": { + "sheet": "Table_3_7", + "count_column_index": 1, + "amount_column_index": 2 + }, + "savings_interest_income": { + "sheet": "Table_3_7", + "count_column_index": 4, + "amount_column_index": 5 + }, + "dividend_income": { + "sheet": "Table_3_7", + "count_column_index": 7, + "amount_column_index": 8 + }, + "other_investment_income": { + "sheet": "Table_3_7", + "count_column_index": 10, + "amount_column_index": 11 + } + }, + "required_band_lower_bounds_gbp": [ + 12570, + 15000, + 20000, + 30000, + 40000, + 50000, + 70000, + 100000, + 150000, + 200000, + 300000, + 500000, + 1000000 + ], + "required_measures": [ + "count", + "amount" + ], + "fail_on_missing_sheet": true, + "fail_on_missing_component": true, + "fail_on_missing_band": true, + "fail_on_non_numeric_value": true + }, + { + "kind": "classify_hmrc_income_facts_with_reviewed_fences", + "target_operation": "materialize_hmrc_income_bands_fail_closed", + "components": [ + "employment_income", + "self_employment_income", + "state_pension", + "private_pension_income", + "property_income", + "savings_interest_income", + "dividend_income", + "other_investment_income" + ], + "breakdown_dependency": "hmrc_spi_assessable_income", + "frs_breakdown_status": "unavailable_full_measure", + "input_weight_kind": "importance", + "output_weight_kind": "importance", + "calibration_permitted": false, + "required_fact_count": 208, + "outcome_counts": { + "exact_pass": 0, + "exact_fail": 0, + "directional_pass": 0, + "directional_fail": 0, + "excluded_with_fence": 208 + }, + "classification_rationale": "Every published fact uses non-overlapping total-income bands. The FRS channel cannot materialize full TEI, and omitted income can move a person between bands, so neither an exact fact nor a per-band directional bound is valid.", + "reviewed_fences": [ + { + "fence_id": "frs_epb_source_absent", + "constituents": [ + "EPB" + ], + "raw_sources_searched": [ + "JOB.EXPBEN01-EXPBEN13", + "JOB.CARVAL", + "JOB.CARAMT", + "JOB.FUELAMT", + "JOB.VCHAMT", + "JOB.CHVAMT" + ], + "finding": "Missing. EXPBEN* are receipt flags, and the amount fields cover only selected benefits; they cannot produce complete taxable expenses payments and benefits.", + "mass_implication": "12.9485464% of certified-candidate FRS effective person mass has at least one receipt flag, but this is not monetary support.", + "rationale": "Receipt flags and selected benefit amounts cannot be promoted to the SPI EPB monetary concept without an imputation or proxy.", + "dependent_fence_ids": [] + }, + { + "fence_id": "frs_exps_source_absent", + "constituents": [ + "EXPS" + ], + "raw_sources_searched": [ + "JOB.EXPBEN04/EXPBEN05", + "JOB.MILEAMT/JOB.MOTAMT", + "JOB.UMILEAMT/JOB.UMOTAMT", + "JOB.DEDUC1-DEDUC9", + "JOB.UDEDUC1-UDEDUC9" + ], + "finding": "Missing. These fields describe reimbursements or payroll deductions, not the complete tax-deductible employment-expense amount required by SPI.", + "mass_implication": "5.1302528% of certified-candidate FRS effective person mass has an adjacent reimbursement flag; the true EXPS mass is not estimable.", + "rationale": "The nearby fields do not measure the required deductible amount, and EXPS enters the employed-income identity with a negative sign.", + "dependent_fence_ids": [] + }, + { + "fence_id": "frs_taxterm_source_absent", + "constituents": [ + "TAXTERM" + ], + "raw_sources_searched": [ + "ADULT.REDAMT", + "ADULT and JOB taxable-termination split search" + ], + "finding": "Missing. REDAMT is gross redundancy pay and has neither the taxable amount nor non-redundancy termination pay.", + "mass_implication": "0.3746084% of certified-candidate FRS effective person mass has positive gross redundancy pay; taxable mass is unknown.", + "rationale": "Gross redundancy pay cannot be relabeled as taxable termination pay.", + "dependent_fence_ids": [] + }, + { + "fence_id": "frs_mothinc_source_absent", + "constituents": [ + "MOTHINC" + ], + "raw_sources_searched": [ + "ODDJOB.OJAMT/ODDJOB.OJNOW", + "ADULT.ALLPAY2", + "ADULT.ROYYR2-ROYYR4", + "JOB.OWNOTHER" + ], + "finding": "Missing. The fields are heterogeneous and belong to distinct income concepts; assigning their union to SPI miscellaneous employment income would be a proxy.", + "mass_implication": "Odd-job-only effective person mass is 0.1724207%; the broader unresolved miscellaneous pool is 1.4650566%.", + "rationale": "The FRS instrument cannot separate the SPI miscellaneous-employment concept source-faithfully.", + "dependent_fence_ids": [] + }, + { + "fence_id": "frs_otherinc_source_absent", + "constituents": [ + "OTHERINC" + ], + "raw_sources_searched": [ + "ADULT, ODDJOB, and JOB miscellaneous fields", + "PENSION", + "ACCOUNTS", + "ASSETS", + "BENEFITS" + ], + "finding": "Missing. No person-level raw FRS variable has SPI OTHERINC semantics, and the miscellaneous pool cannot be split between MOTHINC and OTHERINC from source evidence.", + "mass_implication": "No separable mass estimate exists; the unresolved miscellaneous pool is 1.4650566% of certified-candidate FRS effective person mass.", + "rationale": "A union of heterogeneous residual fields would be a new proxy, not a retained source constituent.", + "dependent_fence_ids": [] + }, + { + "fence_id": "frs_ossben_identifiable_subset", + "constituents": [ + "OSSBEN", + "ossben_identifiable_subset" + ], + "raw_sources_searched": [ + "BENEFITS.BENAMT", + "BENEFITS.BENEFIT", + "BENEFITS.VAR2", + "BENEFITS codes 13, 16, 6, and 30" + ], + "finding": "Incomplete. Carer's Allowance and contribution-based ESA form an identifiable subset, but code 6 mixes tax treatments and code 30 is an undifferentiated catch-all, so the complete taxable family cannot be emitted.", + "mass_implication": "1.8045088% of certified-candidate FRS effective person mass carries the identifiable lower-bound subset; it is not full OSSBEN support.", + "rationale": "The retained column must remain explicitly named as a subset and cannot satisfy the full SPI concept.", + "dependent_fence_ids": [] + }, + { + "fence_id": "frs_srp_regular_code5_subset", + "constituents": [ + "SRP", "srp_regular_code5" - ], - "notes": "Retains the adjudicated source-faithful FRS HMRC leaf columns before the SPI income rebuild: full PAY, UBISJA, and INCPBEN, plus explicitly named OSSBEN and SRP subsets. The runtime verifies the certified candidate before retaining these leaves." - }, - { - "stage": "hmrc_spi_income", - "survey": "Survey of Personal Incomes Public Use Tape 2022-23 and HMRC Personal Incomes Tables 3.6/3.7 2023-24", - "source": "https://assets.publishing.service.gov.uk/media/69f1f12d2fae53a03709682f/Collated_Tables_3_1_to_3_11_2324.ods", - "grain": "person", - "artifacts": [ - { - "role": "qrf_donor", - "kind": "private_microdata", - "format": "tab_delimited", - "survey": "Survey of Personal Incomes Public Use Tape 2022-23", - "vintage": "2022-23", - "tax_year_start": 2022, - "ukds_study_number": "SN 9422", - "doi": "10.5255/UKDA-SN-9422-1", - "filename": "put2223uk.tab", - "sha256": "5ef829461060c91a2a47be59ad541d9b519fc3976d66ca80d4920f711bb96f66", - "size_bytes": 141323762, - "reviewed_source": "PolicyEngine licensed UKDS mirror (private Hugging Face repository), spi_2022_23.zip", - "access": "private_local_input", - "locator": "caller-supplied local input", - "runtime_sha256_required": true - }, - { - "role": "published_fact_surface", - "kind": "administrative_table", - "format": "ods", - "survey": "HMRC Personal Incomes Tables 3.6 and 3.7", - "publication": "https://www.gov.uk/government/statistics/personal-incomes-statistics-for-the-tax-year-2023-to-2024", - "vintage": "2023-24", - "tax_year_start": 2023, - "locator": "https://assets.publishing.service.gov.uk/media/69f1f12d2fae53a03709682f/Collated_Tables_3_1_to_3_11_2324.ods", - "sha256": "ad063b06b2bdeef8600dbbb09d48153337a4966f8c7eea50df7a2e0304ebd73e", - "size_bytes": 166693, - "mime_type": "application/vnd.oasis.opendocument.spreadsheet", - "sheets": [ - "Table_3_6", - "Table_3_7" - ], - "mapped_build_period": 2024, - "period_mapping": "latest_published_tax_year", - "runtime_sha256_required": true - } - ], - "operations": [ - { - "kind": "verify_pinned_hmrc_source_pair", - "artifact_roles": [ - "qrf_donor", - "published_fact_surface" - ], - "require_before_source_read": true, - "runtime_sha256_required": true, - "fail_on_mismatch": true - }, - { - "kind": "replace_zero_weight_spi_support", - "existing_channel": "spi", - "require_existing_weight": 0, - "replacement_strata": [ - "clone_index", - "household_is_capital_gains_clone", - "region" - ], - "spi_prior_national_household_mass_share": 0.5, - "output_weight_kind": "importance", - "preserve_total_household_mass": true, - "require_mass_change_record": true, - "mass_change_reason": "Allocate 50% of certified UK national household prior mass to the rebuilt 2022-23 SPI support channel; total national mass is conserved.", - "fail_on_live_existing_spi_mass": true - }, - { - "kind": "strict_read_private_table", - "artifact_role": "qrf_donor", - "filename": "put2223uk.tab", - "delimiter": "\t", - "weight": "FACT", - "required_columns": [ - "AGERANGE", - "GORCODE", - "SEX", - "FACT", - "PAY", - "EPB", - "EXPS", - "TAXTERM", - "INCPBEN", - "OSSBEN", - "UBISJA", - "MOTHINC", - "OTHERINC", - "PROFITS", - "CAPALL", - "LOSSBF", - "SRP", - "INCBBS", - "DIVIDENDS", - "PENSION", - "INCPROP", - "OTHERINV", - "GIFTAID", - "GIFTINV", - "TEI", - "TII", - "TI" - ], - "runtime_sha256_required": true, - "fail_on_missing_file": true, - "fail_on_missing_columns": true, - "fail_on_invalid_weight": true - }, - { - "kind": "fit_weighted_qrf_stage1", - "training_artifact_role": "qrf_donor", - "predictors": [ - "age", - "gender", - "region" - ], - "categorical_predictors": [ - "gender", - "region" - ], - "source_sampling_weight": "FACT", - "sample_size": 100000, - "sample_with_replacement": true, - "post_sample_fit_weight": "uniform", - "fit_weight_kind": "design", - "double_apply_source_weight": false, - "source_columns": { - "self_employment_income": [ - "PROFITS", - "CAPALL", - "LOSSBF" - ], - "savings_interest_income": [ - "INCBBS" - ], - "dividend_income": [ - "DIVIDENDS" - ], - "private_pension_income": [ - "PENSION" - ], - "property_income": [ - "INCPROP" - ], - "other_investment_income": [ - "OTHERINV" - ], - "gift_aid": [ - "GIFTAID" - ], - "charitable_investment_gifts": [ - "GIFTINV" - ], - "hmrc_spi_pay": [ - "PAY" - ], - "hmrc_spi_employment_benefits": [ - "EPB" - ], - "hmrc_spi_employment_expenses": [ - "EXPS" - ], - "hmrc_spi_incapacity_benefit_income": [ - "INCPBEN" - ], - "hmrc_spi_other_social_security_income": [ - "OSSBEN" - ], - "hmrc_spi_taxable_termination_pay": [ - "TAXTERM" - ], - "hmrc_spi_unemployment_benefit_income": [ - "UBISJA" - ], - "hmrc_spi_miscellaneous_employment_income": [ - "MOTHINC" - ], - "hmrc_spi_other_income": [ - "OTHERINC" - ], - "hmrc_spi_state_pension_income": [ - "SRP" - ] - }, - "derived_policyengine_outputs": { - "employment_income": { - "source_columns": [ - "PAY", - "EPB", - "TAXTERM" - ], - "formula": "hmrc_spi_pay + hmrc_spi_employment_benefits + hmrc_spi_taxable_termination_pay", - "derive_after_draw": true - } - }, - "outputs": [ - "self_employment_income", - "savings_interest_income", - "dividend_income", - "private_pension_income", - "property_income", - "other_investment_income", - "gift_aid", - "charitable_investment_gifts", - "hmrc_spi_pay", - "hmrc_spi_employment_benefits", - "hmrc_spi_employment_expenses", - "hmrc_spi_incapacity_benefit_income", - "hmrc_spi_other_social_security_income", - "hmrc_spi_taxable_termination_pay", - "hmrc_spi_unemployment_benefit_income", - "hmrc_spi_miscellaneous_employment_income", - "hmrc_spi_other_income", - "hmrc_spi_state_pension_income" - ], - "joint_draw": true, - "savings_interest_source_semantics": "INCBBS is taxable bank/building-society interest before reconstruction to the PolicyEngine gross input", - "employment_income_source_semantics": "PolicyEngine input = PAY + EPB + TAXTERM, matching the pinned enhanced-FRS pipeline; it is not the Table 3.6 measure", - "hmrc_employed_income_source_semantics": "Derived after each draw as max(0, PAY + EPB - EXPS) + INCPBEN + OSSBEN + TAXTERM + UBISJA + MOTHINC, using normalized leaves identically on FRS and SPI channels", - "self_employment_income_source_semantics": "max(0, PROFITS - CAPALL - LOSSBF)", - "assessable_income_source_semantics": "QRF draws leaves only; TEI, TII, and TI are deterministic post-draw accounting aggregates and TI equals TEI + TII exactly", - "source_ti_identity_fields": [ - "TI", - "TEI", - "TII" - ], - "source_leaf_reconciliation": { - "documentation_url": "https://doc.ukdataservice.ac.uk/doc/9422/mrdoc/pdf/9422_put_2223_full_documentation.pdf", - "composite_indicator": "AGERANGE == -1", - "formulas": { - "TEI": "max(0, PAY + EPB - EXPS) + INCPBEN + OSSBEN + TAXTERM + UBISJA + MOTHINC + OTHERINC + SRP + PENSION + max(0, PROFITS - CAPALL - LOSSBF)", - "TII": "OTHERINV + DIVIDENDS + INCPROP + INCBBS", - "TI": "TEI + TII" - }, - "maximum_absolute_difference_gbp": { - "ordinary": { - "TEI": 15, - "TII": 10, - "TI": 20 - }, - "composite": { - "TEI": 180, - "TII": 10, - "TI": 180 - } - }, - "rationale": "The official PUT rounds source fields, averages documented composite records, then rounds remaining income fields to GBP 5. These are the observed envelopes in the exact sha-pinned donor; post-draw synthetic identities remain exact." - }, - "ti_identity_absolute_tolerance_gbp": 5, - "stochastic_aggregates_forbidden": [ - "hmrc_spi_employed_income", - "hmrc_spi_total_earned_income", - "hmrc_spi_total_investment_income", - "hmrc_spi_assessable_income" - ], - "require_all_predictors": true, - "require_all_outputs": true - }, - { - "kind": "fit_weighted_qrf_stage2", - "training_population": "certified_microcosm_uk_candidate_base_channel", - "target_population": "rebuilt_spi_support_channel", - "predictors": [ - "age", - "gender", - "region", - "employment_income", - "self_employment_income", - "savings_interest_income", - "dividend_income", - "private_pension_income", - "property_income" - ], - "reviewed_absent_predictors": { - "other_investment_income": "This remains a stage-1 SPI draw and an official HMRC fact component, but it is not an FRS-only stage-2 predictor: the incumbent UK data build's frs_only.py defines exactly six income predictors and the certified Microcosm UK base candidate has no other_investment_income column." - }, - "categorical_predictors": [ - "gender", - "region" - ], - "weight": "household_weight", - "weight_mapping": "household_to_person", - "outputs": [ - "employee_pension_contributions", - "employer_pension_contributions", - "personal_pension_contributions", - "pension_contributions_via_salary_sacrifice", - "tax_free_savings_income", - "universal_credit_reported", - "pension_credit_reported", - "child_benefit_reported", - "housing_benefit_reported", - "income_support_reported", - "working_tax_credit_reported", - "child_tax_credit_reported", - "attendance_allowance_reported", - "state_pension_reported", - "dla_sc_reported", - "dla_m_reported", - "pip_m_reported", - "pip_dl_reported", - "sda_reported", - "carers_allowance_reported", - "iidb_reported", - "afcs_reported", - "bsp_reported", - "winter_fuel_allowance_reported", - "council_tax_benefit_reported", - "jsa_contrib_reported", - "jsa_income_reported", - "esa_contrib_reported", - "esa_income_reported" - ], - "reviewed_absent_outputs": { - "incapacity_benefit_reported": "Absent/all-default on the pinned enhanced-FRS export and certified Microcosm UK base; not a populated loader layer.", - "maternity_allowance_reported": "Absent from the pinned enhanced-FRS export and certified Microcosm UK base; no training source can be materialized for this stage." - }, - "postprocess": { - "gross_savings_interest_income": "stage1 INCBBS draw + stage2 tax_free_savings_income", - "refresh_disability_categories": [ - "aa_category", - "dla_sc_category", - "dla_m_category", - "pip_m_category", - "pip_dl_category" - ], - "refresh_disability_flags": [ - "is_disabled_for_benefits", - "is_enhanced_disabled_for_benefits", - "is_severely_disabled_for_benefits" - ] - }, - "joint_draw": true, - "require_all_predictors": true, - "require_all_materializable_outputs": true, - "require_all_outputs": false - }, - { - "kind": "materialize_hmrc_income_bands_fail_closed", - "artifact_role": "published_fact_surface", - "mapped_build_period": 2024, - "period_mapping": "latest_published_tax_year", - "column_index_base": 0, - "data_row_start_index": 5, - "stop_label": "All ranges", - "count_unit_multiplier": 1000, - "amount_unit_multiplier": 1000000, - "component_columns": { - "employment_income": { - "sheet": "Table_3_6", - "count_column_index": 4, - "amount_column_index": 5 - }, - "self_employment_income": { - "sheet": "Table_3_6", - "count_column_index": 1, - "amount_column_index": 2 - }, - "state_pension": { - "sheet": "Table_3_6", - "count_column_index": 7, - "amount_column_index": 8 - }, - "private_pension_income": { - "sheet": "Table_3_6", - "count_column_index": 10, - "amount_column_index": 11 - }, - "property_income": { - "sheet": "Table_3_7", - "count_column_index": 1, - "amount_column_index": 2 - }, - "savings_interest_income": { - "sheet": "Table_3_7", - "count_column_index": 4, - "amount_column_index": 5 - }, - "dividend_income": { - "sheet": "Table_3_7", - "count_column_index": 7, - "amount_column_index": 8 - }, - "other_investment_income": { - "sheet": "Table_3_7", - "count_column_index": 10, - "amount_column_index": 11 - } - }, - "required_band_lower_bounds_gbp": [ - 12570, - 15000, - 20000, - 30000, - 40000, - 50000, - 70000, - 100000, - 150000, - 200000, - 300000, - 500000, - 1000000 - ], - "required_measures": [ - "count", - "amount" - ], - "fail_on_missing_sheet": true, - "fail_on_missing_component": true, - "fail_on_missing_band": true, - "fail_on_non_numeric_value": true - }, - { - "kind": "classify_hmrc_income_facts_with_reviewed_fences", - "target_operation": "materialize_hmrc_income_bands_fail_closed", - "components": [ - "employment_income", - "self_employment_income", - "state_pension", - "private_pension_income", - "property_income", - "savings_interest_income", - "dividend_income", - "other_investment_income" - ], - "breakdown_dependency": "hmrc_spi_assessable_income", - "frs_breakdown_status": "unavailable_full_measure", - "input_weight_kind": "importance", - "output_weight_kind": "importance", - "calibration_permitted": false, - "required_fact_count": 208, - "outcome_counts": { - "exact_pass": 0, - "exact_fail": 0, - "directional_pass": 0, - "directional_fail": 0, - "excluded_with_fence": 208 - }, - "classification_rationale": "Every published fact uses non-overlapping total-income bands. The FRS channel cannot materialize full TEI, and omitted income can move a person between bands, so neither an exact fact nor a per-band directional bound is valid.", - "reviewed_fences": [ - { - "fence_id": "frs_epb_source_absent", - "constituents": [ - "EPB" - ], - "raw_sources_searched": [ - "JOB.EXPBEN01-EXPBEN13", - "JOB.CARVAL", - "JOB.CARAMT", - "JOB.FUELAMT", - "JOB.VCHAMT", - "JOB.CHVAMT" - ], - "finding": "Missing. EXPBEN* are receipt flags, and the amount fields cover only selected benefits; they cannot produce complete taxable expenses payments and benefits.", - "mass_implication": "12.9485464% of certified-candidate FRS effective person mass has at least one receipt flag, but this is not monetary support.", - "rationale": "Receipt flags and selected benefit amounts cannot be promoted to the SPI EPB monetary concept without an imputation or proxy.", - "dependent_fence_ids": [] - }, - { - "fence_id": "frs_exps_source_absent", - "constituents": [ - "EXPS" - ], - "raw_sources_searched": [ - "JOB.EXPBEN04/EXPBEN05", - "JOB.MILEAMT/JOB.MOTAMT", - "JOB.UMILEAMT/JOB.UMOTAMT", - "JOB.DEDUC1-DEDUC9", - "JOB.UDEDUC1-UDEDUC9" - ], - "finding": "Missing. These fields describe reimbursements or payroll deductions, not the complete tax-deductible employment-expense amount required by SPI.", - "mass_implication": "5.1302528% of certified-candidate FRS effective person mass has an adjacent reimbursement flag; the true EXPS mass is not estimable.", - "rationale": "The nearby fields do not measure the required deductible amount, and EXPS enters the employed-income identity with a negative sign.", - "dependent_fence_ids": [] - }, - { - "fence_id": "frs_taxterm_source_absent", - "constituents": [ - "TAXTERM" - ], - "raw_sources_searched": [ - "ADULT.REDAMT", - "ADULT and JOB taxable-termination split search" - ], - "finding": "Missing. REDAMT is gross redundancy pay and has neither the taxable amount nor non-redundancy termination pay.", - "mass_implication": "0.3746084% of certified-candidate FRS effective person mass has positive gross redundancy pay; taxable mass is unknown.", - "rationale": "Gross redundancy pay cannot be relabeled as taxable termination pay.", - "dependent_fence_ids": [] - }, - { - "fence_id": "frs_mothinc_source_absent", - "constituents": [ - "MOTHINC" - ], - "raw_sources_searched": [ - "ODDJOB.OJAMT/ODDJOB.OJNOW", - "ADULT.ALLPAY2", - "ADULT.ROYYR2-ROYYR4", - "JOB.OWNOTHER" - ], - "finding": "Missing. The fields are heterogeneous and belong to distinct income concepts; assigning their union to SPI miscellaneous employment income would be a proxy.", - "mass_implication": "Odd-job-only effective person mass is 0.1724207%; the broader unresolved miscellaneous pool is 1.4650566%.", - "rationale": "The FRS instrument cannot separate the SPI miscellaneous-employment concept source-faithfully.", - "dependent_fence_ids": [] - }, - { - "fence_id": "frs_otherinc_source_absent", - "constituents": [ - "OTHERINC" - ], - "raw_sources_searched": [ - "ADULT, ODDJOB, and JOB miscellaneous fields", - "PENSION", - "ACCOUNTS", - "ASSETS", - "BENEFITS" - ], - "finding": "Missing. No person-level raw FRS variable has SPI OTHERINC semantics, and the miscellaneous pool cannot be split between MOTHINC and OTHERINC from source evidence.", - "mass_implication": "No separable mass estimate exists; the unresolved miscellaneous pool is 1.4650566% of certified-candidate FRS effective person mass.", - "rationale": "A union of heterogeneous residual fields would be a new proxy, not a retained source constituent.", - "dependent_fence_ids": [] - }, - { - "fence_id": "frs_ossben_identifiable_subset", - "constituents": [ - "OSSBEN", - "ossben_identifiable_subset" - ], - "raw_sources_searched": [ - "BENEFITS.BENAMT", - "BENEFITS.BENEFIT", - "BENEFITS.VAR2", - "BENEFITS codes 13, 16, 6, and 30" - ], - "finding": "Incomplete. Carer's Allowance and contribution-based ESA form an identifiable subset, but code 6 mixes tax treatments and code 30 is an undifferentiated catch-all, so the complete taxable family cannot be emitted.", - "mass_implication": "1.8045088% of certified-candidate FRS effective person mass carries the identifiable lower-bound subset; it is not full OSSBEN support.", - "rationale": "The retained column must remain explicitly named as a subset and cannot satisfy the full SPI concept.", - "dependent_fence_ids": [] - }, - { - "fence_id": "frs_srp_regular_code5_subset", - "constituents": [ - "SRP", - "srp_regular_code5" - ], - "raw_sources_searched": [ - "BENEFITS.BENAMT where BENEFIT == 5", - "BENEFITS codes 6 and 9" - ], - "finding": "Incomplete. Code 5 supplies regular State Pension, but the FRS source does not identify the full SPI combination of State Pension lump sums and widow's pension; code 6 mixes benefits and code 9 is tax-free War Widow's Pension.", - "mass_implication": "18.1567916% of certified-candidate FRS effective person mass carries regular code-5 State Pension; it is not complete SRP support.", - "rationale": "The retained column must remain explicitly named as a subset and cannot be reported as the full published state-pension measure.", - "dependent_fence_ids": [] - }, - { - "fence_id": "full_frs_tei_band_unavailable", - "constituents": [ - "EPB", - "EXPS", - "TAXTERM", - "MOTHINC", - "OTHERINC", - "OSSBEN", - "SRP" - ], - "raw_sources_searched": [], - "finding": "The complete FRS TEI measure cannot be materialized from retained source constituents, so exact HMRC total-income band assignment is unavailable on the FRS channel.", - "mass_implication": "Every one of the 208 published facts is banded by total income and therefore depends on this unavailable like-for-like measure.", - "rationale": "A component-level subset does not imply a per-band lower bound: omitted income can move a taxpayer into or out of any non-overlapping published band. Biased partial bands are not emitted as estimates.", - "dependent_fence_ids": [ - "frs_epb_source_absent", - "frs_exps_source_absent", - "frs_taxterm_source_absent", - "frs_mothinc_source_absent", - "frs_otherinc_source_absent", - "frs_ossben_identifiable_subset", - "frs_srp_regular_code5_subset" - ] - } - ], - "fact_fence_id": "full_frs_tei_band_unavailable", - "blocked_dependency": "hmrc_spi_assessable_income", - "fail_on_unfenced_exclusion": true, - "fail_on_fact_count_mismatch": true, - "forbid_biased_estimate_or_delta": true - }, - { - "kind": "gate_distributional_effective_mass", - "columns": [ - "gift_aid", - "charitable_investment_gifts" - ], - "weight": "household_weight", - "weight_mapping": "household_to_person", - "support_channel_column": "person_support_channel", - "required_support_channel": "spi", - "mass_share_denominator": "all_person_effective_mass", - "minimum_nondefault_mass_share": 0.000001, - "fail_below_floor": true - } - ], - "official_table_components": [ - "employment_income", - "self_employment_income", - "state_pension", - "private_pension_income", - "property_income", - "savings_interest_income", - "dividend_income", - "other_investment_income" - ], - "donor_relief_outputs": [ - "gift_aid", - "charitable_investment_gifts" - ], - "outputs": [ - "employment_income", - "self_employment_income", - "hmrc_spi_state_pension_income", - "private_pension_income", - "property_income", - "savings_interest_income", - "dividend_income", - "other_investment_income", - "gift_aid", - "charitable_investment_gifts", - "hmrc_spi_employed_income", - "hmrc_spi_total_earned_income", - "hmrc_spi_total_investment_income", - "hmrc_spi_assessable_income" - ], - "notes": "Current-source adjudicated replay contract: the private 2022-23 SPI donor and public 2023-24 HMRC ODS are pinned by reviewed SHA-256 and size and verified together before either is opened. The QRF draws source leaves; HMRC employed income, TEI, TII, and TI are deterministic post-draw aggregates on the SPI channel, with TI exactly equal to TEI + TII. PolicyEngine employment_income remains the narrow PAY + EPB + TAXTERM input on SPI rows. Stage 2 mirrors the incumbent UK data build's frs_only.py exactly: its income predictors are employment, self-employment, savings interest, dividends, private pension, and property income. Other investment income remains a stage-1 SPI draw and official HMRC fact component, but is excluded from stage 2 because the certified FRS candidate does not carry it. The FRS channel retains source-faithful full PAY, UBISJA, and INCPBEN plus explicitly named ossben_identifiable_subset and srp_regular_code5; EPB, EXPS, TAXTERM, MOTHINC, OTHERINC, full OSSBEN, and full SRP remain forbidden. Because the missing legs prevent a complete FRS TEI measure, none of the 208 non-overlapping total-income-band facts is exact or directional. Every fact is an excluded-with-fence record, no calibration is performed, and weights remain importance-kind. Gift Aid restoration still requires the rebuilt positive-mass SPI channel to clear the reviewed 1ppm effective-mass floor." + ], + "raw_sources_searched": [ + "BENEFITS.BENAMT where BENEFIT == 5", + "BENEFITS codes 6 and 9" + ], + "finding": "Incomplete. Code 5 supplies regular State Pension, but the FRS source does not identify the full SPI combination of State Pension lump sums and widow's pension; code 6 mixes benefits and code 9 is tax-free War Widow's Pension.", + "mass_implication": "18.1567916% of certified-candidate FRS effective person mass carries regular code-5 State Pension; it is not complete SRP support.", + "rationale": "The retained column must remain explicitly named as a subset and cannot be reported as the full published state-pension measure.", + "dependent_fence_ids": [] + }, + { + "fence_id": "full_frs_tei_band_unavailable", + "constituents": [ + "EPB", + "EXPS", + "TAXTERM", + "MOTHINC", + "OTHERINC", + "OSSBEN", + "SRP" + ], + "raw_sources_searched": [], + "finding": "The complete FRS TEI measure cannot be materialized from retained source constituents, so exact HMRC total-income band assignment is unavailable on the FRS channel.", + "mass_implication": "Every one of the 208 published facts is banded by total income and therefore depends on this unavailable like-for-like measure.", + "rationale": "A component-level subset does not imply a per-band lower bound: omitted income can move a taxpayer into or out of any non-overlapping published band. Biased partial bands are not emitted as estimates.", + "dependent_fence_ids": [ + "frs_epb_source_absent", + "frs_exps_source_absent", + "frs_taxterm_source_absent", + "frs_mothinc_source_absent", + "frs_otherinc_source_absent", + "frs_ossben_identifiable_subset", + "frs_srp_regular_code5_subset" + ] + } + ], + "fact_fence_id": "full_frs_tei_band_unavailable", + "blocked_dependency": "hmrc_spi_assessable_income", + "fail_on_unfenced_exclusion": true, + "fail_on_fact_count_mismatch": true, + "forbid_biased_estimate_or_delta": true + }, + { + "kind": "gate_distributional_effective_mass", + "columns": [ + "gift_aid", + "charitable_investment_gifts" + ], + "weight": "household_weight", + "weight_mapping": "household_to_person", + "support_channel_column": "person_support_channel", + "required_support_channel": "spi", + "mass_share_denominator": "all_person_effective_mass", + "minimum_nondefault_mass_share": 1e-06, + "fail_below_floor": true + } + ], + "outputs": [ + "other_investment_income", + "gift_aid", + "charitable_investment_gifts", + "hmrc_spi_employment_benefits", + "hmrc_spi_employment_expenses", + "hmrc_spi_other_social_security_income", + "hmrc_spi_taxable_termination_pay", + "hmrc_spi_miscellaneous_employment_income", + "hmrc_spi_other_income", + "hmrc_spi_state_pension_income", + "hmrc_spi_employed_income", + "hmrc_spi_total_earned_income", + "hmrc_spi_total_investment_income", + "hmrc_spi_assessable_income" + ], + "nonnegative_outputs": [ + "other_investment_income", + "gift_aid", + "charitable_investment_gifts", + "hmrc_spi_employment_benefits", + "hmrc_spi_employment_expenses", + "hmrc_spi_other_social_security_income", + "hmrc_spi_taxable_termination_pay", + "hmrc_spi_other_income", + "hmrc_spi_state_pension_income", + "hmrc_spi_employed_income", + "hmrc_spi_total_earned_income", + "hmrc_spi_total_investment_income", + "hmrc_spi_assessable_income" + ], + "rewrites": [ + "employment_income", + "self_employment_income", + "savings_interest_income", + "dividend_income", + "private_pension_income", + "property_income", + "employee_pension_contributions", + "employer_pension_contributions", + "personal_pension_contributions", + "pension_contributions_via_salary_sacrifice", + "tax_free_savings_income", + "universal_credit_reported", + "pension_credit_reported", + "child_benefit_reported", + "housing_benefit_reported", + "income_support_reported", + "working_tax_credit_reported", + "child_tax_credit_reported", + "attendance_allowance_reported", + "state_pension_reported", + "dla_sc_reported", + "dla_m_reported", + "pip_m_reported", + "pip_dl_reported", + "sda_reported", + "carers_allowance_reported", + "iidb_reported", + "afcs_reported", + "bsp_reported", + "winter_fuel_allowance_reported", + "council_tax_benefit_reported", + "jsa_contrib_reported", + "jsa_income_reported", + "esa_contrib_reported", + "esa_income_reported", + "hmrc_spi_pay", + "hmrc_spi_unemployment_benefit_income", + "hmrc_spi_incapacity_benefit_income" + ], + "notes": "Runs the SPI-trained income QRFs on the raw-spine support channel, initializes FRS charity columns to zero, trains FRS-only stage 2 before redrawing base-channel dividends, and emits a sidecar-only 208-fact replay report for the spine path." + }, + { + "stage": "cgt_incidence_clone", + "survey": "Family Resources Survey 2024-25, SPI synthetic support, and Advani-Summers capital-gains incidence", + "source": "Advani and Summers (2020), Capital Gains and UK Inequality, CAGE Working Paper 465", + "grain": "household", + "artifacts": [ + { + "role": "capital_gains_incidence_and_quantiles", + "kind": "public_aggregate_reference", + "resource": "advani_summers_capital_gains_distribution.json", + "format": "json", + "runtime_sha256_required": true + } + ], + "operations": [ + { + "kind": "clone_records", + "entity": "household", + "copies": 2, + "flag_column": "household_is_capital_gains_clone", + "original_flag": false, + "clone_flag": true, + "mass_split": 0.5, + "weight_kind_out": "importance", + "conservation": "exact_total", + "id_remapping": "id_multiplier_for_values", + "declared_factor": 1.0, + "reason": "Capital-gains incidence clone splits every household's mass equally across original and clone records; total household mass is conserved." + }, + { + "kind": "draw_capital_gains_prior_from_banded_quantiles", + "resource": "advani_summers_capital_gains_distribution.json", + "income_proxy_components": [ + "employment_income", + "self_employment_income", + "state_pension_reported", + "private_pension_income", + "property_income", + "savings_interest_income", + "dividend_income", + "miscellaneous_income" + ], + "allowance_subtraction": false, + "carrier": "oldest adult; person_id ascending breaks age ties", + "adult_minimum_age": 16, + "quantile_points": [ + 0.05, + 0.1, + 0.25, + 0.5, + 0.75, + 0.9, + 0.95 + ], + "spline_degree": 1, + "extrapolation": "ext=0", + "keep_negative_draws": true, + "seed": 0, + "salt": "cgt_prior_amount" + } + ], + "outputs": [ + "household_is_capital_gains_clone", + "capital_gains" + ], + "rewrites": [ + "capital_gains" + ], + "notes": "Spine-only equal-mass incidence clone. The A&S prior fixes the gainer set and order for the following HMRC Table 3 redraw; negative extrapolated draws remain loss-makers." + }, + { + "stage": "cgt_band_donors", + "survey": "HMRC Capital Gains Tax statistics Table 2.1a and Advani-Summers capital-gains incidence", + "source": "HMRC Capital Gains Tax statistics, July 2025, Table 2.1a", + "grain": "household", + "artifacts": [ + { + "role": "hmrc_cgt_size_bands", + "kind": "public_aggregate_reference", + "resource": "hmrc_cgt_size_bands.json", + "format": "json", + "runtime_sha256_required": true + }, + { + "role": "capital_gains_incidence", + "kind": "public_aggregate_reference", + "resource": "advani_summers_capital_gains_distribution.json", + "format": "json", + "runtime_sha256_required": true + } + ], + "operations": [ + { + "kind": "stack_band_donor_households", + "size_band_resource": "hmrc_cgt_size_bands.json", + "incidence_resource": "advani_summers_capital_gains_distribution.json", + "minimum_band_lower": 12300, + "donors_per_band": 30, + "expected_band_count": 9, + "expected_donor_count": 270, + "candidate_order": "household_id ascending", + "draw": "weighted_without_replacement", + "propensity": "Advani-Summers percent_with_gains at oldest-adult component-sum income", + "seed": 1, + "flag_column": "household_is_cgt_band_donor", + "carrier": "oldest adult; person_id ascending breaks age ties", + "initial_weight": "published band taxpayers / donors_per_band", + "never_zero_weight": true, + "weight_kind_out": "importance", + "reason": "Stack 30 positive-weight HMRC Table 2.1a support households per retained gain band; published donor mass is added explicitly." + } + ], + "outputs": [ + "household_is_cgt_band_donor", + "capital_gains" + ], + "rewrites": [ + "capital_gains" + ], + "notes": "Adds 270 positive-weight band donors. Rows below GBP 12,300 are excluded because they mix annual-exempt-amount regimes and the spline body already supplies that support." + }, + { + "stage": "hmrc_cgt_gains_spine", + "survey": "HMRC Capital Gains Tax statistics table 3 (size of gain by taxable income), 2020-21 to 2023-24", + "source": "https://assets.publishing.service.gov.uk/media/6878ac62760bf6cedaf5bd93/Table_3_2025_Size_of_gain_by_income.ods", + "grain": "person", + "artifacts": [ + { + "role": "cgt_published_fact_surface", + "kind": "administrative_table", + "format": "ods", + "survey": "HMRC Capital Gains Tax statistics table 3", + "publication": "https://www.gov.uk/government/statistics/capital-gains-tax-statistics", + "vintage": "2023-24", + "tax_year_start": 2023, + "locator": "https://assets.publishing.service.gov.uk/media/6878ac62760bf6cedaf5bd93/Table_3_2025_Size_of_gain_by_income.ods", + "sha256": "8e75c00bab949348a7238fea6d995f626c85e5d02813b46606dd7fea85e9d0c3", + "size_bytes": 11996, + "mime_type": "application/vnd.oasis.opendocument.spreadsheet", + "sheets": [ + "3_1_2023-24", + "3_2_2022-23", + "3_3_2021-22", + "3_4_2020-21" + ], + "mapped_build_period": 2024, + "period_mapping": "latest_published_tax_year", + "runtime_sha256_required": true + }, + { + "role": "policy_parameters", + "kind": "versioned_parameter_tree", + "dependency": "policyengine-uk>=2.88 via microcosm-build[uk]", + "parameters": [ + "gov.hmrc.income_tax.allowances.personal_allowance.amount", + "gov.hmrc.income_tax.allowances.personal_allowance.maximum_ANI", + "gov.hmrc.income_tax.allowances.personal_allowance.reduction_rate", + "gov.hmrc.cgt.annual_exempt_amount" + ], + "instant_rule": "raw dated parameter files evaluated at 1 June of the build period's tax year", + "runtime_sha256_required": false, + "dependency_discipline": "deferred inside uk_cgt_policy_parameters; the base package never imports policyengine-uk at import time" + } + ], + "operations": [ + { + "kind": "verify_pinned_cgt_ods", + "artifact_role": "cgt_published_fact_surface", + "require_before_source_read": true, + "runtime_sha256_required": true, + "fail_on_mismatch": true + }, + { + "kind": "taxable_income_proxy", + "components": [ + "employment_income", + "self_employment_income", + "state_pension_reported", + "private_pension_income", + "property_income", + "savings_interest_income", + "dividend_income", + "miscellaneous_income" + ], + "components_semantics": "Persisted leaves of the model's total_income concept (ITA 2007 s.23); state_pension_reported stands in for social_security_income, whose other taxable benefits are not persisted; reliefs such as pension contributions and Gift Aid are not deducted.", + "allowance": "tapered Personal Allowance from the policy_parameters artifact", + "fail_on_missing_component": true + }, + { + "kind": "rank_preserving_allocation", + "within": "income band", + "ordering": "existing gains descending, person_id ascending on ties", + "band_order": "highest gain band first", + "suppressed_cell_allocation": "count implied by the cell's published gains at the band-total mean", + "column_reconciliation": "every income column rescales onto its published All-row taxpayer total", + "shortfall_policy": "proportional scale-down when the population holds less gainer mass than published taxpayers", + "minimum_allocation_people": 1, + "weights": "household_weight mapped to persons; no person splits across bands" + }, + { + "kind": "within_band_draws", + "bounded_band_family": "truncated exponential matched to the cell's published mean", + "open_band_family": "Pareto with alpha = mean / (mean - lower bound)", + "mean_repair_margin": 0.02, + "mean_repair_reason": "Published counts round to the nearest thousand and amounts to the nearest million; four cells of the 2023-24 table imply a mean outside their own band, and repaired means clamp just inside the violated boundary.", + "bottom_band_floor": "annual exempt amount plus one pound", + "seed_base": 552, + "seed_mixing": "seed combined with the build period; draws ordered by allocation rank", + "deterministic": true + }, + { + "kind": "sub_aea_remainder", + "policy": "gainers beyond the published taxpayer mass keep their existing amounts capped at the annual exempt amount", + "rationale": "Table 3 covers only individuals with a CGT liability; remaining gainers are treated as sub-AEA gainers rather than invented into the liability distribution or deleted." + }, + { + "kind": "record_mass_conservation_receipt", + "entity": "household", + "reason": "Amounts-only capital gains redraw: household weights pass through unchanged and total household mass is conserved.", + "declared_factor": 1.0, + "gate_coupling": "The terminal family gate requires a valid mass-conserving MassChangeRecord carrying exactly this reason." + }, + { + "kind": "classify_cgt_band_facts_with_reviewed_fence", + "calibration_permitted": false, + "fact_fence_id": "cgt_band_facts_policy_endogenous_proxy_conditioned", + "fenced_fact_count": 76, + "fenced_fact_composition": "60 joint cells, 10 gain-band row totals, 6 income-column totals", + "classification_rationale": "The taxpayer count is endogenous to policy, the income conditioning is an arithmetic proxy, and the published surface needs rounding and suppression reconciliation before any per-band fact is exact.", + "calibrated_facts_unchanged": "The two aggregate facts in UK_CGT_TARGET_SPECS remain the only calibrated CGT facts.", + "promotion_path": "A separately reviewed target profile may lift specific band facts after the reconciliation and proxy adequacy are adjudicated.", + "adjudication": "https://github.com/PolicyEngine/microcosm/issues/552" + } + ], + "outputs": [ + "capital_gains" + ], + "rewrites": [ + "capital_gains" + ], + "notes": "Spine manifest projection of the merged HMRC Table 3 amounts stage. It deliberately omits base_candidate and verify_certified_candidate, which belong only to the certified-H5 path." + }, + { + "stage": "salary_sacrifice", + "survey": "Family Resources Survey 2024-25 salary-sacrifice respondents and HMRC salary-sacrifice reform analysis", + "source": "HMRC, Salary sacrifice reform for pension contributions effective from 6 April 2029", + "grain": "person", + "artifacts": [ + { + "role": "salary_sacrifice_anchor", + "kind": "public_aggregate_reference", + "resource": "salary_sacrifice_anchor.json", + "format": "json", + "runtime_sha256_required": true + } + ], + "operations": [ + { + "kind": "fit_weighted_qrf", + "training_population": "support_channel == frs and not capital-gains clone and not CGT band donor and salary_sacrifice_asked == 1", + "target_population": "salary_sacrifice_asked != 1 frame-wide", + "predictors": [ + "age", + "employment_income" + ], + "targets": [ + "pension_contributions_via_salary_sacrifice" + ], + "weights": "household_weight", + "weight_mapping": "household_to_person", + "seed": 42, + "n_estimators": 100, + "clamp_minimum": 0, + "preserve_asked_rows": true, + "cache": false + }, + { + "kind": "convert_donors_to_target_stock", + "resource": "salary_sacrifice_anchor.json", + "target": 5400000, + "donor_pool": "employee_pension_contributions > 0 and pension_contributions_via_salary_sacrifice == 0 and employment_income > 0", + "rate_cap": 0.5, + "move": "full employee_pension_contributions to pension_contributions_via_salary_sacrifice; source zeroed", + "seed": 2024, + "salt": "salary_sacrifice_conversion", + "receipt": "weighted_headcount", + "reason": "Salary-sacrifice support stage rewrites pension columns only; household rows and typed household weights pass through and total household mass is conserved." + } + ], + "outputs": [ + "pension_contributions_via_salary_sacrifice", + "employee_pension_contributions" + ], + "nonnegative_outputs": [ + "pension_contributions_via_salary_sacrifice", + "employee_pension_contributions" + ], + "rewrites": [ + "pension_contributions_via_salary_sacrifice", + "employee_pension_contributions" + ], + "notes": "The QRF trains only on the 2024-25 FRS asked subset. The second arm creates support toward the reviewed 5.4m staging target by moving contributors' full pension amounts." + }, + { + "stage": "student_loans", + "survey": "Family Resources Survey 2024-25 and Student Loans Company borrower forecasts for England", + "source": "Explore Education Statistics Table 6a, Higher education total", + "grain": "person", + "artifacts": [ + { + "role": "frs_release", + "kind": "public_aggregate_reference", + "resource": "frs_release.json", + "format": "json", + "runtime_sha256_required": true + }, + { + "role": "slc_liable_stocks", + "kind": "public_aggregate_reference", + "resource": "slc_liable_stocks.json", + "format": "json", + "runtime_sha256_required": true + } + ], + "operations": [ + { + "kind": "assign_student_loan_plan_cohorts", + "year_rule": "calibration_year", + "start_year_formula": "year - age + 18", + "reported_repayment_test": "student_loan_repayments > 0", + "reported_country_gate": false, + "plan_1_before": 2012, + "plan_5_from": 2023, + "enum_domain": [ + "NONE", + "PLAN_1", + "PLAN_2", + "PLAN_5" + ], + "plan_4_imputation": false + }, + { + "kind": "top_up_to_stock", + "plan": "PLAN_5", + "priority": 1, + "resource": "slc_liable_stocks.json", + "stock_series": "plan_5.liable", + "year_rule": "calibration_year", + "age_min": 18, + "age_max": 25, + "cohort_start_min": 2023, + "eligible_region_exclusions": [ + "SCOTLAND", + "WALES", + "NORTHERN_IRELAND" + ], + "highest_education": "TERTIARY", + "seed": 42, + "salt": "student_loan_plan_5" + }, + { + "kind": "top_up_to_stock", + "plan": "PLAN_2", + "priority": 2, + "resource": "slc_liable_stocks.json", + "stock_series": "plan_2.liable", + "year_rule": "calibration_year", + "age_min": 21, + "age_max": 55, + "cohort_start_min": 2012, + "cohort_start_max_exclusive": 2023, + "eligible_region_exclusions": [ + "SCOTLAND", + "WALES", + "NORTHERN_IRELAND" + ], + "highest_education": "TERTIARY", + "seed": 42, + "salt": "student_loan_plan_2", + "reason": "Student-loan plan assignment writes an enum column only; household rows and typed household weights pass through and total household mass is conserved." + } + ], + "outputs": [ + "student_loan_plan" + ], + "rewrites": [ + "student_loan_plan" + ], + "notes": "Reported PAYE repayers are classified without a country gate. England tertiary cohorts are then topped up PLAN_5 first and PLAN_2 second to the pinned liable stocks at the FRS release calibration year; PLAN_4 is never imputed." + }, + { + "stage": "frs_hmrc_retained_leaves", + "survey": "Family Resources Survey 2024-25", + "source": "Department for Work and Pensions Family Resources Survey 2024-25 raw adult.tab and benefits.tab, caller-supplied local input", + "grain": "person", + "artifacts": [], + "operations": [ + { + "kind": "verify_certified_candidate", + "artifact": "base_candidate", + "runtime_sha256_required": true, + "fail_on_mismatch": true + }, + { + "kind": "retain_adjudicated_frs_hmrc_leaves", + "population": "certified_microcosm_uk_candidate_base_channel", + "source_vintage": "2024-25", + "mapped_build_period": 2024, + "annualization": "weekly raw FRS amounts * (365.25 / 7)", + "status": "adjudicated_partial_replay", + "retained_full_constituents": { + "hmrc_spi_pay": { + "spi_concept": "PAY", + "scope": "full", + "raw_sources": [ + "ADULT.INEARNS" + ], + "formula": "max(0, ADULT.INEARNS) * (365.25 / 7)" + }, + "hmrc_spi_unemployment_benefit_income": { + "spi_concept": "UBISJA", + "scope": "full", + "raw_sources": [ + "BENEFITS.BENEFIT=14:BENAMT", + "BENEFITS.BENEFIT=19:BENAMT" + ], + "formula": "sum(BENAMT where BENEFIT in {14, 19}) * (365.25 / 7)" + }, + "hmrc_spi_incapacity_benefit_income": { + "spi_concept": "INCPBEN", + "scope": "full", + "raw_sources": [ + "BENEFITS.BENEFIT=17:BENAMT" + ], + "formula": "sum(BENAMT where BENEFIT == 17) * (365.25 / 7)", + "observed_support": "structural zero in the audited 2023-24 FRS; retained so future vintages flow" + } + }, + "retained_named_subsets": { + "ossben_identifiable_subset": { + "spi_concept": "OSSBEN", + "raw_sources": [ + "BENEFITS.BENEFIT=13:BENAMT", + "BENEFITS.BENEFIT=16,VAR2 in {1,3}:BENAMT" + ], + "formula": "sum(BENAMT where BENEFIT == 13 or (BENEFIT == 16 and VAR2 in {1, 3})) * (365.25 / 7)", + "scope": "identifiable_subset" + }, + "srp_regular_code5": { + "spi_concept": "SRP", + "raw_sources": [ + "BENEFITS.BENEFIT=5:BENAMT" + ], + "formula": "sum(BENAMT where BENEFIT == 5) * (365.25 / 7)", + "scope": "regular_code5_subset" + } + }, + "source_absent_full_constituents": [ + "EPB", + "EXPS", + "TAXTERM", + "MOTHINC", + "OTHERINC" + ], + "full_concepts_forbidden_on_frs": [ + "hmrc_spi_employment_benefits", + "hmrc_spi_employment_expenses", + "hmrc_spi_taxable_termination_pay", + "hmrc_spi_miscellaneous_employment_income", + "hmrc_spi_other_income", + "hmrc_spi_other_social_security_income", + "hmrc_spi_state_pension_income" + ], + "forbid_proxy_substitution": [ + "employment_income", + "miscellaneous_income" + ], + "fail_on_missing_retained_constituent": true, + "fail_on_full_concept_alias": true + } + ], + "outputs": [ + "hmrc_spi_pay", + "hmrc_spi_unemployment_benefit_income", + "hmrc_spi_incapacity_benefit_income", + "ossben_identifiable_subset", + "srp_regular_code5" + ], + "notes": "Retains the adjudicated source-faithful FRS HMRC leaf columns before the SPI income rebuild: full PAY, UBISJA, and INCPBEN, plus explicitly named OSSBEN and SRP subsets. The runtime verifies the certified candidate before retaining these leaves." + }, + { + "stage": "hmrc_spi_income", + "survey": "Survey of Personal Incomes Public Use Tape 2022-23 and HMRC Personal Incomes Tables 3.6/3.7 2023-24", + "source": "https://assets.publishing.service.gov.uk/media/69f1f12d2fae53a03709682f/Collated_Tables_3_1_to_3_11_2324.ods", + "grain": "person", + "artifacts": [ + { + "role": "qrf_donor", + "kind": "private_microdata", + "format": "tab_delimited", + "survey": "Survey of Personal Incomes Public Use Tape 2022-23", + "vintage": "2022-23", + "tax_year_start": 2022, + "ukds_study_number": "SN 9422", + "doi": "10.5255/UKDA-SN-9422-1", + "filename": "put2223uk.tab", + "sha256": "5ef829461060c91a2a47be59ad541d9b519fc3976d66ca80d4920f711bb96f66", + "size_bytes": 141323762, + "reviewed_source": "PolicyEngine licensed UKDS mirror (private Hugging Face repository), spi_2022_23.zip", + "access": "private_local_input", + "locator": "caller-supplied local input", + "runtime_sha256_required": true + }, + { + "role": "published_fact_surface", + "kind": "administrative_table", + "format": "ods", + "survey": "HMRC Personal Incomes Tables 3.6 and 3.7", + "publication": "https://www.gov.uk/government/statistics/personal-incomes-statistics-for-the-tax-year-2023-to-2024", + "vintage": "2023-24", + "tax_year_start": 2023, + "locator": "https://assets.publishing.service.gov.uk/media/69f1f12d2fae53a03709682f/Collated_Tables_3_1_to_3_11_2324.ods", + "sha256": "ad063b06b2bdeef8600dbbb09d48153337a4966f8c7eea50df7a2e0304ebd73e", + "size_bytes": 166693, + "mime_type": "application/vnd.oasis.opendocument.spreadsheet", + "sheets": [ + "Table_3_6", + "Table_3_7" + ], + "mapped_build_period": 2024, + "period_mapping": "latest_published_tax_year", + "runtime_sha256_required": true + } + ], + "operations": [ + { + "kind": "verify_pinned_hmrc_source_pair", + "artifact_roles": [ + "qrf_donor", + "published_fact_surface" + ], + "require_before_source_read": true, + "runtime_sha256_required": true, + "fail_on_mismatch": true + }, + { + "kind": "replace_zero_weight_spi_support", + "existing_channel": "spi", + "require_existing_weight": 0, + "replacement_strata": [ + "clone_index", + "household_is_capital_gains_clone", + "region" + ], + "spi_prior_national_household_mass_share": 0.5, + "output_weight_kind": "importance", + "preserve_total_household_mass": true, + "require_mass_change_record": true, + "mass_change_reason": "Allocate 50% of certified UK national household prior mass to the rebuilt 2022-23 SPI support channel; total national mass is conserved.", + "fail_on_live_existing_spi_mass": true + }, + { + "kind": "strict_read_private_table", + "artifact_role": "qrf_donor", + "filename": "put2223uk.tab", + "delimiter": "\t", + "weight": "FACT", + "required_columns": [ + "AGERANGE", + "GORCODE", + "SEX", + "FACT", + "PAY", + "EPB", + "EXPS", + "TAXTERM", + "INCPBEN", + "OSSBEN", + "UBISJA", + "MOTHINC", + "OTHERINC", + "PROFITS", + "CAPALL", + "LOSSBF", + "SRP", + "INCBBS", + "DIVIDENDS", + "PENSION", + "INCPROP", + "OTHERINV", + "GIFTAID", + "GIFTINV", + "TEI", + "TII", + "TI" + ], + "runtime_sha256_required": true, + "fail_on_missing_file": true, + "fail_on_missing_columns": true, + "fail_on_invalid_weight": true + }, + { + "kind": "fit_weighted_qrf_stage1", + "training_artifact_role": "qrf_donor", + "predictors": [ + "age", + "gender", + "region" + ], + "categorical_predictors": [ + "gender", + "region" + ], + "source_sampling_weight": "FACT", + "sample_size": 100000, + "sample_with_replacement": true, + "post_sample_fit_weight": "uniform", + "fit_weight_kind": "design", + "double_apply_source_weight": false, + "source_columns": { + "self_employment_income": [ + "PROFITS", + "CAPALL", + "LOSSBF" + ], + "savings_interest_income": [ + "INCBBS" + ], + "dividend_income": [ + "DIVIDENDS" + ], + "private_pension_income": [ + "PENSION" + ], + "property_income": [ + "INCPROP" + ], + "other_investment_income": [ + "OTHERINV" + ], + "gift_aid": [ + "GIFTAID" + ], + "charitable_investment_gifts": [ + "GIFTINV" + ], + "hmrc_spi_pay": [ + "PAY" + ], + "hmrc_spi_employment_benefits": [ + "EPB" + ], + "hmrc_spi_employment_expenses": [ + "EXPS" + ], + "hmrc_spi_incapacity_benefit_income": [ + "INCPBEN" + ], + "hmrc_spi_other_social_security_income": [ + "OSSBEN" + ], + "hmrc_spi_taxable_termination_pay": [ + "TAXTERM" + ], + "hmrc_spi_unemployment_benefit_income": [ + "UBISJA" + ], + "hmrc_spi_miscellaneous_employment_income": [ + "MOTHINC" + ], + "hmrc_spi_other_income": [ + "OTHERINC" + ], + "hmrc_spi_state_pension_income": [ + "SRP" + ] + }, + "derived_policyengine_outputs": { + "employment_income": { + "source_columns": [ + "PAY", + "EPB", + "TAXTERM" + ], + "formula": "hmrc_spi_pay + hmrc_spi_employment_benefits + hmrc_spi_taxable_termination_pay", + "derive_after_draw": true + } + }, + "outputs": [ + "self_employment_income", + "savings_interest_income", + "dividend_income", + "private_pension_income", + "property_income", + "other_investment_income", + "gift_aid", + "charitable_investment_gifts", + "hmrc_spi_pay", + "hmrc_spi_employment_benefits", + "hmrc_spi_employment_expenses", + "hmrc_spi_incapacity_benefit_income", + "hmrc_spi_other_social_security_income", + "hmrc_spi_taxable_termination_pay", + "hmrc_spi_unemployment_benefit_income", + "hmrc_spi_miscellaneous_employment_income", + "hmrc_spi_other_income", + "hmrc_spi_state_pension_income" + ], + "joint_draw": true, + "savings_interest_source_semantics": "INCBBS is taxable bank/building-society interest before reconstruction to the PolicyEngine gross input", + "employment_income_source_semantics": "PolicyEngine input = PAY + EPB + TAXTERM, matching the pinned enhanced-FRS pipeline; it is not the Table 3.6 measure", + "hmrc_employed_income_source_semantics": "Derived after each draw as max(0, PAY + EPB - EXPS) + INCPBEN + OSSBEN + TAXTERM + UBISJA + MOTHINC, using normalized leaves identically on FRS and SPI channels", + "self_employment_income_source_semantics": "max(0, PROFITS - CAPALL - LOSSBF)", + "assessable_income_source_semantics": "QRF draws leaves only; TEI, TII, and TI are deterministic post-draw accounting aggregates and TI equals TEI + TII exactly", + "source_ti_identity_fields": [ + "TI", + "TEI", + "TII" + ], + "source_leaf_reconciliation": { + "documentation_url": "https://doc.ukdataservice.ac.uk/doc/9422/mrdoc/pdf/9422_put_2223_full_documentation.pdf", + "composite_indicator": "AGERANGE == -1", + "formulas": { + "TEI": "max(0, PAY + EPB - EXPS) + INCPBEN + OSSBEN + TAXTERM + UBISJA + MOTHINC + OTHERINC + SRP + PENSION + max(0, PROFITS - CAPALL - LOSSBF)", + "TII": "OTHERINV + DIVIDENDS + INCPROP + INCBBS", + "TI": "TEI + TII" + }, + "maximum_absolute_difference_gbp": { + "ordinary": { + "TEI": 15, + "TII": 10, + "TI": 20 + }, + "composite": { + "TEI": 180, + "TII": 10, + "TI": 180 + } + }, + "rationale": "The official PUT rounds source fields, averages documented composite records, then rounds remaining income fields to GBP 5. These are the observed envelopes in the exact sha-pinned donor; post-draw synthetic identities remain exact." + }, + "ti_identity_absolute_tolerance_gbp": 5, + "stochastic_aggregates_forbidden": [ + "hmrc_spi_employed_income", + "hmrc_spi_total_earned_income", + "hmrc_spi_total_investment_income", + "hmrc_spi_assessable_income" + ], + "require_all_predictors": true, + "require_all_outputs": true + }, + { + "kind": "fit_weighted_qrf_stage2", + "training_population": "certified_microcosm_uk_candidate_base_channel", + "target_population": "rebuilt_spi_support_channel", + "predictors": [ + "age", + "gender", + "region", + "employment_income", + "self_employment_income", + "savings_interest_income", + "dividend_income", + "private_pension_income", + "property_income" + ], + "reviewed_absent_predictors": { + "other_investment_income": "This remains a stage-1 SPI draw and an official HMRC fact component, but it is not an FRS-only stage-2 predictor: the incumbent UK data build's frs_only.py defines exactly six income predictors and the certified Microcosm UK base candidate has no other_investment_income column." + }, + "categorical_predictors": [ + "gender", + "region" + ], + "weight": "household_weight", + "weight_mapping": "household_to_person", + "outputs": [ + "employee_pension_contributions", + "employer_pension_contributions", + "personal_pension_contributions", + "pension_contributions_via_salary_sacrifice", + "tax_free_savings_income", + "universal_credit_reported", + "pension_credit_reported", + "child_benefit_reported", + "housing_benefit_reported", + "income_support_reported", + "working_tax_credit_reported", + "child_tax_credit_reported", + "attendance_allowance_reported", + "state_pension_reported", + "dla_sc_reported", + "dla_m_reported", + "pip_m_reported", + "pip_dl_reported", + "sda_reported", + "carers_allowance_reported", + "iidb_reported", + "afcs_reported", + "bsp_reported", + "winter_fuel_allowance_reported", + "council_tax_benefit_reported", + "jsa_contrib_reported", + "jsa_income_reported", + "esa_contrib_reported", + "esa_income_reported" + ], + "reviewed_absent_outputs": { + "incapacity_benefit_reported": "Absent/all-default on the pinned enhanced-FRS export and certified Microcosm UK base; not a populated loader layer.", + "maternity_allowance_reported": "Absent from the pinned enhanced-FRS export and certified Microcosm UK base; no training source can be materialized for this stage." + }, + "postprocess": { + "gross_savings_interest_income": "stage1 INCBBS draw + stage2 tax_free_savings_income", + "refresh_disability_categories": [ + "aa_category", + "dla_sc_category", + "dla_m_category", + "pip_m_category", + "pip_dl_category" + ], + "refresh_disability_flags": [ + "is_disabled_for_benefits", + "is_enhanced_disabled_for_benefits", + "is_severely_disabled_for_benefits" + ] + }, + "joint_draw": true, + "require_all_predictors": true, + "require_all_materializable_outputs": true, + "require_all_outputs": false + }, + { + "kind": "materialize_hmrc_income_bands_fail_closed", + "artifact_role": "published_fact_surface", + "mapped_build_period": 2024, + "period_mapping": "latest_published_tax_year", + "column_index_base": 0, + "data_row_start_index": 5, + "stop_label": "All ranges", + "count_unit_multiplier": 1000, + "amount_unit_multiplier": 1000000, + "component_columns": { + "employment_income": { + "sheet": "Table_3_6", + "count_column_index": 4, + "amount_column_index": 5 + }, + "self_employment_income": { + "sheet": "Table_3_6", + "count_column_index": 1, + "amount_column_index": 2 + }, + "state_pension": { + "sheet": "Table_3_6", + "count_column_index": 7, + "amount_column_index": 8 + }, + "private_pension_income": { + "sheet": "Table_3_6", + "count_column_index": 10, + "amount_column_index": 11 + }, + "property_income": { + "sheet": "Table_3_7", + "count_column_index": 1, + "amount_column_index": 2 + }, + "savings_interest_income": { + "sheet": "Table_3_7", + "count_column_index": 4, + "amount_column_index": 5 + }, + "dividend_income": { + "sheet": "Table_3_7", + "count_column_index": 7, + "amount_column_index": 8 + }, + "other_investment_income": { + "sheet": "Table_3_7", + "count_column_index": 10, + "amount_column_index": 11 + } + }, + "required_band_lower_bounds_gbp": [ + 12570, + 15000, + 20000, + 30000, + 40000, + 50000, + 70000, + 100000, + 150000, + 200000, + 300000, + 500000, + 1000000 + ], + "required_measures": [ + "count", + "amount" + ], + "fail_on_missing_sheet": true, + "fail_on_missing_component": true, + "fail_on_missing_band": true, + "fail_on_non_numeric_value": true + }, + { + "kind": "classify_hmrc_income_facts_with_reviewed_fences", + "target_operation": "materialize_hmrc_income_bands_fail_closed", + "components": [ + "employment_income", + "self_employment_income", + "state_pension", + "private_pension_income", + "property_income", + "savings_interest_income", + "dividend_income", + "other_investment_income" + ], + "breakdown_dependency": "hmrc_spi_assessable_income", + "frs_breakdown_status": "unavailable_full_measure", + "input_weight_kind": "importance", + "output_weight_kind": "importance", + "calibration_permitted": false, + "required_fact_count": 208, + "outcome_counts": { + "exact_pass": 0, + "exact_fail": 0, + "directional_pass": 0, + "directional_fail": 0, + "excluded_with_fence": 208 + }, + "classification_rationale": "Every published fact uses non-overlapping total-income bands. The FRS channel cannot materialize full TEI, and omitted income can move a person between bands, so neither an exact fact nor a per-band directional bound is valid.", + "reviewed_fences": [ + { + "fence_id": "frs_epb_source_absent", + "constituents": [ + "EPB" + ], + "raw_sources_searched": [ + "JOB.EXPBEN01-EXPBEN13", + "JOB.CARVAL", + "JOB.CARAMT", + "JOB.FUELAMT", + "JOB.VCHAMT", + "JOB.CHVAMT" + ], + "finding": "Missing. EXPBEN* are receipt flags, and the amount fields cover only selected benefits; they cannot produce complete taxable expenses payments and benefits.", + "mass_implication": "12.9485464% of certified-candidate FRS effective person mass has at least one receipt flag, but this is not monetary support.", + "rationale": "Receipt flags and selected benefit amounts cannot be promoted to the SPI EPB monetary concept without an imputation or proxy.", + "dependent_fence_ids": [] + }, + { + "fence_id": "frs_exps_source_absent", + "constituents": [ + "EXPS" + ], + "raw_sources_searched": [ + "JOB.EXPBEN04/EXPBEN05", + "JOB.MILEAMT/JOB.MOTAMT", + "JOB.UMILEAMT/JOB.UMOTAMT", + "JOB.DEDUC1-DEDUC9", + "JOB.UDEDUC1-UDEDUC9" + ], + "finding": "Missing. These fields describe reimbursements or payroll deductions, not the complete tax-deductible employment-expense amount required by SPI.", + "mass_implication": "5.1302528% of certified-candidate FRS effective person mass has an adjacent reimbursement flag; the true EXPS mass is not estimable.", + "rationale": "The nearby fields do not measure the required deductible amount, and EXPS enters the employed-income identity with a negative sign.", + "dependent_fence_ids": [] + }, + { + "fence_id": "frs_taxterm_source_absent", + "constituents": [ + "TAXTERM" + ], + "raw_sources_searched": [ + "ADULT.REDAMT", + "ADULT and JOB taxable-termination split search" + ], + "finding": "Missing. REDAMT is gross redundancy pay and has neither the taxable amount nor non-redundancy termination pay.", + "mass_implication": "0.3746084% of certified-candidate FRS effective person mass has positive gross redundancy pay; taxable mass is unknown.", + "rationale": "Gross redundancy pay cannot be relabeled as taxable termination pay.", + "dependent_fence_ids": [] + }, + { + "fence_id": "frs_mothinc_source_absent", + "constituents": [ + "MOTHINC" + ], + "raw_sources_searched": [ + "ODDJOB.OJAMT/ODDJOB.OJNOW", + "ADULT.ALLPAY2", + "ADULT.ROYYR2-ROYYR4", + "JOB.OWNOTHER" + ], + "finding": "Missing. The fields are heterogeneous and belong to distinct income concepts; assigning their union to SPI miscellaneous employment income would be a proxy.", + "mass_implication": "Odd-job-only effective person mass is 0.1724207%; the broader unresolved miscellaneous pool is 1.4650566%.", + "rationale": "The FRS instrument cannot separate the SPI miscellaneous-employment concept source-faithfully.", + "dependent_fence_ids": [] + }, + { + "fence_id": "frs_otherinc_source_absent", + "constituents": [ + "OTHERINC" + ], + "raw_sources_searched": [ + "ADULT, ODDJOB, and JOB miscellaneous fields", + "PENSION", + "ACCOUNTS", + "ASSETS", + "BENEFITS" + ], + "finding": "Missing. No person-level raw FRS variable has SPI OTHERINC semantics, and the miscellaneous pool cannot be split between MOTHINC and OTHERINC from source evidence.", + "mass_implication": "No separable mass estimate exists; the unresolved miscellaneous pool is 1.4650566% of certified-candidate FRS effective person mass.", + "rationale": "A union of heterogeneous residual fields would be a new proxy, not a retained source constituent.", + "dependent_fence_ids": [] + }, + { + "fence_id": "frs_ossben_identifiable_subset", + "constituents": [ + "OSSBEN", + "ossben_identifiable_subset" + ], + "raw_sources_searched": [ + "BENEFITS.BENAMT", + "BENEFITS.BENEFIT", + "BENEFITS.VAR2", + "BENEFITS codes 13, 16, 6, and 30" + ], + "finding": "Incomplete. Carer's Allowance and contribution-based ESA form an identifiable subset, but code 6 mixes tax treatments and code 30 is an undifferentiated catch-all, so the complete taxable family cannot be emitted.", + "mass_implication": "1.8045088% of certified-candidate FRS effective person mass carries the identifiable lower-bound subset; it is not full OSSBEN support.", + "rationale": "The retained column must remain explicitly named as a subset and cannot satisfy the full SPI concept.", + "dependent_fence_ids": [] + }, + { + "fence_id": "frs_srp_regular_code5_subset", + "constituents": [ + "SRP", + "srp_regular_code5" + ], + "raw_sources_searched": [ + "BENEFITS.BENAMT where BENEFIT == 5", + "BENEFITS codes 6 and 9" + ], + "finding": "Incomplete. Code 5 supplies regular State Pension, but the FRS source does not identify the full SPI combination of State Pension lump sums and widow's pension; code 6 mixes benefits and code 9 is tax-free War Widow's Pension.", + "mass_implication": "18.1567916% of certified-candidate FRS effective person mass carries regular code-5 State Pension; it is not complete SRP support.", + "rationale": "The retained column must remain explicitly named as a subset and cannot be reported as the full published state-pension measure.", + "dependent_fence_ids": [] + }, + { + "fence_id": "full_frs_tei_band_unavailable", + "constituents": [ + "EPB", + "EXPS", + "TAXTERM", + "MOTHINC", + "OTHERINC", + "OSSBEN", + "SRP" + ], + "raw_sources_searched": [], + "finding": "The complete FRS TEI measure cannot be materialized from retained source constituents, so exact HMRC total-income band assignment is unavailable on the FRS channel.", + "mass_implication": "Every one of the 208 published facts is banded by total income and therefore depends on this unavailable like-for-like measure.", + "rationale": "A component-level subset does not imply a per-band lower bound: omitted income can move a taxpayer into or out of any non-overlapping published band. Biased partial bands are not emitted as estimates.", + "dependent_fence_ids": [ + "frs_epb_source_absent", + "frs_exps_source_absent", + "frs_taxterm_source_absent", + "frs_mothinc_source_absent", + "frs_otherinc_source_absent", + "frs_ossben_identifiable_subset", + "frs_srp_regular_code5_subset" + ] + } + ], + "fact_fence_id": "full_frs_tei_band_unavailable", + "blocked_dependency": "hmrc_spi_assessable_income", + "fail_on_unfenced_exclusion": true, + "fail_on_fact_count_mismatch": true, + "forbid_biased_estimate_or_delta": true + }, + { + "kind": "gate_distributional_effective_mass", + "columns": [ + "gift_aid", + "charitable_investment_gifts" + ], + "weight": "household_weight", + "weight_mapping": "household_to_person", + "support_channel_column": "person_support_channel", + "required_support_channel": "spi", + "mass_share_denominator": "all_person_effective_mass", + "minimum_nondefault_mass_share": 1e-06, + "fail_below_floor": true } - ] + ], + "official_table_components": [ + "employment_income", + "self_employment_income", + "state_pension", + "private_pension_income", + "property_income", + "savings_interest_income", + "dividend_income", + "other_investment_income" + ], + "donor_relief_outputs": [ + "gift_aid", + "charitable_investment_gifts" + ], + "outputs": [ + "employment_income", + "self_employment_income", + "hmrc_spi_state_pension_income", + "private_pension_income", + "property_income", + "savings_interest_income", + "dividend_income", + "other_investment_income", + "gift_aid", + "charitable_investment_gifts", + "hmrc_spi_employed_income", + "hmrc_spi_total_earned_income", + "hmrc_spi_total_investment_income", + "hmrc_spi_assessable_income" + ], + "notes": "Current-source adjudicated replay contract: the private 2022-23 SPI donor and public 2023-24 HMRC ODS are pinned by reviewed SHA-256 and size and verified together before either is opened. The QRF draws source leaves; HMRC employed income, TEI, TII, and TI are deterministic post-draw aggregates on the SPI channel, with TI exactly equal to TEI + TII. PolicyEngine employment_income remains the narrow PAY + EPB + TAXTERM input on SPI rows. Stage 2 mirrors the incumbent UK data build's frs_only.py exactly: its income predictors are employment, self-employment, savings interest, dividends, private pension, and property income. Other investment income remains a stage-1 SPI draw and official HMRC fact component, but is excluded from stage 2 because the certified FRS candidate does not carry it. The FRS channel retains source-faithful full PAY, UBISJA, and INCPBEN plus explicitly named ossben_identifiable_subset and srp_regular_code5; EPB, EXPS, TAXTERM, MOTHINC, OTHERINC, full OSSBEN, and full SRP remain forbidden. Because the missing legs prevent a complete FRS TEI measure, none of the 208 non-overlapping total-income-band facts is exact or directional. Every fact is an excluded-with-fence record, no calibration is performed, and weights remain importance-kind. Gift Aid restoration still requires the rebuilt positive-mass SPI channel to clear the reviewed 1ppm effective-mass floor." + } + ] } diff --git a/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml b/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml index 3787b830..8897b43e 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml +++ b/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml @@ -237,6 +237,10 @@ stages: - student_loans - access_fund - education_grants + - healthy_start_vouchers + - free_school_breakfasts + - free_school_fruit_veg + - free_school_meals - council_tax_benefit_reported - maintenance_expenses - childcare_expenses diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/frs_spine.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/frs_spine.py index 2bed6cc7..3b890057 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/frs_spine.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/frs_spine.py @@ -172,6 +172,10 @@ "student_loans", "access_fund", "education_grants", + "healthy_start_vouchers", + "free_school_breakfasts", + "free_school_fruit_veg", + "free_school_meals", "council_tax_benefit_reported", "maintenance_expenses", "childcare_expenses", @@ -525,6 +529,20 @@ def _add_person_income( pe_person["education_grants"] = np.maximum( _number(person, "grtdir1") + _number(person, "grtdir2"), 0 ) + # In-kind benefits recorded per person on the FRS tapes. Each is a direct + # weeklyised amount with no derivation: healthy-start vouchers appear on + # both the adult and child tapes, the three school ones only on the child + # tape, so absent columns read as zero for adults through `_number`. + pe_person["healthy_start_vouchers"] = ( + _positive(person, "heartval") * WEEKS_IN_YEAR + ) + pe_person["free_school_breakfasts"] = ( + _positive(person, "fsbval") * WEEKS_IN_YEAR + ) + pe_person["free_school_fruit_veg"] = ( + _positive(person, "fsfvval") * WEEKS_IN_YEAR + ) + pe_person["free_school_meals"] = _positive(person, "fsmval") * WEEKS_IN_YEAR def _odd_job_income(person: pd.DataFrame, oddjob: pd.DataFrame) -> np.ndarray: diff --git a/packages/microcosm-build/tests/test_spec_engine_country_bundles.py b/packages/microcosm-build/tests/test_spec_engine_country_bundles.py index 0ac0adcc..86e1663c 100644 --- a/packages/microcosm-build/tests/test_spec_engine_country_bundles.py +++ b/packages/microcosm-build/tests/test_spec_engine_country_bundles.py @@ -42,7 +42,7 @@ ), ( "uk", - "PLACEHOLDER_RECUT_AT_END", + "eb8e52c075de3f5c6983ad2fedb461c1cc77db0074c9e76aa22c4c492a5524a5", { "benunit.benunit_id", "household.household_id", diff --git a/packages/microcosm-build/tests/test_uk_frs_spine.py b/packages/microcosm-build/tests/test_uk_frs_spine.py index f8864cc7..8960b22c 100644 --- a/packages/microcosm-build/tests/test_uk_frs_spine.py +++ b/packages/microcosm-build/tests/test_uk_frs_spine.py @@ -154,6 +154,8 @@ def _fixture_tables() -> dict[str, list[dict[str, object]]]: "ACCSSAMT": 1.0, "GRTDIR1": 2.0, "GRTDIR2": 3.0, + # heartval is on the adult tape too; the three school columns are not. + "HEARTVAL": 5.0, } adult_2 = {**adult_1, "SERNUM": 2, "PERSON": 1, "SEX": 2, "HRPID": 1} child_1 = { @@ -173,6 +175,10 @@ def _fixture_tables() -> dict[str, list[dict[str, object]]]: "TRAIN": 9, "EMAAMT": 0.0, "CHEMAAMT": 1.0, + "FSMVAL": 3.0, + "FSFVVAL": 1.0, + "FSBVAL": 2.0, + "HEARTVAL": 4.0, } return { "adult": [adult_2, adult_1], @@ -1691,3 +1697,37 @@ def test_retired_cells_cannot_reintroduce_the_incumbent_zeroing(self) -> None: scottish_water_and_sewerage_weekly(absent).iloc[0] ) assert result > 0 + + +def test_in_kind_benefits_map_from_the_raw_person_tapes(tmp_path: Path) -> None: + """The four in-kind benefit columns, ported at #686. + + They were absent from the spine while the incumbent mapped them straight + off the person tapes, so the parity screen reported them as columns the + candidate did not produce. Each is a plain weeklyised amount. + """ + + stage = _write_fixture(tmp_path) + + frame = build_uk_frs_spine_frame(tmp_path, stage=stage) + person = frame.table("person").set_index("person_id") + + children = person.loc[person["age"] < 16] + assert len(children) == 1 + child = children.iloc[0] + assert child["free_school_meals"] == pytest.approx(3.0 * WEEKS_IN_YEAR) + assert child["free_school_fruit_veg"] == pytest.approx(1.0 * WEEKS_IN_YEAR) + assert child["free_school_breakfasts"] == pytest.approx(2.0 * WEEKS_IN_YEAR) + assert child["healthy_start_vouchers"] == pytest.approx(4.0 * WEEKS_IN_YEAR) + + # heartval is on the adult tape as well; the school columns are child-only + # and must read as zero for adults rather than propagating NaN. + adults = person.loc[person["age"] > 16] + assert (adults["healthy_start_vouchers"] > 0).all() + for column in ( + "free_school_meals", + "free_school_fruit_veg", + "free_school_breakfasts", + ): + assert (adults[column] == 0).all() + assert person[column].notna().all() diff --git a/tools/verify_uk_identity_stability.py b/tools/verify_uk_identity_stability.py index dfd47c1a..74f92f6b 100644 --- a/tools/verify_uk_identity_stability.py +++ b/tools/verify_uk_identity_stability.py @@ -482,6 +482,139 @@ def recompute(person_t, benunit_t, household_t) -> dict[str, pd.DataFrame]: } +def e7_identity_receipt( + frame, + *, + permutation_seed: int, +) -> dict[str, object]: + """Receipt E7 deterministic layers under row permutation by entity id. + + Covered: the support-channel labelling the SPI stack introduces — each + entity's channel and clone index, the composite source key, and the + propagation of a household's channel down to its persons and benefit + units. These are pure functions of the synthetic flag and the entity ids, + so they are bitwise both under permutation and against the store, and they + are the layer E7 actually contributes to the artifact. + + Not covered here, and deliberately so: + + * the stage-1/stage-2 QRF fits and the dividend redraw, which twin-build + determinism covers — the e6 and e8 precedent for QRF surfaces; + * the ``employer_pension_contributions = 3 * employee_pension_contributions`` + derive, which is a genuine E7 deterministic layer but which E8's + salary_sacrifice rewrites in place afterwards. Measured on the E8 + roster the relation survives on only 95.9% of survey-channel persons, + so the stage-time relation is not reconstructible from the final + artifact. That is the #721 rewrites-provenance class, not a defect, and + asserting it here would fail for the wrong reason. + """ + + def recompute(person_t, benunit_t, household_t) -> dict[str, pd.DataFrame]: + household_out = pd.DataFrame(index=household_t["household_id"].to_numpy()) + person_out = pd.DataFrame(index=person_t["person_id"].to_numpy()) + benunit_out = pd.DataFrame(index=benunit_t["benunit_id"].to_numpy()) + + if "household_is_spi_synthetic" not in household_t.columns: + return {} + synthetic = household_t["household_is_spi_synthetic"].astype(bool).to_numpy() + channel = np.where(synthetic, "spi", "frs") + household_out["household_support_channel"] = channel + household_out["household_support_clone_index"] = np.where(synthetic, 1, 0) + + if {"source_year", "source_household_id"} <= set(household_t.columns): + household_out["source_household_key"] = [ + f"{int(year)}:{int(source)}" + for year, source in zip( + household_t["source_year"].to_numpy(), + household_t["source_household_id"].to_numpy(), + strict=True, + ) + ] + + # The channel is a household property; persons and benefit units + # inherit it through membership, never redraw it. + by_household = pd.Series(channel, index=household_t["household_id"].to_numpy()) + person_channel = ( + person_t["person_household_id"].map(by_household).to_numpy(dtype=object) + ) + person_out["person_support_channel"] = person_channel + by_benunit = pd.Series( + person_channel, index=person_t["person_benunit_id"].to_numpy() + ) + by_benunit = by_benunit[~by_benunit.index.duplicated(keep="first")] + benunit_out["benunit_support_channel"] = ( + benunit_t["benunit_id"].map(by_benunit).to_numpy(dtype=object) + ) + return { + "household": household_out, + "person": person_out, + "benunit": benunit_out, + } + + person = frame.table("person") + benunit = frame.table("benunit") + household = frame.table("household") + + original = recompute(person, benunit, household) + rng = np.random.default_rng(permutation_seed) + permuted = recompute( + person.iloc[rng.permutation(len(person))].reset_index(drop=True), + benunit.iloc[rng.permutation(len(benunit))].reset_index(drop=True), + household.iloc[rng.permutation(len(household))].reset_index(drop=True), + ) + + stored_tables = { + "person": person.set_index("person_id"), + "benunit": benunit.set_index("benunit_id"), + "household": household.set_index("household_id"), + } + mismatches: dict[str, list[str]] = {} + stored_mismatches: dict[str, list[str]] = {} + # Labels and integer indices: bitwise on both surfaces, no tolerance. + for entity, values in original.items(): + for column in values.columns: + left = values[column] + right = permuted[entity][column].reindex(left.index) + if not np.array_equal( + left.to_numpy().astype(str), right.to_numpy().astype(str) + ): + mismatches.setdefault(entity, []).append(column) + stored_table = stored_tables[entity] + if column in stored_table.columns: + kept = stored_table[column].reindex(left.index) + if not np.array_equal( + left.to_numpy().astype(str), kept.to_numpy().astype(str) + ): + stored_mismatches.setdefault(entity, []).append(column) + return { + "check": "uk_e7_identity_stability", + "permutation_seed": permutation_seed, + "identical_under_permutation": not mismatches, + "permutation_mismatches": mismatches, + "matches_stored_columns": not stored_mismatches, + "stored_column_mismatches": stored_mismatches, + "tolerance_policy": ( + "bitwise on both surfaces: the support channel, clone index and " + "source key are labels and integer indices, so no float " + "tolerance applies" + ), + "columns_by_entity": { + entity: list(values.columns) for entity, values in original.items() + }, + "qrf_draw_columns_scope": ( + "excluded: the stage-1/stage-2 QRF fits and the dividend redraw " + "are covered by twin-build determinism (the e6 and e8 precedent)" + ), + "rewritten_layer_scope": ( + "excluded: employer_pension_contributions = 3 x " + "employee_pension_contributions is an E7 derive, but E8 " + "salary_sacrifice rewrites the multiplicand in place afterwards, " + "so the stage-time relation is not reconstructible from the " + "final artifact (#721 rewrites-provenance class)" + ), + } + + def e8_identity_receipt( frame, *, @@ -722,7 +855,9 @@ def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--input-h5", type=Path, required=True) parser.add_argument("--output", type=Path, required=True) - parser.add_argument("--check", choices=("e4", "e5", "e6", "e8"), default="e4") + parser.add_argument( + "--check", choices=("e4", "e5", "e6", "e7", "e8"), default="e4" + ) parser.add_argument("--permutation-seed", type=int, default=123) args = parser.parse_args() @@ -763,6 +898,17 @@ def main() -> int: ok = bool( receipt["identical_under_permutation"] and receipt["matches_stored_columns"] ) + elif args.check == "e7": + # E7's own layer is the support-channel stack, which is defined over + # the whole frame including the rows it stacks — so unlike e4/e5 this + # receipt deliberately does NOT scope to the unstacked rows. + receipt = e7_identity_receipt( + frame, + permutation_seed=args.permutation_seed, + ) + ok = bool( + receipt["identical_under_permutation"] and receipt["matches_stored_columns"] + ) elif args.check == "e6": receipt = e6_identity_receipt( frame, From a4d3509f5812527b766fd54cd7764704181b862d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Sun, 23 Aug 2026 00:25:14 +0200 Subject: [PATCH 13/28] Re-derive the spec and manifest digests on the rebased base The branch now sits directly on main rather than on the #623 stack, so every derived value is recomputed from its producer against that base: the UK spec bundle sha, and the release input-coverage manifest's source-manifest hashes. source_stages.json and spec/sources.yaml are rebuilt from main's copies with the four in-kind outputs re-applied to both, so the lockstep projection agrees. The four gate-battery mirrors in the data shard were already correct at this base and are unchanged. Coverage stays 145 required, 0 exclusions. Co-Authored-By: Claude Fable 5 --- .../uk/release_input_coverage_manifest.json | 24 +++++++++---------- .../src/microcosm/build/uk/source_stages.json | 4 ++-- .../tests/test_spec_engine_country_bundles.py | 2 +- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json b/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json index a52bd9ba..88ff9db0 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json +++ b/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json @@ -473,7 +473,7 @@ "capital_gains" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "922d01407a3cb36d47903bab3b30ef25fd35e1d53674aec8f8c96bc7247932ba", + "source_manifest_sha256": "2aa227b9731361989c1c44ae5c670ea73f877d779031fa7739e83f8c65568857", "source_vintages": { "source": "HMRC Capital Gains Tax statistics, July 2025, Table 2.1a", "survey": "HMRC Capital Gains Tax statistics Table 2.1a and Advani-Summers capital-gains incidence" @@ -496,7 +496,7 @@ "capital_gains" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "922d01407a3cb36d47903bab3b30ef25fd35e1d53674aec8f8c96bc7247932ba", + "source_manifest_sha256": "2aa227b9731361989c1c44ae5c670ea73f877d779031fa7739e83f8c65568857", "source_vintages": { "source": "Advani and Summers (2020), Capital Gains and UK Inequality, CAGE Working Paper 465", "survey": "Family Resources Survey 2024-25, SPI synthetic support, and Advani-Summers capital-gains incidence" @@ -525,7 +525,7 @@ "required_mass_change_reason": "E5 source-stage transform preserves household rows and typed household weights; total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "922d01407a3cb36d47903bab3b30ef25fd35e1d53674aec8f8c96bc7247932ba", + "source_manifest_sha256": "2aa227b9731361989c1c44ae5c670ea73f877d779031fa7739e83f8c65568857", "source_vintages": { "source": "UK Data Service SN 8856 Effects of Taxes and Benefits household tab, DfT rail fare index, and public NHS activity/cost table.", "survey": "Effects of Taxes and Benefits 1977-2024 and NHS age-gender public table" @@ -545,7 +545,7 @@ "required_mass_change_reason": "E5 source-stage transform preserves household rows and typed household weights; total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "922d01407a3cb36d47903bab3b30ef25fd35e1d53674aec8f8c96bc7247932ba", + "source_manifest_sha256": "2aa227b9731361989c1c44ae5c670ea73f877d779031fa7739e83f8c65568857", "source_vintages": { "source": "UK Data Service SN 8856 Effects of Taxes and Benefits household tab and cited VAT anchor resource.", "survey": "Effects of Taxes and Benefits 1977-2024" @@ -585,12 +585,12 @@ "outputs": [ "capital_gains" ], - "required_mass_change_reason": "Amounts-only capital gains redraw: household weights pass through unchanged and total household mass is conserved.", + "required_mass_change_reason": "Amounts-only capital gains redraw on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [ "capital_gains" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "922d01407a3cb36d47903bab3b30ef25fd35e1d53674aec8f8c96bc7247932ba", + "source_manifest_sha256": "2aa227b9731361989c1c44ae5c670ea73f877d779031fa7739e83f8c65568857", "source_vintages": { "hmrc_surface": "2023-24", "mapped_build_period": "2024" @@ -604,7 +604,7 @@ "base_candidate_tier": "frs", "calibration_permitted": false, "canonical_source_manifest": "source_stages.json", - "canonical_source_manifest_sha256": "922d01407a3cb36d47903bab3b30ef25fd35e1d53674aec8f8c96bc7247932ba", + "canonical_source_manifest_sha256": "2aa227b9731361989c1c44ae5c670ea73f877d779031fa7739e83f8c65568857", "effective_mass_requirements": { "charitable_investment_gifts": { "mass_share_denominator": "all_person_effective_mass", @@ -715,7 +715,7 @@ "required_mass_change_reason": "E5 source-stage transform preserves household rows and typed household weights; total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "922d01407a3cb36d47903bab3b30ef25fd35e1d53674aec8f8c96bc7247932ba", + "source_manifest_sha256": "2aa227b9731361989c1c44ae5c670ea73f877d779031fa7739e83f8c65568857", "source_vintages": { "source": "UK Data Service SN 9468 Living Costs and Food Survey 2023-24 household/person tabs, NEED 2023 headline energy tables, Ofgem Q2 2026 unit rates, and WAS round-8 bridge donor.", "survey": "Living Costs and Food Survey 2023-24" @@ -736,7 +736,7 @@ "property_wealth" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "922d01407a3cb36d47903bab3b30ef25fd35e1d53674aec8f8c96bc7247932ba", + "source_manifest_sha256": "2aa227b9731361989c1c44ae5c670ea73f877d779031fa7739e83f8c65568857", "source_vintages": { "source": "MHCLG dwellings and ONS UK House Price Index December 2025 regional average prices.", "survey": "Public regional property reference" @@ -760,7 +760,7 @@ "employee_pension_contributions" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "922d01407a3cb36d47903bab3b30ef25fd35e1d53674aec8f8c96bc7247932ba", + "source_manifest_sha256": "2aa227b9731361989c1c44ae5c670ea73f877d779031fa7739e83f8c65568857", "source_vintages": { "source": "HMRC, Salary sacrifice reform for pension contributions effective from 6 April 2029", "survey": "Family Resources Survey 2024-25 salary-sacrifice respondents and HMRC salary-sacrifice reform analysis" @@ -782,7 +782,7 @@ "student_loan_plan" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "922d01407a3cb36d47903bab3b30ef25fd35e1d53674aec8f8c96bc7247932ba", + "source_manifest_sha256": "2aa227b9731361989c1c44ae5c670ea73f877d779031fa7739e83f8c65568857", "source_vintages": { "source": "Explore Education Statistics Table 6a, Higher education total", "survey": "Family Resources Survey 2024-25 and Student Loans Company borrower forecasts for England" @@ -814,7 +814,7 @@ "required_mass_change_reason": "E5 source-stage transform preserves household rows and typed household weights; total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "922d01407a3cb36d47903bab3b30ef25fd35e1d53674aec8f8c96bc7247932ba", + "source_manifest_sha256": "2aa227b9731361989c1c44ae5c670ea73f877d779031fa7739e83f8c65568857", "source_vintages": { "source": "Office for National Statistics Wealth and Assets Survey, UK Data Service SN 7215, DOI 10.5255/UKDA-SN-7215-20; local licensed 2006-22 household tab.", "survey": "Wealth and Assets Survey round 8" diff --git a/packages/microcosm-build/src/microcosm/build/uk/source_stages.json b/packages/microcosm-build/src/microcosm/build/uk/source_stages.json index 3e8dc10a..b878bbdc 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/source_stages.json +++ b/packages/microcosm-build/src/microcosm/build/uk/source_stages.json @@ -2620,9 +2620,9 @@ { "kind": "record_mass_conservation_receipt", "entity": "household", - "reason": "Amounts-only capital gains redraw: household weights pass through unchanged and total household mass is conserved.", + "reason": "Amounts-only capital gains redraw on the source spine: household weights pass through unchanged and total household mass is conserved.", "declared_factor": 1.0, - "gate_coupling": "The terminal family gate requires a valid mass-conserving MassChangeRecord carrying exactly this reason." + "gate_coupling": "The terminal family gate requires a valid mass-conserving MassChangeRecord carrying exactly this spine-specific reason." }, { "kind": "classify_cgt_band_facts_with_reviewed_fence", diff --git a/packages/microcosm-build/tests/test_spec_engine_country_bundles.py b/packages/microcosm-build/tests/test_spec_engine_country_bundles.py index 86e1663c..b9e44938 100644 --- a/packages/microcosm-build/tests/test_spec_engine_country_bundles.py +++ b/packages/microcosm-build/tests/test_spec_engine_country_bundles.py @@ -42,7 +42,7 @@ ), ( "uk", - "eb8e52c075de3f5c6983ad2fedb461c1cc77db0074c9e76aa22c4c492a5524a5", + "e091f4b012d28fe865681a8e166d5d2a9043955e9113e9097cd5161bc0433690", { "benunit.benunit_id", "household.household_id", From 38a81ba7febdf652225c4ded003bd576db599f7e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Sun, 23 Aug 2026 11:23:14 +0200 Subject: [PATCH 14/28] Commit the comparison ledger as a reviewable Markdown rendition MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The living adjudication packet is a Claude artifact, which is private by default — a reviewer clicking through from the PR hits an access wall. The packet therefore also lives in the repo, where GitHub renders it in the review itself and it travels with the branch. Same content as the interactive page: what the shares measure and their limits, the three incumbents, per-class share tables with donor or raw-source truth, the levels table with the taxable-to-taxable SPI comparison, coverage, diagnostics, and the open items. Adds the UKDS EUL clause 11-12 citations and the disclosure-control statement, which apply wherever these aggregates are posted. Co-Authored-By: Claude Fable 5 --- experiments/686-uk-spine-comparison-ledger.md | 207 ++++++++++++++++++ 1 file changed, 207 insertions(+) create mode 100644 experiments/686-uk-spine-comparison-ledger.md diff --git a/experiments/686-uk-spine-comparison-ledger.md b/experiments/686-uk-spine-comparison-ledger.md new file mode 100644 index 00000000..2874b2a3 --- /dev/null +++ b/experiments/686-uk-spine-comparison-ledger.md @@ -0,0 +1,207 @@ +# UK spine comparison ledger — microcosm#686 · PR #747 + +Every variable the E workstream manipulates, measured against the incumbent, +with the mechanism and evidence behind each difference — and what is still +unruled. This is the committed rendition of the living adjudication packet; +the interactive version is a Claude artifact (private; ask María for access). +Updated as adjudications land. + +**Candidate**: `spine-c.h5` · 52,846 hh / 61,211 bu / 113,649 p · 147 input layers +**Reference**: `enhanced_frs_2024_25.h5` @ 1.56.16 (`a9e52499`) + +## What the numbers are + +Every figure in the share tables is a **nonzero share**: the fraction of the +column's owning-entity rows carrying a non-zero value, 0–1. `0.6072` for +savings means 60.7% of households report some savings — it is not an amount. +The parity screen measures *incidence*, never level. That limit is real: a +column can match perfectly on share while being badly wrong on level — the +Scottish water fix produces an identical share of 0.878377 under both the +retired and corrected mappings while the amount moved from ~£185 to ~£395 per +Scottish household. Figures that are levels carry a £ sign and sit in the +Levels section, as conditional means (mean over nonzero carriers) with the +carrier count alongside. + +## Which incumbent + +Three artifacts get called "the incumbent"; they answer different questions. + +| | artifact | role | +|---|---|---| +| A | `enhanced_frs_2024_25.h5` @ 1.56.16 | the published post-calibration artifact; every share delta here is against it | +| B | `incumbent_{base,wealth,consumption}_ebf733c.h5` | partial pipelines rebuilt locally at 1.56.14 for E3/E5/E6 method head-to-heads; one release behind, carries the benunit-sort defect | +| C | WAS R8 · LCFS 2023-24 · ETB · SPI 2022-23 | the source surveys — ground truth neither build controls | + +The 1.56.14→1.56.16 re-pin moved no reference share by more than 0.0023 +(E7: exactly zero); the deltas below are robust to it. + +## Gate status + +| leg | state | +|---|---| +| L0 build + determinism | complete — 3 rungs clean, twins payload-identical, record identity exact at 52,846 | +| L1 identity receipts | e4 e5 e6 e7 e8 green (e7 written at #686; ladder complete) | +| L2 whole-spine parity | measured — 26 beyond ±0.02, verdict `defect` until the queue is ruled | +| L3 baselines | measured — 177 input-mass totals, 47 QRF tail grids | + +## E6 · consumption — needs ruling + +Measured fresh against the donors. On the eleven LCFS columns **ours is closer +on seven**, the incumbent on three, one tie — weaker than "the incumbent +collapses zero-inflated targets" (which stays true and dramatic on education: +incumbent 0.0003 vs donor 0.0476) but honest. Petrol/diesel are partly by +design (our `has_fuel` gate zeroes non-fuel households); on petrol the *level* +favours us while the share favours the incumbent — see Levels. + +| column | donor truth (share) | incumbent | ours | closer | +|---|---|---|---|---| +| dfe_education_spending | see ETB note | 0.0003 | 0.2258 | undecidable | +| bus_subsidy_spending | see ETB note | 0.3167 | 0.5554 | undecidable | +| rail_subsidy_spending | see ETB note | 0.1277 | 0.1422 | undecidable | +| restaurants_and_hotels_consumption | 0.7651 | 0.6324 | 0.7903 | ours | +| petrol_spending | 0.3911 | 0.4446 | 0.3002 | incumbent | +| education_consumption | 0.0476 | 0.1256 | 0.0170 | ours | +| household_furnishings_consumption | 0.9104 | 0.8136 | 0.9216 | ours | +| miscellaneous_consumption | 0.9728 | 0.9003 | 0.9918 | ours | +| communication_consumption | 0.8696 | 0.7974 | 0.8814 | ours | +| alcohol_and_tobacco_consumption | 0.5383 | 0.5603 | 0.5150 | tie | +| domestic_energy_consumption | 0.9879 | 0.9549 | 0.9952 | ours | +| diesel_spending | 0.2040 | 0.1910 | 0.1580 | incumbent | +| transport_consumption | 0.8702 | 0.8668 | 0.8934 | incumbent | +| health_consumption | 0.5426 | 0.5128 | 0.5386 | ours | + +**ETB truth is not yet decidable**: the donor share moves with the weight +basis (education 0.0943 unweighted / 0.1327 household / 0.2150 individual) +and none reconcile with the E6 receipt's 0.2546. Until the stage's weight +convention is pinned, quoting one as truth would be false precision. + +## E5 · wealth — signed at E5 · carry-forward? + +Adjudicated 2026-08-19 ("E5 is not required to reproduce the incumbent's +inflated totals; donor-benchmark evidence"). Fresh measurement corroborates. + +| column | donor truth · WAS R8 (share) | incumbent | ours | closer | +|---|---|---|---|---| +| savings | 0.6072 | 0.6620 | 0.6107 | ours (0.0035 off) | +| property_wealth | 0.6433 | 0.7081 | 0.6607 | ours | +| other_residential_property_value | 0.0363 | 0.0763 | 0.0367 | ours (0.0004 off) | +| main_residence_value | 0.6236 | 0.6747 | 0.6356 | ours | +| corporate_wealth | not measured (fold of several donor columns) | 0.8222 | 0.7792 | — | +| student_loan_balance | not measured (separate source) | 0.0197 | 0.0493 | — | + +`owned_land` is **not** in this list; its reviewed exclusion expiring +2026-09-20 is a separate question. + +## E7 · SPI channel — evidence gap closed, favours the spine + +The three columns have three different sources of truth. Measured against +each, **ours is closer on all three**; the direction that looked "uniformly +one-way and unexplained" at #717 is uniformly toward the source. Ruling here +closes #717's open item. + +| column | truth (share) | source of truth | incumbent | ours | closer | +|---|---|---|---|---|---| +| savings_interest_income | 0.3960 | SPI donor `INCBBS`, FACT-weighted | 0.4250 | 0.3946 | ours (0.0014 off) | +| tax_free_savings_income | 0.1540 | raw FRS at the `frs_spine` stage | 0.1897 | 0.1352 | ours | +| employer_pension_contributions | 0.2587 | the 3× derive at `frs_hmrc_spine_leaves` | 0.3149 | 0.2682 | ours | + +Three of the six columns #717 flagged have since fallen inside the band: +dividend_income (−0.0021), gift_aid (−0.0097), +pension_contributions_via_salary_sacrifice (−0.0035). + +## E8 and entity counts — signed at #684 + +| surface | reference | ours | delta | mechanism | +|---|---|---|---|---| +| employee_pension_contributions | 0.2735 | 0.2246 | −0.0488 | salsac conversion depth; incumbent's conversion step was inert | +| person rows | 113,617 | 113,649 | +32 | donor-selection RNG over id-sorted candidates | +| benunit rows | 61,223 | 61,211 | −12 | same draw | +| household rows | 52,846 | 52,846 | exact | identity closes — proves selection, not miscount | + +## Column coverage + +| column | status | origin | disposition | +|---|---|---|---| +| free_school_meals | in band | raw `fsmval` · child.tab | ported; ref 0.034220 → ours 0.034131 (−0.000089) | +| free_school_fruit_veg | in band | raw `fsfvval` · child.tab | ported; −0.000001 | +| healthy_start_vouchers | in band | raw `heartval` · child + adult | ported; +0.000008 | +| free_school_breakfasts | outside contract | raw `fsbval` | ported; not engine-known, never enters the 145-column surface | +| num_bedrooms | extra | `frs_spine` | net-new; predictor-quality question open on #145 | +| other_investment_income | extra | `hmrc_spi_income_spine` | declared by the incumbent's own restoration — ahead, not diverging | + +These four were mislabelled "E9 derived-benefit class" in the #723 receipt; +#685 (E9) is UC deduction attributes, bus fares and WAS debt. This was a port +omission from a merged increment, now closed — parity reports **0 missing**. + +## Levels — annual £ per carrier + +Conditional means over nonzero rows; carrier counts withheld here for brevity +but recorded in the interactive ledger and reproducible from the licensed +evidence dir; no cell is dominated by a single record; all cells ≥10 carriers. +Donor figures survey-weighted and annualised where the stage annualises. + +| column | donor truth | incumbent | ours | closer on level | +|---|---|---|---|---| +| **E5 · WAS** | | | | | +| savings | 27,860 | 85,294 | 52,061 | ours (1.9× donor vs 3.1×) | +| property_wealth | 387,430 | 311,208 | 341,415 | ours | +| other_residential_property_value | 259,375 | 493,846 | 291,791 | ours (1.1× vs 1.9×) | +| main_residence_value | 340,534 | 286,378 | 280,812 | incumbent | +| **E6 · LCFS** | | | | | +| education_consumption | 5,543 | 10,783 | 6,134 | ours | +| restaurants_and_hotels_consumption | 2,995 | 5,304 | 3,473 | ours | +| miscellaneous_consumption | 2,441 | 4,228 | 2,777 | ours | +| petrol_spending | 1,614 | 2,429 | 1,713 | ours — opposite of the share | +| diesel_spending | 1,911 | 2,389 | 2,065 | ours | +| transport_consumption | 5,278 | 7,588 | 6,656 | ours | +| health_consumption | 726 | 1,249 | 1,110 | ours | +| alcohol_and_tobacco_consumption | 1,075 | 1,091 | 1,373 | incumbent | +| household_furnishings_consumption | 2,217 | 3,501 | 3,589 | incumbent | +| communication_consumption | 773 | 836 | 863 | incumbent | +| **E7 and raw mappings** | | | | | +| savings_interest_income (taxable part, SPI channel) | 339 | 41 | 187 | ours (0.55× donor vs 0.12×) | +| tax_free_savings_income | — | 602 | 747 | no donor | +| employer_pension_contributions | — | 6,357 | 6,347 | ≈ equal | +| water_and_sewerage_charges | — | 483 | 475 | ≈ equal — but the incumbent's is England+Wales-only (it zeroes Scotland); coincidence of averaging, not agreement | +| free_school_meals | — | 629 | 629 | exact | + +The SPI comparison must be taxable-to-taxable on the SPI channel: donor +`INCBBS` is interest *as reported for tax* (no ISA, nothing under the personal +savings allowance) while the spine's column is gross (taxable draw + FRS-side +tax-free part, identity-guarded). The residual 0.55× gap is the right +direction — SPI is a taxpayer population, the recipient channel is broader. + +## Diagnostics · both spines + +| check | result | +|---|---| +| twin determinism | pass — two independent full builds payload-identical; bytes differ only from HDF5 write-time stamps | +| record-count identity | exact — (16,288 + 10,000) × 2 + 270 = 52,846 | +| rung ladder | pass — f001 / f010 / full, each landing a Logbook row | +| identity receipts e4–e8 | pass, on both the post-E8 and pre-E8 spines | +| weighted-integrity baselines | measured — 177 input-mass totals, 47 QRF tail grids, 12 household QRF tail | +| Scottish water fix | verified — share 0.878377 at every rung; £390.80 / £375.08 / £405.34 per Scottish household vs ~£185 under the retired mapping | + +## Open + +- **ETB weight basis** — pins the truth for three columns. +- **E6 is not one class** — a single entry over all fifteen would sign columns where we are further from the donor than the incumbent is. +- **Entity-count entry scope** — surface-wide vs scoped to the two entities. +- **Stale donor ratios** in the E5/E6 prose receipts were measured at 1.56.14; the tables above are fresh at the current pin. +- **Upstream defect filed** as policyengine-uk-data#467 (no prescriptive fix); our side is correct and signed. + +## Disclosure and citation + +All figures are aggregates under disclosure control (CD171 §5.2.1): no unit +records, minimum cell count 10, thin columns suppressed, dominance checked. +Data collections used, cited and acknowledged per UKDS EUL clauses 11–12: + +- DWP, *Family Resources Survey 2024-25*, UKDS SN 9563, DOI 10.5255/UKDA-SN-9563-1 +- DWP/ONS, *Living Costs and Food Survey 2023-24*, UKDS SN 9468, DOI 10.5255/UKDA-SN-9468-3 +- ONS, *Wealth and Assets Survey, Round 8*, UKDS SN 7215, DOI 10.5255/UKDA-SN-7215-20 +- HMRC, *Survey of Personal Incomes 2022-23 Public Use Tape*, UKDS SN 9422 +- ONS, *Effects of Taxes and Benefits on Household Income 1977–2024*, UKDS SN 8856, DOI 10.5255/UKDA-SN-8856-4 + +Crown copyright material is reproduced with the permission of the Controller +of HMSO and the King's Printer for Scotland. The original data creators, +depositors and funders bear no responsibility for this analysis. From e136a0f00b8ade757046d2519fab20806e5170f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Sun, 23 Aug 2026 21:48:12 +0200 Subject: [PATCH 15/28] Add the Universal Credit pre-calibration health check to the ledger MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit UC had no dedicated row despite being what the armed calibration binds. Measured modeled universal_credit through the engine on both artifacts: unweighted the spine carries 9% more UC-positive benunits than the incumbent at an equal per-recipient level, and the reported column is exact against the raw tab to every digit. The halved weighted caseload is the calibration boundary itself, not a spine defect — the incumbent's weights already embody a UC caseload target since 1.56.15, so the comparison is before-medicine to after-medicine. Recorded watch-item: would_claim_uc frozen at 0.55 (U8, uk-data#452), the pre-registered lever if the armed run's UC fit is strained. Co-Authored-By: Claude Fable 5 --- experiments/686-uk-spine-comparison-ledger.md | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/experiments/686-uk-spine-comparison-ledger.md b/experiments/686-uk-spine-comparison-ledger.md index 2874b2a3..97de999a 100644 --- a/experiments/686-uk-spine-comparison-ledger.md +++ b/experiments/686-uk-spine-comparison-ledger.md @@ -182,6 +182,40 @@ direction — SPI is a taxpayer population, the recipient channel is broader. | weighted-integrity baselines | measured — 177 input-mass totals, 47 QRF tail grids, 12 household QRF tail | | Scottish water fix | verified — share 0.878377 at every rung; £390.80 / £375.08 / £405.34 per Scottish household vs ~£185 under the retired mapping | +## Universal Credit — pre-calibration health: healthy + +UC is what the armed calibration binds (`dwp.uc.households`, CY-2025 avg +≈6.76m), so its health check is about the raw material the solve receives. +Modeled `universal_credit` materialized through the engine on both artifacts: + +| measure | spine (pre-calibration) | incumbent (post-calibration) | read | +|---|---|---|---| +| modeled UC benunits, unweighted | 5,323 (8.70%) | 4,869 (7.95%) | ours richer, ratio 1.09 | +| modeled UC per recipient, annual £ | 10,285 | 10,130 | equal within 1.5% | +| reported UC share at `frs_spine` | 0.063261 | — | **exact** vs raw tab 0.063261 | +| reported UC per recipient, annual £ | 11,436 | 10,780 | admin per-unit ≈ £11.3k (#731) | +| modeled caseload, weighted | 3.15m | 6.30m | see below | +| modeled UC annual total, weighted | £30.6bn | £74.7bn | see below | + +**Why the weighted rows halve, and why that is the boundary, not a bug.** The +spine carries design grossing weights (29.2m households); the incumbent's +weights are *calibrated*, and since 1.56.15 that calibration includes a UC +caseload target — its solve moved roughly 2× mass onto UC-positive benunits. +Comparing our design-weighted caseload to their calibrated one compares +before-medicine to after-medicine. Like-for-like (unweighted) the spine hands +calibration **more** raw material than the incumbent had, at an equal +per-recipient level; reaching 6.76m means lifting ~2.1× mass onto 8.7% of +benunits — the same order the incumbent's own solve performed — policed at the +armed run by the ESS and weight-ratio gates. + +**Watch-item:** `would_claim_uc` is frozen at 0.55 by the U8 adjudication for +incumbent parity, with uk-data#452 (the take-up vintage behind #731's June UC +diagnosis) as the recorded follow-up. The freeze caps the eligible pool; it is +the pre-registered lever if the armed run's UC fit is strained. The +final-surface share gap on `universal_credit_reported` (+0.0069) is the E7 +SPI-rewrite class — both builds rewrite synthetic rows, with different QRF +implementations. + ## Open - **ETB weight basis** — pins the truth for three columns. From 1d5ea2424c26e97c94948297ffba508aa01f0ba2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Mon, 24 Aug 2026 11:14:06 +0200 Subject: [PATCH 16/28] Re-derive the UK spec digest after rebasing onto main Main moved uk/target_references.json and uk/target_reference_membership.json, both of which the UK spec bundle hashes, so the pinned bundle digest in the country-bundle test moves with them. Re-cut against the rebased base; the contract mirrors, the parity reference and the coverage manifest all verify unchanged. Co-Authored-By: Claude Opus 5 --- .../microcosm-build/tests/test_spec_engine_country_bundles.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/microcosm-build/tests/test_spec_engine_country_bundles.py b/packages/microcosm-build/tests/test_spec_engine_country_bundles.py index b9e44938..a150a100 100644 --- a/packages/microcosm-build/tests/test_spec_engine_country_bundles.py +++ b/packages/microcosm-build/tests/test_spec_engine_country_bundles.py @@ -42,7 +42,7 @@ ), ( "uk", - "e091f4b012d28fe865681a8e166d5d2a9043955e9113e9097cd5161bc0433690", + "57855b0cea72a99558ac1f204541fccd899fa7bf312e36e399f2ffec078598a2", { "benunit.benunit_id", "household.household_id", From 689019721110be3adf9b13fd924ce07076f8c90b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Mon, 24 Aug 2026 11:29:46 +0200 Subject: [PATCH 17/28] Sign the twenty-six beyond-band spine-vs-incumbent divergences Takes the whole-spine parity verdict from defect to signed_parity with nothing unsigned and no strict failure. Two changes make that honest rather than merely green. The register grows to thirteen entries, split so that no entry covers both columns where the spine is closer to its donor and columns where the incumbent is. Donor evidence is re-measured through each stage's own committed cleaning function on the survey-weighted basis, which reproduces the E6 acceptance receipt's education figure exactly; that settles the ETB weight-basis question and exposes the incumbent's dfe_education_spending as degenerate at fourteen nonzero households in 52,846. The instrument's share surface moves from the reference's six-decimal grain to the #723 acceptance band. Ninety columns sit inside it on third-decimal drift, and signing those would have been the blanket amnesty the register is built to prevent. Nothing is hidden: in-band differences are reported under their own key with the in-band maximum, --share-band 0 restores the exact check, and structural differences stay outside the band's reach at any band. Co-Authored-By: Claude Opus 5 --- .../686-uk-parity-acceptance-band.changed.md | 1 + ...k-spine-swap-signed-differences.changed.md | 1 + .../uk/spine_swap_signed_differences.json | 177 ++++++++++++++- .../tests/test_uk_signed_differences.py | 48 +++++ .../tests/test_uk_spine_parity_instrument.py | 203 ++++++++++++++++++ tools/verify_uk_spine_parity.py | 85 +++++++- 6 files changed, 505 insertions(+), 10 deletions(-) create mode 100644 changelog.d/686-uk-parity-acceptance-band.changed.md create mode 100644 changelog.d/686-uk-spine-swap-signed-differences.changed.md diff --git a/changelog.d/686-uk-parity-acceptance-band.changed.md b/changelog.d/686-uk-parity-acceptance-band.changed.md new file mode 100644 index 00000000..a81b327c --- /dev/null +++ b/changelog.d/686-uk-parity-acceptance-band.changed.md @@ -0,0 +1 @@ +Hold the whole-spine parity instrument's share surface to the #723 acceptance band rather than to the reference's six-decimal grain. The spine re-runs every stochastic stage, so ninety of the file's columns land inside the band on third-decimal drift; requiring a permanent adjudication for each would have filled the register with entries describing noise and blanket-covering the columns they name, which is the failure mode the register exists to prevent. The band governs only which magnitudes must be adjudicated, never what is reported: every difference down to the six-decimal grain still appears in the receipt, now partitioned into `differing` and `within_band` with the in-band maximum carried alongside, and `--share-band 0` restores the exact-grain check. Structural differences are deliberately outside the band's reach — a column appearing or vanishing, and every entity count, still signs exactly, at any band. Alongside this, `--strict` now separates a register entry that matched nothing on a surface this run compared from one whose surface was never examined: the weighted-totals surface stays unexamined until there is a calibrated candidate to compare, so an entry scoped to it is reported as dormant instead of failing the swap-acceptance posture, and becomes an ordinary unused entry again as soon as a run supplies the sidecars. diff --git a/changelog.d/686-uk-spine-swap-signed-differences.changed.md b/changelog.d/686-uk-spine-swap-signed-differences.changed.md new file mode 100644 index 00000000..234c7bda --- /dev/null +++ b/changelog.d/686-uk-spine-swap-signed-differences.changed.md @@ -0,0 +1 @@ +Sign the twenty-six beyond-band divergences between the microcosm-built UK spine and the re-pinned 1.56.16 incumbent (#686), taking the whole-spine parity verdict from `defect` to `signed_parity` with nothing unsigned. The register grows from two entries to thirteen, and the split between them is the substance: no entry covers both columns where the spine is closer to its donor and columns where the incumbent is, because the direction of the evidence is part of what is being signed. So the LCFS consumption class is three entries rather than one — ten columns where the regime-gated draw lands closer to the donor on incidence, `petrol_spending` and `diesel_spending` where the `has_fuel` gate under-places incidence and the entry says so, and `transport_consumption` alone where the incumbent is marginally closer on share and the spine closer on level. All donor evidence is re-measured through each stage's own committed cleaning function over its own pinned tab, on the survey-weighted basis, which is the convention that reproduces the E6 acceptance receipt's education figure exactly; that measurement also settles the ETB weight-basis question that was blocking three rows, and shows the incumbent's `dfe_education_spending` to be degenerate — fourteen nonzero households in 52,846 against a donor incidence of 0.28 — so those two columns are signed as a defect fix on the incumbent side rather than as a method preference. `student_loan_balance` is scoped alone because no like-for-like donor benchmark exists for it, which the entry states rather than papers over. Entity counts are signed scoped to `person` and `benunit` rather than surface-wide, so that a future divergence in the household count stays a defect. diff --git a/packages/microcosm-build/src/microcosm/build/uk/spine_swap_signed_differences.json b/packages/microcosm-build/src/microcosm/build/uk/spine_swap_signed_differences.json index 0f83bd48..7c50fd3d 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/spine_swap_signed_differences.json +++ b/packages/microcosm-build/src/microcosm/build/uk/spine_swap_signed_differences.json @@ -1,6 +1,6 @@ { "schema_version": 1, - "scope_note": "Adjudicated intentional differences between the microcosm-built UK spine and the frozen enhanced-FRS incumbent (#686). The whole-spine comparison treats anything differing that is not signed here as a defect, so every entry is scoped to the exact surface and columns where the difference is expected to appear: a too-broad entry would sign a real defect. Entries are permanent adjudications and carry no expiry; time-limited per-gate suppressions belong in input_mass_reviewed_exclusions.json, qrf_tail_reviewed_exclusions.json or degenerate_reviewed_exclusions.json instead, and an entry here points at one through its evidence field when both descend from the same adjudication. The register is completed during the whole-spine parity loop: the E4-E8 method classes recorded in the licensed per-increment acceptance receipts, and the columns the #723 screen placed beyond the parity band, are transcribed here as each is re-measured against the re-pinned 1.56.16 reference and scoped to the divergence actually observed, rather than carried across on their prose classification.", + "scope_note": "Adjudicated intentional differences between the microcosm-built UK spine and the frozen enhanced-FRS incumbent (#686). The whole-spine comparison treats any difference beyond the #723 acceptance band that is not signed here as a defect, so every entry is scoped to the exact surface and columns where the difference is expected to appear: a too-broad entry would sign a real defect. Entries are permanent adjudications and carry no expiry; time-limited per-gate suppressions belong in input_mass_reviewed_exclusions.json, qrf_tail_reviewed_exclusions.json or degenerate_reviewed_exclusions.json instead, and an entry here points at one through its evidence field when both descend from the same adjudication. The twenty-six beyond-band divergences measured against the re-pinned 1.56.16 reference are signed below, each scoped to the divergence actually observed rather than carried across on its prose classification, and each deliberately split so that no entry covers both columns where the spine is closer to its donor and columns where the incumbent is: the direction of the evidence is part of what is being signed. Donor figures quoted as magnitude evidence are survey-weighted shares and population means computed on each stage's own committed cleaning function over its own pinned donor tab, which is the convention that reproduces the E6 acceptance receipt's education figure exactly.", "differences": [ { "id": "scottish-water-incumbent-nan-zeroing", @@ -29,6 +29,181 @@ "evidence": "experiments/686-uk-spine-swap-receipts.md#r1-scottish-water-and-sewerage-charges-736-item-13", "adjudicator": "juaristi22", "adjudicated_on": "2026-08-22" + }, + { + "id": "lcfs-consumption-regime-gated-incidence", + "class": "mechanism_change", + "scope": { + "surface": "nonzero_shares", + "columns": [ + "alcohol_and_tobacco_consumption", + "communication_consumption", + "domestic_energy_consumption", + "education_consumption", + "electricity_consumption", + "gas_consumption", + "health_consumption", + "household_furnishings_consumption", + "miscellaneous_consumption", + "restaurants_and_hotels_consumption" + ], + "entities": ["household"] + }, + "expectation": "column_differs", + "magnitude_evidence": "The spine draws LCFS consumption through a regime-gated QRF that carries the donor's zero mass as a modelled incidence, where the incumbent's plain QRF regresses a zero-inflated target toward its conditional mean. On these ten columns the spine's unweighted share is closer to the LCFS 2023-24 survey-weighted donor share than the incumbent's is, on the stage's own cleaned donor frame of 4,202 households: education_consumption donor 0.0476 against incumbent 0.1258 and ours 0.0170; restaurants_and_hotels donor 0.7651 against 0.6305 and 0.7903; miscellaneous donor 0.9728 against 0.8975 and 0.9918; domestic_energy donor 0.9835 against 0.9541 and 0.9953, with its electricity and gas components moving the same way. Population mean per household is closer for the spine on nine of the ten, the exception being communication_consumption at 1.20x the donor against the incumbent's 1.16x; the incumbent runs 1.7x to 5.0x the donor mean on the rest. Every cell is above the disclosure floor, the thinnest being 193 donor carriers on education_consumption.", + "evidence": "experiments/686-uk-spine-comparison-ledger.md#e6-consumption", + "adjudicator": "juaristi22", + "adjudicated_on": "2026-08-24" + }, + { + "id": "lcfs-fuel-consumption-incidence-gate", + "class": "mechanism_change", + "scope": { + "surface": "nonzero_shares", + "columns": ["diesel_spending", "petrol_spending"], + "entities": ["household"] + }, + "expectation": "column_differs", + "magnitude_evidence": "These two are signed with the evidence pointing the other way on incidence, and that is the point of scoping them apart from the rest of the LCFS class. The spine gates fuel spending on a has_fuel draw, so it places incidence on fewer households than either the donor or the incumbent: petrol donor 0.3911 against incumbent 0.4442 and ours 0.3002, diesel donor 0.2040 against 0.1903 and 0.1580. The incumbent is closer on both shares. On level the ordering reverses decisively — population mean per household is 0.85x the donor for petrol and 0.81x for diesel against the incumbent's 2.06x and 2.29x — so the gate is under-placing incidence while the incumbent is over-stating amounts by roughly a factor of two. Signed as the accepted cost of the fuel gate, not as a claim that the spine is closer here; the incidence rate of the gate is the pre-registered lever if this surface needs to move.", + "evidence": "experiments/686-uk-spine-comparison-ledger.md#e6-consumption", + "adjudicator": "juaristi22", + "adjudicated_on": "2026-08-24" + }, + { + "id": "lcfs-transport-aggregate-incidence", + "class": "mechanism_change", + "scope": { + "surface": "nonzero_shares", + "columns": ["transport_consumption"], + "entities": ["household"] + }, + "expectation": "column_differs", + "magnitude_evidence": "Scoped alone because the incumbent is marginally closer on this share and the aggregate sits above the fuel columns that the has_fuel gate moves. Donor 0.8702 against incumbent 0.8623 and ours 0.8934: the spine overshoots the donor by 0.0232 where the incumbent undershoots by 0.0079. On level the spine is closer, at 1.37x the donor population mean per household against the incumbent's 1.63x. Signed as an accepted incidence cost of the regime-gated draw, with the direction recorded so it can be re-examined on its own rather than under a class verdict it does not share.", + "evidence": "experiments/686-uk-spine-comparison-ledger.md#e6-consumption", + "adjudicator": "juaristi22", + "adjudicated_on": "2026-08-24" + }, + { + "id": "etb-services-regime-gated-incidence", + "class": "defect_fix", + "scope": { + "surface": "nonzero_shares", + "columns": ["bus_subsidy_spending", "dfe_education_spending"], + "entities": ["household"] + }, + "expectation": "column_differs", + "magnitude_evidence": "The incumbent's state-education column is degenerate and the spine's is not, which makes this a defect fix on the incumbent side rather than a method preference. Measured on the ETB services stage's own cleaned donor frame — SN 8856, year 2023, complete cases on the thirteen-column services subset, 4,199 rows, weighted by hhold_adj_weight — dfe_education_spending has a donor share of 0.2794 weighted (0.2546 unweighted, which reproduces the E6 acceptance receipt's figure exactly) and a donor population mean of GBP 3,461 per household. The incumbent carries 14 nonzero households out of 52,846, a share of 0.000265 and a population mean of GBP 2 per household; the spine carries 11,934, a share of 0.2258 and GBP 3,111, or 0.90x the donor. bus_subsidy_spending moves the same way: donor share 0.5255 weighted and GBP 87 per household, against incumbent 0.3167 and GBP 114 (1.30x) and ours 0.5554 and GBP 89 (1.02x). The spine is closer on both the share and the level of both columns. This resolves the ETB weight-basis question that was previously recorded as blocking these rows: the stage's convention is the household grossing weight, and the verdict holds on either basis.", + "evidence": "experiments/686-uk-spine-comparison-ledger.md#e6-consumption", + "adjudicator": "juaristi22", + "adjudicated_on": "2026-08-24" + }, + { + "id": "was-wealth-qrf-incidence", + "class": "qrf_implementation", + "scope": { + "surface": "nonzero_shares", + "columns": [ + "corporate_wealth", + "main_residence_value", + "other_residential_property_value", + "property_wealth", + "savings" + ], + "entities": ["household"] + }, + "expectation": "column_differs", + "magnitude_evidence": "Carries forward the E5 adjudication of 2026-08-19 — that the wealth stage is not required to reproduce the incumbent's inflated totals — now scoped to the five household columns that actually diverge beyond the band, and re-measured against WAS Round 8 on the stage's own cleaning of the pinned tab (15,128 rows, weighted by R8xshhwgt). The spine is closer than the incumbent on all five survey-weighted donor shares: savings donor 0.6072 against incumbent 0.6621 and ours 0.6107; other_residential_property_value donor 0.0363 against 0.0750 and 0.0367; property_wealth donor 0.6433 against 0.7082 and 0.6607; main_residence_value donor 0.6236 against 0.6761 and 0.6356; corporate_wealth donor 0.7629 against 0.8225 and 0.7792. The level evidence behind the original adjudication reproduces and is the more dramatic surface: population mean per household runs 5.70x the donor for the incumbent's savings against 1.97x for ours, and 6.18x against 1.26x for other residential property. main_residence_value is the one column where the incumbent's level is closer, at 1.02x against our 0.88x. Note that the unweighted donor shares tell the opposite story on incidence, because WAS oversamples wealth-holders by design; the weighted basis is the population one and is the basis quoted here throughout.", + "evidence": "experiments/686-uk-spine-comparison-ledger.md#e5-wealth", + "adjudicator": "juaristi22", + "adjudicated_on": "2026-08-24" + }, + { + "id": "was-student-loan-balance-fold", + "class": "qrf_implementation", + "scope": { + "surface": "nonzero_shares", + "columns": ["student_loan_balance"], + "entities": ["household"] + }, + "expectation": "column_differs", + "magnitude_evidence": "Scoped apart from the benchmarked wealth columns because it has no donor benchmark to quote: the column is a fold of two WAS aggregates (total loans less total loans excluding Student Loans Company debt) and lands at a different entity grain from the household columns beside it, so no like-for-like donor share exists on the stage's cleaned frame. The observed divergence is +0.0296 on the unweighted share, incumbent 0.0197 against ours 0.0493 — the spine places student debt on about two and a half times as many carriers. Signed under the standing E5 adjudication as part of the same correlated-rank draw, with the absence of a benchmark stated rather than papered over; if the wealth stage is revisited, this is the column whose direction is unevidenced.", + "evidence": "experiments/686-uk-spine-comparison-ledger.md#e5-wealth", + "adjudicator": "juaristi22", + "adjudicated_on": "2026-08-24" + }, + { + "id": "spi-channel-qrf-incidence", + "class": "qrf_implementation", + "scope": { + "surface": "nonzero_shares", + "columns": [ + "employer_pension_contributions", + "savings_interest_income", + "tax_free_savings_income" + ], + "entities": ["person"] + }, + "expectation": "column_differs", + "magnitude_evidence": "The three columns rewritten on the SPI channel, each measured against its own source of truth rather than against one another, which is what closes the open item #717 left. savings_interest_income against the SPI donor INCBBS, FACT-weighted: truth 0.3960, incumbent 0.4250, ours 0.3946. tax_free_savings_income against the raw FRS at the frs_spine stage: truth 0.1540, incumbent 0.1897, ours 0.1352. employer_pension_contributions against the 3x derive at frs_hmrc_spine_leaves: truth 0.2587, incumbent 0.3149, ours 0.2682. The spine is closer on all three, so the divergence #717 recorded as uniformly one-way and unexplained is uniformly toward the source. Attribution is to the last stage that rewrites, not the stage that produces: the first two originate in frs_spine and are rewritten by hmrc_spi_income_spine, and attributing them to their producer would report them as raw-mapping defects, which is the one signature that indicates a genuine port defect.", + "evidence": "experiments/686-uk-spine-comparison-ledger.md#e7-spi-channel", + "adjudicator": "juaristi22", + "adjudicated_on": "2026-08-24" + }, + { + "id": "salary-sacrifice-conversion-depth", + "class": "mechanism_change", + "scope": { + "surface": "nonzero_shares", + "columns": ["employee_pension_contributions"], + "entities": ["person"] + }, + "expectation": "column_differs", + "magnitude_evidence": "Signed at #684 and transcribed here against the re-pinned reference. The spine converts salary-sacrificed pension contributions at the depth the mechanism specifies, where the incumbent's conversion step was inert, so contributions that should have moved out of the employee column stayed in it. Unweighted share 0.2735 for the incumbent against 0.2246 for the spine, a difference of -0.0488 on the person entity. The counterpart column pension_contributions_via_salary_sacrifice sits inside the acceptance band at -0.0035 and is therefore not signed.", + "evidence": "experiments/686-uk-spine-comparison-ledger.md#e8-and-entity-counts", + "adjudicator": "juaristi22", + "adjudicated_on": "2026-08-24" + }, + { + "id": "donor-selection-rng-entity-counts", + "class": "rng_stream", + "scope": { + "surface": "entity_counts", + "columns": ["benunit", "person"], + "entities": ["benunit", "person"] + }, + "expectation": "count_differs", + "magnitude_evidence": "The CGT band-donor selection draws over id-sorted candidate households, so the 270 donors it picks are not the 270 the incumbent picked, and the two sets carry different numbers of people and benefit units. Persons 113,617 in the reference against 113,649 in the spine, a difference of +32; benefit units 61,223 against 61,211, a difference of -12. Households are 52,846 on both sides and match exactly, which is what proves this is a selection difference rather than a miscount: the record-count identity (16,288 raw FRS plus 10,000 SPI) times two for the capital-gains clone, plus 270 band donors, closes on the nose. This entry is deliberately scoped to the two entities that differ rather than written surface-wide, so that any future divergence in the household count is still a defect.", + "evidence": "experiments/686-uk-spine-swap-receipts.md#r3-the-rebuilt-spine", + "adjudicator": "juaristi22", + "adjudicated_on": "2026-08-24" + }, + { + "id": "num-bedrooms-net-new-column", + "class": "net_new_column", + "scope": { + "surface": "nonzero_shares", + "columns": ["num_bedrooms"], + "entities": ["household"] + }, + "expectation": "column_missing_in_reference", + "magnitude_evidence": "The spine populates num_bedrooms at the frs_spine stage from the raw household tape; the pinned incumbent does not populate it at all, so the column is present in the candidate and absent from the reference. This is coverage the spine adds rather than a divergence in a shared column, and it cannot be measured as a share difference. Its predictor quality is a separate question tracked on #145, not a parity matter.", + "evidence": "experiments/686-uk-spine-comparison-ledger.md#column-coverage", + "adjudicator": "juaristi22", + "adjudicated_on": "2026-08-24" + }, + { + "id": "other-investment-income-net-new-column", + "class": "net_new_column", + "scope": { + "surface": "nonzero_shares", + "columns": ["other_investment_income"], + "entities": ["person"] + }, + "expectation": "column_missing_in_reference", + "magnitude_evidence": "The spine populates other_investment_income at the hmrc_spi_income_spine stage. The column is declared by the incumbent's own national restoration but is not populated in the pinned artifact, so the spine is ahead of the reference here rather than diverging from it. Present in the candidate, absent from the reference, and not measurable as a share difference.", + "evidence": "experiments/686-uk-spine-comparison-ledger.md#column-coverage", + "adjudicator": "juaristi22", + "adjudicated_on": "2026-08-24" } ] } diff --git a/packages/microcosm-build/tests/test_uk_signed_differences.py b/packages/microcosm-build/tests/test_uk_signed_differences.py index 2081d14f..af5cf598 100644 --- a/packages/microcosm-build/tests/test_uk_signed_differences.py +++ b/packages/microcosm-build/tests/test_uk_signed_differences.py @@ -8,6 +8,7 @@ from __future__ import annotations import json +from importlib.resources import files from pathlib import Path import pytest @@ -76,6 +77,53 @@ def test_committed_evidence_anchors_point_at_a_real_file(self) -> None: f"{difference.id} cites missing evidence file {relative}" ) + def test_every_signed_column_exists_on_the_surface_it_signs(self) -> None: + # A typo in a column name is the quiet failure mode here: the entry + # matches nothing, the real divergence stays unsigned, and the only + # symptom is a defect verdict nobody can trace back to the typo. + reference = json.loads( + files("microcosm.build.uk") + .joinpath("efrs_parity_reference.json") + .read_text(encoding="utf-8") + ) + known = set(reference["nonzero_shares"]) + entities = set(reference["entity_stats"]) + # Columns the spine adds are signed precisely because the reference + # does not carry them, so they are checked against the expectation + # rather than against the reference's column set. + net_new = { + column + for difference in load_uk_spine_swap_signed_differences().differences + if difference.expectation == "column_missing_in_reference" + for column in difference.columns + } + for difference in load_uk_spine_swap_signed_differences().differences: + for column in difference.columns: + if difference.surface == "entity_counts": + assert column in entities, ( + f"{difference.id} signs entity {column!r}, which the " + "reference does not carry." + ) + elif difference.surface == "nonzero_shares" and column not in net_new: + assert column in known, ( + f"{difference.id} signs column {column!r}, which is not " + "on the reference share surface — likely a typo, which " + "would leave the real divergence unsigned." + ) + + def test_the_register_signs_no_column_twice(self) -> None: + # Two entries covering one column on one surface make the adjudication + # ambiguous: the reader cannot tell which rationale is the live one. + seen: dict[tuple[str, str], str] = {} + for difference in load_uk_spine_swap_signed_differences().differences: + for column in difference.columns: + key = (difference.surface, column) + assert key not in seen, ( + f"{column!r} on {difference.surface} is signed by both " + f"{seen[key]} and {difference.id}." + ) + seen[key] = difference.id + def test_no_committed_entry_expires(self) -> None: payload = json.loads( ( diff --git a/packages/microcosm-build/tests/test_uk_spine_parity_instrument.py b/packages/microcosm-build/tests/test_uk_spine_parity_instrument.py index 1f70c0bf..77a11b6b 100644 --- a/packages/microcosm-build/tests/test_uk_spine_parity_instrument.py +++ b/packages/microcosm-build/tests/test_uk_spine_parity_instrument.py @@ -300,6 +300,81 @@ def test_strict_fails_an_unused_register_entry(self, tmp_path: Path) -> None: == 0 ) + def test_strict_treats_a_dormant_surface_entry_as_dormant_not_unused( + self, tmp_path: Path + ) -> None: + # The weighted-totals surface stays unexamined until there is a + # calibrated candidate, so an entry scoped to it has had no chance to + # match. Failing --strict for that would make the swap-acceptance + # posture impossible to satisfy before calibration. + tool = _load_tool() + candidate = _write(tmp_path / "c.json", _candidate_from_reference()) + register = _register( + tmp_path, + _entry("totals-only", surface="weighted_totals", columns=["col"]), + ) + receipt = tmp_path / "receipt.json" + + assert ( + tool.main( + [ + "--candidate-json", + str(candidate), + "--register", + str(register), + "--strict", + "--receipt-json", + str(receipt), + ] + ) + == 0 + ) + report = json.loads(receipt.read_text(encoding="utf-8")) + assert report["strict_failure"] is False + assert report["register"]["unused_ids"] == [] + assert report["register"]["dormant_ids"] == ["totals-only"] + assert "weighted_totals" not in report["register"]["compared_surfaces"] + + def test_dormancy_is_not_a_loophole_once_the_surface_is_compared( + self, tmp_path: Path + ) -> None: + # Same entry, same register — but this run supplies the sidecars, so + # the surface is examined and a signature that matches nothing on it is + # register rot again. + tool = _load_tool() + candidate = _write(tmp_path / "c.json", _candidate_from_reference()) + left = _write(tmp_path / "ref.json", {"identity": {}, "totals": {"col": 100.0}}) + right = _write( + tmp_path / "cand.json", {"identity": {}, "totals": {"col": 100.0}} + ) + register = _register( + tmp_path, + _entry("totals-only", surface="weighted_totals", columns=["col"]), + ) + receipt = tmp_path / "receipt.json" + + assert ( + tool.main( + [ + "--candidate-json", + str(candidate), + "--register", + str(register), + "--reference-weighted-totals", + str(left), + "--candidate-weighted-totals", + str(right), + "--strict", + "--receipt-json", + str(receipt), + ] + ) + == 1 + ) + report = json.loads(receipt.read_text(encoding="utf-8")) + assert report["register"]["unused_ids"] == ["totals-only"] + assert report["register"]["dormant_ids"] == [] + def test_one_sided_weighted_totals_is_refused(self, tmp_path: Path) -> None: tool = _load_tool() candidate = _write(tmp_path / "c.json", _candidate_from_reference()) @@ -422,3 +497,131 @@ def test_unsigned_weighted_divergence_is_a_defect(self, tmp_path: Path) -> None: ) == 1 ) + + +class TestAcceptanceBand: + """The band decides what must be adjudicated, never what is reported.""" + + def test_an_in_band_difference_needs_no_signature(self, tmp_path: Path) -> None: + tool = _load_tool() + column = _first_household_column() + payload = _candidate_from_reference() + payload["nonzero_shares"][column] += 0.01 + candidate = _write(tmp_path / "c.json", payload) + receipt = tmp_path / "receipt.json" + + code = tool.main( + [ + "--candidate-json", + str(candidate), + "--register", + str(_register(tmp_path)), + "--receipt-json", + str(receipt), + ] + ) + + assert code == 0 + report = json.loads(receipt.read_text(encoding="utf-8")) + assert report["verdict"] == "parity" + assert report["unsigned_differences"] == [] + # Reported, not dropped: the reader still sees the movement. + assert column in report["nonzero_shares"]["within_band"] + assert column not in report["nonzero_shares"]["differing"] + assert report["nonzero_shares"]["within_band"][column]["delta"] == pytest.approx( + 0.01 + ) + assert report["nonzero_shares"]["within_band_max_abs_delta"] == pytest.approx( + 0.01 + ) + + def test_a_difference_just_beyond_the_band_still_signs( + self, tmp_path: Path + ) -> None: + tool = _load_tool() + column = _first_household_column() + payload = _candidate_from_reference() + payload["nonzero_shares"][column] += 0.021 + candidate = _write(tmp_path / "c.json", payload) + + assert ( + tool.main( + [ + "--candidate-json", + str(candidate), + "--register", + str(_register(tmp_path)), + ] + ) + == 1 + ) + + def test_a_zero_band_restores_the_exact_grain_check(self, tmp_path: Path) -> None: + tool = _load_tool() + column = _first_household_column() + payload = _candidate_from_reference() + payload["nonzero_shares"][column] += 0.01 + candidate = _write(tmp_path / "c.json", payload) + + assert ( + tool.main( + [ + "--candidate-json", + str(candidate), + "--register", + str(_register(tmp_path)), + "--share-band", + "0", + ] + ) + == 1 + ) + + def test_the_band_never_covers_a_structural_difference( + self, tmp_path: Path + ) -> None: + # A column that appears or vanishes is not a magnitude, so no band can + # absorb it; nor can one absorb an entity-count difference. + tool = _load_tool() + payload = _candidate_from_reference() + payload["nonzero_shares"]["a_column_the_incumbent_never_had"] = 0.000001 + payload["entity_stats"]["household"]["records"] += 1 + candidate = _write(tmp_path / "c.json", payload) + receipt = tmp_path / "receipt.json" + + assert ( + tool.main( + [ + "--candidate-json", + str(candidate), + "--register", + str(_register(tmp_path)), + "--share-band", + "0.9", + "--receipt-json", + str(receipt), + ] + ) + == 1 + ) + report = json.loads(receipt.read_text(encoding="utf-8")) + assert "a_column_the_incumbent_never_had" in report["unsigned_differences"] + assert "household" in report["unsigned_differences"] + + def test_an_out_of_range_band_yields_no_verdict(self, tmp_path: Path) -> None: + tool = _load_tool() + candidate = _write(tmp_path / "c.json", _candidate_from_reference()) + for bad in ("-0.01", "1.0", "5"): + assert ( + tool.main( + [ + "--candidate-json", + str(candidate), + "--register", + str(_register(tmp_path)), + "--share-band", + bad, + ] + ) + == 2 + ) diff --git a/tools/verify_uk_spine_parity.py b/tools/verify_uk_spine_parity.py index a3021d10..cb13c75d 100644 --- a/tools/verify_uk_spine_parity.py +++ b/tools/verify_uk_spine_parity.py @@ -10,9 +10,10 @@ * ``entity_counts`` — the record-count identity, exactly. The spine and the pinned incumbent are both pre-clone, so these must match to the row. -* ``nonzero_shares`` — per-column unweighted owning-entity nonzero share, at the - reference's own 6-decimal grain, plus the column-set difference in both - directions. +* ``nonzero_shares`` — per-column unweighted owning-entity nonzero share, + plus the column-set difference in both directions. Magnitudes are held to the + #723 acceptance band: every difference down to the reference's own 6-decimal + grain is reported, and those beyond the band are the ones that must be signed. * ``weighted_totals`` — optional, and licensed. Supplied as the two register sidecars, compared as relative deltas only. @@ -20,7 +21,10 @@ always the committed instrument, never anything derived from the candidate; and the tool refuses inputs that alias each other, so a candidate cannot be compared against itself. Under ``--strict`` an unused register entry is also a failure, -so the register cannot rot into a blanket amnesty as the spine changes. +so the register cannot rot into a blanket amnesty as the spine changes — an +entry counts as unused only on a surface this run actually compared, so entries +scoped to the optional weighted-totals surface are reported as dormant rather +than failing a run that did not supply it. Disclosure control: output is column names, counts, shares already carried by the committed reference, and relative deltas — never unit-record values. The @@ -56,6 +60,15 @@ #: extracted by the same producer agrees to that grain or it genuinely differs. SHARE_EPSILON = 1e-6 +#: The acceptance band the UK parity screen has used on this surface since +#: #723. A share difference inside it is reported but does not require a +#: signature: the spine re-runs every stochastic stage, so demanding a +#: permanent adjudication for third-decimal drift would fill the register with +#: entries that describe noise and blanket-cover the columns they name. The +#: band applies only to magnitudes on the share surface — a column appearing or +#: vanishing, and every entity count, is structural and still signs exactly. +SHARE_PARITY_BAND = 0.02 + #: Weighted totals ride calibrated weights; a relative delta below this is #: numerical noise rather than a divergence to sign. TOTALS_EPSILON = 1e-9 @@ -116,23 +129,33 @@ def _compare_shares( candidate_shares: Mapping[str, float], entities: Mapping[str, str], register: UKSignedDifferenceRegister, + band: float = SHARE_PARITY_BAND, ) -> tuple[dict[str, Any], list[str]]: unsigned: list[str] = [] differing: dict[str, Any] = {} + within_band: dict[str, Any] = {} compared = sorted(set(reference_shares) & set(candidate_shares)) for column in compared: expected = float(reference_shares[column]) observed = float(candidate_shares[column]) - if abs(observed - expected) <= SHARE_EPSILON: + delta = observed - expected + if abs(delta) <= SHARE_EPSILON: continue - signed = register.matching(surface="nonzero_shares", column=column) - differing[column] = { + record = { "entity": entities.get(column), "reference": expected, "candidate": observed, - "delta": observed - expected, - "signed_id": signed.id if signed else None, + "delta": delta, } + signed = register.matching(surface="nonzero_shares", column=column) + if abs(delta) <= band: + # Reported, never dropped: the band decides what must be + # adjudicated, not what the reader is allowed to see. + record["signed_id"] = signed.id if signed else None + within_band[column] = record + continue + record["signed_id"] = signed.id if signed else None + differing[column] = record if signed is None: unsigned.append(column) @@ -151,7 +174,14 @@ def _missing(names: list[str], expectation: str) -> dict[str, Any]: report = { "compared": len(compared), + "band": band, "differing": differing, + "within_band": within_band, + "within_band_max_abs_delta": ( + max(abs(entry["delta"]) for entry in within_band.values()) + if within_band + else 0.0 + ), "missing_in_candidate": _missing( list(set(reference_shares) - set(candidate_shares)), "column_missing_in_candidate", @@ -209,6 +239,7 @@ def verify_uk_spine_parity( reference_weighted_totals: Path | None = None, candidate_weighted_totals: Path | None = None, strict: bool = False, + share_band: float = SHARE_PARITY_BAND, ) -> dict[str, Any]: """Compare a candidate extraction against the committed reference.""" @@ -250,6 +281,7 @@ def verify_uk_spine_parity( {name: float(value) for name, value in candidate_shares.items()}, reference.input_entities, register, + band=share_band, ) matched_ids = { @@ -307,16 +339,32 @@ def verify_uk_spine_parity( if entry.get("signed_id") ) + # An entry can only rot on a surface this run actually looked at. The + # weighted-totals surface is optional and stays unexamined until there is a + # calibrated candidate to compare — an entry scoped to it is dormant then, + # not unused, and --strict must not read the two as the same thing. + compared_surfaces = {"entity_counts", "nonzero_shares"} + if "weighted_totals" in report: + compared_surfaces.add("weighted_totals") unused = sorted( difference.id for difference in register.differences if difference.id not in matched_ids + and difference.surface in compared_surfaces + ) + dormant = sorted( + difference.id + for difference in register.differences + if difference.id not in matched_ids + and difference.surface not in compared_surfaces ) report["register"] = { "resource": "spine_swap_signed_differences.json", "entries": len(register.differences), + "compared_surfaces": sorted(compared_surfaces), "matched_ids": sorted(matched_ids), "unused_ids": unused, + "dormant_ids": dormant, } report["unsigned_differences"] = sorted(set(unsigned)) @@ -369,6 +417,17 @@ def _parser() -> argparse.ArgumentParser: default=None, help="Licensed weighted-totals sidecar for the candidate spine.", ) + parser.add_argument( + "--share-band", + type=float, + default=SHARE_PARITY_BAND, + help=( + "Acceptance band on the nonzero-share surface (default " + f"{SHARE_PARITY_BAND}). Differences inside it are reported under " + "'within_band' and need no signature; pass 0 to require one for " + "every difference at the reference's six-decimal grain." + ), + ) parser.add_argument( "--strict", action="store_true", @@ -390,6 +449,13 @@ def _parser() -> argparse.ArgumentParser: def main(argv: list[str] | None = None) -> int: args = _parser().parse_args(argv) + if not 0.0 <= args.share_band < 1.0: + print( + "error: --share-band must be a share magnitude in [0, 1).", + file=sys.stderr, + ) + return 2 + totals = (args.reference_weighted_totals, args.candidate_weighted_totals) if any(totals) and not all(totals): print( @@ -422,6 +488,7 @@ def main(argv: list[str] | None = None) -> int: reference_weighted_totals=args.reference_weighted_totals, candidate_weighted_totals=args.candidate_weighted_totals, strict=args.strict, + share_band=args.share_band, ) rendered = json.dumps(report, indent=2, sort_keys=True, allow_nan=False) except Exception as error: # noqa: BLE001 - message is ours, not the data's From 8ccdc450a6625a7ec62f4b4d972c4d7021df12c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Mon, 24 Aug 2026 11:31:02 +0200 Subject: [PATCH 18/28] Record the signed queue and the settled donor evidence in the ledger Re-measures every donor comparison through each stage's own committed cleaning function on the survey-weighted basis, adds the level ratios alongside the shares, and replaces the E6 "needs ruling" and E5 "carry-forward?" sections with what the evidence now shows. The ETB rows are no longer undecidable: the stage cleans a thirteen-column subset of one year and weights by hhold_adj_weight, and on that frame the incumbent's education column is degenerate at fourteen nonzero households in 52,846. The incumbent's per-head division is recorded as a second upstream defect, observed but not filed. Also records the correction that a mid-review unweighted re-measurement of the wealth columns appeared to overturn the standing E5 adjudication and did not: WAS oversamples wealth-holders, so the weighted basis is the population one, and on it the original reading stands. Co-Authored-By: Claude Opus 5 --- experiments/686-uk-spine-comparison-ledger.md | 193 +++++++++++++----- 1 file changed, 146 insertions(+), 47 deletions(-) diff --git a/experiments/686-uk-spine-comparison-ledger.md b/experiments/686-uk-spine-comparison-ledger.md index 97de999a..33e7a986 100644 --- a/experiments/686-uk-spine-comparison-ledger.md +++ b/experiments/686-uk-spine-comparison-ledger.md @@ -41,56 +41,145 @@ The 1.56.14→1.56.16 re-pin moved no reference share by more than 0.0023 |---|---| | L0 build + determinism | complete — 3 rungs clean, twins payload-identical, record identity exact at 52,846 | | L1 identity receipts | e4 e5 e6 e7 e8 green (e7 written at #686; ladder complete) | -| L2 whole-spine parity | measured — 26 beyond ±0.02, verdict `defect` until the queue is ruled | +| L2 whole-spine parity | **signed_parity** — 26 beyond ±0.02, all signed; 0 unsigned; `--strict` clean | | L3 baselines | measured — 177 input-mass totals, 47 QRF tail grids | -## E6 · consumption — needs ruling +## What signing did, and what the band means -Measured fresh against the donors. On the eleven LCFS columns **ours is closer -on seven**, the incumbent on three, one tie — weaker than "the incumbent -collapses zero-inflated targets" (which stays true and dramatic on education: -incumbent 0.0003 vs donor 0.0476) but honest. Petrol/diesel are partly by -design (our `has_fuel` gate zeroes non-fuel households); on petrol the *level* -favours us while the share favours the incumbent — see Levels. +`spine_swap_signed_differences.json` now holds **13 entries covering 26 +beyond-band share columns, 2 entity counts and 2 net-new columns**. The +instrument reads it and returns `signed_parity` with nothing unsigned. -| column | donor truth (share) | incumbent | ours | closer | -|---|---|---|---|---| -| dfe_education_spending | see ETB note | 0.0003 | 0.2258 | undecidable | -| bus_subsidy_spending | see ETB note | 0.3167 | 0.5554 | undecidable | -| rail_subsidy_spending | see ETB note | 0.1277 | 0.1422 | undecidable | -| restaurants_and_hotels_consumption | 0.7651 | 0.6324 | 0.7903 | ours | -| petrol_spending | 0.3911 | 0.4446 | 0.3002 | incumbent | -| education_consumption | 0.0476 | 0.1256 | 0.0170 | ours | -| household_furnishings_consumption | 0.9104 | 0.8136 | 0.9216 | ours | -| miscellaneous_consumption | 0.9728 | 0.9003 | 0.9918 | ours | -| communication_consumption | 0.8696 | 0.7974 | 0.8814 | ours | -| alcohol_and_tobacco_consumption | 0.5383 | 0.5603 | 0.5150 | tie | -| domestic_energy_consumption | 0.9879 | 0.9549 | 0.9952 | ours | -| diesel_spending | 0.2040 | 0.1910 | 0.1580 | incumbent | -| transport_consumption | 0.8702 | 0.8668 | 0.8934 | incumbent | -| health_consumption | 0.5426 | 0.5128 | 0.5386 | ours | - -**ETB truth is not yet decidable**: the donor share moves with the weight -basis (education 0.0943 unweighted / 0.1327 household / 0.2150 individual) -and none reconcile with the E6 receipt's 0.2546. Until the stage's weight -convention is pinned, quoting one as truth would be false precision. - -## E5 · wealth — signed at E5 · carry-forward? +| entry | class | covers | +|---|---|---| +| `lcfs-consumption-regime-gated-incidence` | mechanism_change | 10 LCFS columns where ours is closer to the donor | +| `lcfs-fuel-consumption-incidence-gate` | mechanism_change | petrol, diesel — incumbent closer on share, ours on level | +| `lcfs-transport-aggregate-incidence` | mechanism_change | transport — incumbent marginally closer on share | +| `etb-services-regime-gated-incidence` | **defect_fix** | dfe_education, bus_subsidy — incumbent degenerate | +| `was-wealth-qrf-incidence` | qrf_implementation | 5 benchmarked wealth columns | +| `was-student-loan-balance-fold` | qrf_implementation | student_loan_balance — no benchmark | +| `spi-channel-qrf-incidence` | qrf_implementation | the 3 SPI-rewritten columns | +| `salary-sacrifice-conversion-depth` | mechanism_change | employee_pension_contributions | +| `donor-selection-rng-entity-counts` | rng_stream | person +32, benunit −12 (household stays exact) | +| `num-bedrooms-net-new-column` | net_new_column | spine adds it | +| `other-investment-income-net-new-column` | net_new_column | spine adds it | +| `scottish-water-incumbent-nan-zeroing` | defect_fix | water share | +| `scottish-water-sewerage-successor-level` | mechanism_change | water + council_tax **levels** (dormant until calibration) | + +**Three entries where the incumbent is closer** — petrol, diesel, transport — +are scoped apart on purpose. A single LCFS class entry would have signed them +under a verdict they do not share; that was your objection and it is what +drove the split. Each says in its own text that the evidence runs the other +way on incidence, so re-opening one does not require re-opening the class. + +**The band.** The instrument now holds the share surface to the ±0.02 #723 +band rather than the reference's 6-decimal grain. Without that, 90 further +columns — third-decimal drift from re-running every stochastic stage — would +each have needed a permanent adjudication, which is precisely the blanket +amnesty the register exists to stop. Nothing is hidden: all 90 are still in +the receipt under `within_band` (max 0.019), `--share-band 0` restores the +exact check, and **structural differences ignore the band entirely** — a +column appearing or vanishing, and every entity count, signs exactly at any +band. + +## E6 · consumption — signed + +Re-measured against each donor through the **stage's own committed cleaning +function**, on the **survey-weighted** basis. That convention matters: it +reproduces the E6 acceptance receipt's education figure (0.2546 unweighted on +the ETB services frame) exactly, which is what confirms it is the house +convention rather than one of several defensible choices. + +On the thirteen LCFS columns **ours is closer on ten shares and twelve +levels**. Petrol and diesel are the `has_fuel` gate under-placing incidence — +signed as that, with the direction stated. Level is population mean per +household, as a ratio to the donor's. + +| column | donor (share) | incumbent | ours | closer | donor £/hh | inc × | our × | closer | +|---|---|---|---|---|---|---|---|---| +| restaurants_and_hotels_consumption | 0.7651 | 0.6305 | 0.7903 | ours | 2,291 | 1.75 | 1.25 | ours | +| education_consumption | 0.0476 | 0.1258 | 0.0170 | ours | 264 | 5.00 | 0.41 | ours | +| household_furnishings_consumption | 0.9104 | 0.8126 | 0.9216 | ours | 2,019 | 1.68 | 1.63 | ours | +| electricity_consumption | 0.9228 | 0.8614 | 0.9644 | ours | 845 | 1.06 | 0.99 | ours | +| gas_consumption | 0.9815 | 0.9523 | 0.9952 | ours | 651 | 1.07 | 1.00 | ours | +| miscellaneous_consumption | 0.9728 | 0.8975 | 0.9918 | ours | 2,375 | 1.81 | 1.21 | ours | +| communication_consumption | 0.8696 | 0.7951 | 0.8814 | ours | 672 | 1.16 | 1.20 | incumbent | +| alcohol_and_tobacco_consumption | 0.5383 | 0.5630 | 0.5150 | ours | 579 | 1.25 | 1.24 | ours | +| domestic_energy_consumption | 0.9835 | 0.9541 | 0.9953 | ours | 1,496 | 1.06 | 1.00 | ours | +| health_consumption | 0.5426 | 0.5128 | 0.5386 | ours | 394 | 1.90 | 1.59 | ours | +| transport_consumption | 0.8702 | 0.8623 | 0.8934 | **incumbent** | 4,593 | 1.63 | 1.37 | ours | +| petrol_spending | 0.3911 | 0.4442 | 0.3002 | **incumbent** | 631 | 2.06 | 0.85 | ours | +| diesel_spending | 0.2040 | 0.1903 | 0.1580 | **incumbent** | 390 | 2.29 | 0.81 | ours | + +### ETB — the weight-basis question is closed + +It was never undecidable; it was measured on the wrong frame. The stage's own +convention is explicit in code: donor SN 8856, **year 2023 only**, complete +cases on the **thirteen-column services subset** (4,199 rows), weighted by +**`hhold_adj_weight`**. The incumbent cleans an eighteen-column subset, so a +"donor share" taken off its frame has a different denominator — that +mismatch, not a weighting ambiguity, produced the three irreconcilable +candidates recorded earlier. + +| column | donor (share) | incumbent | ours | closer | donor £/hh | incumbent | ours | closer | +|---|---|---|---|---|---|---|---|---| +| dfe_education_spending | 0.2794 | 0.000265 | 0.2258 | ours | 3,461 | 2 | 3,111 | ours | +| bus_subsidy_spending | 0.5255 | 0.3167 | 0.5554 | ours | 87 | 114 | 89 | ours | +| rail_subsidy_spending | 0.1652 | 0.1277 | 0.1422 | ours | 225 | 691 | 211 | ours | + +The incumbent's education column is **degenerate**: 14 nonzero households in +52,846, £2 per household against a donor £3,461. That is decidable on any +weight basis, so these are signed as a **defect fix on the incumbent side**, +not a method preference. Rail is inside the band and needs no signature; it is +shown because it moves the same way. + +The incumbent additionally divides each household total by household size and +stores the per-head figure in a household-entity column +(`policyengine_uk_data/datasets/imputations/services/etb.py`). That is a +second, independent defect — but it does **not** reconcile the levels (rail is +3.07× the donor even after it), so it is recorded as an observation rather +than as the explanation. **Worth an upstream issue alongside #467; not filed, +pending your call.** + +## E5 · wealth — signed, carried forward Adjudicated 2026-08-19 ("E5 is not required to reproduce the incumbent's -inflated totals; donor-benchmark evidence"). Fresh measurement corroborates. - -| column | donor truth · WAS R8 (share) | incumbent | ours | closer | -|---|---|---|---|---| -| savings | 0.6072 | 0.6620 | 0.6107 | ours (0.0035 off) | -| property_wealth | 0.6433 | 0.7081 | 0.6607 | ours | -| other_residential_property_value | 0.0363 | 0.0763 | 0.0367 | ours (0.0004 off) | -| main_residence_value | 0.6236 | 0.6747 | 0.6356 | ours | -| corporate_wealth | not measured (fold of several donor columns) | 0.8222 | 0.7792 | — | -| student_loan_balance | not measured (separate source) | 0.0197 | 0.0493 | — | +inflated totals; donor-benchmark evidence"). Re-measured through +`clean_was_household_table` over the pinned WAS R8 tab (15,128 rows, weighted +by `R8xshhwgt`), the ruling holds and the level evidence behind it is the more +dramatic surface. + +| column | donor (share) | incumbent | ours | closer | donor £/hh | inc × | our × | closer | +|---|---|---|---|---|---|---|---|---| +| savings | 0.6072 | 0.6621 | 0.6107 | ours (0.0035 off) | 16,918 | 5.70 | 1.97 | ours | +| property_wealth | 0.6433 | 0.7082 | 0.6607 | ours | 249,247 | 1.03 | 0.98 | ours | +| other_residential_property_value | 0.0363 | 0.0750 | 0.0367 | ours (0.0004 off) | 9,422 | 6.18 | 1.26 | ours | +| main_residence_value | 0.6236 | 0.6761 | 0.6356 | ours | 212,344 | 1.02 | 0.88 | incumbent | +| corporate_wealth | 0.7629 | 0.8225 | 0.7792 | ours | 160,010 | 1.85 | 1.45 | ours | +| student_loan_balance | no like-for-like benchmark | 0.0197 | 0.0493 | — | — | — | — | — | + +Ours is closer on **five of five** benchmarked shares and four of five levels. +`corporate_wealth`, previously "not measured", now has a benchmark — it is the +committed fold of five WAS aggregates, so it can be reconstructed on the donor +frame exactly as the stage builds it. + +**A caveat that matters, and a correction.** The *unweighted* donor shares tell +the opposite story on incidence — the incumbent looks closer on all five. WAS +oversamples wealth-holders by design, so the unweighted frame is not a +population; the weighted basis is, and it is the basis used throughout this +document. The earlier "ours closer on 4/4" reading was on the weighted basis +and stands; a mid-review unweighted re-measurement briefly appeared to +overturn it and did not. + +`student_loan_balance` is signed **on its own** rather than inside the wealth +class: it is a fold of two WAS loan aggregates at a different entity grain, +with no like-for-like donor share available. It is the one E5 column whose +direction is unevidenced, and the register says so. `owned_land` is **not** in this list; its reviewed exclusion expiring -2026-09-20 is a separate question. +2026-09-20 is a separate question. For the record it is in band (−0.0026) and +ours is closer to the donor on both share (0.0071 donor · 0.0143 incumbent · +0.0119 ours) and level (9.28× against 2.44×). ## E7 · SPI channel — evidence gap closed, favours the spine @@ -218,11 +307,21 @@ implementations. ## Open -- **ETB weight basis** — pins the truth for three columns. -- **E6 is not one class** — a single entry over all fifteen would sign columns where we are further from the donor than the incumbent is. -- **Entity-count entry scope** — surface-wide vs scoped to the two entities. -- **Stale donor ratios** in the E5/E6 prose receipts were measured at 1.56.14; the tables above are fresh at the current pin. -- **Upstream defect filed** as policyengine-uk-data#467 (no prescriptive fix); our side is correct and signed. +- **The queue is signed.** All 26 beyond-band divergences carry register + entries; whole-spine parity is `signed_parity`, `--strict` clean. The two + items that blocked it are closed: the ETB weight basis (above) and the + entity-count scope (signed to `person` and `benunit` only, so a future + household-count divergence stays a defect). +- **The incumbent's ETB per-head division** — a second upstream defect of the + #467 family, observed but not filed. Your call. +- **`communication_consumption`** is the one LCFS column where the incumbent is + closer on level (1.16× against our 1.20×) while we are closer on share. It is + signed inside the donor-faithful class; if that bothers you it should be + scoped out, and that is a one-line register change. +- **`student_loan_balance`** carries no donor benchmark — signed on the + standing E5 adjudication, direction unevidenced. +- **Upstream defect filed** as policyengine-uk-data#467 (no prescriptive fix); + our side is correct and signed. ## Disclosure and citation From 39f49b5abc5e0197f73e195caade357d2ec6f1b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Mon, 24 Aug 2026 11:33:55 +0200 Subject: [PATCH 19/28] Record the signing run and the two measurement corrections behind it R5 receipts the signed_parity verdict and, more usefully, the two ways the donor measurements had been wrong before it. Both had the same cause: a donor truth was computed on a frame the stage does not use. Once against the ETB services frame, which produced a false blocker and had been recorded as an unpinnable weight basis; once against the WAS frame unweighted, which briefly appeared to overturn a standing adjudication. Calling each stage's own cleaning function reproduces the E6 acceptance receipt's education figure to four decimals, which is the check that the convention is the right one. Co-Authored-By: Claude Opus 5 --- experiments/686-uk-spine-swap-receipts.md | 115 ++++++++++++++++++++++ 1 file changed, 115 insertions(+) diff --git a/experiments/686-uk-spine-swap-receipts.md b/experiments/686-uk-spine-swap-receipts.md index 965e9fd8..cc1cd764 100644 --- a/experiments/686-uk-spine-swap-receipts.md +++ b/experiments/686-uk-spine-swap-receipts.md @@ -546,3 +546,118 @@ The re-pin does not by itself re-validate the #723 acceptance screen: the reference. Because no reference share moved by more than 0.0046, that classification is expected to survive intact, but it is re-measured against the new reference in the L2 whole-spine parity loop rather than assumed. + +--- + +## R5 — signing the queue, and the two measurement corrections that preceded it + +Run on the rebased branch (rebase onto `origin/main` after #740 and 239 other +commits; the only derived value that moved was the UK spec bundle digest, +because main touched `uk/target_references.json` and +`uk/target_reference_membership.json`). + +### The verdict + +``` +verify_uk_spine_parity.py --candidate-json candidate_extraction_spine_c.json --strict +``` + +| field | value | +|---|---| +| verdict | **`signed_parity`** | +| unsigned differences | **0** | +| strict failure | **false** | +| columns compared | 145 · band 0.02 | +| beyond band | 26 — all signed | +| within band | 90 · max abs delta 0.019037 | +| register | 13 entries · 12 matched · 0 unused · 1 dormant | +| entity counts | household **exact**; person +32 and benunit −12 both signed | + +Receipt at `data/ukds/acceptance/686-spine-swap/parity_receipt_spine_c_signed.json`. + +### Correction 1 — the ETB rows were never undecidable + +The three ETB columns were recorded as blocked on an unpinnable weight basis: +the donor education share moved across 0.0943 / 0.1327 / 0.2150 by weighting +and none reconciled with the E6 acceptance receipt's 0.2546. + +The weight basis was not the problem. The stage's convention is explicit in +code — `train["weight"] = data["hhold_adj_weight"]`, `weights="weight"` at both +fit sites — and the *frame* is the part that had been wrong: the services stage +cleans **year 2023 only** on complete cases over a **thirteen-column** subset +(4,199 rows), while the incumbent's services imputation drops on an +**eighteen-column** subset. A "donor share" taken off the incumbent's frame has +a different denominator. + +Measured on the stage's own cleaned frame, `(v != 0).mean()` returns +**0.2546** — the acceptance receipt's figure, to four decimals. The receipt was +right the whole time; the re-measurement had been reading a different +population. + +With the frame fixed, the row decides cleanly and against the incumbent: + +| | donor | incumbent | ours | +|---|---|---|---| +| `dfe_education_spending` share | 0.2794 | **0.000265** | 0.2258 | +| `dfe_education_spending` £/household | 3,461 | **2** | 3,111 | + +Fourteen nonzero households in 52,846, at £2 per household against a donor +£3,461. That is degenerate on any weight basis, which is why these are signed +`defect_fix` rather than as a method preference. + +**A second incumbent defect, observed and not filed.** The incumbent divides +each household total by household size and stores the per-head figure in a +household-entity column (`imputations/services/etb.py`). It does not reconcile +the levels — rail is 3.07× the donor even after it — so it is recorded as an +observation, not as the explanation. It is a #467-family upstream issue and +wants a ruling before filing. + +### Correction 2 — a reversal that was itself wrong + +Re-measuring the E5 wealth columns through `clean_was_household_table` +initially appeared to overturn the standing 2026-08-19 adjudication: the +incumbent looked closer on all five shares, where the ledger had recorded ours +closer on four of four. + +It did not. The new pass was unweighted and the ledger's was survey-weighted. +WAS **oversamples wealth-holders by design**, so its unweighted frame is not a +population and its unweighted share is not a truth. On the weighted basis the +original reading reproduces exactly — savings 0.6072, property_wealth 0.6433, +other_residential 0.0363, main_residence 0.6236 — and ours is closer on five +of five, `corporate_wealth` included, which previously had no benchmark. + +**Standing lesson, and it is the same one both times.** A donor "truth" is not +a property of the donor file; it is a property of the *frame and weighting the +stage itself uses*. Measuring it any other way produces numbers that look +authoritative and are not — once in the direction of a false blocker, once in +the direction of a false reversal. Every donor figure in the ledger is now +computed by calling the stage's own committed cleaning function over its own +pinned tab, on the survey-weighted basis, which is the convention that +reproduces the acceptance figure. + +### Why the band, and why it is not a loosening + +Holding the share surface to the reference's 6-decimal grain left 90 in-band +columns unsigned, so the verdict could only be cleared by writing 90 further +register entries for third-decimal drift. That would have been the blanket +amnesty the `--strict` fence exists to prevent: 90 permanent adjudications +describing noise, each blanket-covering the column it names against any future +real regression. + +The instrument now holds the share surface to the ±0.02 band the #723 screen +already used. What that changes is only *which magnitudes require an +adjudication*. Every difference down to the 6-decimal grain is still in the +receipt, under `within_band`, with the in-band maximum alongside; +`--share-band 0` restores the exact check; and structural differences are +outside the band's reach at any band — a column appearing or vanishing, and +every entity count, still signs exactly. There is a test for that last +property specifically, run at `--share-band 0.9`. + +`--strict` additionally separates an entry that matched nothing on a surface +the run *compared* from one whose surface was never *examined*. The +weighted-totals surface stays unexamined until there is a calibrated candidate +— comparing our design weights to the incumbent's calibrated ones would flag +all 131 columns, which is the same before-medicine/after-medicine error the UC +check documents. The water-level entry is therefore reported as **dormant**, +and a test pins that dormancy stops being available the moment a run supplies +the sidecars. From d0ffadab7bed3b88445880003910c6d655f58efe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Mon, 24 Aug 2026 11:36:29 +0200 Subject: [PATCH 20/28] Distinguish the two level statistics in the ledger The signing tables report population mean per household; the Levels section reports conditional mean per carrier. They can disagree on which side is closer, so the section now says which one the signing used and why: the population mean is what a calibration target binds and what survives a difference in incidence. Co-Authored-By: Claude Opus 5 --- experiments/686-uk-spine-comparison-ledger.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/experiments/686-uk-spine-comparison-ledger.md b/experiments/686-uk-spine-comparison-ledger.md index 33e7a986..b2346066 100644 --- a/experiments/686-uk-spine-comparison-ledger.md +++ b/experiments/686-uk-spine-comparison-ledger.md @@ -224,6 +224,17 @@ omission from a merged increment, now closed — parity reports **0 missing**. ## Levels — annual £ per carrier +> **Two different level statistics — read the label.** This section is the +> **conditional mean**: £ per *carrier*, over nonzero rows only. The E5 and E6 +> tables above are the **population mean per household**, over every household +> including zeros, as a ratio to the donor's. They answer different questions +> and can disagree on which side is closer — a build can put too little on each +> carrier while putting carriers on too many households, and come out right in +> aggregate. **The signing used the population mean**, because that is what a +> calibration target binds and what survives a difference in incidence. Where a +> verdict here differs from one above, the one above is operative. + + Conditional means over nonzero rows; carrier counts withheld here for brevity but recorded in the interactive ledger and reproducible from the licensed evidence dir; no cell is dominated by a single record; all cells ≥10 carriers. From 160bb5bb5bf36f5e906410e1a5177b7ffd1268e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Mon, 24 Aug 2026 11:38:04 +0200 Subject: [PATCH 21/28] Apply ruff formatting to the parity instrument and its tests Co-Authored-By: Claude Opus 5 --- .../tests/test_uk_spine_parity_instrument.py | 6 +++--- tools/verify_uk_spine_parity.py | 3 +-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/packages/microcosm-build/tests/test_uk_spine_parity_instrument.py b/packages/microcosm-build/tests/test_uk_spine_parity_instrument.py index 77a11b6b..6ad8a949 100644 --- a/packages/microcosm-build/tests/test_uk_spine_parity_instrument.py +++ b/packages/microcosm-build/tests/test_uk_spine_parity_instrument.py @@ -528,9 +528,9 @@ def test_an_in_band_difference_needs_no_signature(self, tmp_path: Path) -> None: # Reported, not dropped: the reader still sees the movement. assert column in report["nonzero_shares"]["within_band"] assert column not in report["nonzero_shares"]["differing"] - assert report["nonzero_shares"]["within_band"][column]["delta"] == pytest.approx( - 0.01 - ) + assert report["nonzero_shares"]["within_band"][column][ + "delta" + ] == pytest.approx(0.01) assert report["nonzero_shares"]["within_band_max_abs_delta"] == pytest.approx( 0.01 ) diff --git a/tools/verify_uk_spine_parity.py b/tools/verify_uk_spine_parity.py index cb13c75d..5d07313c 100644 --- a/tools/verify_uk_spine_parity.py +++ b/tools/verify_uk_spine_parity.py @@ -349,8 +349,7 @@ def verify_uk_spine_parity( unused = sorted( difference.id for difference in register.differences - if difference.id not in matched_ids - and difference.surface in compared_surfaces + if difference.id not in matched_ids and difference.surface in compared_surfaces ) dormant = sorted( difference.id From 9a6e3602b272d269e263cb202d70559d4c5f6193 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Mon, 24 Aug 2026 11:45:51 +0200 Subject: [PATCH 22/28] Measure the incumbent side on the pinned artifact, not the 1.56.14 one MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An earlier pass read the only full incumbent H5 on disk, which is the 1.56.14 published artifact rather than the pinned 1.56.16 one. The two differ by at most 0.0045 on any share, but that was enough to flip one verdict: alcohol_and_tobacco_consumption reads as ours at 1.56.14 and as the incumbent at the pin. It moves out of the donor-faithful entry into a two-column entry with transport_consumption, which shares its evidence shape exactly — incumbent closer on share, ours closer on level. The pinned artifact is now fetched and verified by digest before measuring, and the correction is recorded in the ledger rather than quietly applied. Also rewrites "What the numbers are", which still claimed the ledger measures incidence and never level. That is true of the committed instrument and is precisely why the ledger exists; the document itself evaluates both, and the section now says which quantity is operative when they disagree. Co-Authored-By: Claude Opus 5 --- experiments/686-uk-spine-comparison-ledger.md | 110 +++++++++++------ .../uk/spine_swap_signed_differences.json | 115 +++++++++++++----- 2 files changed, 154 insertions(+), 71 deletions(-) diff --git a/experiments/686-uk-spine-comparison-ledger.md b/experiments/686-uk-spine-comparison-ledger.md index b2346066..b7b3a180 100644 --- a/experiments/686-uk-spine-comparison-ledger.md +++ b/experiments/686-uk-spine-comparison-ledger.md @@ -11,16 +11,34 @@ Updated as adjudications land. ## What the numbers are -Every figure in the share tables is a **nonzero share**: the fraction of the -column's owning-entity rows carrying a non-zero value, 0–1. `0.6072` for -savings means 60.7% of households report some savings — it is not an amount. -The parity screen measures *incidence*, never level. That limit is real: a -column can match perfectly on share while being badly wrong on level — the -Scottish water fix produces an identical share of 0.878377 under both the -retired and corrected mappings while the amount moved from ~£185 to ~£395 per -Scottish household. Figures that are levels carry a £ sign and sit in the -Levels section, as conditional means (mean over nonzero carriers) with the -carrier count alongside. +**Two quantities are evaluated here, and both matter.** + +**Shares** are *nonzero shares*: the fraction of the column's owning-entity +rows carrying a non-zero value, 0–1. `0.6072` for savings means 60.7% of +households report some savings — it is not an amount. + +**Levels** are amounts in £, and they appear in two forms. Throughout the E5, +E6 and ETB tables the level is the **population mean per household** — the +total spread over *every* household including the zeros — quoted as a ratio to +the donor's (`1.25×` means we are 25% above the donor). The separate Levels +section reports the **conditional mean**: £ per *carrier*, over nonzero rows +only. + +**Why both, and why it is not optional.** The committed parity instrument +measures incidence and nothing else. That is a real limit, and it is the +reason this document exists: a column can match perfectly on share while being +badly wrong on level. The Scottish water fix is the proof — an identical share +of 0.878377 under both the retired and the corrected mapping, while the amount +moved from ~£185 to ~£395 per Scottish household. The instrument would have +called that parity. + +So the instrument decides *what must be adjudicated*, and the level evidence +decides *which way*. Several columns come out differently on the two: petrol +and diesel are further from the donor than the incumbent on incidence and much +closer on amount, and that split is signed explicitly rather than averaged +away. **Where a share verdict and a level verdict disagree, the population +mean is the operative one** — it is what a calibration target binds, and it is +the one that survives a difference in incidence. ## Which incumbent @@ -32,8 +50,12 @@ Three artifacts get called "the incumbent"; they answer different questions. | B | `incumbent_{base,wealth,consumption}_ebf733c.h5` | partial pipelines rebuilt locally at 1.56.14 for E3/E5/E6 method head-to-heads; one release behind, carries the benunit-sort defect | | C | WAS R8 · LCFS 2023-24 · ETB · SPI 2022-23 | the source surveys — ground truth neither build controls | -The 1.56.14→1.56.16 re-pin moved no reference share by more than 0.0023 -(E7: exactly zero); the deltas below are robust to it. +The 1.56.14→1.56.16 re-pin moved no reference share by more than 0.0046 (R0). +A direct check across the nineteen E5/E6 columns puts the largest at 0.0045, +on `transport_consumption` — consistent, and small, but not negligible: it is +exactly the size that flipped the `alcohol_and_tobacco_consumption` verdict +when an earlier pass measured against the wrong vintage. Every incumbent +figure here is measured on artifact **A** at the pin. ## Gate status @@ -52,9 +74,9 @@ instrument reads it and returns `signed_parity` with nothing unsigned. | entry | class | covers | |---|---|---| -| `lcfs-consumption-regime-gated-incidence` | mechanism_change | 10 LCFS columns where ours is closer to the donor | +| `lcfs-consumption-regime-gated-incidence` | mechanism_change | 9 LCFS columns where ours is closer to the donor | | `lcfs-fuel-consumption-incidence-gate` | mechanism_change | petrol, diesel — incumbent closer on share, ours on level | -| `lcfs-transport-aggregate-incidence` | mechanism_change | transport — incumbent marginally closer on share | +| `lcfs-aggregate-incidence-incumbent-closer` | mechanism_change | transport, alcohol — incumbent closer on share, ours on level | | `etb-services-regime-gated-incidence` | **defect_fix** | dfe_education, bus_subsidy — incumbent degenerate | | `was-wealth-qrf-incidence` | qrf_implementation | 5 benchmarked wealth columns | | `was-student-loan-balance-fold` | qrf_implementation | student_loan_balance — no benchmark | @@ -66,7 +88,8 @@ instrument reads it and returns `signed_parity` with nothing unsigned. | `scottish-water-incumbent-nan-zeroing` | defect_fix | water share | | `scottish-water-sewerage-successor-level` | mechanism_change | water + council_tax **levels** (dormant until calibration) | -**Three entries where the incumbent is closer** — petrol, diesel, transport — +**Two entries covering four columns where the incumbent is closer on share** — +petrol, diesel, transport, alcohol_and_tobacco — are scoped apart on purpose. A single LCFS class entry would have signed them under a verdict they do not share; that was your objection and it is what drove the split. Each says in its own text that the evidence runs the other @@ -90,26 +113,35 @@ reproduces the E6 acceptance receipt's education figure (0.2546 unweighted on the ETB services frame) exactly, which is what confirms it is the house convention rather than one of several defensible choices. -On the thirteen LCFS columns **ours is closer on ten shares and twelve +On the thirteen LCFS columns **ours is closer on nine shares and twelve levels**. Petrol and diesel are the `has_fuel` gate under-placing incidence — signed as that, with the direction stated. Level is population mean per household, as a ratio to the donor's. | column | donor (share) | incumbent | ours | closer | donor £/hh | inc × | our × | closer | |---|---|---|---|---|---|---|---|---| -| restaurants_and_hotels_consumption | 0.7651 | 0.6305 | 0.7903 | ours | 2,291 | 1.75 | 1.25 | ours | -| education_consumption | 0.0476 | 0.1258 | 0.0170 | ours | 264 | 5.00 | 0.41 | ours | -| household_furnishings_consumption | 0.9104 | 0.8126 | 0.9216 | ours | 2,019 | 1.68 | 1.63 | ours | -| electricity_consumption | 0.9228 | 0.8614 | 0.9644 | ours | 845 | 1.06 | 0.99 | ours | -| gas_consumption | 0.9815 | 0.9523 | 0.9952 | ours | 651 | 1.07 | 1.00 | ours | -| miscellaneous_consumption | 0.9728 | 0.8975 | 0.9918 | ours | 2,375 | 1.81 | 1.21 | ours | -| communication_consumption | 0.8696 | 0.7951 | 0.8814 | ours | 672 | 1.16 | 1.20 | incumbent | -| alcohol_and_tobacco_consumption | 0.5383 | 0.5630 | 0.5150 | ours | 579 | 1.25 | 1.24 | ours | -| domestic_energy_consumption | 0.9835 | 0.9541 | 0.9953 | ours | 1,496 | 1.06 | 1.00 | ours | -| health_consumption | 0.5426 | 0.5128 | 0.5386 | ours | 394 | 1.90 | 1.59 | ours | -| transport_consumption | 0.8702 | 0.8623 | 0.8934 | **incumbent** | 4,593 | 1.63 | 1.37 | ours | -| petrol_spending | 0.3911 | 0.4442 | 0.3002 | **incumbent** | 631 | 2.06 | 0.85 | ours | -| diesel_spending | 0.2040 | 0.1903 | 0.1580 | **incumbent** | 390 | 2.29 | 0.81 | ours | +| restaurants_and_hotels_consumption | 0.7651 | 0.6324 | 0.7903 | ours | 2,291 | 1.74 | 1.25 | ours | +| education_consumption | 0.0476 | 0.1256 | 0.0170 | ours | 264 | 4.80 | 0.41 | ours | +| household_furnishings_consumption | 0.9104 | 0.8136 | 0.9216 | ours | 2,019 | 1.67 | 1.63 | ours | +| electricity_consumption | 0.9228 | 0.8621 | 0.9644 | ours | 845 | 1.06 | 0.99 | ours | +| gas_consumption | 0.9815 | 0.9532 | 0.9952 | ours | 651 | 1.06 | 1.00 | ours | +| miscellaneous_consumption | 0.9728 | 0.9003 | 0.9918 | ours | 2,375 | 1.80 | 1.21 | ours | +| communication_consumption | 0.8696 | 0.7974 | 0.8814 | ours | 672 | 1.15 | 1.20 | incumbent | +| domestic_energy_consumption | 0.9835 | 0.9549 | 0.9953 | ours | 1,496 | 1.06 | 1.00 | ours | +| health_consumption | 0.5426 | 0.5128 | 0.5386 | ours | 394 | 1.88 | 1.59 | ours | +| alcohol_and_tobacco_consumption | 0.5383 | 0.5603 | 0.5150 | **incumbent** | 579 | 1.26 | 1.24 | ours | +| transport_consumption | 0.8702 | 0.8668 | 0.8934 | **incumbent** | 4,593 | 1.62 | 1.37 | ours | +| petrol_spending | 0.3911 | 0.4446 | 0.3002 | **incumbent** | 631 | 2.05 | 0.85 | ours | +| diesel_spending | 0.2040 | 0.1910 | 0.1580 | **incumbent** | 390 | 2.31 | 0.81 | ours | + +> **A correction, recorded rather than quietly fixed.** An earlier pass +> measured the incumbent side against the **1.56.14** published artifact — the +> only full incumbent H5 on disk — instead of the pinned **1.56.16** one. The +> two differ by at most 0.0045 on any share, but that was enough to flip one +> verdict: `alcohol_and_tobacco_consumption` read as ours at 1.56.14 and reads +> as the incumbent at the pin. Every incumbent figure in this document is now +> measured on the pinned artifact itself (sha256 `e433e532`, fetched at +> revision `a9e52499`), and it is signed accordingly. ### ETB — the weight-basis question is closed @@ -124,8 +156,8 @@ candidates recorded earlier. | column | donor (share) | incumbent | ours | closer | donor £/hh | incumbent | ours | closer | |---|---|---|---|---|---|---|---|---| | dfe_education_spending | 0.2794 | 0.000265 | 0.2258 | ours | 3,461 | 2 | 3,111 | ours | -| bus_subsidy_spending | 0.5255 | 0.3167 | 0.5554 | ours | 87 | 114 | 89 | ours | -| rail_subsidy_spending | 0.1652 | 0.1277 | 0.1422 | ours | 225 | 691 | 211 | ours | +| bus_subsidy_spending | 0.5255 | 0.3167 | 0.5554 | ours | 87 | 113 | 89 | ours | +| rail_subsidy_spending | 0.1652 | 0.1277 | 0.1422 | ours | 225 | 689 | 211 | ours | The incumbent's education column is **degenerate**: 14 nonzero households in 52,846, £2 per household against a donor £3,461. That is decidable on any @@ -137,7 +169,7 @@ The incumbent additionally divides each household total by household size and stores the per-head figure in a household-entity column (`policyengine_uk_data/datasets/imputations/services/etb.py`). That is a second, independent defect — but it does **not** reconcile the levels (rail is -3.07× the donor even after it), so it is recorded as an observation rather +3.06× the donor even after it), so it is recorded as an observation rather than as the explanation. **Worth an upstream issue alongside #467; not filed, pending your call.** @@ -151,11 +183,11 @@ dramatic surface. | column | donor (share) | incumbent | ours | closer | donor £/hh | inc × | our × | closer | |---|---|---|---|---|---|---|---|---| -| savings | 0.6072 | 0.6621 | 0.6107 | ours (0.0035 off) | 16,918 | 5.70 | 1.97 | ours | -| property_wealth | 0.6433 | 0.7082 | 0.6607 | ours | 249,247 | 1.03 | 0.98 | ours | -| other_residential_property_value | 0.0363 | 0.0750 | 0.0367 | ours (0.0004 off) | 9,422 | 6.18 | 1.26 | ours | +| savings | 0.6072 | 0.6620 | 0.6107 | ours (0.0035 off) | 16,918 | 5.66 | 1.97 | ours | +| property_wealth | 0.6433 | 0.7081 | 0.6607 | ours | 249,247 | 1.04 | 0.98 | ours | +| other_residential_property_value | 0.0363 | 0.0763 | 0.0367 | ours (0.0004 off) | 9,422 | 6.12 | 1.26 | ours | | main_residence_value | 0.6236 | 0.6761 | 0.6356 | ours | 212,344 | 1.02 | 0.88 | incumbent | -| corporate_wealth | 0.7629 | 0.8225 | 0.7792 | ours | 160,010 | 1.85 | 1.45 | ours | +| corporate_wealth | 0.7629 | 0.8222 | 0.7792 | ours | 160,010 | 1.84 | 1.45 | ours | | student_loan_balance | no like-for-like benchmark | 0.0197 | 0.0493 | — | — | — | — | — | Ours is closer on **five of five** benchmarked shares and four of five levels. @@ -178,8 +210,8 @@ direction is unevidenced, and the register says so. `owned_land` is **not** in this list; its reviewed exclusion expiring 2026-09-20 is a separate question. For the record it is in band (−0.0026) and -ours is closer to the donor on both share (0.0071 donor · 0.0143 incumbent · -0.0119 ours) and level (9.28× against 2.44×). +ours is closer to the donor on both share (0.0071 donor · 0.0145 incumbent · +0.0119 ours) and level (9.19× against 2.44×). ## E7 · SPI channel — evidence gap closed, favours the spine @@ -326,7 +358,7 @@ implementations. - **The incumbent's ETB per-head division** — a second upstream defect of the #467 family, observed but not filed. Your call. - **`communication_consumption`** is the one LCFS column where the incumbent is - closer on level (1.16× against our 1.20×) while we are closer on share. It is + closer on level (1.15× against our 1.20×) while we are closer on share. It is signed inside the donor-faithful class; if that bothers you it should be scoped out, and that is a one-line register change. - **`student_loan_balance`** carries no donor benchmark — signed on the diff --git a/packages/microcosm-build/src/microcosm/build/uk/spine_swap_signed_differences.json b/packages/microcosm-build/src/microcosm/build/uk/spine_swap_signed_differences.json index 7c50fd3d..4d30b0f3 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/spine_swap_signed_differences.json +++ b/packages/microcosm-build/src/microcosm/build/uk/spine_swap_signed_differences.json @@ -1,14 +1,18 @@ { "schema_version": 1, - "scope_note": "Adjudicated intentional differences between the microcosm-built UK spine and the frozen enhanced-FRS incumbent (#686). The whole-spine comparison treats any difference beyond the #723 acceptance band that is not signed here as a defect, so every entry is scoped to the exact surface and columns where the difference is expected to appear: a too-broad entry would sign a real defect. Entries are permanent adjudications and carry no expiry; time-limited per-gate suppressions belong in input_mass_reviewed_exclusions.json, qrf_tail_reviewed_exclusions.json or degenerate_reviewed_exclusions.json instead, and an entry here points at one through its evidence field when both descend from the same adjudication. The twenty-six beyond-band divergences measured against the re-pinned 1.56.16 reference are signed below, each scoped to the divergence actually observed rather than carried across on its prose classification, and each deliberately split so that no entry covers both columns where the spine is closer to its donor and columns where the incumbent is: the direction of the evidence is part of what is being signed. Donor figures quoted as magnitude evidence are survey-weighted shares and population means computed on each stage's own committed cleaning function over its own pinned donor tab, which is the convention that reproduces the E6 acceptance receipt's education figure exactly.", + "scope_note": "Adjudicated intentional differences between the microcosm-built UK spine and the frozen enhanced-FRS incumbent (#686). The whole-spine comparison treats any difference beyond the #723 acceptance band that is not signed here as a defect, so every entry is scoped to the exact surface and columns where the difference is expected to appear: a too-broad entry would sign a real defect. Entries are permanent adjudications and carry no expiry; time-limited per-gate suppressions belong in input_mass_reviewed_exclusions.json, qrf_tail_reviewed_exclusions.json or degenerate_reviewed_exclusions.json instead, and an entry here points at one through its evidence field when both descend from the same adjudication. The twenty-six beyond-band divergences measured against the re-pinned 1.56.16 reference are signed below, each scoped to the divergence actually observed rather than carried across on its prose classification, and each deliberately split so that no entry covers both columns where the spine is closer to its donor and columns where the incumbent is: the direction of the evidence is part of what is being signed. Donor figures quoted as magnitude evidence are survey-weighted shares and population means computed on each stage's own committed cleaning function over its own pinned donor tab, which is the convention that reproduces the E6 acceptance receipt's education figure exactly. The incumbent side of every figure is measured on the pinned 1.56.16 artifact itself (sha256 e433e532), not on a local rebuild: an earlier pass read the 1.56.14 published artifact, whose shares differ by up to 0.0045 and which flipped one verdict.", "differences": [ { "id": "scottish-water-incumbent-nan-zeroing", "class": "defect_fix", "scope": { "surface": "nonzero_shares", - "columns": ["water_and_sewerage_charges"], - "entities": ["household"] + "columns": [ + "water_and_sewerage_charges" + ], + "entities": [ + "household" + ] }, "expectation": "column_differs", "magnitude_evidence": "The spine's unweighted nonzero share exceeds the incumbent's by about +0.10 on the whole file. FRS 2024-25 retired CWATAMT/CSEWAMT: the headers survive but carry no data in any of the 16,288 households. The incumbent adds the retired CSEWAMT before filling, so NaN propagates and the charge is zeroed for every Scottish household that has one; the spine fills per column and they stand. Reproduced on the raw tab as 12,644 nonzero households for the incumbent formula against 14,307 for ours, a +0.1021 share gap whose 1,663 differing households are all Scottish, with England, Wales and Northern Ireland identical under both. Latent since at least 2023-24, where 378 Scottish households were already affected. The defect is on the incumbent side and survives at 1.56.16.", @@ -21,8 +25,13 @@ "class": "mechanism_change", "scope": { "surface": "weighted_totals", - "columns": ["water_and_sewerage_charges", "council_tax"], - "entities": ["household"] + "columns": [ + "water_and_sewerage_charges", + "council_tax" + ], + "entities": [ + "household" + ] }, "expectation": "column_differs", "magnitude_evidence": "The Scottish charge is assembled from the successors FRS 2024-25 published for the cells it retired, so its level rises against both the incumbent and our own earlier build. CWATAMTD is the water charge alone; CSEWAMT1 supplies the sewerage side and is discounted at the household's own observed CWATAMTD/CWATAMT1 factor, which keeps the retired cells' after-discount meaning rather than switching to a gross basis. Weighted annual per Scottish household moves from about GBP 185 on water alone to about GBP 395, against roughly GBP 490 for England and Wales on WATSEWRT; the incumbent sits at zero because of the separate NaN defect. The same amount is netted from council_tax, so that column moves by the same construction. The nonzero share is unaffected, so this entry deliberately does not sign the share surface.", @@ -36,7 +45,6 @@ "scope": { "surface": "nonzero_shares", "columns": [ - "alcohol_and_tobacco_consumption", "communication_consumption", "domestic_energy_consumption", "education_consumption", @@ -47,10 +55,12 @@ "miscellaneous_consumption", "restaurants_and_hotels_consumption" ], - "entities": ["household"] + "entities": [ + "household" + ] }, "expectation": "column_differs", - "magnitude_evidence": "The spine draws LCFS consumption through a regime-gated QRF that carries the donor's zero mass as a modelled incidence, where the incumbent's plain QRF regresses a zero-inflated target toward its conditional mean. On these ten columns the spine's unweighted share is closer to the LCFS 2023-24 survey-weighted donor share than the incumbent's is, on the stage's own cleaned donor frame of 4,202 households: education_consumption donor 0.0476 against incumbent 0.1258 and ours 0.0170; restaurants_and_hotels donor 0.7651 against 0.6305 and 0.7903; miscellaneous donor 0.9728 against 0.8975 and 0.9918; domestic_energy donor 0.9835 against 0.9541 and 0.9953, with its electricity and gas components moving the same way. Population mean per household is closer for the spine on nine of the ten, the exception being communication_consumption at 1.20x the donor against the incumbent's 1.16x; the incumbent runs 1.7x to 5.0x the donor mean on the rest. Every cell is above the disclosure floor, the thinnest being 193 donor carriers on education_consumption.", + "magnitude_evidence": "The spine draws LCFS consumption through a regime-gated QRF that carries the donor's zero mass as a modelled incidence, where the incumbent's plain QRF regresses a zero-inflated target toward its conditional mean. On these nine columns the spine's unweighted share is closer to the LCFS 2023-24 survey-weighted donor share than the incumbent's is, measured on the stage's own cleaned donor frame of 4,202 households against the pinned 1.56.16 artifact (sha256 e433e532): education_consumption donor 0.0476 against incumbent 0.1256 and ours 0.0170; restaurants_and_hotels donor 0.7651 against 0.6324 and 0.7903; miscellaneous donor 0.9728 against 0.9003 and 0.9918; domestic_energy donor 0.9835 against 0.9549 and 0.9953, with its electricity and gas components moving the same way. Population mean per household is closer for the spine on eight of the nine, the exception being communication_consumption at 1.20x the donor against the incumbent's 1.15x; the incumbent runs 1.7x to 4.8x the donor mean on the rest. Every cell is above the disclosure floor, the thinnest being 193 donor carriers on education_consumption.", "evidence": "experiments/686-uk-spine-comparison-ledger.md#e6-consumption", "adjudicator": "juaristi22", "adjudicated_on": "2026-08-24" @@ -60,25 +70,35 @@ "class": "mechanism_change", "scope": { "surface": "nonzero_shares", - "columns": ["diesel_spending", "petrol_spending"], - "entities": ["household"] + "columns": [ + "diesel_spending", + "petrol_spending" + ], + "entities": [ + "household" + ] }, "expectation": "column_differs", - "magnitude_evidence": "These two are signed with the evidence pointing the other way on incidence, and that is the point of scoping them apart from the rest of the LCFS class. The spine gates fuel spending on a has_fuel draw, so it places incidence on fewer households than either the donor or the incumbent: petrol donor 0.3911 against incumbent 0.4442 and ours 0.3002, diesel donor 0.2040 against 0.1903 and 0.1580. The incumbent is closer on both shares. On level the ordering reverses decisively — population mean per household is 0.85x the donor for petrol and 0.81x for diesel against the incumbent's 2.06x and 2.29x — so the gate is under-placing incidence while the incumbent is over-stating amounts by roughly a factor of two. Signed as the accepted cost of the fuel gate, not as a claim that the spine is closer here; the incidence rate of the gate is the pre-registered lever if this surface needs to move.", + "magnitude_evidence": "These two are signed with the evidence pointing the other way on incidence, and that is the point of scoping them apart from the rest of the LCFS class. The spine gates fuel spending on a has_fuel draw, so it places incidence on fewer households than either the donor or the incumbent: petrol donor 0.3911 against incumbent 0.4446 and ours 0.3002, diesel donor 0.2040 against 0.1910 and 0.1580. The incumbent is closer on both shares. On level the ordering reverses decisively - population mean per household is 0.85x the donor for petrol and 0.81x for diesel against the incumbent's 2.05x and 2.31x - so the gate is under-placing incidence while the incumbent is over-stating amounts by roughly a factor of two. Signed as the accepted cost of the fuel gate, not as a claim that the spine is closer here; the incidence rate of the gate is the pre-registered lever if this surface needs to move.", "evidence": "experiments/686-uk-spine-comparison-ledger.md#e6-consumption", "adjudicator": "juaristi22", "adjudicated_on": "2026-08-24" }, { - "id": "lcfs-transport-aggregate-incidence", + "id": "lcfs-aggregate-incidence-incumbent-closer", "class": "mechanism_change", "scope": { "surface": "nonzero_shares", - "columns": ["transport_consumption"], - "entities": ["household"] + "columns": [ + "alcohol_and_tobacco_consumption", + "transport_consumption" + ], + "entities": [ + "household" + ] }, "expectation": "column_differs", - "magnitude_evidence": "Scoped alone because the incumbent is marginally closer on this share and the aggregate sits above the fuel columns that the has_fuel gate moves. Donor 0.8702 against incumbent 0.8623 and ours 0.8934: the spine overshoots the donor by 0.0232 where the incumbent undershoots by 0.0079. On level the spine is closer, at 1.37x the donor population mean per household against the incumbent's 1.63x. Signed as an accepted incidence cost of the regime-gated draw, with the direction recorded so it can be re-examined on its own rather than under a class verdict it does not share.", + "magnitude_evidence": "Scoped apart from the rest of the LCFS class because on these two the incumbent is closer on the share and the spine is closer on the level, so a class verdict would misstate both. transport_consumption: donor 0.8702 against incumbent 0.8668 and ours 0.8934, so the spine overshoots by 0.0232 where the incumbent undershoots by 0.0034; on level the spine is at 1.37x the donor population mean per household against the incumbent's 1.62x. alcohol_and_tobacco_consumption: donor 0.5383 against incumbent 0.5603 and ours 0.5150, so the incumbent is off by 0.0220 and the spine by 0.0233 - close enough that it read as a tie against the previous 1.56.14 pin and resolves to the incumbent against the pinned 1.56.16 artifact; on level the two are within a point of each other, 1.24x for the spine against 1.26x. Signed as accepted incidence costs of the regime-gated draw, with the direction recorded so each can be re-examined on its own rather than under a class verdict it does not share.", "evidence": "experiments/686-uk-spine-comparison-ledger.md#e6-consumption", "adjudicator": "juaristi22", "adjudicated_on": "2026-08-24" @@ -88,11 +108,16 @@ "class": "defect_fix", "scope": { "surface": "nonzero_shares", - "columns": ["bus_subsidy_spending", "dfe_education_spending"], - "entities": ["household"] + "columns": [ + "bus_subsidy_spending", + "dfe_education_spending" + ], + "entities": [ + "household" + ] }, "expectation": "column_differs", - "magnitude_evidence": "The incumbent's state-education column is degenerate and the spine's is not, which makes this a defect fix on the incumbent side rather than a method preference. Measured on the ETB services stage's own cleaned donor frame — SN 8856, year 2023, complete cases on the thirteen-column services subset, 4,199 rows, weighted by hhold_adj_weight — dfe_education_spending has a donor share of 0.2794 weighted (0.2546 unweighted, which reproduces the E6 acceptance receipt's figure exactly) and a donor population mean of GBP 3,461 per household. The incumbent carries 14 nonzero households out of 52,846, a share of 0.000265 and a population mean of GBP 2 per household; the spine carries 11,934, a share of 0.2258 and GBP 3,111, or 0.90x the donor. bus_subsidy_spending moves the same way: donor share 0.5255 weighted and GBP 87 per household, against incumbent 0.3167 and GBP 114 (1.30x) and ours 0.5554 and GBP 89 (1.02x). The spine is closer on both the share and the level of both columns. This resolves the ETB weight-basis question that was previously recorded as blocking these rows: the stage's convention is the household grossing weight, and the verdict holds on either basis.", + "magnitude_evidence": "The incumbent's state-education column is degenerate and the spine's is not, which makes this a defect fix on the incumbent side rather than a method preference. Measured on the ETB services stage's own cleaned donor frame \u2014 SN 8856, year 2023, complete cases on the thirteen-column services subset, 4,199 rows, weighted by hhold_adj_weight \u2014 dfe_education_spending has a donor share of 0.2794 weighted (0.2546 unweighted, which reproduces the E6 acceptance receipt's figure exactly) and a donor population mean of GBP 3,461 per household. The incumbent carries 14 nonzero households out of 52,846, a share of 0.000265 and a population mean of GBP 2 per household; the spine carries 11,934, a share of 0.2258 and GBP 3,111, or 0.90x the donor. bus_subsidy_spending moves the same way: donor share 0.5255 weighted and GBP 87 per household, against incumbent 0.3167 and GBP 113 (1.30x) and ours 0.5554 and GBP 89 (1.02x). The spine is closer on both the share and the level of both columns. This resolves the ETB weight-basis question that was previously recorded as blocking these rows: the stage's convention is the household grossing weight, and the verdict holds on either basis.", "evidence": "experiments/686-uk-spine-comparison-ledger.md#e6-consumption", "adjudicator": "juaristi22", "adjudicated_on": "2026-08-24" @@ -109,10 +134,12 @@ "property_wealth", "savings" ], - "entities": ["household"] + "entities": [ + "household" + ] }, "expectation": "column_differs", - "magnitude_evidence": "Carries forward the E5 adjudication of 2026-08-19 — that the wealth stage is not required to reproduce the incumbent's inflated totals — now scoped to the five household columns that actually diverge beyond the band, and re-measured against WAS Round 8 on the stage's own cleaning of the pinned tab (15,128 rows, weighted by R8xshhwgt). The spine is closer than the incumbent on all five survey-weighted donor shares: savings donor 0.6072 against incumbent 0.6621 and ours 0.6107; other_residential_property_value donor 0.0363 against 0.0750 and 0.0367; property_wealth donor 0.6433 against 0.7082 and 0.6607; main_residence_value donor 0.6236 against 0.6761 and 0.6356; corporate_wealth donor 0.7629 against 0.8225 and 0.7792. The level evidence behind the original adjudication reproduces and is the more dramatic surface: population mean per household runs 5.70x the donor for the incumbent's savings against 1.97x for ours, and 6.18x against 1.26x for other residential property. main_residence_value is the one column where the incumbent's level is closer, at 1.02x against our 0.88x. Note that the unweighted donor shares tell the opposite story on incidence, because WAS oversamples wealth-holders by design; the weighted basis is the population one and is the basis quoted here throughout.", + "magnitude_evidence": "Carries forward the E5 adjudication of 2026-08-19 \u2014 that the wealth stage is not required to reproduce the incumbent's inflated totals \u2014 now scoped to the five household columns that actually diverge beyond the band, and re-measured against WAS Round 8 on the stage's own cleaning of the pinned tab (15,128 rows, weighted by R8xshhwgt). The spine is closer than the incumbent on all five survey-weighted donor shares: savings donor 0.6072 against incumbent 0.6621 and ours 0.6107; other_residential_property_value donor 0.0363 against 0.0763 and 0.0367; property_wealth donor 0.6433 against 0.7082 and 0.6607; main_residence_value donor 0.6236 against 0.6761 and 0.6356; corporate_wealth donor 0.7629 against 0.8225 and 0.7792. The level evidence behind the original adjudication reproduces and is the more dramatic surface: population mean per household runs 5.66x the donor for the incumbent's savings against 1.97x for ours, and 6.12x against 1.26x for other residential property. main_residence_value is the one column where the incumbent's level is closer, at 1.02x against our 0.88x. Note that the unweighted donor shares tell the opposite story on incidence, because WAS oversamples wealth-holders by design; the weighted basis is the population one and is the basis quoted here throughout.", "evidence": "experiments/686-uk-spine-comparison-ledger.md#e5-wealth", "adjudicator": "juaristi22", "adjudicated_on": "2026-08-24" @@ -122,11 +149,15 @@ "class": "qrf_implementation", "scope": { "surface": "nonzero_shares", - "columns": ["student_loan_balance"], - "entities": ["household"] + "columns": [ + "student_loan_balance" + ], + "entities": [ + "household" + ] }, "expectation": "column_differs", - "magnitude_evidence": "Scoped apart from the benchmarked wealth columns because it has no donor benchmark to quote: the column is a fold of two WAS aggregates (total loans less total loans excluding Student Loans Company debt) and lands at a different entity grain from the household columns beside it, so no like-for-like donor share exists on the stage's cleaned frame. The observed divergence is +0.0296 on the unweighted share, incumbent 0.0197 against ours 0.0493 — the spine places student debt on about two and a half times as many carriers. Signed under the standing E5 adjudication as part of the same correlated-rank draw, with the absence of a benchmark stated rather than papered over; if the wealth stage is revisited, this is the column whose direction is unevidenced.", + "magnitude_evidence": "Scoped apart from the benchmarked wealth columns because it has no donor benchmark to quote: the column is a fold of two WAS aggregates (total loans less total loans excluding Student Loans Company debt) and lands at a different entity grain from the household columns beside it, so no like-for-like donor share exists on the stage's cleaned frame. The observed divergence is +0.0296 on the unweighted share, incumbent 0.0197 against ours 0.0493 \u2014 the spine places student debt on about two and a half times as many carriers. Signed under the standing E5 adjudication as part of the same correlated-rank draw, with the absence of a benchmark stated rather than papered over; if the wealth stage is revisited, this is the column whose direction is unevidenced.", "evidence": "experiments/686-uk-spine-comparison-ledger.md#e5-wealth", "adjudicator": "juaristi22", "adjudicated_on": "2026-08-24" @@ -141,7 +172,9 @@ "savings_interest_income", "tax_free_savings_income" ], - "entities": ["person"] + "entities": [ + "person" + ] }, "expectation": "column_differs", "magnitude_evidence": "The three columns rewritten on the SPI channel, each measured against its own source of truth rather than against one another, which is what closes the open item #717 left. savings_interest_income against the SPI donor INCBBS, FACT-weighted: truth 0.3960, incumbent 0.4250, ours 0.3946. tax_free_savings_income against the raw FRS at the frs_spine stage: truth 0.1540, incumbent 0.1897, ours 0.1352. employer_pension_contributions against the 3x derive at frs_hmrc_spine_leaves: truth 0.2587, incumbent 0.3149, ours 0.2682. The spine is closer on all three, so the divergence #717 recorded as uniformly one-way and unexplained is uniformly toward the source. Attribution is to the last stage that rewrites, not the stage that produces: the first two originate in frs_spine and are rewritten by hmrc_spi_income_spine, and attributing them to their producer would report them as raw-mapping defects, which is the one signature that indicates a genuine port defect.", @@ -154,8 +187,12 @@ "class": "mechanism_change", "scope": { "surface": "nonzero_shares", - "columns": ["employee_pension_contributions"], - "entities": ["person"] + "columns": [ + "employee_pension_contributions" + ], + "entities": [ + "person" + ] }, "expectation": "column_differs", "magnitude_evidence": "Signed at #684 and transcribed here against the re-pinned reference. The spine converts salary-sacrificed pension contributions at the depth the mechanism specifies, where the incumbent's conversion step was inert, so contributions that should have moved out of the employee column stayed in it. Unweighted share 0.2735 for the incumbent against 0.2246 for the spine, a difference of -0.0488 on the person entity. The counterpart column pension_contributions_via_salary_sacrifice sits inside the acceptance band at -0.0035 and is therefore not signed.", @@ -168,8 +205,14 @@ "class": "rng_stream", "scope": { "surface": "entity_counts", - "columns": ["benunit", "person"], - "entities": ["benunit", "person"] + "columns": [ + "benunit", + "person" + ], + "entities": [ + "benunit", + "person" + ] }, "expectation": "count_differs", "magnitude_evidence": "The CGT band-donor selection draws over id-sorted candidate households, so the 270 donors it picks are not the 270 the incumbent picked, and the two sets carry different numbers of people and benefit units. Persons 113,617 in the reference against 113,649 in the spine, a difference of +32; benefit units 61,223 against 61,211, a difference of -12. Households are 52,846 on both sides and match exactly, which is what proves this is a selection difference rather than a miscount: the record-count identity (16,288 raw FRS plus 10,000 SPI) times two for the capital-gains clone, plus 270 band donors, closes on the nose. This entry is deliberately scoped to the two entities that differ rather than written surface-wide, so that any future divergence in the household count is still a defect.", @@ -182,8 +225,12 @@ "class": "net_new_column", "scope": { "surface": "nonzero_shares", - "columns": ["num_bedrooms"], - "entities": ["household"] + "columns": [ + "num_bedrooms" + ], + "entities": [ + "household" + ] }, "expectation": "column_missing_in_reference", "magnitude_evidence": "The spine populates num_bedrooms at the frs_spine stage from the raw household tape; the pinned incumbent does not populate it at all, so the column is present in the candidate and absent from the reference. This is coverage the spine adds rather than a divergence in a shared column, and it cannot be measured as a share difference. Its predictor quality is a separate question tracked on #145, not a parity matter.", @@ -196,8 +243,12 @@ "class": "net_new_column", "scope": { "surface": "nonzero_shares", - "columns": ["other_investment_income"], - "entities": ["person"] + "columns": [ + "other_investment_income" + ], + "entities": [ + "person" + ] }, "expectation": "column_missing_in_reference", "magnitude_evidence": "The spine populates other_investment_income at the hmrc_spi_income_spine stage. The column is declared by the incumbent's own national restoration but is not populated in the pinned artifact, so the spine is ahead of the reference here rather than diverging from it. Present in the candidate, absent from the reference, and not measurable as a share difference.", From f1d013fb538ddf4f45d2d882b02b9ddad089ba9e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Mon, 24 Aug 2026 11:48:09 +0200 Subject: [PATCH 23/28] Record the wrong-artifact correction as R5 correction 3 A cross-check of every register-quoted number against its measured source found the incumbent side had been measured on the 1.56.14 artifact staged during #723 rather than the pinned 1.56.16 one. The instrument itself was never wrong, since it reads the committed reference; the hand measurements beside it were. Co-Authored-By: Claude Opus 5 --- experiments/686-uk-spine-swap-receipts.md | 33 +++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/experiments/686-uk-spine-swap-receipts.md b/experiments/686-uk-spine-swap-receipts.md index cc1cd764..b7b54ea7 100644 --- a/experiments/686-uk-spine-swap-receipts.md +++ b/experiments/686-uk-spine-swap-receipts.md @@ -661,3 +661,36 @@ all 131 columns, which is the same before-medicine/after-medicine error the UC check documents. The water-level entry is therefore reported as **dormant**, and a test pins that dormancy stops being available the moment a run supplies the sidecars. + +### Correction 3 — the incumbent side was measured on the wrong artifact + +Caught by a cross-check that compared every number quoted in a register entry +against the measured source. The only full incumbent H5 on disk is +`populace-723/.codex-work/licensed/enhanced_frs_2024_25.h5`, 126,579,434 bytes, +sha256 `97a07f9c…` — that is the **1.56.14** artifact staged during #723's +phase-0, not the re-pinned **1.56.16** one (126,553,300 bytes, sha256 +`e433e532…`). Every incumbent share and level in the first pass came from it. + +The pinned artifact was fetched from the private model repo at revision +`a9e52499` and digest-verified before re-measuring. Across the nineteen E5/E6 +columns the two vintages differ by at most **0.0045**, on +`transport_consumption` — consistent with R0's "no reference share moved by +more than 0.0046". + +Small, but not harmless: it flipped one verdict. +`alcohol_and_tobacco_consumption` reads donor 0.5383 · incumbent 0.5630 · ours +0.5150 at 1.56.14, where ours is nearer, and donor 0.5383 · incumbent 0.5603 · +ours 0.5150 at the pin, where the incumbent is nearer by 0.0013. The column +moved out of `lcfs-consumption-regime-gated-incidence` and into a two-column +entry with `transport_consumption`, which shares its evidence shape exactly — +incumbent closer on share, spine closer on level. + +Level ratios moved in the third digit throughout (education 5.00× → 4.80×, +savings 5.70× → 5.66×, owned_land 9.28× → 9.19×); no other verdict changed. + +**Standing lesson.** The parity instrument could not have caught this: it reads +the *committed reference*, which was always at the correct pin, so its verdict +was right the whole time. What was wrong was the side evidence measured by hand +outside the instrument. **A file on disk with the right name is not the pinned +artifact — check the digest before measuring against it**, exactly as the +instrument does before comparing against it. From 17513773b8bc1e73fa6b6be3e9c1d4c4c9dd113f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Mon, 24 Aug 2026 11:49:44 +0200 Subject: [PATCH 24/28] Correct three wealth shares still quoted at the 1.56.14 vintage The earlier pin correction updated the level ratios and the other-residential share but left savings, property_wealth and corporate_wealth quoting the 1.56.14 incumbent. Caught by re-running the cross-check that compares every register-quoted number against its source. Co-Authored-By: Claude Opus 5 --- .../src/microcosm/build/uk/spine_swap_signed_differences.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/microcosm-build/src/microcosm/build/uk/spine_swap_signed_differences.json b/packages/microcosm-build/src/microcosm/build/uk/spine_swap_signed_differences.json index 4d30b0f3..35da2151 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/spine_swap_signed_differences.json +++ b/packages/microcosm-build/src/microcosm/build/uk/spine_swap_signed_differences.json @@ -139,7 +139,7 @@ ] }, "expectation": "column_differs", - "magnitude_evidence": "Carries forward the E5 adjudication of 2026-08-19 \u2014 that the wealth stage is not required to reproduce the incumbent's inflated totals \u2014 now scoped to the five household columns that actually diverge beyond the band, and re-measured against WAS Round 8 on the stage's own cleaning of the pinned tab (15,128 rows, weighted by R8xshhwgt). The spine is closer than the incumbent on all five survey-weighted donor shares: savings donor 0.6072 against incumbent 0.6621 and ours 0.6107; other_residential_property_value donor 0.0363 against 0.0763 and 0.0367; property_wealth donor 0.6433 against 0.7082 and 0.6607; main_residence_value donor 0.6236 against 0.6761 and 0.6356; corporate_wealth donor 0.7629 against 0.8225 and 0.7792. The level evidence behind the original adjudication reproduces and is the more dramatic surface: population mean per household runs 5.66x the donor for the incumbent's savings against 1.97x for ours, and 6.12x against 1.26x for other residential property. main_residence_value is the one column where the incumbent's level is closer, at 1.02x against our 0.88x. Note that the unweighted donor shares tell the opposite story on incidence, because WAS oversamples wealth-holders by design; the weighted basis is the population one and is the basis quoted here throughout.", + "magnitude_evidence": "Carries forward the E5 adjudication of 2026-08-19 \u2014 that the wealth stage is not required to reproduce the incumbent's inflated totals \u2014 now scoped to the five household columns that actually diverge beyond the band, and re-measured against WAS Round 8 on the stage's own cleaning of the pinned tab (15,128 rows, weighted by R8xshhwgt). The spine is closer than the incumbent on all five survey-weighted donor shares: savings donor 0.6072 against incumbent 0.6620 and ours 0.6107; other_residential_property_value donor 0.0363 against 0.0763 and 0.0367; property_wealth donor 0.6433 against 0.7081 and 0.6607; main_residence_value donor 0.6236 against 0.6761 and 0.6356; corporate_wealth donor 0.7629 against 0.8222 and 0.7792. The level evidence behind the original adjudication reproduces and is the more dramatic surface: population mean per household runs 5.66x the donor for the incumbent's savings against 1.97x for ours, and 6.12x against 1.26x for other residential property. main_residence_value is the one column where the incumbent's level is closer, at 1.02x against our 0.88x. Note that the unweighted donor shares tell the opposite story on incidence, because WAS oversamples wealth-holders by design; the weighted basis is the population one and is the basis quoted here throughout.", "evidence": "experiments/686-uk-spine-comparison-ledger.md#e5-wealth", "adjudicator": "juaristi22", "adjudicated_on": "2026-08-24" From effedbdd3436f5ff6451504743676b6f1ebee063 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Mon, 24 Aug 2026 11:52:45 +0200 Subject: [PATCH 25/28] Repoint the register's evidence anchors and test that they resolve MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Eleven of the thirteen evidence fragments did not match any heading in the file they cite — two of them since before this branch, the rest because the ledger's section titles changed when the queue was signed. A dead fragment fails silently in a browser, so the rot only surfaces when a reviewer clicks and lands nowhere, which is exactly when the pointer needed to work. The new test derives GitHub's own anchor form from each cited file's headings and requires the fragment to be among them, so renaming a section now breaks CI instead of breaking an audit trail. Co-Authored-By: Claude Opus 5 --- .../uk/spine_swap_signed_differences.json | 22 +++++------ .../tests/test_uk_signed_differences.py | 37 +++++++++++++++++++ 2 files changed, 48 insertions(+), 11 deletions(-) diff --git a/packages/microcosm-build/src/microcosm/build/uk/spine_swap_signed_differences.json b/packages/microcosm-build/src/microcosm/build/uk/spine_swap_signed_differences.json index 35da2151..8a424046 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/spine_swap_signed_differences.json +++ b/packages/microcosm-build/src/microcosm/build/uk/spine_swap_signed_differences.json @@ -16,7 +16,7 @@ }, "expectation": "column_differs", "magnitude_evidence": "The spine's unweighted nonzero share exceeds the incumbent's by about +0.10 on the whole file. FRS 2024-25 retired CWATAMT/CSEWAMT: the headers survive but carry no data in any of the 16,288 households. The incumbent adds the retired CSEWAMT before filling, so NaN propagates and the charge is zeroed for every Scottish household that has one; the spine fills per column and they stand. Reproduced on the raw tab as 12,644 nonzero households for the incumbent formula against 14,307 for ours, a +0.1021 share gap whose 1,663 differing households are all Scottish, with England, Wales and Northern Ireland identical under both. Latent since at least 2023-24, where 378 Scottish households were already affected. The defect is on the incumbent side and survives at 1.56.16.", - "evidence": "experiments/686-uk-spine-swap-receipts.md#r1-scottish-water-and-sewerage-charges-736-item-13", + "evidence": "experiments/686-uk-spine-swap-receipts.md#r1--scottish-water-and-sewerage-charges-736-item-13", "adjudicator": "juaristi22", "adjudicated_on": "2026-08-22" }, @@ -35,7 +35,7 @@ }, "expectation": "column_differs", "magnitude_evidence": "The Scottish charge is assembled from the successors FRS 2024-25 published for the cells it retired, so its level rises against both the incumbent and our own earlier build. CWATAMTD is the water charge alone; CSEWAMT1 supplies the sewerage side and is discounted at the household's own observed CWATAMTD/CWATAMT1 factor, which keeps the retired cells' after-discount meaning rather than switching to a gross basis. Weighted annual per Scottish household moves from about GBP 185 on water alone to about GBP 395, against roughly GBP 490 for England and Wales on WATSEWRT; the incumbent sits at zero because of the separate NaN defect. The same amount is netted from council_tax, so that column moves by the same construction. The nonzero share is unaffected, so this entry deliberately does not sign the share surface.", - "evidence": "experiments/686-uk-spine-swap-receipts.md#r1-scottish-water-and-sewerage-charges-736-item-13", + "evidence": "experiments/686-uk-spine-swap-receipts.md#r1--scottish-water-and-sewerage-charges-736-item-13", "adjudicator": "juaristi22", "adjudicated_on": "2026-08-22" }, @@ -61,7 +61,7 @@ }, "expectation": "column_differs", "magnitude_evidence": "The spine draws LCFS consumption through a regime-gated QRF that carries the donor's zero mass as a modelled incidence, where the incumbent's plain QRF regresses a zero-inflated target toward its conditional mean. On these nine columns the spine's unweighted share is closer to the LCFS 2023-24 survey-weighted donor share than the incumbent's is, measured on the stage's own cleaned donor frame of 4,202 households against the pinned 1.56.16 artifact (sha256 e433e532): education_consumption donor 0.0476 against incumbent 0.1256 and ours 0.0170; restaurants_and_hotels donor 0.7651 against 0.6324 and 0.7903; miscellaneous donor 0.9728 against 0.9003 and 0.9918; domestic_energy donor 0.9835 against 0.9549 and 0.9953, with its electricity and gas components moving the same way. Population mean per household is closer for the spine on eight of the nine, the exception being communication_consumption at 1.20x the donor against the incumbent's 1.15x; the incumbent runs 1.7x to 4.8x the donor mean on the rest. Every cell is above the disclosure floor, the thinnest being 193 donor carriers on education_consumption.", - "evidence": "experiments/686-uk-spine-comparison-ledger.md#e6-consumption", + "evidence": "experiments/686-uk-spine-comparison-ledger.md#e6--consumption--signed", "adjudicator": "juaristi22", "adjudicated_on": "2026-08-24" }, @@ -80,7 +80,7 @@ }, "expectation": "column_differs", "magnitude_evidence": "These two are signed with the evidence pointing the other way on incidence, and that is the point of scoping them apart from the rest of the LCFS class. The spine gates fuel spending on a has_fuel draw, so it places incidence on fewer households than either the donor or the incumbent: petrol donor 0.3911 against incumbent 0.4446 and ours 0.3002, diesel donor 0.2040 against 0.1910 and 0.1580. The incumbent is closer on both shares. On level the ordering reverses decisively - population mean per household is 0.85x the donor for petrol and 0.81x for diesel against the incumbent's 2.05x and 2.31x - so the gate is under-placing incidence while the incumbent is over-stating amounts by roughly a factor of two. Signed as the accepted cost of the fuel gate, not as a claim that the spine is closer here; the incidence rate of the gate is the pre-registered lever if this surface needs to move.", - "evidence": "experiments/686-uk-spine-comparison-ledger.md#e6-consumption", + "evidence": "experiments/686-uk-spine-comparison-ledger.md#e6--consumption--signed", "adjudicator": "juaristi22", "adjudicated_on": "2026-08-24" }, @@ -99,7 +99,7 @@ }, "expectation": "column_differs", "magnitude_evidence": "Scoped apart from the rest of the LCFS class because on these two the incumbent is closer on the share and the spine is closer on the level, so a class verdict would misstate both. transport_consumption: donor 0.8702 against incumbent 0.8668 and ours 0.8934, so the spine overshoots by 0.0232 where the incumbent undershoots by 0.0034; on level the spine is at 1.37x the donor population mean per household against the incumbent's 1.62x. alcohol_and_tobacco_consumption: donor 0.5383 against incumbent 0.5603 and ours 0.5150, so the incumbent is off by 0.0220 and the spine by 0.0233 - close enough that it read as a tie against the previous 1.56.14 pin and resolves to the incumbent against the pinned 1.56.16 artifact; on level the two are within a point of each other, 1.24x for the spine against 1.26x. Signed as accepted incidence costs of the regime-gated draw, with the direction recorded so each can be re-examined on its own rather than under a class verdict it does not share.", - "evidence": "experiments/686-uk-spine-comparison-ledger.md#e6-consumption", + "evidence": "experiments/686-uk-spine-comparison-ledger.md#e6--consumption--signed", "adjudicator": "juaristi22", "adjudicated_on": "2026-08-24" }, @@ -118,7 +118,7 @@ }, "expectation": "column_differs", "magnitude_evidence": "The incumbent's state-education column is degenerate and the spine's is not, which makes this a defect fix on the incumbent side rather than a method preference. Measured on the ETB services stage's own cleaned donor frame \u2014 SN 8856, year 2023, complete cases on the thirteen-column services subset, 4,199 rows, weighted by hhold_adj_weight \u2014 dfe_education_spending has a donor share of 0.2794 weighted (0.2546 unweighted, which reproduces the E6 acceptance receipt's figure exactly) and a donor population mean of GBP 3,461 per household. The incumbent carries 14 nonzero households out of 52,846, a share of 0.000265 and a population mean of GBP 2 per household; the spine carries 11,934, a share of 0.2258 and GBP 3,111, or 0.90x the donor. bus_subsidy_spending moves the same way: donor share 0.5255 weighted and GBP 87 per household, against incumbent 0.3167 and GBP 113 (1.30x) and ours 0.5554 and GBP 89 (1.02x). The spine is closer on both the share and the level of both columns. This resolves the ETB weight-basis question that was previously recorded as blocking these rows: the stage's convention is the household grossing weight, and the verdict holds on either basis.", - "evidence": "experiments/686-uk-spine-comparison-ledger.md#e6-consumption", + "evidence": "experiments/686-uk-spine-comparison-ledger.md#etb--the-weight-basis-question-is-closed", "adjudicator": "juaristi22", "adjudicated_on": "2026-08-24" }, @@ -140,7 +140,7 @@ }, "expectation": "column_differs", "magnitude_evidence": "Carries forward the E5 adjudication of 2026-08-19 \u2014 that the wealth stage is not required to reproduce the incumbent's inflated totals \u2014 now scoped to the five household columns that actually diverge beyond the band, and re-measured against WAS Round 8 on the stage's own cleaning of the pinned tab (15,128 rows, weighted by R8xshhwgt). The spine is closer than the incumbent on all five survey-weighted donor shares: savings donor 0.6072 against incumbent 0.6620 and ours 0.6107; other_residential_property_value donor 0.0363 against 0.0763 and 0.0367; property_wealth donor 0.6433 against 0.7081 and 0.6607; main_residence_value donor 0.6236 against 0.6761 and 0.6356; corporate_wealth donor 0.7629 against 0.8222 and 0.7792. The level evidence behind the original adjudication reproduces and is the more dramatic surface: population mean per household runs 5.66x the donor for the incumbent's savings against 1.97x for ours, and 6.12x against 1.26x for other residential property. main_residence_value is the one column where the incumbent's level is closer, at 1.02x against our 0.88x. Note that the unweighted donor shares tell the opposite story on incidence, because WAS oversamples wealth-holders by design; the weighted basis is the population one and is the basis quoted here throughout.", - "evidence": "experiments/686-uk-spine-comparison-ledger.md#e5-wealth", + "evidence": "experiments/686-uk-spine-comparison-ledger.md#e5--wealth--signed-carried-forward", "adjudicator": "juaristi22", "adjudicated_on": "2026-08-24" }, @@ -158,7 +158,7 @@ }, "expectation": "column_differs", "magnitude_evidence": "Scoped apart from the benchmarked wealth columns because it has no donor benchmark to quote: the column is a fold of two WAS aggregates (total loans less total loans excluding Student Loans Company debt) and lands at a different entity grain from the household columns beside it, so no like-for-like donor share exists on the stage's cleaned frame. The observed divergence is +0.0296 on the unweighted share, incumbent 0.0197 against ours 0.0493 \u2014 the spine places student debt on about two and a half times as many carriers. Signed under the standing E5 adjudication as part of the same correlated-rank draw, with the absence of a benchmark stated rather than papered over; if the wealth stage is revisited, this is the column whose direction is unevidenced.", - "evidence": "experiments/686-uk-spine-comparison-ledger.md#e5-wealth", + "evidence": "experiments/686-uk-spine-comparison-ledger.md#e5--wealth--signed-carried-forward", "adjudicator": "juaristi22", "adjudicated_on": "2026-08-24" }, @@ -178,7 +178,7 @@ }, "expectation": "column_differs", "magnitude_evidence": "The three columns rewritten on the SPI channel, each measured against its own source of truth rather than against one another, which is what closes the open item #717 left. savings_interest_income against the SPI donor INCBBS, FACT-weighted: truth 0.3960, incumbent 0.4250, ours 0.3946. tax_free_savings_income against the raw FRS at the frs_spine stage: truth 0.1540, incumbent 0.1897, ours 0.1352. employer_pension_contributions against the 3x derive at frs_hmrc_spine_leaves: truth 0.2587, incumbent 0.3149, ours 0.2682. The spine is closer on all three, so the divergence #717 recorded as uniformly one-way and unexplained is uniformly toward the source. Attribution is to the last stage that rewrites, not the stage that produces: the first two originate in frs_spine and are rewritten by hmrc_spi_income_spine, and attributing them to their producer would report them as raw-mapping defects, which is the one signature that indicates a genuine port defect.", - "evidence": "experiments/686-uk-spine-comparison-ledger.md#e7-spi-channel", + "evidence": "experiments/686-uk-spine-comparison-ledger.md#e7--spi-channel--evidence-gap-closed-favours-the-spine", "adjudicator": "juaristi22", "adjudicated_on": "2026-08-24" }, @@ -196,7 +196,7 @@ }, "expectation": "column_differs", "magnitude_evidence": "Signed at #684 and transcribed here against the re-pinned reference. The spine converts salary-sacrificed pension contributions at the depth the mechanism specifies, where the incumbent's conversion step was inert, so contributions that should have moved out of the employee column stayed in it. Unweighted share 0.2735 for the incumbent against 0.2246 for the spine, a difference of -0.0488 on the person entity. The counterpart column pension_contributions_via_salary_sacrifice sits inside the acceptance band at -0.0035 and is therefore not signed.", - "evidence": "experiments/686-uk-spine-comparison-ledger.md#e8-and-entity-counts", + "evidence": "experiments/686-uk-spine-comparison-ledger.md#e8-and-entity-counts--signed-at-684", "adjudicator": "juaristi22", "adjudicated_on": "2026-08-24" }, @@ -216,7 +216,7 @@ }, "expectation": "count_differs", "magnitude_evidence": "The CGT band-donor selection draws over id-sorted candidate households, so the 270 donors it picks are not the 270 the incumbent picked, and the two sets carry different numbers of people and benefit units. Persons 113,617 in the reference against 113,649 in the spine, a difference of +32; benefit units 61,223 against 61,211, a difference of -12. Households are 52,846 on both sides and match exactly, which is what proves this is a selection difference rather than a miscount: the record-count identity (16,288 raw FRS plus 10,000 SPI) times two for the capital-gains clone, plus 270 band donors, closes on the nose. This entry is deliberately scoped to the two entities that differ rather than written surface-wide, so that any future divergence in the household count is still a defect.", - "evidence": "experiments/686-uk-spine-swap-receipts.md#r3-the-rebuilt-spine", + "evidence": "experiments/686-uk-spine-swap-receipts.md#r3--the-spine-rebuilt-and-the-l2-adjudication-queue", "adjudicator": "juaristi22", "adjudicated_on": "2026-08-24" }, diff --git a/packages/microcosm-build/tests/test_uk_signed_differences.py b/packages/microcosm-build/tests/test_uk_signed_differences.py index af5cf598..d3a5df00 100644 --- a/packages/microcosm-build/tests/test_uk_signed_differences.py +++ b/packages/microcosm-build/tests/test_uk_signed_differences.py @@ -8,6 +8,7 @@ from __future__ import annotations import json +import re from importlib.resources import files from pathlib import Path @@ -50,6 +51,24 @@ def _write(tmp_path: Path, payload: object) -> str: return str(path) +def _github_anchors(path: Path) -> set[str]: + """The fragment ids GitHub derives from a Markdown file's headings. + + Lower-case, punctuation dropped, whitespace to hyphens — so an em-dash + surrounded by spaces yields a doubled hyphen, which is the detail that + makes hand-written anchors get this wrong. + """ + + anchors: set[str] = set() + for line in path.read_text(encoding="utf-8").splitlines(): + matched = re.match(r"^#{1,6}\s+(.*?)\s*$", line) + if matched is None: + continue + text = re.sub(r"[^\w\s-]", "", matched.group(1).lower()) + anchors.add(re.sub(r"\s", "-", text)) + return anchors + + class TestCommittedRegister: def test_committed_register_loads(self) -> None: register = load_uk_spine_swap_signed_differences() @@ -77,6 +96,24 @@ def test_committed_evidence_anchors_point_at_a_real_file(self) -> None: f"{difference.id} cites missing evidence file {relative}" ) + def test_committed_evidence_anchors_resolve_to_a_real_heading(self) -> None: + # The evidence pointer is what makes an adjudication auditable. A + # fragment that no longer resolves fails silently in a browser, so the + # rot only shows up when a reviewer clicks and lands nowhere. + register = load_uk_spine_swap_signed_differences() + headings: dict[str, set[str]] = {} + for difference in register.differences: + relative, _, fragment = difference.evidence.partition("#") + if not fragment: + continue + if relative not in headings: + headings[relative] = _github_anchors(REPO_ROOT / relative) + assert fragment in headings[relative], ( + f"{difference.id} cites {relative}#{fragment}, which is not a " + f"heading in that file. Available: " + f"{sorted(headings[relative])}" + ) + def test_every_signed_column_exists_on_the_surface_it_signs(self) -> None: # A typo in a column name is the quiet failure mode here: the entry # matches nothing, the real divergence stays unsigned, and the only From 1664cca03307f2b6a2b47d27f9ee506ba0b0d6f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Mon, 24 Aug 2026 13:13:01 +0200 Subject: [PATCH 26/28] Fix the spine defects and the proofs that certify it, before the swap MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two lanes of fixes that belong together: the review found the spine's construction held up and its proof machinery did not, and the first armed calibration campaign found two spine defects it had to work around outside the build. Both are cheaper to fix now than after a swap. Spine defects. The SPI stage left twelve full-concept income columns NaN on the FRS channel, which made the artifact unloadable in practice — the engine refuses NaN inputs and any finiteness fence on the calibration path refuses the frame — so the campaign zero-filled them outside the build. Zero is the adjudicated stage-time semantics for a concept the instrument never asked about. Separately, the licensed FRS records no age above 80, so the 85+ population targets were structurally unbindable and the 80-84 band carried the whole 80+ population; the new age_tail stage disperses the pile from a sex-specific inverse CDF over committed ONS band populations, keyed on person_source_id so clone twins agree, running last so nothing that conditions on age sees a different input. Proof machinery. Three findings could change what a proof means: the register and the payload comparator spoke different surface vocabularies, so --structure-only could only ever fail; `expectation` was validated and then never consulted, so a column's signed *appearance* also signed its values; and the E7 receipt reported green over an empty recomputation. Four quieter ones: a candidate omitting its source identity passed the aliasing fence vacuously, a zero reference total produced float("inf") that the encoder refused, --strict false-failed on within-band signed columns, and the Scottish water discount fallback rested on an unasserted vintage claim. Whole-spine parity re-verified after the register semantics changed: signed_parity, 0 unsigned, --strict clean. Co-Authored-By: Claude Opus 5 --- .../686-uk-proof-machinery-review.fixed.md | 1 + .../686-uk-spine-defect-batch.fixed.md | 1 + .../src/microcosm/build/source_manifest.py | 1 + .../spec_engine/schema/sources.schema.json | 1029 +++++++++++++---- .../microcosm/build/uk/country_package.json | 5 + .../uk/ons_age_tail_band_populations.json | 17 + .../src/microcosm/build/uk/source_stages.json | 52 + .../src/microcosm/build/uk/spec/sources.yaml | 36 + .../microcosm/build/uk_runtime/__init__.py | 16 + .../microcosm/build/uk_runtime/age_tail.py | 306 +++++ .../microcosm/build/uk_runtime/frs_spine.py | 28 +- .../build/uk_runtime/signed_differences.py | 48 +- .../microcosm/build/uk_runtime/spi_income.py | 24 +- .../tests/test_country_spec.py | 8 +- .../tests/test_spec_engine_country_bundles.py | 2 +- .../microcosm-build/tests/test_uk_age_tail.py | 280 +++++ .../tests/test_uk_frs_spine.py | 11 + .../tests/test_uk_h5_payload_compare.py | 106 +- .../test_uk_identity_stability_receipts.py | 127 ++ .../tests/test_uk_signed_differences.py | 128 +- .../tests/test_uk_spi_income.py | 55 +- .../tests/test_uk_spi_spine.py | 5 +- .../tests/test_uk_spine_parity_instrument.py | 145 +++ tools/build_uk_frs_spine.py | 15 +- tools/compare_uk_h5_payload.py | 15 +- tools/verify_uk_identity_stability.py | 51 +- tools/verify_uk_spine_parity.py | 56 +- 27 files changed, 2243 insertions(+), 325 deletions(-) create mode 100644 changelog.d/686-uk-proof-machinery-review.fixed.md create mode 100644 changelog.d/686-uk-spine-defect-batch.fixed.md create mode 100644 packages/microcosm-build/src/microcosm/build/uk/ons_age_tail_band_populations.json create mode 100644 packages/microcosm-build/src/microcosm/build/uk_runtime/age_tail.py create mode 100644 packages/microcosm-build/tests/test_uk_age_tail.py create mode 100644 packages/microcosm-build/tests/test_uk_identity_stability_receipts.py diff --git a/changelog.d/686-uk-proof-machinery-review.fixed.md b/changelog.d/686-uk-proof-machinery-review.fixed.md new file mode 100644 index 00000000..c17e9c9d --- /dev/null +++ b/changelog.d/686-uk-proof-machinery-review.fixed.md @@ -0,0 +1 @@ +Fix seven defects in the machinery that certifies the UK spine (#686 review). Three could change what a proof means. The signed-differences register and the payload comparator spoke different surface vocabularies — every committed entry is scoped to `nonzero_shares`, `weighted_totals` or `entity_counts`, while `--structure-only` looked entries up as `payload_column`, so the swap-acceptance verdict could only ever fail while reporting all thirteen entries as unused; lookup now bridges the value-bearing surfaces to the payload surface, because both instruments read the same adjudicated fact through different measurements. `expectation` was validated at load and then never consulted, so an entry signing a column's *appearance* also silently signed an arbitrarily large *value* divergence in it — the failure mode the register's own scope note names; every lookup now matches on expectation, and structural expectations never excuse a value difference. The E7 identity receipt returned an empty recomputation when the support-channel layer was absent, so its mismatch loops never ran and it reported green on an artifact where nothing had been checked — it now refuses, names the columns it compared, and treats a certified column missing from the store as a mismatch rather than a narrower comparison. Four more were quieter: the anti-self-comparison fence passed vacuously for a candidate extraction that omitted its source identity, which is now required; a zero reference total produced `float("inf")`, which `allow_nan=False` then refused, turning a real divergence into "no verdict possible", and is now reported as an explicit flag; `--strict` counted an entry matching a within-band column as unused and false-failed on it; and the Scottish water helper's discount fallback rested on an unasserted claim about the vintage's domain, which now raises at build time rather than silently paying sewerage at gross into `council_tax`. diff --git a/changelog.d/686-uk-spine-defect-batch.fixed.md b/changelog.d/686-uk-spine-defect-batch.fixed.md new file mode 100644 index 00000000..aaf86a4d --- /dev/null +++ b/changelog.d/686-uk-spine-defect-batch.fixed.md @@ -0,0 +1 @@ +Fix two spine defects the first armed calibration campaign surfaced, before the swap rather than after it (#686). **The SPI channel no longer ships NaN.** Twelve person columns are full income concepts the FRS instrument does not measure, and the SPI stage left them NaN on the FRS channel — assessment-era honesty that made the artifact unloadable in practice: PolicyEngine-UK's `validate()` refuses NaN inputs, and any finiteness fence on the calibration path refuses the frame, so the campaign had to zero-fill them outside the build before it could calibrate at all. Zero is the adjudicated stage-time semantics for a concept the instrument never asked about, the auxiliary-crosswalk guard already stops the QRF mistaking the fill for measured data, and a regression test now asserts the stage leaves no NaN in any float column. **The FRS age top-code is disaggregated.** The licensed delivery records no age above 80, so every 80+ person arrives piled at exactly 80: the 85-89 and 90+ population targets are structurally unbindable and the 80-84 band starts at roughly double its target carrying the whole 80+ population — a defect the incumbent shares. The new `age_tail` stage reassigns each piled person a band drawn from a sex-specific inverse CDF over the ONS mid-year populations and a uniform integer age within it, keyed on `person_source_id` so a household and its capital-gains clone twin receive the same age, deterministic under a declared seed with no global RNG. The six band populations ship as a committed resource in which every cell records the calibration target id it must agree with, so the imputation source and the target denominators cannot drift apart. It runs last in the spine plan, downstream of every stage that conditions on age, so imputation conditioning is unchanged and the stage's whole effect is the pile's dispersal; ages are assigned toward the ONS distribution while calibration still owns the totals. diff --git a/packages/microcosm-build/src/microcosm/build/source_manifest.py b/packages/microcosm-build/src/microcosm/build/source_manifest.py index fca25b48..ebad0f61 100644 --- a/packages/microcosm-build/src/microcosm/build/source_manifest.py +++ b/packages/microcosm-build/src/microcosm/build/source_manifest.py @@ -71,6 +71,7 @@ "derive_childcare_inputs", "derive_child_support_inputs", "derive_disability_benefits", + "disaggregate_top_coded_ages", "derive_energy_subsidy", "derive_education_inputs", "derive_eligibility_inputs", diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/schema/sources.schema.json b/packages/microcosm-build/src/microcosm/build/spec_engine/schema/sources.schema.json index 9d29c809..c6f16826 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/schema/sources.schema.json +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/schema/sources.schema.json @@ -2235,6 +2235,73 @@ "weight" ] }, + { + "type": "object", + "additionalProperties": false, + "properties": { + "bands": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "additionalProperties": false, + "properties": { + "low": { + "type": "number" + }, + "name": { + "type": "string" + }, + "width": { + "type": "number" + } + }, + "required": [ + "low", + "name", + "width" + ] + } + }, + "draw_key": { + "type": "string" + }, + "kind": { + "const": "disaggregate_top_coded_ages" + }, + "output": { + "type": "string" + }, + "reason": { + "type": "string" + }, + "resource": { + "type": "string" + }, + "salt_streams": { + "type": "array", + "items": { + "type": "string" + } + }, + "seed": { + "type": "number" + }, + "top_code": { + "type": "number" + } + }, + "required": [ + "bands", + "draw_key", + "kind", + "output", + "resource", + "salt_streams", + "seed", + "top_code" + ] + }, { "type": "object", "additionalProperties": false, @@ -5813,7 +5880,9 @@ { "type": "object", "additionalProperties": false, - "required": ["kind"], + "required": [ + "kind" + ], "properties": { "kind": { "enum": [ @@ -5826,199 +5895,136 @@ "zero_when_false" ] }, - "age_bands": {"type": "string"}, - "annualization_weeks": {"type": "number"}, - "budget_resource": {"type": "string"}, - "categorical_predictors": {"type": "array", "items": {"type": "string"}}, - "chain_order": {"type": "array", "items": {"type": "string"}}, - "columns": {"type": "array", "items": {"type": "string"}}, - "condition": {"type": "string"}, - "denominator_key": {"type": "string"}, - "denominator_resource": {"type": "string"}, - "donor_weight": {"type": "string"}, - "exempt": {"type": "array", "items": {"type": "string"}}, - "fail_loud_on_missing_rate": {"type": "boolean"}, - "iterations": {"type": "integer", "minimum": 1}, - "lossy_mappings": {"type": "array", "items": {"type": "string"}}, - "logged_dropna_row_count": {"type": "boolean"}, - "margins": {"type": "array", "items": {"type": "string"}}, - "n_estimators": {"type": "integer", "minimum": 1}, - "numerator": {"type": "string"}, - "output": {"type": "string"}, - "predictors": {"type": "array", "items": {"type": "string"}}, - "rate_key": {"type": "string"}, - "range": {"type": "string"}, - "reduced_rate_share": {"type": "number"}, - "resource": {"type": "string"}, - "seed": {"type": "integer"}, - "source": {"type": "string"}, - "standard_rate": {"type": "number"}, - "target": {"type": "string"}, - "targets": {"type": "array", "items": {"type": "string"}}, - "top_band_fold_in": {"type": "string"}, - "weights": {"type": "string"}, - "weighted": {"type": "boolean"}, - "year": {"type": ["string", "integer"]}, - "lowercase_columns": {"type": "boolean"} - } - }, - { - "type": "object", - "additionalProperties": false, - "required": ["kind", "predictors", "targets", "seed"], - "properties": { - "kind": {"const": "fit_weighted_qrf"}, - "predictors": {"type": "array", "items": {"type": "string"}}, - "targets": {"type": "array", "items": {"type": "string"}}, - "weights": {"type": "string"}, - "n_estimators": {"type": "integer", "minimum": 1}, - "seed": {"type": "integer"}, - "training_population": {"type": "string"}, - "target_population": {"type": "string"}, - "weight_mapping": {"type": "string"}, - "clamp_minimum": {"type": "number"}, - "preserve_asked_rows": {"type": "boolean"}, - "cache": {"type": "boolean"} - } - }, - { - "type": "object", - "additionalProperties": false, - "required": ["kind", "predictors", "targets", "seed"], - "properties": { - "kind": {"const": "fit_weighted_qrf_chain"}, - "predictors": {"type": "array", "items": {"type": "string"}}, - "targets": {"type": "array", "items": {"type": "string"}}, - "categorical_predictors": {"type": "array", "items": {"type": "string"}}, - "weights": {"type": "string"}, - "n_estimators": {"type": "integer", "minimum": 1}, - "seed": {"type": "integer"} - } - }, - { - "type": "object", - "additionalProperties": false, - "required": ["kind", "range", "exempt"], - "properties": { - "kind": {"const": "support_clip"}, - "range": {"type": "string"}, - "exempt": {"type": "array", "items": {"type": "string"}} - } - }, - { - "type": "object", - "additionalProperties": false, - "required": ["kind", "predictors", "derived_predictors"], - "properties": { - "kind": {"const": "materialize_rules_engine_predictors"}, - "predictors": {"type": "array", "items": {"type": "string"}}, - "derived_predictors": {"type": "object", "additionalProperties": {"type": "string"}} - } - }, - { - "type": "object", - "additionalProperties": false, - "required": ["kind", "output", "inputs"], - "properties": { - "kind": {"const": "fold_into"}, - "output": {"type": "string"}, - "inputs": {"type": "array", "items": {"type": "string"}}, - "drop_inputs": {"type": "boolean"} - } - }, - { - "type": "object", - "additionalProperties": false, - "required": [ - "kind", - "artifact_role", - "filename", - "delimiter", - "required_columns", - "weight", - "seed" - ], - "properties": { - "kind": { - "const": "strict_read_private_table" - }, - "artifact_role": { + "age_bands": { "type": "string" }, - "filename": { - "type": "string" + "annualization_weeks": { + "type": "number" }, - "delimiter": { + "budget_resource": { "type": "string" }, - "required_columns": { + "categorical_predictors": { "type": "array", "items": { "type": "string" } }, - "weight": { + "chain_order": { + "type": "array", + "items": { + "type": "string" + } + }, + "columns": { + "type": "array", + "items": { + "type": "string" + } + }, + "condition": { "type": "string" }, - "seed": { - "type": "integer" + "denominator_key": { + "type": "string" }, - "runtime_sha256_required": { - "type": "boolean" + "denominator_resource": { + "type": "string" }, - "fail_on_missing_file": { - "type": "boolean" + "donor_weight": { + "type": "string" }, - "fail_on_missing_columns": { + "exempt": { + "type": "array", + "items": { + "type": "string" + } + }, + "fail_loud_on_missing_rate": { "type": "boolean" }, - "fail_on_invalid_weight": { + "iterations": { + "type": "integer", + "minimum": 1 + }, + "lossy_mappings": { + "type": "array", + "items": { + "type": "string" + } + }, + "logged_dropna_row_count": { "type": "boolean" - } - } - }, - { - "type": "object", - "additionalProperties": false, - "required": [ - "kind", - "channels", - "count", - "draw", - "flag_column", - "seed" - ], - "properties": { - "kind": { - "const": "stack_zero_weight_donors" }, - "count": { - "type": "integer" + "margins": { + "type": "array", + "items": { + "type": "string" + } }, - "draw": { + "n_estimators": { + "type": "integer", + "minimum": 1 + }, + "numerator": { "type": "string" }, - "flag_column": { + "output": { + "type": "string" + }, + "predictors": { + "type": "array", + "items": { + "type": "string" + } + }, + "rate_key": { + "type": "string" + }, + "range": { + "type": "string" + }, + "reduced_rate_share": { + "type": "number" + }, + "resource": { "type": "string" }, "seed": { "type": "integer" }, - "channels": { - "type": "object", - "additionalProperties": false, - "required": [ - "base", - "synthetic" - ], - "properties": { - "base": { - "type": "string" - }, - "synthetic": { - "type": "string" - } + "source": { + "type": "string" + }, + "standard_rate": { + "type": "number" + }, + "target": { + "type": "string" + }, + "targets": { + "type": "array", + "items": { + "type": "string" } + }, + "top_band_fold_in": { + "type": "string" + }, + "weights": { + "type": "string" + }, + "weighted": { + "type": "boolean" + }, + "year": { + "type": [ + "string", + "integer" + ] + }, + "lowercase_columns": { + "type": "boolean" } } }, @@ -6027,22 +6033,291 @@ "additionalProperties": false, "required": [ "kind", - "gate", - "declarations" + "predictors", + "targets", + "seed" ], "properties": { "kind": { - "const": "gate_zero_weight_strata" + "const": "fit_weighted_qrf" }, - "gate": { - "type": "string" + "predictors": { + "type": "array", + "items": { + "type": "string" + } }, - "declarations": { + "targets": { "type": "array", - "minItems": 1, "items": { - "type": "object", - "additionalProperties": false, + "type": "string" + } + }, + "weights": { + "type": "string" + }, + "n_estimators": { + "type": "integer", + "minimum": 1 + }, + "seed": { + "type": "integer" + }, + "training_population": { + "type": "string" + }, + "target_population": { + "type": "string" + }, + "weight_mapping": { + "type": "string" + }, + "clamp_minimum": { + "type": "number" + }, + "preserve_asked_rows": { + "type": "boolean" + }, + "cache": { + "type": "boolean" + } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "kind", + "predictors", + "targets", + "seed" + ], + "properties": { + "kind": { + "const": "fit_weighted_qrf_chain" + }, + "predictors": { + "type": "array", + "items": { + "type": "string" + } + }, + "targets": { + "type": "array", + "items": { + "type": "string" + } + }, + "categorical_predictors": { + "type": "array", + "items": { + "type": "string" + } + }, + "weights": { + "type": "string" + }, + "n_estimators": { + "type": "integer", + "minimum": 1 + }, + "seed": { + "type": "integer" + } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "kind", + "range", + "exempt" + ], + "properties": { + "kind": { + "const": "support_clip" + }, + "range": { + "type": "string" + }, + "exempt": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "kind", + "predictors", + "derived_predictors" + ], + "properties": { + "kind": { + "const": "materialize_rules_engine_predictors" + }, + "predictors": { + "type": "array", + "items": { + "type": "string" + } + }, + "derived_predictors": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "kind", + "output", + "inputs" + ], + "properties": { + "kind": { + "const": "fold_into" + }, + "output": { + "type": "string" + }, + "inputs": { + "type": "array", + "items": { + "type": "string" + } + }, + "drop_inputs": { + "type": "boolean" + } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "kind", + "artifact_role", + "filename", + "delimiter", + "required_columns", + "weight", + "seed" + ], + "properties": { + "kind": { + "const": "strict_read_private_table" + }, + "artifact_role": { + "type": "string" + }, + "filename": { + "type": "string" + }, + "delimiter": { + "type": "string" + }, + "required_columns": { + "type": "array", + "items": { + "type": "string" + } + }, + "weight": { + "type": "string" + }, + "seed": { + "type": "integer" + }, + "runtime_sha256_required": { + "type": "boolean" + }, + "fail_on_missing_file": { + "type": "boolean" + }, + "fail_on_missing_columns": { + "type": "boolean" + }, + "fail_on_invalid_weight": { + "type": "boolean" + } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "kind", + "channels", + "count", + "draw", + "flag_column", + "seed" + ], + "properties": { + "kind": { + "const": "stack_zero_weight_donors" + }, + "count": { + "type": "integer" + }, + "draw": { + "type": "string" + }, + "flag_column": { + "type": "string" + }, + "seed": { + "type": "integer" + }, + "channels": { + "type": "object", + "additionalProperties": false, + "required": [ + "base", + "synthetic" + ], + "properties": { + "base": { + "type": "string" + }, + "synthetic": { + "type": "string" + } + } + } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "kind", + "gate", + "declarations" + ], + "properties": { + "kind": { + "const": "gate_zero_weight_strata" + }, + "gate": { + "type": "string" + }, + "declarations": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "additionalProperties": false, "required": [ "name", "selector", @@ -6563,118 +6838,360 @@ { "type": "object", "additionalProperties": false, - "required": ["kind", "entity", "copies", "flag_column", "mass_split", "weight_kind_out", "conservation", "id_remapping", "declared_factor", "reason"], + "required": [ + "kind", + "entity", + "copies", + "flag_column", + "mass_split", + "weight_kind_out", + "conservation", + "id_remapping", + "declared_factor", + "reason" + ], "properties": { - "kind": {"const": "clone_records"}, - "entity": {"type": "string"}, - "copies": {"type": "integer", "minimum": 2}, - "flag_column": {"type": "string"}, - "original_flag": {"type": "boolean"}, - "clone_flag": {"type": "boolean"}, - "mass_split": {"type": "number"}, - "weight_kind_out": {"type": "string"}, - "conservation": {"type": "string"}, - "id_remapping": {"type": "string"}, - "declared_factor": {"type": "number"}, - "reason": {"type": "string"} + "kind": { + "const": "clone_records" + }, + "entity": { + "type": "string" + }, + "copies": { + "type": "integer", + "minimum": 2 + }, + "flag_column": { + "type": "string" + }, + "original_flag": { + "type": "boolean" + }, + "clone_flag": { + "type": "boolean" + }, + "mass_split": { + "type": "number" + }, + "weight_kind_out": { + "type": "string" + }, + "conservation": { + "type": "string" + }, + "id_remapping": { + "type": "string" + }, + "declared_factor": { + "type": "number" + }, + "reason": { + "type": "string" + } } }, { "type": "object", "additionalProperties": false, - "required": ["kind", "resource", "income_proxy_components", "allowance_subtraction", "carrier", "adult_minimum_age", "quantile_points", "spline_degree", "extrapolation", "keep_negative_draws", "seed", "salt"], + "required": [ + "kind", + "resource", + "income_proxy_components", + "allowance_subtraction", + "carrier", + "adult_minimum_age", + "quantile_points", + "spline_degree", + "extrapolation", + "keep_negative_draws", + "seed", + "salt" + ], "properties": { - "kind": {"const": "draw_capital_gains_prior_from_banded_quantiles"}, - "resource": {"type": "string"}, - "income_proxy_components": {"type": "array", "items": {"type": "string"}}, - "allowance_subtraction": {"type": "boolean"}, - "carrier": {"type": "string"}, - "adult_minimum_age": {"type": "integer"}, - "quantile_points": {"type": "array", "items": {"type": "number"}}, - "spline_degree": {"type": "integer"}, - "extrapolation": {"type": "string"}, - "keep_negative_draws": {"type": "boolean"}, - "seed": {"type": "integer"}, - "salt": {"type": "string"} + "kind": { + "const": "draw_capital_gains_prior_from_banded_quantiles" + }, + "resource": { + "type": "string" + }, + "income_proxy_components": { + "type": "array", + "items": { + "type": "string" + } + }, + "allowance_subtraction": { + "type": "boolean" + }, + "carrier": { + "type": "string" + }, + "adult_minimum_age": { + "type": "integer" + }, + "quantile_points": { + "type": "array", + "items": { + "type": "number" + } + }, + "spline_degree": { + "type": "integer" + }, + "extrapolation": { + "type": "string" + }, + "keep_negative_draws": { + "type": "boolean" + }, + "seed": { + "type": "integer" + }, + "salt": { + "type": "string" + } } }, { "type": "object", "additionalProperties": false, - "required": ["kind", "size_band_resource", "incidence_resource", "minimum_band_lower", "donors_per_band", "expected_band_count", "expected_donor_count", "candidate_order", "draw", "propensity", "seed", "flag_column", "carrier", "initial_weight", "never_zero_weight", "weight_kind_out", "reason"], + "required": [ + "kind", + "size_band_resource", + "incidence_resource", + "minimum_band_lower", + "donors_per_band", + "expected_band_count", + "expected_donor_count", + "candidate_order", + "draw", + "propensity", + "seed", + "flag_column", + "carrier", + "initial_weight", + "never_zero_weight", + "weight_kind_out", + "reason" + ], "properties": { - "kind": {"const": "stack_band_donor_households"}, - "size_band_resource": {"type": "string"}, - "incidence_resource": {"type": "string"}, - "minimum_band_lower": {"type": "number"}, - "donors_per_band": {"type": "integer", "minimum": 1}, - "expected_band_count": {"type": "integer", "minimum": 1}, - "expected_donor_count": {"type": "integer", "minimum": 1}, - "candidate_order": {"type": "string"}, - "draw": {"type": "string"}, - "propensity": {"type": "string"}, - "seed": {"type": "integer"}, - "flag_column": {"type": "string"}, - "carrier": {"type": "string"}, - "initial_weight": {"type": "string"}, - "never_zero_weight": {"type": "boolean"}, - "weight_kind_out": {"type": "string"}, - "reason": {"type": "string"} + "kind": { + "const": "stack_band_donor_households" + }, + "size_band_resource": { + "type": "string" + }, + "incidence_resource": { + "type": "string" + }, + "minimum_band_lower": { + "type": "number" + }, + "donors_per_band": { + "type": "integer", + "minimum": 1 + }, + "expected_band_count": { + "type": "integer", + "minimum": 1 + }, + "expected_donor_count": { + "type": "integer", + "minimum": 1 + }, + "candidate_order": { + "type": "string" + }, + "draw": { + "type": "string" + }, + "propensity": { + "type": "string" + }, + "seed": { + "type": "integer" + }, + "flag_column": { + "type": "string" + }, + "carrier": { + "type": "string" + }, + "initial_weight": { + "type": "string" + }, + "never_zero_weight": { + "type": "boolean" + }, + "weight_kind_out": { + "type": "string" + }, + "reason": { + "type": "string" + } } }, { "type": "object", "additionalProperties": false, - "required": ["kind", "resource", "target", "donor_pool", "rate_cap", "move", "seed", "salt", "receipt"], + "required": [ + "kind", + "resource", + "target", + "donor_pool", + "rate_cap", + "move", + "seed", + "salt", + "receipt" + ], "properties": { - "kind": {"const": "convert_donors_to_target_stock"}, - "reason": {"type": "string"}, - "resource": {"type": "string"}, - "target": {"type": "number"}, - "donor_pool": {"type": "string"}, - "rate_cap": {"type": "number"}, - "move": {"type": "string"}, - "seed": {"type": "integer"}, - "salt": {"type": "string"}, - "receipt": {"type": "string"} + "kind": { + "const": "convert_donors_to_target_stock" + }, + "reason": { + "type": "string" + }, + "resource": { + "type": "string" + }, + "target": { + "type": "number" + }, + "donor_pool": { + "type": "string" + }, + "rate_cap": { + "type": "number" + }, + "move": { + "type": "string" + }, + "seed": { + "type": "integer" + }, + "salt": { + "type": "string" + }, + "receipt": { + "type": "string" + } } }, { "type": "object", "additionalProperties": false, - "required": ["kind", "year_rule", "start_year_formula", "reported_repayment_test", "reported_country_gate", "plan_1_before", "plan_5_from", "enum_domain", "plan_4_imputation"], + "required": [ + "kind", + "year_rule", + "start_year_formula", + "reported_repayment_test", + "reported_country_gate", + "plan_1_before", + "plan_5_from", + "enum_domain", + "plan_4_imputation" + ], "properties": { - "kind": {"const": "assign_student_loan_plan_cohorts"}, - "year_rule": {"type": "string"}, - "start_year_formula": {"type": "string"}, - "reported_repayment_test": {"type": "string"}, - "reported_country_gate": {"type": "boolean"}, - "plan_1_before": {"type": "integer"}, - "plan_5_from": {"type": "integer"}, - "enum_domain": {"type": "array", "items": {"type": "string"}}, - "plan_4_imputation": {"type": "boolean"} + "kind": { + "const": "assign_student_loan_plan_cohorts" + }, + "year_rule": { + "type": "string" + }, + "start_year_formula": { + "type": "string" + }, + "reported_repayment_test": { + "type": "string" + }, + "reported_country_gate": { + "type": "boolean" + }, + "plan_1_before": { + "type": "integer" + }, + "plan_5_from": { + "type": "integer" + }, + "enum_domain": { + "type": "array", + "items": { + "type": "string" + } + }, + "plan_4_imputation": { + "type": "boolean" + } } }, { "type": "object", "additionalProperties": false, - "required": ["kind", "plan", "priority", "resource", "stock_series", "year_rule", "age_min", "age_max", "cohort_start_min", "eligible_region_exclusions", "highest_education", "seed", "salt"], + "required": [ + "kind", + "plan", + "priority", + "resource", + "stock_series", + "year_rule", + "age_min", + "age_max", + "cohort_start_min", + "eligible_region_exclusions", + "highest_education", + "seed", + "salt" + ], "properties": { - "kind": {"const": "top_up_to_stock"}, - "reason": {"type": "string"}, - "plan": {"type": "string"}, - "priority": {"type": "integer"}, - "resource": {"type": "string"}, - "stock_series": {"type": "string"}, - "year_rule": {"type": "string"}, - "age_min": {"type": "integer"}, - "age_max": {"type": "integer"}, - "cohort_start_min": {"type": "integer"}, - "cohort_start_max_exclusive": {"type": "integer"}, - "eligible_region_exclusions": {"type": "array", "items": {"type": "string"}}, - "highest_education": {"type": "string"}, - "seed": {"type": "integer"}, - "salt": {"type": "string"} + "kind": { + "const": "top_up_to_stock" + }, + "reason": { + "type": "string" + }, + "plan": { + "type": "string" + }, + "priority": { + "type": "integer" + }, + "resource": { + "type": "string" + }, + "stock_series": { + "type": "string" + }, + "year_rule": { + "type": "string" + }, + "age_min": { + "type": "integer" + }, + "age_max": { + "type": "integer" + }, + "cohort_start_min": { + "type": "integer" + }, + "cohort_start_max_exclusive": { + "type": "integer" + }, + "eligible_region_exclusions": { + "type": "array", + "items": { + "type": "string" + } + }, + "highest_education": { + "type": "string" + }, + "seed": { + "type": "integer" + }, + "salt": { + "type": "string" + } } }, { diff --git a/packages/microcosm-build/src/microcosm/build/uk/country_package.json b/packages/microcosm-build/src/microcosm/build/uk/country_package.json index 195c42a8..7883de48 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/country_package.json +++ b/packages/microcosm-build/src/microcosm/build/uk/country_package.json @@ -132,6 +132,11 @@ "kind": "legacy_json", "schema_id": "legacy_json" }, + { + "path": "ons_age_tail_band_populations.json", + "kind": "legacy_json", + "schema_id": "legacy_json" + }, { "path": "lcfs_consumption_support_bounds.json", "kind": "legacy_json", diff --git a/packages/microcosm-build/src/microcosm/build/uk/ons_age_tail_band_populations.json b/packages/microcosm-build/src/microcosm/build/uk/ons_age_tail_band_populations.json new file mode 100644 index 00000000..28b9a6af --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk/ons_age_tail_band_populations.json @@ -0,0 +1,17 @@ +{ + "schema_version": 1, + "description": "ONS mid-year population estimates for the 80+ age bands, split by sex, used by the age_tail stage to disaggregate the FRS age top-code. These are the same facts the calibration register binds as its ons.population.{sex}_{band} targets, recorded here so the imputation source and the target denominators share one source of truth; the values were read from the compiled national register at calibration year 2025 and the target_id recorded on each cell is the drift check.", + "source": "Office for National Statistics, mid-year population estimates, via the chronicle ledger facts compiled into the UK national calibration register (target family ons_population).", + "bands": { + "MALE": { + "80_84": {"population": 815910.0, "target_id": "ons.population.male_80_84"}, + "85_89": {"population": 459511.0, "target_id": "ons.population.male_85_89"}, + "90_plus": {"population": 210520.0, "target_id": "ons.population.male_90_plus"} + }, + "FEMALE": { + "80_84": {"population": 1024480.0, "target_id": "ons.population.female_80_84"}, + "85_89": {"population": 665267.0, "target_id": "ons.population.female_85_89"}, + "90_plus": {"population": 414716.0, "target_id": "ons.population.female_90_plus"} + } + } +} diff --git a/packages/microcosm-build/src/microcosm/build/uk/source_stages.json b/packages/microcosm-build/src/microcosm/build/uk/source_stages.json index b878bbdc..27417362 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/source_stages.json +++ b/packages/microcosm-build/src/microcosm/build/uk/source_stages.json @@ -2792,6 +2792,58 @@ ], "notes": "Reported PAYE repayers are classified without a country gate. England tertiary cohorts are then topped up PLAN_5 first and PLAN_2 second to the pinned liable stocks at the FRS release calibration year; PLAN_4 is never imputed." }, + { + "stage": "age_tail", + "survey": "Family Resources Survey 2024-25 and ONS mid-year population estimates", + "source": "Office for National Statistics mid-year population estimates for the 80+ age bands by sex, via the chronicle ledger facts the calibration register binds (target family ons_population).", + "grain": "person", + "artifacts": [ + { + "format": "json", + "kind": "public_aggregate_reference", + "resource": "ons_age_tail_band_populations.json", + "role": "age_tail_band_populations", + "runtime_sha256_required": true + } + ], + "operations": [ + { + "bands": [ + { + "low": 80, + "name": "80_84", + "width": 5 + }, + { + "low": 85, + "name": "85_89", + "width": 5 + }, + { + "low": 90, + "name": "90_plus", + "width": 8 + } + ], + "draw_key": "person_source_id", + "kind": "disaggregate_top_coded_ages", + "output": "age", + "reason": "Age disaggregation rewrites an existing person column in place; no rows move, no weights change, and total household mass is conserved.", + "resource": "ons_age_tail_band_populations.json", + "salt_streams": [ + "band", + "within" + ], + "seed": 0, + "top_code": 80 + } + ], + "outputs": [], + "notes": "The licensed FRS records no age above 80, so every 80+ person arrives piled at exactly 80 and the 85-89 and 90+ population targets are structurally unbindable; the incumbent shares the defect. Each piled person draws a band from the sex-specific ONS inverse CDF and a uniform integer age within it, keyed on person_source_id so a household and its capital-gains clone twin receive the same age. Runs after student_loans \u2014 downstream of every stage that conditions on age, the position the", + "rewrites": [ + "age" + ] + }, { "stage": "frs_hmrc_retained_leaves", "survey": "Family Resources Survey 2024-25", diff --git a/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml b/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml index 8897b43e..d30a9419 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml +++ b/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml @@ -2225,6 +2225,42 @@ stages: rewrites: - student_loan_plan notes: Reported PAYE repayers are classified without a country gate. England tertiary cohorts are then topped up PLAN_5 first and PLAN_2 second to the pinned liable stocks at the FRS release calibration year; PLAN_4 is never imputed. +- stage: age_tail + survey: Family Resources Survey 2024-25 and ONS mid-year population estimates + source: Office for National Statistics mid-year population estimates for the 80+ age bands by sex, via the chronicle ledger facts the calibration register binds (target family ons_population). + grain: person + artifacts: + - role: age_tail_band_populations + kind: public_aggregate_reference + resource: ons_age_tail_band_populations.json + format: json + runtime_sha256_required: true + operations: + - kind: disaggregate_top_coded_ages + output: age + resource: ons_age_tail_band_populations.json + top_code: 80 + bands: + - name: '80_84' + low: 80 + width: 5 + - name: '85_89' + low: 85 + width: 5 + - name: '90_plus' + low: 90 + width: 8 + seed: 0 + draw_key: person_source_id + salt_streams: + - band + - within + reason: Age disaggregation rewrites an existing person column in place; no rows + move, no weights change, and total household mass is conserved. + outputs: [] + rewrites: + - age + notes: The licensed FRS records no age above 80, so every 80+ person arrives piled at exactly 80 and the 85-89 and 90+ population targets are structurally unbindable; the incumbent shares the defect. Each piled person draws a band from the sex-specific ONS inverse CDF and a uniform integer age within it, keyed on person_source_id so a household and its capital-gains clone twin receive the same age. Runs after student_loans — downstream of every stage that conditions on age, the position the #623 calibration campaign proved — so imputation conditioning is unchanged. Ages are assigned toward the ONS distribution; calibration still owns the totals. - stage: frs_hmrc_retained_leaves survey: Family Resources Survey 2024-25 source: Department for Work and Pensions Family Resources Survey 2024-25 raw adult.tab and benefits.tab, caller-supplied local input diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/__init__.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/__init__.py index 04c1e4dd..ba309d80 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/__init__.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/__init__.py @@ -1,5 +1,14 @@ """UK build helpers for Microcosm-owned local-geography artifacts.""" +from microcosm.build.uk_runtime.age_tail import ( + UK_AGE_TAIL_BAND_POPULATIONS_RESOURCE, + UK_AGE_TAIL_BANDS, + UK_AGE_TAIL_DECLARED_SEEDS, + UK_AGE_TOP_CODE, + UKAgeTailStageTransform, + disaggregate_uk_age_top_code, + load_uk_age_tail_band_populations, +) from microcosm.build.uk_runtime.battery_bindings import ( UK_GATE_REGISTRY, UKGateBinding, @@ -473,6 +482,13 @@ ) __all__ = [ + "UK_AGE_TAIL_BANDS", + "UK_AGE_TAIL_BAND_POPULATIONS_RESOURCE", + "UK_AGE_TAIL_DECLARED_SEEDS", + "UK_AGE_TOP_CODE", + "UKAgeTailStageTransform", + "disaggregate_uk_age_top_code", + "load_uk_age_tail_band_populations", "ARTIFACT_CLONE_INDEX_COLUMN", "UKGateBinding", "UKRowwiseDoctrineSolve", diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/age_tail.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/age_tail.py new file mode 100644 index 00000000..7ca222d5 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/age_tail.py @@ -0,0 +1,306 @@ +"""Disaggregate the FRS age top-code so 85+ population targets can bind. + +The licensed FRS delivery records no age above 80: the raw ``AGE`` column is +blanked and ``age80`` caps at 80, so every person aged 80 or older arrives as +exactly 80. That single pile produces two distortions at once — the 85-89 and +90+ population targets are structurally unbindable (estimate 0), and the +80-84 band starts roughly double its target because it carries the entire +80+ population. The incumbent has the identical defect: its own registry +estimates 0 on 85+. + +This stage reassigns each piled person an age drawn from the ONS mid-year +band populations (the same chronicle facts the calibration targets bind, so +the imputation source and the target denominators cannot drift apart). The +draw is: + +- keyed on ``person_source_id``, so a household and its capital-gains clone + twin receive the same age and the payload-identity discipline holds; +- deterministic under a declared seed (sha256 counter stream, no global RNG); +- sex-specific, using the MALE/FEMALE 80-84 / 85-89 / 90+ populations as an + inverse CDF, with a uniform integer age within the drawn band. + +Ages are assigned, not weighted, toward the ONS distribution: the achieved +weighted shares land near the ONS shares by construction and calibration +still owns the totals, so the age-band targets remain honest constraints +rather than tautologies. + +Written for the #623 assessment runner, proven over its nine-run calibration +campaign, and ported here as the declarative WS-E source stage its docstring +always said it would become. In the spine it runs after ``student_loans`` — +the position the campaign exercised, downstream of everything that consumes +``age``, so imputation conditioning is unchanged and the stage's whole +effect is the pile's disaggregation. The band populations come from the +committed ``ons_age_tail_band_populations.json`` resource, each cell +carrying the register target id it must agree with. +""" + +from __future__ import annotations + +import hashlib +import json +from collections.abc import Mapping +from dataclasses import dataclass, field +from importlib.resources import files +from pathlib import Path +from typing import Any + +import numpy as np +import pandas as pd + +from microcosm.build.source_manifest import SourceStageSpec + +__all__ = [ + "UK_AGE_TAIL_BANDS", + "UK_AGE_TAIL_BAND_POPULATIONS_RESOURCE", + "UK_AGE_TAIL_DECLARED_SEEDS", + "UK_AGE_TOP_CODE", + "UKAgeTailStageTransform", + "disaggregate_uk_age_top_code", + "load_uk_age_tail_band_populations", +] + +UK_AGE_TOP_CODE = 80 + +UK_AGE_TAIL_DECLARED_SEEDS = {"age": 0} + +UK_AGE_TAIL_BAND_POPULATIONS_RESOURCE = "ons_age_tail_band_populations.json" + +_UK_PACKAGE = "microcosm.build.uk" + +# Band name -> (lowest assigned age, number of integer ages drawn). +# 90+ is drawn over 90-97: wide enough to be demographically honest, narrow +# enough that no simulated rule changes past 90 are being invented. +UK_AGE_TAIL_BANDS: tuple[tuple[str, int, int], ...] = ( + ("80_84", 80, 5), + ("85_89", 85, 5), + ("90_plus", 90, 8), +) + + +def _unit_draw(source_id: object, seed: int, stream: str) -> float: + """Deterministic uniform in [0, 1) keyed on a stable identity.""" + + digest = hashlib.sha256( + f"uk_age_tail:{stream}:{seed}:{source_id}".encode() + ).digest() + return int.from_bytes(digest[:8], "big") / float(1 << 64) + + +def load_uk_age_tail_band_populations( + resource: str = UK_AGE_TAIL_BAND_POPULATIONS_RESOURCE, +) -> dict[tuple[str, str], float]: + """Load the six committed ONS band populations, validated closed-world.""" + + candidate = Path(resource) + if candidate.exists(): + text = candidate.read_text(encoding="utf-8") + else: + text = files(_UK_PACKAGE).joinpath(resource).read_text(encoding="utf-8") + payload = json.loads(text) + if payload.get("schema_version") != 1: + raise ValueError( + f"{resource}: unsupported schema_version " + f"{payload.get('schema_version')!r}; expected 1." + ) + bands = payload.get("bands") + if not isinstance(bands, Mapping): + raise ValueError(f"{resource}: 'bands' must be an object.") + band_names = [name for name, _, _ in UK_AGE_TAIL_BANDS] + populations: dict[tuple[str, str], float] = {} + for gender in ("MALE", "FEMALE"): + cells = bands.get(gender) + if not isinstance(cells, Mapping) or set(cells) != set(band_names): + raise ValueError( + f"{resource}: bands.{gender} must carry exactly {band_names}." + ) + for band in band_names: + cell = cells[band] + value = cell.get("population") if isinstance(cell, Mapping) else None + expected_target = f"ons.population.{gender.lower()}_{band}" + if ( + not isinstance(cell, Mapping) + or cell.get("target_id") != expected_target + ): + raise ValueError( + f"{resource}: bands.{gender}.{band} must record " + f"target_id {expected_target!r} — the register drift check." + ) + if not isinstance(value, (int, float)) or not np.isfinite(value): + raise ValueError( + f"{resource}: bands.{gender}.{band}.population must be a " + f"finite number, got {value!r}." + ) + populations[(gender, band)] = float(value) + return populations + + +def disaggregate_uk_age_top_code( + frame: Any, + *, + band_populations: Mapping[tuple[str, str], float], + seed: int = 0, + top_code: int = UK_AGE_TOP_CODE, +) -> dict[str, Any]: + """Reassign top-coded ages in place and return the receipt. + + ``band_populations`` maps ``(gender, band)`` — gender in MALE/FEMALE, + band in 80_84/85_89/90_plus — to the ONS mid-year population. All six + cells are required; a missing cell aborts. + """ + + person = frame.table("person") + ages = pd.to_numeric(person["age"], errors="raise").to_numpy(dtype=float) + if (ages > top_code).any(): + raise ValueError( + f"input already has ages above {top_code}; refusing to " + "disaggregate a surface that is not top-coded." + ) + piled = ages == float(top_code) + if not piled.any(): + raise ValueError(f"no persons at the top-code age {top_code}.") + + genders = person["gender"].astype(str).to_numpy() + observed = set(np.unique(genders[piled])) + if not observed <= {"MALE", "FEMALE"}: + raise ValueError(f"unexpected gender labels in the pile: {observed}") + + band_names = [name for name, _, _ in UK_AGE_TAIL_BANDS] + cdf: dict[str, np.ndarray] = {} + for gender in ("MALE", "FEMALE"): + populations = [] + for band in band_names: + value = band_populations.get((gender, band)) + if value is None or not np.isfinite(value) or value <= 0: + raise ValueError( + f"band population ({gender}, {band}) is missing or " + f"non-positive: {value!r}" + ) + populations.append(float(value)) + shares = np.asarray(populations) / sum(populations) + cdf[gender] = np.cumsum(shares) + + source_ids = person["person_source_id"].to_numpy() + new_ages = ages.copy() + assigned_counts: dict[tuple[str, str], int] = {} + for index in np.flatnonzero(piled): + gender = genders[index] + band_draw = _unit_draw(source_ids[index], seed, "band") + band_index = int(np.searchsorted(cdf[gender], band_draw, side="right")) + band_index = min(band_index, len(band_names) - 1) + name, low, width = UK_AGE_TAIL_BANDS[band_index] + within_draw = _unit_draw(source_ids[index], seed, "within") + new_ages[index] = low + int(within_draw * width) + key = (gender, name) + assigned_counts[key] = assigned_counts.get(key, 0) + 1 + + person["age"] = new_ages + check = pd.to_numeric(frame.table("person")["age"], errors="raise") + if int((check == float(top_code)).sum()) >= int(piled.sum()): + raise RuntimeError("age disaggregation did not persist on the frame.") + + weights = np.asarray(frame.weights_for("household").values, dtype=float) + household = frame.table("household") + weight_by_household = pd.Series(weights, index=household["household_id"].to_numpy()) + person_weights = weight_by_household.loc[ + person["person_household_id"].to_numpy() + ].to_numpy() + + achieved: dict[str, dict[str, float]] = {} + for gender in ("MALE", "FEMALE"): + gender_rows: dict[str, float] = {} + for band, low, width in UK_AGE_TAIL_BANDS: + mask = ( + piled + & (genders == gender) + & (new_ages >= low) + & (new_ages < low + width) + ) + gender_rows[band] = float(person_weights[mask].sum()) + achieved[gender] = gender_rows + + return { + "stage": "uk_age_tail_disaggregation", + "seed": seed, + "top_code": top_code, + "piled_persons": int(piled.sum()), + "assigned_unweighted": { + f"{gender}:{band}": count + for (gender, band), count in sorted(assigned_counts.items()) + }, + "achieved_weighted": achieved, + "band_populations": { + f"{gender}:{band}": float(value) + for (gender, band), value in sorted(band_populations.items()) + }, + "draw_key": "person_source_id (clone-twin consistent)", + } + + +def _assert_age_tail_stage_parameters(stage: SourceStageSpec) -> None: + """Closed-world drift assert: the manifest declares exactly this stage.""" + + operations = list(stage.operations) + if len(operations) != 1: + raise ValueError( + f"age_tail stage must declare exactly one operation, got {len(operations)}." + ) + parameters = dict(operations[0].parameters) + expected = { + "kind": "disaggregate_top_coded_ages", + "output": "age", + "resource": UK_AGE_TAIL_BAND_POPULATIONS_RESOURCE, + "top_code": UK_AGE_TOP_CODE, + "bands": [ + {"name": name, "low": low, "width": width} + for name, low, width in UK_AGE_TAIL_BANDS + ], + "seed": UK_AGE_TAIL_DECLARED_SEEDS["age"], + "draw_key": "person_source_id", + "salt_streams": ["band", "within"], + } + declared = {"kind": operations[0].kind, **parameters} + for key, value in expected.items(): + if declared.get(key) != value: + raise ValueError( + f"age_tail stage parameter drift: {key!r} declares " + f"{declared.get(key)!r} but the runtime implements {value!r}." + ) + extra = set(declared) - set(expected) - {"reason"} + if extra: + raise ValueError( + f"age_tail stage declares parameter(s) {sorted(extra)} that the " + "runtime does not implement." + ) + + +@dataclass +class UKAgeTailStageTransform: + """Whole-stage transform for the FRS age top-code disaggregation.""" + + stage: SourceStageSpec + band_populations: Mapping[tuple[str, str], float] | None = None + last_result: dict[str, Any] | None = field(default=None, init=False) + + def __call__(self, frame: Any) -> Any: + _assert_age_tail_stage_parameters(self.stage) + populations = ( + dict(self.band_populations) + if self.band_populations is not None + else load_uk_age_tail_band_populations() + ) + receipt = disaggregate_uk_age_top_code( + frame, + band_populations=populations, + seed=UK_AGE_TAIL_DECLARED_SEEDS["age"], + ) + self.last_result = receipt + return frame + + @staticmethod + def output_columns() -> tuple[str, ...]: + return () + + def checkpoint_metadata(self) -> dict[str, object]: + if self.last_result is None: + raise RuntimeError("checkpoint metadata requires a completed stage run.") + return {"evidence": self.last_result} diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/frs_spine.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/frs_spine.py index 3b890057..2da68ccf 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/frs_spine.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/frs_spine.py @@ -533,15 +533,9 @@ def _add_person_income( # weeklyised amount with no derivation: healthy-start vouchers appear on # both the adult and child tapes, the three school ones only on the child # tape, so absent columns read as zero for adults through `_number`. - pe_person["healthy_start_vouchers"] = ( - _positive(person, "heartval") * WEEKS_IN_YEAR - ) - pe_person["free_school_breakfasts"] = ( - _positive(person, "fsbval") * WEEKS_IN_YEAR - ) - pe_person["free_school_fruit_veg"] = ( - _positive(person, "fsfvval") * WEEKS_IN_YEAR - ) + pe_person["healthy_start_vouchers"] = _positive(person, "heartval") * WEEKS_IN_YEAR + pe_person["free_school_breakfasts"] = _positive(person, "fsbval") * WEEKS_IN_YEAR + pe_person["free_school_fruit_veg"] = _positive(person, "fsfvval") * WEEKS_IN_YEAR pe_person["free_school_meals"] = _positive(person, "fsmval") * WEEKS_IN_YEAR @@ -860,6 +854,22 @@ def scottish_water_and_sewerage_weekly(household: pd.DataFrame) -> pd.Series: water_gross = _positive(household, "cwatamt1") sewerage_gross = _positive(household, "csewamt1") billed = water_gross > 0 + # The correctness of the discount fallback rests on the domain claim + # above: a household with no gross water bill carries no sewerage gross + # either, so the fallback factor of 1.0 can never re-introduce the gross + # basis this helper exists to avoid. That claim is a property of the + # vintage, not of the code — so it is asserted, and a vintage that breaks + # it refuses at build time instead of silently paying gross sewerage. + undiscountable = ~billed & (sewerage_gross > 0) + if bool(undiscountable.any()): + raise ValueError( + "Scottish water assembly: " + f"{int(undiscountable.sum())} household(s) carry a gross sewerage " + "charge (CSEWAMT1 > 0) with no gross water bill (CWATAMT1 == 0), " + "so no discount factor is observable for them. Adding sewerage at " + "gross would silently change the charge's after-discount meaning; " + "this vintage needs an adjudicated rule for these households." + ) discount = np.where(billed, water / water_gross.where(billed, 1.0), 1.0) return water + sewerage_gross * discount diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/signed_differences.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/signed_differences.py index 1d38b328..3f4acb03 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/signed_differences.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/signed_differences.py @@ -86,6 +86,15 @@ _ID = re.compile(r"^[a-z0-9]+(?:-[a-z0-9]+)*$") _ISO_DATE = re.compile(r"^\d{4}-\d{2}-\d{2}$") +#: Surfaces whose ``column_differs`` adjudication also covers a payload-level +#: value difference on the same column. A share or weighted-total divergence +#: and a payload value mismatch are one fact measured by two instruments, so +#: an entry adjudicated on either value surface covers both readings. No other +#: cross-surface coverage exists: structural expectations (a column appearing +#: or vanishing, a count moving) never excuse a value difference, and vice +#: versa. +_PAYLOAD_BRIDGE_SURFACES = frozenset({"nonzero_shares", "weighted_totals"}) + @dataclass(frozen=True) class UKSignedDifference: @@ -102,18 +111,35 @@ class UKSignedDifference: adjudicator: str adjudicated_on: str - def covers(self, *, surface: str, column: str) -> bool: - """Whether this entry signs ``column`` on ``surface``. + def covers(self, *, surface: str, column: str, expectation: str) -> bool: + """Whether this entry signs the observed difference. + + A difference is observed as ``(surface, column, expectation)`` and an + entry signs it only when the expectation matches — an entry adjudicated + for a column appearing (``column_missing_in_reference``) never excuses + that column's *values* diverging, and vice versa. The expectation is + therefore consulted at every lookup, not just validated at load. An empty ``columns`` tuple is a surface-wide entry (used by ``entity_counts``, where the "column" is an entity name). + + One deliberate cross-surface rule: a ``column_differs`` entry on a + value-bearing surface (``nonzero_shares``, ``weighted_totals``) also + covers a ``payload_column`` value mismatch on the same column, because + both readings measure the same adjudicated fact. """ - if surface != self.surface: + if expectation != self.expectation: return False - if not self.columns: - return True - return column in self.columns + if surface == self.surface: + return not self.columns or column in self.columns + if ( + surface == "payload_column" + and expectation == "column_differs" + and self.surface in _PAYLOAD_BRIDGE_SURFACES + ): + return not self.columns or column in self.columns + return False @dataclass(frozen=True) @@ -140,11 +166,15 @@ def by_id(self, identifier: str) -> UKSignedDifference | None: return difference return None - def matching(self, *, surface: str, column: str) -> UKSignedDifference | None: - """The entry signing ``column`` on ``surface``, if any.""" + def matching( + self, *, surface: str, column: str, expectation: str + ) -> UKSignedDifference | None: + """The entry signing the observed difference, if any.""" for difference in self.differences: - if difference.covers(surface=surface, column=column): + if difference.covers( + surface=surface, column=column, expectation=expectation + ): return difference return None diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_income.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_income.py index 4bd613f7..49329bda 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_income.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_income.py @@ -463,10 +463,15 @@ def impute_uk_spi_income_support( raise ValueError("SPI stage-1 produced negative non-negative outputs.") for column in SPI_INCOME_QRF_OUTPUT_COLUMNS: if column not in person: - # These full concepts are unavailable on the FRS instrument. NaN - # is the honest state until the SPI donor overwrites its own rows; - # zero would falsely assert a measured structural zero on FRS. - person[column] = np.nan + # These full concepts are unavailable on the FRS instrument, so + # the FRS channel carries the adjudicated stage-time semantics: + # zero, the value every consumer fills at read time. Shipping NaN + # here was assessment-era honesty that made the artifact + # unloadable — the engine's validate() refuses NaN inputs, and + # the calibration seam's finiteness fence rightly refuses the + # frame — while the auxiliary crosswalk guard above already + # protects the QRF from mistaking the fill for measured data. + person[column] = 0.0 person.loc[spi_people, column] = stage1_draws[column].to_numpy() person = _derive_policyengine_employment_input(person, spi_people=spi_people) @@ -609,7 +614,10 @@ def _initialize_frs_channel_columns( f"initialize_frs_channel_columns[{column!r}] must be finite." ) if column not in result: - result[column] = np.nan + # Belt-and-braces zero: every initialized column has its base + # rows overwritten on the next line, but a NaN default here would + # silently survive any future wiring change. + result[column] = 0.0 result.loc[base_people, column] = numeric return result @@ -865,9 +873,11 @@ def derive_hmrc_income_auxiliaries( result[column] = values else: # A prior full-frame auxiliary is not valid evidence for the partial - # FRS channel. Clear every derived output, then materialize SPI only. + # FRS channel. Clear every derived output to the stage-time zero, + # then materialize SPI only — the mask-restricted arithmetic below + # never reads the cleared rows, and the artifact must not ship NaN. for column, values in derived.items(): - result[column] = np.nan + result[column] = 0.0 result.loc[mask, column] = values if not np.array_equal( result.loc[mask, "hmrc_spi_assessable_income"].to_numpy(dtype=float), diff --git a/packages/microcosm-build/tests/test_country_spec.py b/packages/microcosm-build/tests/test_country_spec.py index 131eea41..fa82cf72 100644 --- a/packages/microcosm-build/tests/test_country_spec.py +++ b/packages/microcosm-build/tests/test_country_spec.py @@ -301,6 +301,7 @@ def test_spi_spine_adds_no_country_package_resources(self) -> None: "etb_policy_anchors.json", "etb_services_anchors.json", "nhs_consumption_by_age_gender.json", + "ons_age_tail_band_populations.json", "lcfs_consumption_support_bounds.json", "etb_vat_support_bounds.json", "etb_services_support_bounds.json", @@ -323,11 +324,13 @@ def test_spi_spine_adds_no_country_package_resources(self) -> None: "target_reference_membership.json", ) - def test_uk_source_manifest_loads_twenty_six_stages(self) -> None: + def test_uk_source_manifest_loads_twenty_seven_stages(self) -> None: spec = load_country_spec("uk") assert spec.sources is not None - assert len(spec.sources.stages) == 26 + # 24 spine stages (age_tail is the newest, #747) plus the two + # certified-pair stages the June path still uses. + assert len(spec.sources.stages) == 27 class TestExistingPackagesGeneralize: @@ -373,6 +376,7 @@ def test_uk_package_loads(self) -> None: "etb_policy_anchors.json", "etb_services_anchors.json", "nhs_consumption_by_age_gender.json", + "ons_age_tail_band_populations.json", "lcfs_consumption_support_bounds.json", "etb_vat_support_bounds.json", "etb_services_support_bounds.json", diff --git a/packages/microcosm-build/tests/test_spec_engine_country_bundles.py b/packages/microcosm-build/tests/test_spec_engine_country_bundles.py index a150a100..5ede29d2 100644 --- a/packages/microcosm-build/tests/test_spec_engine_country_bundles.py +++ b/packages/microcosm-build/tests/test_spec_engine_country_bundles.py @@ -42,7 +42,7 @@ ), ( "uk", - "57855b0cea72a99558ac1f204541fccd899fa7bf312e36e399f2ffec078598a2", + "e3b171bdaae1bdfe842e61c129fd279a6b29001cdef4c18de51823768911ec8c", { "benunit.benunit_id", "household.household_id", diff --git a/packages/microcosm-build/tests/test_uk_age_tail.py b/packages/microcosm-build/tests/test_uk_age_tail.py new file mode 100644 index 00000000..098e1698 --- /dev/null +++ b/packages/microcosm-build/tests/test_uk_age_tail.py @@ -0,0 +1,280 @@ +"""The FRS age top-code disaggregation stage (#686, ported from the #623 campaign). + +The stage exists because the licensed FRS records no age above 80, so the +85+ population targets are structurally unbindable. These tests pin the +properties that make its draw safe to run inside the spine: it is keyed on +the source identity (so clone twins agree), it is deterministic under a +declared seed, it moves nobody out of the 80+ population, and it refuses +rather than guesses when its preconditions do not hold. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import numpy as np +import pandas as pd +import pytest + +from microcosm.build.source_manifest import SourceManifest +from microcosm.build.uk_runtime.age_tail import ( + UK_AGE_TAIL_BANDS, + UK_AGE_TOP_CODE, + UKAgeTailStageTransform, + disaggregate_uk_age_top_code, + load_uk_age_tail_band_populations, +) +from microcosm.build.uk_runtime.national_frame import uk_national_frame +from microcosm.frame import WeightKind + +REPO_ROOT = Path(__file__).resolve().parents[3] + +_BANDS = { + ("MALE", "80_84"): 815910.0, + ("MALE", "85_89"): 459511.0, + ("MALE", "90_plus"): 210520.0, + ("FEMALE", "80_84"): 1024480.0, + ("FEMALE", "85_89"): 665267.0, + ("FEMALE", "90_plus"): 414716.0, +} + + +def _frame(*, n_piled: int = 40, clone_twins: bool = False): + """A minimal UK national frame with a top-code pile.""" + + ages = [30.0, 55.0] + [float(UK_AGE_TOP_CODE)] * n_piled + genders = ["MALE", "FEMALE"] + [ + "MALE" if index % 2 == 0 else "FEMALE" for index in range(n_piled) + ] + count = len(ages) + person_ids = list(range(1, count + 1)) + source_ids = [f"s{index}" for index in person_ids] + if clone_twins: + # A clone twin shares its original's source id *and* its sex — the + # band CDF is sex-specific, so a twin with a different gender would + # legitimately draw a different band. + source_ids[-1] = source_ids[-2] + genders[-1] = genders[-2] + person = pd.DataFrame( + { + "person_id": person_ids, + "person_benunit_id": person_ids, + "person_household_id": person_ids, + "person_source_id": source_ids, + "age": ages, + "gender": genders, + } + ) + benunit = pd.DataFrame({"benunit_id": person_ids}) + household = pd.DataFrame( + {"household_id": person_ids, "household_weight": [2.0] * count} + ) + return uk_national_frame( + person=person, + benunit=benunit, + household=household, + time_period="2024", + weight_kind=WeightKind.DESIGN, + ) + + +class TestDraw: + def test_the_pile_disperses_across_every_band(self) -> None: + frame = _frame(n_piled=400) + receipt = disaggregate_uk_age_top_code(frame, band_populations=_BANDS) + + ages = frame.table("person")["age"].to_numpy() + assert receipt["piled_persons"] == 400 + # Nobody leaves the 80+ population and nobody exceeds the widest band. + assert ages[2:].min() >= UK_AGE_TOP_CODE + assert ages.max() <= 97 + # Every band receives someone, so the 85+ targets can bind at all. + assert {age for age in ages[2:] if age >= 85} + assert {age for age in ages[2:] if age >= 90} + # Ages below the top code are untouched. + assert list(ages[:2]) == [30.0, 55.0] + + def test_the_draw_is_deterministic_under_the_seed(self) -> None: + first = _frame(n_piled=120) + second = _frame(n_piled=120) + disaggregate_uk_age_top_code(first, band_populations=_BANDS, seed=0) + disaggregate_uk_age_top_code(second, band_populations=_BANDS, seed=0) + assert np.array_equal( + first.table("person")["age"].to_numpy(), + second.table("person")["age"].to_numpy(), + ) + + def test_a_different_seed_moves_the_draw(self) -> None: + first = _frame(n_piled=120) + second = _frame(n_piled=120) + disaggregate_uk_age_top_code(first, band_populations=_BANDS, seed=0) + disaggregate_uk_age_top_code(second, band_populations=_BANDS, seed=1) + assert not np.array_equal( + first.table("person")["age"].to_numpy(), + second.table("person")["age"].to_numpy(), + ) + + def test_clone_twins_receive_the_same_age(self) -> None: + # The capital-gains clone shares its original's source id, so the two + # rows must agree or the payload-identity discipline breaks. + frame = _frame(n_piled=40, clone_twins=True) + disaggregate_uk_age_top_code(frame, band_populations=_BANDS) + person = frame.table("person") + by_source = person.groupby("person_source_id")["age"].nunique() + assert int(by_source.max()) == 1 + + def test_the_draw_follows_the_band_populations(self) -> None: + # A degenerate CDF that puts all mass on 90+ must send the whole pile + # there — the shares drive the draw, not a hardcoded split. + bands = dict.fromkeys(_BANDS, 1.0) + bands[("MALE", "90_plus")] = 1e9 + bands[("FEMALE", "90_plus")] = 1e9 + frame = _frame(n_piled=200) + disaggregate_uk_age_top_code(frame, band_populations=bands) + assert frame.table("person")["age"].to_numpy()[2:].min() >= 90 + + def test_the_receipt_records_what_it_did(self) -> None: + frame = _frame(n_piled=60) + receipt = disaggregate_uk_age_top_code(frame, band_populations=_BANDS) + assert receipt["stage"] == "uk_age_tail_disaggregation" + assert receipt["top_code"] == UK_AGE_TOP_CODE + assert sum(receipt["assigned_unweighted"].values()) == 60 + assert set(receipt["achieved_weighted"]) == {"MALE", "FEMALE"} + assert receipt["draw_key"].startswith("person_source_id") + + +class TestRefusals: + def test_an_already_disaggregated_surface_is_refused(self) -> None: + frame = _frame(n_piled=10) + disaggregate_uk_age_top_code(frame, band_populations=_BANDS) + with pytest.raises(ValueError, match="already has ages above"): + disaggregate_uk_age_top_code(frame, band_populations=_BANDS) + + def test_no_pile_is_refused(self) -> None: + frame = _frame(n_piled=0) + with pytest.raises(ValueError, match="no persons at the top-code"): + disaggregate_uk_age_top_code(frame, band_populations=_BANDS) + + @pytest.mark.parametrize("bad", [0.0, -1.0, float("nan")]) + def test_a_non_positive_band_population_is_refused(self, bad: float) -> None: + bands = dict(_BANDS) + bands[("MALE", "85_89")] = bad + with pytest.raises(ValueError, match="missing or"): + disaggregate_uk_age_top_code(_frame(), band_populations=bands) + + def test_a_missing_band_population_is_refused(self) -> None: + bands = {key: value for key, value in _BANDS.items() if key[1] != "90_plus"} + with pytest.raises(ValueError, match="missing or"): + disaggregate_uk_age_top_code(_frame(), band_populations=bands) + + def test_an_unexpected_gender_label_is_refused(self) -> None: + frame = _frame(n_piled=4) + person = frame.table("person") + person.loc[person.index[-1], "gender"] = "OTHER" + with pytest.raises(ValueError, match="unexpected gender labels"): + disaggregate_uk_age_top_code(frame, band_populations=_BANDS) + + +class TestCommittedResource: + def test_the_committed_resource_loads_all_six_cells(self) -> None: + populations = load_uk_age_tail_band_populations() + assert set(populations) == set(_BANDS) + assert all(value > 0 for value in populations.values()) + + def test_every_cell_names_its_register_target(self) -> None: + # The target id is the drift check: it is how a reader confirms the + # imputation source and the calibration denominator are one fact. + payload = json.loads( + ( + REPO_ROOT + / "packages/microcosm-build/src/microcosm/build/uk" + / "ons_age_tail_band_populations.json" + ).read_text(encoding="utf-8") + ) + for gender, cells in payload["bands"].items(): + for band, cell in cells.items(): + assert cell["target_id"] == f"ons.population.{gender.lower()}_{band}" + + def test_a_cell_naming_the_wrong_target_is_refused(self, tmp_path: Path) -> None: + payload = { + "schema_version": 1, + "bands": { + gender: { + band: { + "population": _BANDS[(gender, band)], + "target_id": f"ons.population.{gender.lower()}_{band}", + } + for band, _, _ in UK_AGE_TAIL_BANDS + } + for gender in ("MALE", "FEMALE") + }, + } + payload["bands"]["MALE"]["85_89"]["target_id"] = "ons.population.male_80_84" + path = tmp_path / "bands.json" + path.write_text(json.dumps(payload), encoding="utf-8") + with pytest.raises(ValueError, match="drift check"): + load_uk_age_tail_band_populations(str(path)) + + def test_a_missing_band_in_the_resource_is_refused(self, tmp_path: Path) -> None: + payload = { + "schema_version": 1, + "bands": { + "MALE": {"80_84": {"population": 1.0, "target_id": "x"}}, + "FEMALE": {}, + }, + } + path = tmp_path / "bands.json" + path.write_text(json.dumps(payload), encoding="utf-8") + with pytest.raises(ValueError, match="must carry exactly"): + load_uk_age_tail_band_populations(str(path)) + + +class TestStageTransform: + def _stage(self): + manifest = SourceManifest.from_mapping( + json.loads( + ( + REPO_ROOT + / "packages/microcosm-build/src/microcosm/build/uk" + / "source_stages.json" + ).read_text(encoding="utf-8") + ) + ) + return next(stage for stage in manifest.stages if stage.stage == "age_tail") + + def test_the_committed_stage_runs_and_receipts(self) -> None: + transform = UKAgeTailStageTransform(stage=self._stage()) + frame = _frame(n_piled=50) + returned = transform(frame) + assert returned is frame + assert transform.last_result is not None + assert transform.last_result["piled_persons"] == 50 + assert transform.checkpoint_metadata()["evidence"] is transform.last_result + + def test_the_stage_declares_no_new_columns(self) -> None: + # It rewrites `age`; declaring an output would claim a column the + # manifest already attributes to frs_spine. + assert UKAgeTailStageTransform.output_columns() == () + + def test_parameter_drift_is_refused(self) -> None: + stage = self._stage() + operation = stage.operations[0] + object.__setattr__( + operation, "parameters", {**operation.parameters, "top_code": 75} + ) + transform = UKAgeTailStageTransform(stage=stage) + with pytest.raises(ValueError, match="parameter drift"): + transform(_frame()) + + def test_an_undeclared_parameter_is_refused(self) -> None: + stage = self._stage() + operation = stage.operations[0] + object.__setattr__( + operation, + "parameters", + {**operation.parameters, "unimplemented_knob": 1}, + ) + transform = UKAgeTailStageTransform(stage=stage) + with pytest.raises(ValueError, match="does not implement"): + transform(_frame()) diff --git a/packages/microcosm-build/tests/test_uk_frs_spine.py b/packages/microcosm-build/tests/test_uk_frs_spine.py index 8960b22c..1c72bcdc 100644 --- a/packages/microcosm-build/tests/test_uk_frs_spine.py +++ b/packages/microcosm-build/tests/test_uk_frs_spine.py @@ -1678,6 +1678,17 @@ def test_recorded_water_without_a_gross_bill_cell_is_not_scaled(self) -> None: frame.columns = [c.lower() for c in frame.columns] assert scottish_water_and_sewerage_weekly(frame).iloc[0] == pytest.approx(3.0) + def test_sewerage_without_an_observable_discount_is_refused(self) -> None: + # The domain claim in the docstring — that a household with no gross + # water bill also carries no gross sewerage — is what makes the 1.0 + # fallback safe. A vintage refresh that breaks it must refuse at build + # time rather than silently pay sewerage at gross, which would flow + # into council_tax through the netting. + frame = self._frame(CWATAMTD=3.0, CWATAMT1=0.0, CSEWAMT1=5.0) + frame.columns = [c.lower() for c in frame.columns] + with pytest.raises(ValueError, match="no discount factor is observable"): + scottish_water_and_sewerage_weekly(frame) + def test_household_without_council_tax_cells_is_zero(self) -> None: # 21 Scottish households carry no council-tax cells at all. frame = self._frame(CWATAMTD="", CWATAMT1="", CSEWAMT1="") diff --git a/packages/microcosm-build/tests/test_uk_h5_payload_compare.py b/packages/microcosm-build/tests/test_uk_h5_payload_compare.py index 613af7ea..3d8f899a 100644 --- a/packages/microcosm-build/tests/test_uk_h5_payload_compare.py +++ b/packages/microcosm-build/tests/test_uk_h5_payload_compare.py @@ -280,12 +280,18 @@ def _register(tmp_path: Path, *entries: dict) -> Path: return path -def _entry(identifier: str, *, surface: str, columns: list[str]) -> dict: +def _entry( + identifier: str, + *, + surface: str, + columns: list[str], + expectation: str = "column_differs", +) -> dict: return { "id": identifier, "class": "mechanism_change", "scope": {"surface": surface, "columns": columns, "entities": ["household"]}, - "expectation": "column_differs", + "expectation": expectation, "magnitude_evidence": "disclosure-safe magnitude statement", "evidence": "experiments/686-uk-spine-swap-receipts.md#r0", "adjudicator": "juaristi22", @@ -452,3 +458,99 @@ def test_root_attr_difference_must_be_signed(self, tmp_path: Path) -> None: assert report["signed"]["unsigned_root_attrs"] == [ "populace_household_weight_kind" ] + + +class TestRegisterSurfaceBridge: + """The register's real entries must reach this comparator (#747 review). + + Every committed entry is scoped to a share, totals or counts surface, so + a payload_column-only lookup matched nothing and --structure-only — the + swap-acceptance verdict — could only ever exit 1. + """ + + def test_a_share_scoped_entry_signs_a_payload_value_difference( + self, tmp_path: Path + ) -> None: + pytest.importorskip("tables") + left = _write(tmp_path / "left.h5", _tables()) + right = _write(tmp_path / "right.h5", _tables(weight_two=SENTINEL_VALUE)) + register = _register( + tmp_path, + _entry( + "share-scoped", + surface="nonzero_shares", + columns=["household_weight"], + ), + ) + out = tmp_path / "report.json" + + code = COMPARATOR.main( + [ + str(left), + str(right), + "--structure-only", + "--signed-differences", + str(register), + "--json-out", + str(out), + ] + ) + + report = json.loads(out.read_text(encoding="utf-8")) + assert code == 0 + assert report["structure_only_ok"] is True + assert report["signed"]["matched_ids"] == ["share-scoped"] + assert report["signed"]["unsigned_columns"] == [] + + def test_the_committed_register_signs_a_real_adjudicated_column( + self, tmp_path: Path + ) -> None: + # The end-to-end shape of the finding, against the real committed + # register: `savings` is adjudicated on the share surface, so a + # payload value difference in it must reach a signed verdict. + pytest.importorskip("tables") + left_tables = _tables() + right_tables = _tables() + left_tables["household"]["savings"] = [1000.0, 2000.0] + right_tables["household"]["savings"] = [1000.0, SENTINEL_VALUE] + left = _write(tmp_path / "left.h5", left_tables) + right = _write(tmp_path / "right.h5", right_tables) + out = tmp_path / "report.json" + + code = COMPARATOR.main( + [str(left), str(right), "--structure-only", "--json-out", str(out)] + ) + + report = json.loads(out.read_text(encoding="utf-8")) + assert report["signed"]["unsigned_columns"] == [] + assert "was-wealth-qrf-incidence" in report["signed"]["matched_ids"] + assert code == 0 + + def test_a_structural_expectation_does_not_sign_a_value_difference( + self, tmp_path: Path + ) -> None: + pytest.importorskip("tables") + left = _write(tmp_path / "left.h5", _tables()) + right = _write(tmp_path / "right.h5", _tables(weight_two=SENTINEL_VALUE)) + register = _register( + tmp_path, + _entry( + "net-new-only", + surface="nonzero_shares", + columns=["household_weight"], + expectation="column_missing_in_reference", + ), + ) + + assert ( + COMPARATOR.main( + [ + str(left), + str(right), + "--structure-only", + "--signed-differences", + str(register), + ] + ) + == 1 + ) diff --git a/packages/microcosm-build/tests/test_uk_identity_stability_receipts.py b/packages/microcosm-build/tests/test_uk_identity_stability_receipts.py new file mode 100644 index 00000000..90fa0438 --- /dev/null +++ b/packages/microcosm-build/tests/test_uk_identity_stability_receipts.py @@ -0,0 +1,127 @@ +"""The E7 identity receipt must never pass vacuously (#747 review). + +A receipt's whole value is that it certifies something. The E7 branch +recomputed the support-channel layer only when the synthetic flag was +present and otherwise returned an empty recomputation — over which the +mismatch loops never ran, so the receipt reported +``identical_under_permutation: true`` and ``matches_stored_columns: true`` +with exit 0 on an artifact where nothing had been checked. These tests pin +the refusals that replaced that silence. +""" + +from __future__ import annotations + +import importlib.util +from pathlib import Path + +import pandas as pd +import pytest + +from microcosm.build.uk_runtime.national_frame import uk_national_frame +from microcosm.frame import WeightKind + +_TOOL_PATH = ( + Path(__file__).resolve().parents[3] / "tools" / "verify_uk_identity_stability.py" +) + + +def _load_tool(): + spec = importlib.util.spec_from_file_location( + "verify_uk_identity_stability", _TOOL_PATH + ) + assert spec is not None + assert spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _frame( + *, + synthetic: bool = True, + source_keys: bool = True, + stored_channel: bool = True, +): + """A two-household frame carrying the E7 support-channel layer.""" + + person = pd.DataFrame( + { + "person_id": [1, 2, 3], + "person_benunit_id": [10, 10, 20], + "person_household_id": [100, 100, 200], + } + ) + benunit = pd.DataFrame({"benunit_id": [10, 20]}) + household = pd.DataFrame( + { + "household_id": [100, 200], + "household_weight": [10.0, 20.0], + } + ) + if synthetic: + household["household_is_spi_synthetic"] = [False, True] + if source_keys: + household["source_year"] = [2024, 2024] + household["source_household_id"] = [100, 100] + if stored_channel: + household["household_support_channel"] = ["frs", "spi"] + household["household_support_clone_index"] = [0, 1] + household["source_household_key"] = ["2024:100", "2024:100"] + person["person_support_channel"] = ["frs", "frs", "spi"] + benunit["benunit_support_channel"] = ["frs", "spi"] + return uk_national_frame( + person=person, + benunit=benunit, + household=household, + time_period="2024", + weight_kind=WeightKind.DESIGN, + ) + + +class TestE7Receipt: + def test_a_complete_artifact_receipts_green(self) -> None: + tool = _load_tool() + receipt = tool.e7_identity_receipt(_frame(), permutation_seed=7) + assert receipt["identical_under_permutation"] is True + assert receipt["matches_stored_columns"] is True + # The receipt names what it compared, so a green result is auditable. + assert receipt["columns_compared"]["household"] == [ + "household_support_channel", + "household_support_clone_index", + "source_household_key", + ] + + def test_an_artifact_without_the_e7_layer_is_refused(self) -> None: + # Previously this returned a green receipt over an empty comparison. + tool = _load_tool() + with pytest.raises(ValueError, match="no\\s+household_is_spi_synthetic"): + tool.e7_identity_receipt(_frame(synthetic=False), permutation_seed=7) + + def test_missing_source_keys_are_refused_not_skipped(self) -> None: + # Skipping the source key would silently shrink the receipt's + # coverage while still reporting a pass. + tool = _load_tool() + with pytest.raises(ValueError, match="source key cannot be recomputed"): + tool.e7_identity_receipt(_frame(source_keys=False), permutation_seed=7) + + def test_a_column_absent_from_the_store_is_a_mismatch(self) -> None: + # The store not carrying a column this receipt certifies is a failed + # comparison, not a narrower one. + tool = _load_tool() + receipt = tool.e7_identity_receipt( + _frame(stored_channel=False), permutation_seed=7 + ) + assert receipt["identical_under_permutation"] is True + assert receipt["matches_stored_columns"] is False + assert ( + "household_support_channel" + in receipt["stored_column_mismatches"]["household"] + ) + + def test_a_corrupted_stored_channel_is_caught(self) -> None: + tool = _load_tool() + frame = _frame() + household = frame.table("household") + household.loc[household.index[-1], "household_support_channel"] = "frs" + receipt = tool.e7_identity_receipt(frame, permutation_seed=7) + assert receipt["matches_stored_columns"] is False diff --git a/packages/microcosm-build/tests/test_uk_signed_differences.py b/packages/microcosm-build/tests/test_uk_signed_differences.py index d3a5df00..6f4f7497 100644 --- a/packages/microcosm-build/tests/test_uk_signed_differences.py +++ b/packages/microcosm-build/tests/test_uk_signed_differences.py @@ -177,7 +177,9 @@ class TestLookup: def test_matching_finds_the_signing_entry(self) -> None: register = load_uk_spine_swap_signed_differences() found = register.matching( - surface="nonzero_shares", column="water_and_sewerage_charges" + surface="nonzero_shares", + column="water_and_sewerage_charges", + expectation="column_differs", ) assert found is not None assert found.id == "scottish-water-incumbent-nan-zeroing" @@ -187,16 +189,29 @@ def test_a_column_signed_on_one_surface_is_not_signed_on_another(self) -> None: # the nonzero share, and the share entry is a different adjudication. register = load_uk_spine_swap_signed_differences() weighted = register.matching( - surface="weighted_totals", column="water_and_sewerage_charges" + surface="weighted_totals", + column="water_and_sewerage_charges", + expectation="column_differs", ) assert weighted is not None assert weighted.id == "scottish-water-sewerage-successor-level" - assert register.matching(surface="entity_counts", column="household") is None + assert ( + register.matching( + surface="entity_counts", + column="household", + expectation="column_differs", + ) + is None + ) def test_unsigned_column_returns_none(self) -> None: register = load_uk_spine_swap_signed_differences() assert ( - register.matching(surface="nonzero_shares", column="employment_income") + register.matching( + surface="nonzero_shares", + column="employment_income", + expectation="column_differs", + ) is None ) @@ -213,9 +228,108 @@ def test_surface_wide_entry_covers_any_column(self) -> None: adjudicator="juaristi22", adjudicated_on="2026-08-22", ) - assert entry.covers(surface="entity_counts", column="person") - assert entry.covers(surface="entity_counts", column="benunit") - assert not entry.covers(surface="nonzero_shares", column="person") + assert entry.covers( + surface="entity_counts", column="person", expectation="count_differs" + ) + assert entry.covers( + surface="entity_counts", column="benunit", expectation="count_differs" + ) + assert not entry.covers( + surface="nonzero_shares", column="person", expectation="count_differs" + ) + + +class TestExpectationAwareCoverage: + """The expectation is consulted at lookup, not merely validated at load. + + Without this, an entry adjudicated for a column *appearing* would also + sign an arbitrarily large *value* divergence in that same column — the + register's own scope note names that failure mode. + """ + + def _entry(self, *, surface: str, expectation: str) -> UKSignedDifference: + return UKSignedDifference( + id="entry", + difference_class="net_new_column", + surface=surface, + expectation=expectation, + columns=("num_bedrooms",), + entities=("household",), + magnitude_evidence="evidence", + evidence="experiments/686-uk-spine-swap-receipts.md", + adjudicator="juaristi22", + adjudicated_on="2026-08-22", + ) + + def test_a_net_new_entry_does_not_sign_a_value_divergence(self) -> None: + entry = self._entry( + surface="nonzero_shares", expectation="column_missing_in_reference" + ) + assert entry.covers( + surface="nonzero_shares", + column="num_bedrooms", + expectation="column_missing_in_reference", + ) + assert not entry.covers( + surface="nonzero_shares", + column="num_bedrooms", + expectation="column_differs", + ) + + def test_a_value_entry_does_not_sign_a_structural_difference(self) -> None: + entry = self._entry(surface="nonzero_shares", expectation="column_differs") + assert not entry.covers( + surface="nonzero_shares", + column="num_bedrooms", + expectation="column_missing_in_reference", + ) + + def test_a_share_entry_bridges_to_the_payload_surface(self) -> None: + # The share instrument and the payload comparator read the same + # adjudicated fact through different measurements, so one signature + # covers both — this is what lets --structure-only reach a verdict. + entry = self._entry(surface="nonzero_shares", expectation="column_differs") + assert entry.covers( + surface="payload_column", + column="num_bedrooms", + expectation="column_differs", + ) + + def test_the_bridge_does_not_reach_structural_surfaces(self) -> None: + entry = self._entry( + surface="nonzero_shares", expectation="column_missing_in_reference" + ) + assert not entry.covers( + surface="payload_column", + column="num_bedrooms", + expectation="column_missing_in_reference", + ) + counts = self._entry(surface="entity_counts", expectation="count_differs") + assert not counts.covers( + surface="payload_column", + column="num_bedrooms", + expectation="count_differs", + ) + + def test_the_committed_register_covers_the_payload_surface(self) -> None: + # Every committed value adjudication must be reachable from the + # payload comparator, or --structure-only can never pass. + register = load_uk_spine_swap_signed_differences() + for difference in register.differences: + if ( + difference.surface != "nonzero_shares" + or difference.expectation != "column_differs" + ): + continue + for column in difference.columns: + assert ( + register.matching( + surface="payload_column", + column=column, + expectation="column_differs", + ) + is not None + ) class TestValidation: diff --git a/packages/microcosm-build/tests/test_uk_spi_income.py b/packages/microcosm-build/tests/test_uk_spi_income.py index 8219e03c..fd1dc0b3 100644 --- a/packages/microcosm-build/tests/test_uk_spi_income.py +++ b/packages/microcosm-build/tests/test_uk_spi_income.py @@ -298,7 +298,11 @@ def test_spi_qrf_stages_use_typed_weights_and_restore_gross_savings( SPI_HMRC_TOTAL_INVESTMENT_INCOME_COLUMN, HMRC_SPI_ASSESSABLE_INCOME_COLUMN, ) - assert result.person.loc[base_people, list(unavailable_on_frs)].isna().all().all() + # These full concepts are unmeasured on the FRS instrument, so the FRS + # channel carries the adjudicated stage-time zero rather than NaN — the + # artifact must load through an engine that refuses NaN inputs, and the + # calibration seam's finiteness fence stays fail-loud because of it. + assert result.person.loc[base_people, list(unavailable_on_frs)].eq(0.0).all().all() expected_employed = ( np.maximum( @@ -395,7 +399,10 @@ def test_spi_stage2_does_not_require_frs_other_investment_income( channel = support_channel_column("person") spi_people = result.person[channel] == "spi" - assert result.person.loc[~spi_people, "other_investment_income"].isna().all() + # The FRS channel carries the stage-time zero, not NaN: the column is a + # full concept the FRS instrument does not measure, and the artifact has + # to load through an engine that refuses NaN inputs. + assert result.person.loc[~spi_people, "other_investment_income"].eq(0.0).all() assert result.person.loc[spi_people, "other_investment_income"].eq(25.0).all() @@ -629,3 +636,47 @@ def test_spi_qrf_requires_current_donor_filename(tmp_path) -> None: with pytest.raises(ValueError, match=SPI_DONOR_FILENAME): impute_uk_spi_income_support(support, donor_path) + + +def test_the_spi_channel_ships_no_structural_nan_on_the_frs_channel( + monkeypatch, + tmp_path, +) -> None: + """The FRS channel carries stage-time zero, never NaN (#747). + + The SPI stage populates full-concept income columns the FRS instrument + does not measure. Shipping NaN on the FRS rows was assessment-era + honesty that made the artifact unloadable: the engine's ``validate()`` + refuses NaN inputs, and the calibration seam's finiteness fence refuses + the frame — so the first armed campaign had to zero-fill twelve person + columns outside the build before it could calibrate at all. Zero is the + adjudicated stage-time semantics, and the auxiliary-crosswalk guard + already stops the QRF mistaking the fill for measured data. + """ + + pytest.importorskip("policyengine_uk") + support = _dead_support() + donor_path = tmp_path / SPI_DONOR_FILENAME + _write_donor(donor_path) + monkeypatch.setattr(spi_income, "QRF", _FakeQRF) + _bypass_reviewed_donor_identity(monkeypatch) + + result = impute_uk_spi_income_support( + support, + donor_path, + seed=9, + n_estimators=3, + donor_sample_size=None, + ) + + person = result.person + nan_columns = sorted( + column + for column in person.columns + if person[column].dtype.kind == "f" and bool(person[column].isna().any()) + ) + assert nan_columns == [], ( + f"the SPI stage left NaN in {nan_columns}; the artifact must ship " + "stage-time zeros so the engine can load it and the calibration " + "seam's finiteness fence can stay fail-loud" + ) diff --git a/packages/microcosm-build/tests/test_uk_spi_spine.py b/packages/microcosm-build/tests/test_uk_spi_spine.py index fe8e681c..82569c00 100644 --- a/packages/microcosm-build/tests/test_uk_spi_spine.py +++ b/packages/microcosm-build/tests/test_uk_spi_spine.py @@ -392,7 +392,10 @@ def test_spi_income_zero_initializes_frs_charity_and_redraws_dividends_after_sta assert person.loc[base, "gift_aid"].tolist() == [0.0, 0.0] assert person.loc[base, "charitable_investment_gifts"].tolist() == [0.0, 0.0] - assert person.loc[base, "hmrc_spi_employment_benefits"].isna().all() + # Unmeasured on the FRS instrument, so the base channel carries the + # stage-time zero — the explicit initialization above and this fill are + # now the same semantics, and the artifact ships no NaN. + assert person.loc[base, "hmrc_spi_employment_benefits"].eq(0.0).all() assert person.loc[base, "dividend_income"].tolist() == [100.0, 101.0] assert person.loc[spi, "dividend_income"].tolist() == [100.0, 101.0] assert _FakeQRF.events[1][0] == "stage2" diff --git a/packages/microcosm-build/tests/test_uk_spine_parity_instrument.py b/packages/microcosm-build/tests/test_uk_spine_parity_instrument.py index 6ad8a949..fd74b1b7 100644 --- a/packages/microcosm-build/tests/test_uk_spine_parity_instrument.py +++ b/packages/microcosm-build/tests/test_uk_spine_parity_instrument.py @@ -625,3 +625,148 @@ def test_an_out_of_range_band_yields_no_verdict(self, tmp_path: Path) -> None: ) == 2 ) + + +class TestReviewFindings: + """Regressions for the review findings on the proof machinery (#747). + + Each of these let the instrument reach a wrong verdict — or no verdict — + for reasons unrelated to whether the spine's data is right. + """ + + def test_a_candidate_without_a_source_identity_is_refused( + self, tmp_path: Path + ) -> None: + # The anti-self-comparison fence compares identities, so an anonymous + # candidate would pass it vacuously. + payload = _candidate_from_reference() + del payload["source"] + candidate = _write(tmp_path / "c.json", payload) + assert ( + tool_main := _load_tool().main( + [ + "--candidate-json", + str(candidate), + "--register", + str(_register(tmp_path)), + ] + ) + ) == 2, tool_main + + def test_a_candidate_without_a_sha256_is_refused(self, tmp_path: Path) -> None: + payload = _candidate_from_reference() + payload["source"] = {"filename": "microcosm_uk_2024.h5"} + candidate = _write(tmp_path / "c.json", payload) + assert ( + _load_tool().main( + [ + "--candidate-json", + str(candidate), + "--register", + str(_register(tmp_path)), + ] + ) + == 2 + ) + + def test_a_zero_reference_total_still_reaches_a_verdict( + self, tmp_path: Path + ) -> None: + # A new-in-candidate column with a nonzero weighted total has no + # finite relative delta; reporting float("inf") made json.dumps + # refuse the receipt and turned a real divergence into "no verdict". + tool = _load_tool() + candidate = _write(tmp_path / "c.json", _candidate_from_reference()) + left = _write(tmp_path / "ref.json", {"identity": {}, "totals": {"col": 0.0}}) + right = _write( + tmp_path / "cand.json", {"identity": {}, "totals": {"col": 125.0}} + ) + receipt = tmp_path / "receipt.json" + + code = tool.main( + [ + "--candidate-json", + str(candidate), + "--register", + str(_register(tmp_path)), + "--reference-weighted-totals", + str(left), + "--candidate-weighted-totals", + str(right), + "--receipt-json", + str(receipt), + ] + ) + + assert code == 1 + report = json.loads(receipt.read_text(encoding="utf-8")) + assert report["verdict"] == "defect" + entry = report["weighted_totals"]["differing"]["col"] + assert entry["reference_total_zero"] is True + assert entry["relative_delta"] is None + + def test_strict_accepts_an_entry_matching_a_within_band_column( + self, tmp_path: Path + ) -> None: + # A signed column whose divergence has since shrunk under the band is + # still a matched entry: the difference it adjudicates is real and + # reported. --strict exists to catch entries matching nothing at all. + tool = _load_tool() + column = _first_household_column() + payload = _candidate_from_reference() + payload["nonzero_shares"][column] += 0.01 + candidate = _write(tmp_path / "c.json", payload) + register = _register( + tmp_path, + _entry("shrunk-since-signing", surface="nonzero_shares", columns=[column]), + ) + receipt = tmp_path / "receipt.json" + + code = tool.main( + [ + "--candidate-json", + str(candidate), + "--register", + str(register), + "--strict", + "--receipt-json", + str(receipt), + ] + ) + + assert code == 0 + report = json.loads(receipt.read_text(encoding="utf-8")) + assert report["strict_failure"] is False + assert report["register"]["unused_ids"] == [] + assert report["register"]["matched_ids"] == ["shrunk-since-signing"] + assert column in report["nonzero_shares"]["within_band"] + + def test_a_structural_expectation_does_not_sign_a_value_divergence( + self, tmp_path: Path + ) -> None: + tool = _load_tool() + column = _first_household_column() + payload = _candidate_from_reference() + payload["nonzero_shares"][column] += 0.25 + candidate = _write(tmp_path / "c.json", payload) + register = _register( + tmp_path, + _entry( + "net-new-only", + surface="nonzero_shares", + columns=[column], + expectation="column_missing_in_reference", + ), + ) + + assert ( + tool.main( + [ + "--candidate-json", + str(candidate), + "--register", + str(register), + ] + ) + == 1 + ) diff --git a/tools/build_uk_frs_spine.py b/tools/build_uk_frs_spine.py index 59921589..eb87ba75 100644 --- a/tools/build_uk_frs_spine.py +++ b/tools/build_uk_frs_spine.py @@ -32,6 +32,7 @@ sha256_argument, write_error_receipt, ) +from microcosm.build.uk_runtime.age_tail import UKAgeTailStageTransform from microcosm.build.uk_runtime.cgt_imputation import uk_cgt_spine_stage_transform from microcosm.build.uk_runtime.cgt_structure import ( UKCGTBandDonorStageTransform, @@ -114,6 +115,7 @@ "hmrc_cgt_gains_spine", "salary_sacrifice", "student_loans", + "age_tail", ) @@ -840,6 +842,10 @@ def main(argv: list[str] | None = None) -> int: stage=stages_by_name["student_loans"], calibration_year=frs_release.calibration_year, ) + if "age_tail" in stage_names: + implementations["age_tail"] = UKAgeTailStageTransform( + stage=stages_by_name["age_tail"] + ) plan = country_stage_plan( spec, implementations, @@ -894,11 +900,18 @@ def main(argv: list[str] | None = None) -> int: "cgt_band_donors", "salary_sacrifice", "student_loans", + "age_tail", ): e8_implementation = implementations.get(e8_stage_name) e8_last_result = getattr(e8_implementation, "last_result", None) if e8_last_result is not None: - e8_stage_evidence[e8_stage_name] = e8_last_result.evidence() + # age_tail's receipt is already the evidence mapping; the E8 + # transforms carry a result object that produces one. + e8_stage_evidence[e8_stage_name] = ( + e8_last_result + if isinstance(e8_last_result, dict) + else e8_last_result.evidence() + ) if e8_stage_evidence: sidecar["stage_evidence"] = e8_stage_evidence atomic_write_json(sidecar_path, sidecar) diff --git a/tools/compare_uk_h5_payload.py b/tools/compare_uk_h5_payload.py index 2985824c..b24ca9d0 100644 --- a/tools/compare_uk_h5_payload.py +++ b/tools/compare_uk_h5_payload.py @@ -335,7 +335,12 @@ def apply_structure_only_verdict( The swap comparison expects the control and candidate artifacts to have an identical surface and to differ only where a difference is signed, so this keeps every structural predicate strict and requires each differing - column and root attribute to name a register entry. + column and root attribute to name a register entry. Lookup is + expectation-aware: a value mismatch is covered by a ``column_differs`` + entry on ``payload_column`` or, through the register's payload bridge, on + a value-bearing surface (``nonzero_shares``, ``weighted_totals``) — the + share instrument and this one read the same adjudicated fact. Structural + expectations never excuse a value difference. ``payload_identical`` is left exactly as computed, so a structure-only receipt stays comparable with a full-mode one. @@ -346,7 +351,9 @@ def apply_structure_only_verdict( for key, table in report["tables"].items(): signed_ids: dict[str, str | None] = {} for column in sorted(table["value_mismatch_rows_by_column"]): - entry = register.matching(surface="payload_column", column=column) + entry = register.matching( + surface="payload_column", column=column, expectation="column_differs" + ) signed_ids[column] = entry.id if entry else None if entry is None: unsigned_columns.append(f"{key}.{column}") @@ -357,7 +364,9 @@ def apply_structure_only_verdict( unsigned_attrs: list[str] = [] attr_signed: dict[str, str | None] = {} for name in report["root_attrs"]["attrs_with_differing_values"]: - entry = register.matching(surface="root_attr", column=name) + entry = register.matching( + surface="root_attr", column=name, expectation="column_differs" + ) attr_signed[name] = entry.id if entry else None if entry is None: unsigned_attrs.append(name) diff --git a/tools/verify_uk_identity_stability.py b/tools/verify_uk_identity_stability.py index 74f92f6b..e92ba9e8 100644 --- a/tools/verify_uk_identity_stability.py +++ b/tools/verify_uk_identity_stability.py @@ -514,22 +514,40 @@ def recompute(person_t, benunit_t, household_t) -> dict[str, pd.DataFrame]: person_out = pd.DataFrame(index=person_t["person_id"].to_numpy()) benunit_out = pd.DataFrame(index=benunit_t["benunit_id"].to_numpy()) + # An artifact without the synthetic flag carries no E7 layer, so + # there is nothing this receipt could certify about it. Returning an + # empty receipt here would report a vacuous pass — the mismatch loops + # never run over an empty recomputation — which is the one outcome a + # receipt must never produce. Refuse instead. if "household_is_spi_synthetic" not in household_t.columns: - return {} + raise ValueError( + "e7 identity receipt: the artifact carries no " + "household_is_spi_synthetic column, so the E7 support-channel " + "layer is absent and there is nothing to receipt. Run the " + "check against an artifact built with the SPI channel, or " + "drop --check e7 for this artifact." + ) synthetic = household_t["household_is_spi_synthetic"].astype(bool).to_numpy() channel = np.where(synthetic, "spi", "frs") household_out["household_support_channel"] = channel household_out["household_support_clone_index"] = np.where(synthetic, 1, 0) - if {"source_year", "source_household_id"} <= set(household_t.columns): - household_out["source_household_key"] = [ - f"{int(year)}:{int(source)}" - for year, source in zip( - household_t["source_year"].to_numpy(), - household_t["source_household_id"].to_numpy(), - strict=True, - ) - ] + missing_keys = {"source_year", "source_household_id"} - set(household_t.columns) + if missing_keys: + raise ValueError( + "e7 identity receipt: the artifact carries the synthetic flag " + f"but not {sorted(missing_keys)}; the source key cannot be " + "recomputed, and skipping it would silently shrink the " + "receipt's coverage." + ) + household_out["source_household_key"] = [ + f"{int(year)}:{int(source)}" + for year, source in zip( + household_t["source_year"].to_numpy(), + household_t["source_household_id"].to_numpy(), + strict=True, + ) + ] # The channel is a household property; persons and benefit units # inherit it through membership, never redraw it. @@ -580,7 +598,11 @@ def recompute(person_t, benunit_t, household_t) -> dict[str, pd.DataFrame]: ): mismatches.setdefault(entity, []).append(column) stored_table = stored_tables[entity] - if column in stored_table.columns: + if column not in stored_table.columns: + # The store not carrying a column this receipt certifies is a + # failed comparison, not a narrower one. + stored_mismatches.setdefault(entity, []).append(column) + else: kept = stored_table[column].reindex(left.index) if not np.array_equal( left.to_numpy().astype(str), kept.to_numpy().astype(str) @@ -589,6 +611,9 @@ def recompute(person_t, benunit_t, household_t) -> dict[str, pd.DataFrame]: return { "check": "uk_e7_identity_stability", "permutation_seed": permutation_seed, + "columns_compared": { + entity: sorted(values.columns) for entity, values in original.items() + }, "identical_under_permutation": not mismatches, "permutation_mismatches": mismatches, "matches_stored_columns": not stored_mismatches, @@ -855,9 +880,7 @@ def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--input-h5", type=Path, required=True) parser.add_argument("--output", type=Path, required=True) - parser.add_argument( - "--check", choices=("e4", "e5", "e6", "e7", "e8"), default="e4" - ) + parser.add_argument("--check", choices=("e4", "e5", "e6", "e7", "e8"), default="e4") parser.add_argument("--permutation-seed", type=int, default=123) args = parser.parse_args() diff --git a/tools/verify_uk_spine_parity.py b/tools/verify_uk_spine_parity.py index 5d07313c..29807b9d 100644 --- a/tools/verify_uk_spine_parity.py +++ b/tools/verify_uk_spine_parity.py @@ -93,9 +93,27 @@ def _paths_alias(left: Path, right: Path) -> bool: def _candidate_identity(payload: Mapping[str, Any]) -> dict[str, Any]: + """The candidate's declared source identity, required to be present. + + The anti-self-comparison fence works by comparing this identity against + the pinned reference's, so an extraction that omits it would pass the + fence vacuously. Absence is therefore a refusal, not an empty dict. + """ + source = payload.get("source") if not isinstance(source, Mapping): - return {} + raise ValueError( + "candidate extraction carries no 'source' identity block; " + "the aliasing fence cannot run against an anonymous candidate. " + "Re-extract with build_uk_efrs_parity_reference.py " + "--candidate-h5 --emit-candidate-json." + ) + sha256 = source.get("sha256") + if not isinstance(sha256, str) or len(sha256) != 64: + raise ValueError( + "candidate extraction's source identity carries no sha256; " + "the aliasing fence cannot run without it." + ) return { key: source.get(key) for key in ("filename", "sha256", "size_bytes", "vintage", "period") @@ -116,7 +134,9 @@ def _compare_entity_counts( equal = expected == observed entry = {"reference": expected, "candidate": observed, "equal": equal} if not equal: - signed = register.matching(surface="entity_counts", column=entity) + signed = register.matching( + surface="entity_counts", column=entity, expectation="count_differs" + ) entry["signed_id"] = signed.id if signed else None if signed is None: unsigned.append(entity) @@ -147,7 +167,9 @@ def _compare_shares( "candidate": observed, "delta": delta, } - signed = register.matching(surface="nonzero_shares", column=column) + signed = register.matching( + surface="nonzero_shares", column=column, expectation="column_differs" + ) if abs(delta) <= band: # Reported, never dropped: the band decides what must be # adjudicated, not what the reader is allowed to see. @@ -162,7 +184,9 @@ def _compare_shares( def _missing(names: list[str], expectation: str) -> dict[str, Any]: out: dict[str, Any] = {} for column in sorted(names): - signed = register.matching(surface="nonzero_shares", column=column) + signed = register.matching( + surface="nonzero_shares", column=column, expectation=expectation + ) out[column] = { "entity": entities.get(column), "signed_id": signed.id if signed else None, @@ -207,16 +231,21 @@ def _compare_weighted_totals( observed = float(candidate_totals[column]) if expected == 0.0 and observed == 0.0: continue - if expected == 0.0: - relative = float("inf") - else: - relative = (observed - expected) / expected - if abs(relative) <= TOTALS_EPSILON: + # A zero reference total has no finite relative delta; report the + # fact as a flag rather than as float("inf"), which the JSON encoder + # (allow_nan=False) would refuse — turning a real divergence into + # "no verdict possible" instead of a verdict. + reference_zero = expected == 0.0 + relative = None if reference_zero else (observed - expected) / expected + if relative is not None and abs(relative) <= TOTALS_EPSILON: continue - signed = register.matching(surface="weighted_totals", column=column) + signed = register.matching( + surface="weighted_totals", column=column, expectation="column_differs" + ) # Deltas only: the absolute totals are licensed and stay outside. differing[column] = { "relative_delta": relative, + "reference_total_zero": reference_zero, "signed_id": signed.id if signed else None, } if signed is None: @@ -249,7 +278,7 @@ def verify_uk_spine_parity( candidate_identity = _candidate_identity(candidate) # The reference side must never be derived from the candidate: a copied # reference would make this pass by construction. - if candidate_identity.get("sha256") == reference.source.sha256: + if candidate_identity["sha256"] == reference.source.sha256: raise ValueError( "the candidate extraction names the pinned incumbent's own sha256; " "the reference side must be independent of the candidate." @@ -289,6 +318,11 @@ def verify_uk_spine_parity( } for section in ( shares_report["differing"], + # A signed column whose divergence has since shrunk under the band is + # a matched entry, not register rot: the difference it adjudicates is + # still real and still reported. --strict exists to catch entries + # matching nothing at all. + shares_report["within_band"], shares_report["missing_in_candidate"], shares_report["extra_in_candidate"], ): From ff46edf91aa0918233950434cef0c895fc433ce4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Mon, 24 Aug 2026 15:20:17 +0200 Subject: [PATCH 27/28] Re-pin the roster surfaces the new spine stage moves MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CI caught eight failures I should have: adding age_tail moves three derived surfaces, and I ran targeted suites instead of the shard. The stage roster is pinned in three places, and the E8 contiguity invariant asserted that the E8 block sat immediately before the certified pair — true only because E8 happened to be the last increment. What the invariant is actually protecting is that E8 stays contiguous and the certified pair stays at [-2:], where the frozen-copy lockstep test reads it; both survive a spine tail. The test now says that, with age_tail declared as a POST_E8 block so the next stage after it has to make the same decision deliberately. The release input-coverage manifest carries the source manifest's digest, so it regenerates: 145 required columns and 0 exclusions unchanged, confirming age_tail adds no column — it rewrites `age`, which was already required. The stale digest was also what failed the preflight battery's manifest-current gate. Verified against the packaging gate this time: wheels built, installed into a clean venv, suite run from there. Co-Authored-By: Claude Opus 5 --- .../uk/release_input_coverage_manifest.json | 22 +++++++++---------- .../tests/test_uk_source_stages.py | 22 +++++++++++++++++-- 2 files changed, 31 insertions(+), 13 deletions(-) diff --git a/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json b/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json index 88ff9db0..f6ae04ef 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json +++ b/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json @@ -473,7 +473,7 @@ "capital_gains" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "2aa227b9731361989c1c44ae5c670ea73f877d779031fa7739e83f8c65568857", + "source_manifest_sha256": "84d1c1c85f0172a9a396ef69d445ee33cf1340f62ff73c8653c399af40087bf4", "source_vintages": { "source": "HMRC Capital Gains Tax statistics, July 2025, Table 2.1a", "survey": "HMRC Capital Gains Tax statistics Table 2.1a and Advani-Summers capital-gains incidence" @@ -496,7 +496,7 @@ "capital_gains" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "2aa227b9731361989c1c44ae5c670ea73f877d779031fa7739e83f8c65568857", + "source_manifest_sha256": "84d1c1c85f0172a9a396ef69d445ee33cf1340f62ff73c8653c399af40087bf4", "source_vintages": { "source": "Advani and Summers (2020), Capital Gains and UK Inequality, CAGE Working Paper 465", "survey": "Family Resources Survey 2024-25, SPI synthetic support, and Advani-Summers capital-gains incidence" @@ -525,7 +525,7 @@ "required_mass_change_reason": "E5 source-stage transform preserves household rows and typed household weights; total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "2aa227b9731361989c1c44ae5c670ea73f877d779031fa7739e83f8c65568857", + "source_manifest_sha256": "84d1c1c85f0172a9a396ef69d445ee33cf1340f62ff73c8653c399af40087bf4", "source_vintages": { "source": "UK Data Service SN 8856 Effects of Taxes and Benefits household tab, DfT rail fare index, and public NHS activity/cost table.", "survey": "Effects of Taxes and Benefits 1977-2024 and NHS age-gender public table" @@ -545,7 +545,7 @@ "required_mass_change_reason": "E5 source-stage transform preserves household rows and typed household weights; total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "2aa227b9731361989c1c44ae5c670ea73f877d779031fa7739e83f8c65568857", + "source_manifest_sha256": "84d1c1c85f0172a9a396ef69d445ee33cf1340f62ff73c8653c399af40087bf4", "source_vintages": { "source": "UK Data Service SN 8856 Effects of Taxes and Benefits household tab and cited VAT anchor resource.", "survey": "Effects of Taxes and Benefits 1977-2024" @@ -590,7 +590,7 @@ "capital_gains" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "2aa227b9731361989c1c44ae5c670ea73f877d779031fa7739e83f8c65568857", + "source_manifest_sha256": "84d1c1c85f0172a9a396ef69d445ee33cf1340f62ff73c8653c399af40087bf4", "source_vintages": { "hmrc_surface": "2023-24", "mapped_build_period": "2024" @@ -604,7 +604,7 @@ "base_candidate_tier": "frs", "calibration_permitted": false, "canonical_source_manifest": "source_stages.json", - "canonical_source_manifest_sha256": "2aa227b9731361989c1c44ae5c670ea73f877d779031fa7739e83f8c65568857", + "canonical_source_manifest_sha256": "84d1c1c85f0172a9a396ef69d445ee33cf1340f62ff73c8653c399af40087bf4", "effective_mass_requirements": { "charitable_investment_gifts": { "mass_share_denominator": "all_person_effective_mass", @@ -715,7 +715,7 @@ "required_mass_change_reason": "E5 source-stage transform preserves household rows and typed household weights; total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "2aa227b9731361989c1c44ae5c670ea73f877d779031fa7739e83f8c65568857", + "source_manifest_sha256": "84d1c1c85f0172a9a396ef69d445ee33cf1340f62ff73c8653c399af40087bf4", "source_vintages": { "source": "UK Data Service SN 9468 Living Costs and Food Survey 2023-24 household/person tabs, NEED 2023 headline energy tables, Ofgem Q2 2026 unit rates, and WAS round-8 bridge donor.", "survey": "Living Costs and Food Survey 2023-24" @@ -736,7 +736,7 @@ "property_wealth" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "2aa227b9731361989c1c44ae5c670ea73f877d779031fa7739e83f8c65568857", + "source_manifest_sha256": "84d1c1c85f0172a9a396ef69d445ee33cf1340f62ff73c8653c399af40087bf4", "source_vintages": { "source": "MHCLG dwellings and ONS UK House Price Index December 2025 regional average prices.", "survey": "Public regional property reference" @@ -760,7 +760,7 @@ "employee_pension_contributions" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "2aa227b9731361989c1c44ae5c670ea73f877d779031fa7739e83f8c65568857", + "source_manifest_sha256": "84d1c1c85f0172a9a396ef69d445ee33cf1340f62ff73c8653c399af40087bf4", "source_vintages": { "source": "HMRC, Salary sacrifice reform for pension contributions effective from 6 April 2029", "survey": "Family Resources Survey 2024-25 salary-sacrifice respondents and HMRC salary-sacrifice reform analysis" @@ -782,7 +782,7 @@ "student_loan_plan" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "2aa227b9731361989c1c44ae5c670ea73f877d779031fa7739e83f8c65568857", + "source_manifest_sha256": "84d1c1c85f0172a9a396ef69d445ee33cf1340f62ff73c8653c399af40087bf4", "source_vintages": { "source": "Explore Education Statistics Table 6a, Higher education total", "survey": "Family Resources Survey 2024-25 and Student Loans Company borrower forecasts for England" @@ -814,7 +814,7 @@ "required_mass_change_reason": "E5 source-stage transform preserves household rows and typed household weights; total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "2aa227b9731361989c1c44ae5c670ea73f877d779031fa7739e83f8c65568857", + "source_manifest_sha256": "84d1c1c85f0172a9a396ef69d445ee33cf1340f62ff73c8653c399af40087bf4", "source_vintages": { "source": "Office for National Statistics Wealth and Assets Survey, UK Data Service SN 7215, DOI 10.5255/UKDA-SN-7215-20; local licensed 2006-22 household tab.", "survey": "Wealth and Assets Survey round 8" diff --git a/packages/microcosm-build/tests/test_uk_source_stages.py b/packages/microcosm-build/tests/test_uk_source_stages.py index c00ce208..aaaceec6 100644 --- a/packages/microcosm-build/tests/test_uk_source_stages.py +++ b/packages/microcosm-build/tests/test_uk_source_stages.py @@ -53,6 +53,13 @@ "salary_sacrifice", "student_loans", ] +# Spine stages that land after the E8 block. `age_tail` rewrites `age` and so +# must run downstream of every stage that conditions on it — student_loans +# reads it for cohort start years, the CGT stages for the adult carrier — which +# is the position the #623 calibration campaign exercised. +POST_E8_STAGE_NAMES = [ + "age_tail", +] UK_SOURCE_STAGE_NAMES = [ "frs_spine", *E3_STAGE_NAMES, @@ -61,6 +68,7 @@ *E6_STAGE_NAMES, *E7_STAGE_NAMES, *E8_STAGE_NAMES, + *POST_E8_STAGE_NAMES, "frs_hmrc_retained_leaves", "hmrc_spi_income", ] @@ -71,6 +79,7 @@ *E6_STAGE_NAMES, *E7_STAGE_NAMES, *E8_STAGE_NAMES, + *POST_E8_STAGE_NAMES, ] FROZEN_SOURCE_STAGES_SHA256 = ( "c0341af7166ae3a85a3c1164e7d9e880c4b4aec122f1a8fa90c73b46c596e1ea" @@ -141,12 +150,20 @@ def test_e7_block_sits_between_e6_and_e8(self) -> None: == E7_STAGE_NAMES ) - def test_e8_block_is_contiguous_before_certified_pair(self) -> None: + def test_e8_block_is_contiguous_and_the_certified_pair_stays_last(self) -> None: + # Two invariants, and only two: the E8 stages stay contiguous, and the + # certified pair stays at [-2:] (the frozen-copy lockstep test reads + # them from there). E8 being the *final* spine block was an artifact + # of it having been the last increment — the spine may grow a tail + # after it, as `age_tail` does, without either invariant moving. canonical = _load_json(CANONICAL_SOURCE_STAGES) names = [stage["stage"] for stage in canonical["stages"]] - assert names[-7:-2] == E8_STAGE_NAMES assert names[-2:] == ["frs_hmrc_retained_leaves", "hmrc_spi_income"] + spine = names[:-2] + start = spine.index(E8_STAGE_NAMES[0]) + assert spine[start : start + len(E8_STAGE_NAMES)] == E8_STAGE_NAMES + assert spine[start + len(E8_STAGE_NAMES) :] == POST_E8_STAGE_NAMES def test_copy_is_lockstep_with_frozen_original_except_citation_rewrites( self, @@ -269,6 +286,7 @@ def test_country_stage_plan_assembles_spine_plan(self) -> None: "hmrc_cgt_gains_spine": _identity, "salary_sacrifice": _identity, "student_loans": _identity, + "age_tail": _identity, "frs_hmrc_retained_leaves": _identity, "hmrc_spi_income": _identity, "hmrc_spi_income_fallback": _identity, From 76e39f9b77256766d7608c1f03e12db2f8cef118 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Tue, 25 Aug 2026 11:01:50 +0200 Subject: [PATCH 28/28] Honour the entity scope when the signed-differences register is consulted MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The re-review found the payload bridge made an old gap load-bearing: covers() matched on surface and column only, so a household-scoped adjudication could sign a same-named column on the person or benunit table once a per-table comparator started consulting it. That is an unsigned divergence becoming silently signed — the failure the register exists to prevent, reintroduced by the fix for it. Lookups now carry the entity, the payload comparator passes the store key it is iterating, and the bridge forwards the entity scope rather than widening it. The check earned its keep immediately: student_loan_balance was scoped to household and is a person column. The loader now also refuses a column-surface entry that names no columns or no entities, so the surface-wide form survives only on entity_counts, where the "column" is itself an entity name. Weighted-totals matches were already counted in matched_ids before unused is computed; a test now pins that a totals-scoped entry reads as matched rather than as rot, since the surface is dormant and the accounting would otherwise first be exercised on a calibrated candidate. Co-Authored-By: Claude Opus 5 --- ...uk-signed-difference-entity-scope.fixed.md | 1 + .../uk/spine_swap_signed_differences.json | 4 +- .../build/uk_runtime/signed_differences.py | 136 +++++++++++++----- .../tests/test_uk_h5_payload_compare.py | 101 ++++++++++++- .../tests/test_uk_signed_differences.py | 123 ++++++++++++++++ .../tests/test_uk_spine_parity_instrument.py | 47 ++++++ tools/compare_uk_h5_payload.py | 8 +- tools/verify_uk_spine_parity.py | 22 ++- 8 files changed, 402 insertions(+), 40 deletions(-) create mode 100644 changelog.d/686-uk-signed-difference-entity-scope.fixed.md diff --git a/changelog.d/686-uk-signed-difference-entity-scope.fixed.md b/changelog.d/686-uk-signed-difference-entity-scope.fixed.md new file mode 100644 index 00000000..565ccaae --- /dev/null +++ b/changelog.d/686-uk-signed-difference-entity-scope.fixed.md @@ -0,0 +1 @@ +Honour `scope.entities` when the signed-differences register is consulted (#686 re-review). Every committed entry names the entity its columns belong to, but `covers()` matched on surface and column alone — harmless while only the share surface consulted it, because a Frame's column names are globally unique, and load-bearing the moment the payload bridge let a lookup reach a comparator that iterates one table at a time. A household-scoped adjudication could then sign a same-named column on the person or benunit table: a previously-unsigned divergence becoming silently signed, which is the one failure the register exists to prevent. Lookups now take the entity, the payload comparator passes the store key it is iterating, and the bridge carries the entity scope across rather than widening it; a caller that cannot determine an entity passes none and gets no entity filtering, which is safe on the surfaces where the column namespace is already global. Honouring the scope immediately caught a mis-declared one: `student_loan_balance` was scoped to `household` and is a person column, so it is re-scoped here. Two related holes close with it — the loader now refuses an entry on a column surface that names no columns or no entities, so the surface-wide form stays available only to `entity_counts` where the "column" is itself an entity name and a blanket entry cannot absorb anything unrelated. diff --git a/packages/microcosm-build/src/microcosm/build/uk/spine_swap_signed_differences.json b/packages/microcosm-build/src/microcosm/build/uk/spine_swap_signed_differences.json index 8a424046..8e05de96 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/spine_swap_signed_differences.json +++ b/packages/microcosm-build/src/microcosm/build/uk/spine_swap_signed_differences.json @@ -153,11 +153,11 @@ "student_loan_balance" ], "entities": [ - "household" + "person" ] }, "expectation": "column_differs", - "magnitude_evidence": "Scoped apart from the benchmarked wealth columns because it has no donor benchmark to quote: the column is a fold of two WAS aggregates (total loans less total loans excluding Student Loans Company debt) and lands at a different entity grain from the household columns beside it, so no like-for-like donor share exists on the stage's cleaned frame. The observed divergence is +0.0296 on the unweighted share, incumbent 0.0197 against ours 0.0493 \u2014 the spine places student debt on about two and a half times as many carriers. Signed under the standing E5 adjudication as part of the same correlated-rank draw, with the absence of a benchmark stated rather than papered over; if the wealth stage is revisited, this is the column whose direction is unevidenced.", + "magnitude_evidence": "Scoped apart from the benchmarked wealth columns because it has no donor benchmark to quote: the column is a fold of two WAS aggregates (total loans less total loans excluding Student Loans Company debt) and is a person-entity column where the wealth columns beside it are household-entity, so no like-for-like donor share exists on the stage's cleaned frame. The observed divergence is +0.0296 on the unweighted share, incumbent 0.0197 against ours 0.0493 \u2014 the spine places student debt on about two and a half times as many carriers. Signed under the standing E5 adjudication as part of the same correlated-rank draw, with the absence of a benchmark stated rather than papered over; if the wealth stage is revisited, this is the column whose direction is unevidenced.", "evidence": "experiments/686-uk-spine-comparison-ledger.md#e5--wealth--signed-carried-forward", "adjudicator": "juaristi22", "adjudicated_on": "2026-08-24" diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/signed_differences.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/signed_differences.py index 3f4acb03..fc0fd654 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/signed_differences.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/signed_differences.py @@ -95,6 +95,16 @@ #: versa. _PAYLOAD_BRIDGE_SURFACES = frozenset({"nonzero_shares", "weighted_totals"}) +#: Surfaces whose "column" is a column of a named entity's table. An entry on +#: one of these must name both its columns and its entities: column names are +#: not unique across tables, so an unscoped entry would sign a same-named +#: column on an entity it was never adjudicated for. ``entity_counts`` is +#: excluded because its "column" *is* the entity, and ``root_attr`` because a +#: root attribute belongs to no entity. +_ENTITY_SCOPED_SURFACES = frozenset( + {"nonzero_shares", "weighted_totals", "payload_column"} +) + @dataclass(frozen=True) class UKSignedDifference: @@ -111,35 +121,67 @@ class UKSignedDifference: adjudicator: str adjudicated_on: str - def covers(self, *, surface: str, column: str, expectation: str) -> bool: + def covers( + self, + *, + surface: str, + column: str, + expectation: str, + entity: str | None = None, + ) -> bool: """Whether this entry signs the observed difference. - A difference is observed as ``(surface, column, expectation)`` and an - entry signs it only when the expectation matches — an entry adjudicated - for a column appearing (``column_missing_in_reference``) never excuses - that column's *values* diverging, and vice versa. The expectation is - therefore consulted at every lookup, not just validated at load. + A difference is observed as ``(surface, entity, column, expectation)`` + and an entry signs it only when all of them match. + + ``expectation`` is consulted at every lookup, not merely validated at + load: an entry adjudicated for a column *appearing* + (``column_missing_in_reference``) never excuses that column's *values* + diverging, and vice versa. + + ``entity`` matters because column names are not unique across tables. + A caller that knows which entity's table it is reading passes it, and + an entry only signs a column on an entity it names — so a + household-scoped adjudication cannot sign a same-named person column. + A caller that cannot determine an entity passes ``None`` and gets no + entity filtering; on the surfaces where that is possible the column + namespace is already global, and the payload comparator — the one + surface that genuinely iterates per table — always knows its entity. - An empty ``columns`` tuple is a surface-wide entry (used by - ``entity_counts``, where the "column" is an entity name). + An empty ``columns`` tuple is a surface-wide entry, which the loader + permits only on ``entity_counts``, where the "column" is an entity + name. One deliberate cross-surface rule: a ``column_differs`` entry on a value-bearing surface (``nonzero_shares``, ``weighted_totals``) also covers a ``payload_column`` value mismatch on the same column, because - both readings measure the same adjudicated fact. + both readings measure the same adjudicated fact. The bridge carries + the entity scope with it rather than widening it. """ if expectation != self.expectation: return False - if surface == self.surface: - return not self.columns or column in self.columns - if ( + bridged = ( surface == "payload_column" and expectation == "column_differs" and self.surface in _PAYLOAD_BRIDGE_SURFACES + ) + if surface != self.surface and not bridged: + return False + if self.columns and column not in self.columns: + return False + if not self.columns and surface in _ENTITY_SCOPED_SURFACES: + # Defence in depth: the loader refuses these, so reaching here + # would mean a register built by another path. + return False + if ( + entity is not None + and surface in _ENTITY_SCOPED_SURFACES + and self.entities + and entity not in self.entities ): - return not self.columns or column in self.columns - return False + return False + return True @dataclass(frozen=True) @@ -167,13 +209,21 @@ def by_id(self, identifier: str) -> UKSignedDifference | None: return None def matching( - self, *, surface: str, column: str, expectation: str + self, + *, + surface: str, + column: str, + expectation: str, + entity: str | None = None, ) -> UKSignedDifference | None: """The entry signing the observed difference, if any.""" for difference in self.differences: if difference.covers( - surface=surface, column=column, expectation=expectation + surface=surface, + column=column, + expectation=expectation, + entity=entity, ): return difference return None @@ -287,6 +337,41 @@ def load_uk_spine_swap_signed_differences( "per-gate reviewed-exclusion register instead." ) + surface = _require_member( + raw_scope.get("surface"), + allowed=SIGNED_DIFFERENCE_SURFACES, + field_name=f"{where}.scope.surface", + resource=resource, + ) + columns = _require_str_tuple( + raw_scope.get("columns"), + field_name=f"{where}.scope.columns", + resource=resource, + ) + entities = _require_str_tuple( + raw_scope.get("entities"), + field_name=f"{where}.scope.entities", + resource=resource, + ) + # A column name is not unique across entity tables, so an entry on a + # column surface that names no columns — or no entities — would sign + # divergences it was never adjudicated for. That is the one thing this + # register exists to prevent, so it is refused at load rather than + # left to reviewer vigilance. + if surface in _ENTITY_SCOPED_SURFACES: + if not columns: + raise ValueError( + f"{resource}: {where} signs the {surface!r} surface without " + "naming columns; it would absorb unrelated divergences." + ) + if not entities: + raise ValueError( + f"{resource}: {where} signs the {surface!r} surface without " + "naming entities; column names are not unique across " + "tables, so it could sign a same-named column on an entity " + "it was never adjudicated for." + ) + differences.append( UKSignedDifference( id=identifier, @@ -296,28 +381,15 @@ def load_uk_spine_swap_signed_differences( field_name=f"{where}.class", resource=resource, ), - surface=_require_member( - raw_scope.get("surface"), - allowed=SIGNED_DIFFERENCE_SURFACES, - field_name=f"{where}.scope.surface", - resource=resource, - ), + surface=surface, expectation=_require_member( raw.get("expectation"), allowed=SIGNED_DIFFERENCE_EXPECTATIONS, field_name=f"{where}.expectation", resource=resource, ), - columns=_require_str_tuple( - raw_scope.get("columns"), - field_name=f"{where}.scope.columns", - resource=resource, - ), - entities=_require_str_tuple( - raw_scope.get("entities"), - field_name=f"{where}.scope.entities", - resource=resource, - ), + columns=columns, + entities=entities, magnitude_evidence=_require_str( raw.get("magnitude_evidence"), field_name=f"{where}.magnitude_evidence", diff --git a/packages/microcosm-build/tests/test_uk_h5_payload_compare.py b/packages/microcosm-build/tests/test_uk_h5_payload_compare.py index 3d8f899a..820c4e46 100644 --- a/packages/microcosm-build/tests/test_uk_h5_payload_compare.py +++ b/packages/microcosm-build/tests/test_uk_h5_payload_compare.py @@ -286,11 +286,16 @@ def _entry( surface: str, columns: list[str], expectation: str = "column_differs", + entities: list[str] | None = None, ) -> dict: return { "id": identifier, "class": "mechanism_change", - "scope": {"surface": surface, "columns": columns, "entities": ["household"]}, + "scope": { + "surface": surface, + "columns": columns, + "entities": entities if entities is not None else ["household"], + }, "expectation": expectation, "magnitude_evidence": "disclosure-safe magnitude statement", "evidence": "experiments/686-uk-spine-swap-receipts.md#r0", @@ -554,3 +559,97 @@ def test_a_structural_expectation_does_not_sign_a_value_difference( ) == 1 ) + + +class TestEntityScopeAcrossTables: + """A household adjudication must not sign a person column (#747 re-review). + + The comparator iterates per table, so a column name that exists on two + entities reaches the same register lookup twice. Before the entity scope + was honoured, the household-scoped Scottish-water entry would have signed + a person-table divergence in a same-named column. + """ + + @staticmethod + def _tables_with_shared_column(*, person_value: float) -> dict: + tables = _tables() + tables["household"]["water_and_sewerage_charges"] = [5.0, 6.0] + tables["person"]["water_and_sewerage_charges"] = [1.0, 2.0, person_value] + return tables + + def test_a_person_divergence_is_not_signed_by_a_household_entry( + self, tmp_path: Path + ) -> None: + pytest.importorskip("tables") + left = _write( + tmp_path / "left.h5", self._tables_with_shared_column(person_value=3.0) + ) + right = _write( + tmp_path / "right.h5", + self._tables_with_shared_column(person_value=SENTINEL_VALUE), + ) + register = _register( + tmp_path, + _entry( + "household-water", + surface="nonzero_shares", + columns=["water_and_sewerage_charges"], + entities=["household"], + ), + ) + out = tmp_path / "report.json" + + code = COMPARATOR.main( + [ + str(left), + str(right), + "--structure-only", + "--signed-differences", + str(register), + "--json-out", + str(out), + ] + ) + + report = json.loads(out.read_text(encoding="utf-8")) + assert code == 1 + assert report["signed"]["unsigned_columns"] == [ + "person.water_and_sewerage_charges" + ] + assert report["signed"]["matched_ids"] == [] + + def test_the_same_entry_signs_the_household_divergence( + self, tmp_path: Path + ) -> None: + pytest.importorskip("tables") + left_tables = self._tables_with_shared_column(person_value=3.0) + right_tables = self._tables_with_shared_column(person_value=3.0) + right_tables["household"]["water_and_sewerage_charges"] = [5.0, SENTINEL_VALUE] + left = _write(tmp_path / "left.h5", left_tables) + right = _write(tmp_path / "right.h5", right_tables) + register = _register( + tmp_path, + _entry( + "household-water", + surface="nonzero_shares", + columns=["water_and_sewerage_charges"], + entities=["household"], + ), + ) + out = tmp_path / "report.json" + + code = COMPARATOR.main( + [ + str(left), + str(right), + "--structure-only", + "--signed-differences", + str(register), + "--json-out", + str(out), + ] + ) + + report = json.loads(out.read_text(encoding="utf-8")) + assert code == 0 + assert report["signed"]["matched_ids"] == ["household-water"] diff --git a/packages/microcosm-build/tests/test_uk_signed_differences.py b/packages/microcosm-build/tests/test_uk_signed_differences.py index 6f4f7497..d5733aad 100644 --- a/packages/microcosm-build/tests/test_uk_signed_differences.py +++ b/packages/microcosm-build/tests/test_uk_signed_differences.py @@ -332,6 +332,91 @@ def test_the_committed_register_covers_the_payload_surface(self) -> None: ) +class TestEntityScope: + """A column name is not unique across tables (#747 re-review). + + Every committed entry is entity-scoped, but `covers()` matched on column + alone — harmless while only the share surface consulted it, and + load-bearing the moment the payload bridge let a household adjudication + reach a per-table comparison. + """ + + def _entry(self, *, entities: list[str]) -> UKSignedDifference: + return UKSignedDifference( + id="household-scoped", + difference_class="mechanism_change", + surface="nonzero_shares", + expectation="column_differs", + columns=("water_and_sewerage_charges",), + entities=tuple(entities), + magnitude_evidence="evidence", + evidence="experiments/686-uk-spine-swap-receipts.md", + adjudicator="juaristi22", + adjudicated_on="2026-08-22", + ) + + def test_an_entry_signs_only_the_entity_it_names(self) -> None: + entry = self._entry(entities=["household"]) + assert entry.covers( + surface="nonzero_shares", + column="water_and_sewerage_charges", + expectation="column_differs", + entity="household", + ) + assert not entry.covers( + surface="nonzero_shares", + column="water_and_sewerage_charges", + expectation="column_differs", + entity="person", + ) + + def test_the_payload_bridge_carries_the_entity_scope(self) -> None: + # The bridge must not widen the adjudication it forwards. + entry = self._entry(entities=["household"]) + assert entry.covers( + surface="payload_column", + column="water_and_sewerage_charges", + expectation="column_differs", + entity="household", + ) + assert not entry.covers( + surface="payload_column", + column="water_and_sewerage_charges", + expectation="column_differs", + entity="person", + ) + + def test_a_caller_without_an_entity_gets_no_entity_filtering(self) -> None: + entry = self._entry(entities=["household"]) + assert entry.covers( + surface="nonzero_shares", + column="water_and_sewerage_charges", + expectation="column_differs", + ) + + def test_every_committed_entry_names_the_reference_entity(self) -> None: + # An entry naming the wrong entity now fails to match rather than + # signing across tables, so a mis-declared scope is a defect. + reference = json.loads( + files("microcosm.build.uk") + .joinpath("efrs_parity_reference.json") + .read_text(encoding="utf-8") + ) + entities = reference["input_entities"] + for difference in load_uk_spine_swap_signed_differences().differences: + if difference.surface not in {"nonzero_shares", "weighted_totals"}: + continue + for column in difference.columns: + actual = entities.get(column) + if actual is None: + continue + assert actual in difference.entities, ( + f"{difference.id} scopes {column!r} to " + f"{list(difference.entities)}, but the reference carries " + f"it on {actual!r}." + ) + + class TestValidation: def test_duplicate_ids_are_refused(self, tmp_path: Path) -> None: payload = { @@ -374,6 +459,44 @@ def test_field_validation( with pytest.raises(ValueError, match=match): load_uk_spine_swap_signed_differences(_write(tmp_path, payload)) + def test_a_column_surface_entry_without_columns_is_refused( + self, tmp_path: Path + ) -> None: + # Via the payload bridge an empty columns tuple would blanket-sign + # every column in every table. + entry = _valid_entry() + entry["scope"] = { + "surface": "nonzero_shares", + "columns": [], + "entities": ["household"], + } + payload = {"schema_version": 1, "scope_note": "note", "differences": [entry]} + with pytest.raises(ValueError, match="without naming columns"): + load_uk_spine_swap_signed_differences(_write(tmp_path, payload)) + + def test_a_column_surface_entry_without_entities_is_refused( + self, tmp_path: Path + ) -> None: + entry = _valid_entry() + entry["scope"] = { + "surface": "nonzero_shares", + "columns": ["savings"], + "entities": [], + } + payload = {"schema_version": 1, "scope_note": "note", "differences": [entry]} + with pytest.raises(ValueError, match="without naming entities"): + load_uk_spine_swap_signed_differences(_write(tmp_path, payload)) + + def test_an_entity_counts_entry_may_stay_surface_wide(self, tmp_path: Path) -> None: + # The one surface where a column-less entry is meaningful: its + # "column" is the entity name. + entry = _valid_entry() + entry["scope"] = {"surface": "entity_counts", "columns": [], "entities": []} + entry["expectation"] = "count_differs" + payload = {"schema_version": 1, "scope_note": "note", "differences": [entry]} + register = load_uk_spine_swap_signed_differences(_write(tmp_path, payload)) + assert register.differences[0].columns == () + def test_unknown_surface_is_refused(self, tmp_path: Path) -> None: payload = { "schema_version": 1, diff --git a/packages/microcosm-build/tests/test_uk_spine_parity_instrument.py b/packages/microcosm-build/tests/test_uk_spine_parity_instrument.py index fd74b1b7..6ba3c856 100644 --- a/packages/microcosm-build/tests/test_uk_spine_parity_instrument.py +++ b/packages/microcosm-build/tests/test_uk_spine_parity_instrument.py @@ -770,3 +770,50 @@ def test_a_structural_expectation_does_not_sign_a_value_divergence( ) == 1 ) + + +class TestWeightedTotalsRegisterAccounting: + """A totals-scoped entry counts as matched, not as register rot. + + Latent while the surface is dormant, and it bites exactly when the + surface is un-dormanted for a calibrated candidate — which is the moment + the accounting starts mattering. + """ + + def test_a_totals_entry_that_matched_is_not_reported_unused( + self, tmp_path: Path + ) -> None: + tool = _load_tool() + candidate = _write(tmp_path / "c.json", _candidate_from_reference()) + left = _write(tmp_path / "ref.json", {"identity": {}, "totals": {"col": 100.0}}) + right = _write( + tmp_path / "cand.json", {"identity": {}, "totals": {"col": 125.0}} + ) + register = _register( + tmp_path, + _entry("totals-signed", surface="weighted_totals", columns=["col"]), + ) + receipt = tmp_path / "receipt.json" + + code = tool.main( + [ + "--candidate-json", + str(candidate), + "--register", + str(register), + "--reference-weighted-totals", + str(left), + "--candidate-weighted-totals", + str(right), + "--strict", + "--receipt-json", + str(receipt), + ] + ) + + assert code == 0 + report = json.loads(receipt.read_text(encoding="utf-8")) + assert report["strict_failure"] is False + assert report["register"]["matched_ids"] == ["totals-signed"] + assert report["register"]["unused_ids"] == [] + assert report["register"]["dormant_ids"] == [] diff --git a/tools/compare_uk_h5_payload.py b/tools/compare_uk_h5_payload.py index b24ca9d0..9dc320da 100644 --- a/tools/compare_uk_h5_payload.py +++ b/tools/compare_uk_h5_payload.py @@ -351,8 +351,14 @@ def apply_structure_only_verdict( for key, table in report["tables"].items(): signed_ids: dict[str, str | None] = {} for column in sorted(table["value_mismatch_rows_by_column"]): + # The store key is the entity whose table this column lives in; + # passing it stops a household-scoped adjudication signing a + # same-named person column. entry = register.matching( - surface="payload_column", column=column, expectation="column_differs" + surface="payload_column", + column=column, + expectation="column_differs", + entity=key, ) signed_ids[column] = entry.id if entry else None if entry is None: diff --git a/tools/verify_uk_spine_parity.py b/tools/verify_uk_spine_parity.py index 29807b9d..482429ca 100644 --- a/tools/verify_uk_spine_parity.py +++ b/tools/verify_uk_spine_parity.py @@ -135,7 +135,10 @@ def _compare_entity_counts( entry = {"reference": expected, "candidate": observed, "equal": equal} if not equal: signed = register.matching( - surface="entity_counts", column=entity, expectation="count_differs" + surface="entity_counts", + column=entity, + expectation="count_differs", + entity=entity, ) entry["signed_id"] = signed.id if signed else None if signed is None: @@ -168,7 +171,10 @@ def _compare_shares( "delta": delta, } signed = register.matching( - surface="nonzero_shares", column=column, expectation="column_differs" + surface="nonzero_shares", + column=column, + expectation="column_differs", + entity=entities.get(column), ) if abs(delta) <= band: # Reported, never dropped: the band decides what must be @@ -185,7 +191,10 @@ def _missing(names: list[str], expectation: str) -> dict[str, Any]: out: dict[str, Any] = {} for column in sorted(names): signed = register.matching( - surface="nonzero_shares", column=column, expectation=expectation + surface="nonzero_shares", + column=column, + expectation=expectation, + entity=entities.get(column), ) out[column] = { "entity": entities.get(column), @@ -222,6 +231,7 @@ def _compare_weighted_totals( reference_totals: Mapping[str, float], candidate_totals: Mapping[str, float], register: UKSignedDifferenceRegister, + entities: Mapping[str, str] | None = None, ) -> tuple[dict[str, Any], list[str]]: unsigned: list[str] = [] differing: dict[str, Any] = {} @@ -240,7 +250,10 @@ def _compare_weighted_totals( if relative is not None and abs(relative) <= TOTALS_EPSILON: continue signed = register.matching( - surface="weighted_totals", column=column, expectation="column_differs" + surface="weighted_totals", + column=column, + expectation="column_differs", + entity=(entities or {}).get(column), ) # Deltas only: the absolute totals are licensed and stay outside. differing[column] = { @@ -362,6 +375,7 @@ def verify_uk_spine_parity( {k: float(v) for k, v in left_totals.items()}, {k: float(v) for k, v in right_totals.items()}, register, + reference.input_entities, ) totals_report["reference_identity"] = left.get("identity") totals_report["candidate_identity"] = right.get("identity")