diff --git a/FINAL_REPORT.md b/FINAL_REPORT.md index 923d4e01..7648239b 100644 --- a/FINAL_REPORT.md +++ b/FINAL_REPORT.md @@ -1,119 +1,130 @@ -# Final report: PolicyEngine-US 1.819.0 lock bump +# Final report: stacked-pool to release CD-vintage provenance ## Outcome -The branch is ready for the owner to open the PR. `uv.lock` now resolves -`policyengine-us==1.819.0` and `policyengine-core==3.31.0`; the complete -resolver version movement is exactly: - -```text -policyengine-core 3.26.11 -> 3.31.0 -policyengine-us 1.764.6 -> 1.819.0 -``` - -NumPy remains 2.4.6 and Torch remains 2.12.0. The lock binds the official -PE-US 1.819.0 wheel SHA-256 -`525bdf8b238c3eb11cd60c5f4f7a7b0c57bc7eea5c1cf4346c261241b061be45` -(`uv.lock:1366-1421`). The required upgraded -`uv sync --all-packages --extra us` completed for all five workspace packages -from that lock and task-local official PyPI artifacts. - -## Compatibility repairs - -- WIC's upstream input is now `takes_up_wic_if_eligible`; Microcosm writes - that verified successor while retaining the historical draw salt, and the - six current cross-entity consumers are frozen with direct-receiver and - aggregation ownership - (`packages/microcosm-build/src/microcosm/build/us_runtime/wic_claim.py:106-109,392-409`; - `packages/microcosm-frame/src/microcosm/frame/adapters/_policyengine_us_source_index.py:1694-1731,2004-2090`; - `packages/microcosm-build/tests/test_us_pool_input_consumers.py:235-359`). -- Reported TANF now writes the verified input successor `receives_tanf`, since - PE-US owns `is_tanf_enrolled` as a formula - (`packages/microcosm-build/src/microcosm/build/us_runtime/cps_carried.py:127-160,266-289,485-500`). -- New CA, CO, and NM premium-assistance take-up leaves remain explicit - `engine_default`/`rate_unsourced` inputs because Microcosm has no reviewed - participation source; no rate was invented - (`packages/microcosm-build/src/microcosm/build/us/spec/take_up.yaml:507-563`). -- Mortgage-interest, Oklahoma pension, SNAP proration/missing-hours, and the - expanded AL/NY/OK/PA `weeks_unemployed` consumers were updated or guarded - against the installed graph - (`packages/microcosm-build/tests/test_us_pool_input_consumers.py:491-575`; - `packages/microcosm-build/tests/test_us_child_support.py:448-489`; - `packages/microcosm-build/tests/test_us_weeks_unemployed.py:695-751`). -- The published US dataset remains certified for the prior engine lock. - Certified loading still fails closed before construction on model/core - mismatch; only the live smoke test recognizes that specific typed mismatch - as an expected skip under this new development lock - (`packages/microcosm-data/src/microcosm/data/loader.py:63-64,425-445`; - `packages/microcosm-data/tests/test_loader.py:326-356,403-419`). - -No pool-consumed upstream variable was genuinely removed without a successor, -so the owner-question stop condition did not trigger. The full installed-input -audit and mechanism-by-mechanism code citations are in `_LANE-NOTES.md` under -“verified upstream compatibility repairs.” - -## Identities and compatibility note - -Repository generators refreshed every affected raw-resource, generated-source, -engine-ABI, remaining-input, field-usage, inventory, seed-protocol, authority, -spec, coverage, and golden identity. Source bytes are part of the attested seed -protocol, so those module edits legitimately move dependent identities even -though NumPy and Torch did not move -(`packages/microcosm-build/src/microcosm/build/spec_engine/seeds.py:320-367`; -`packages/microcosm-frame/src/microcosm/frame/adapters/policyengine_us.py:114-152`). -The final US spec is -`3189d90dec95c8ea7090e41b5283fa52b1e6855bed4a776dfa02820f2bd11c62`. - -`_LANE-NOTES.md`, under “final tool-generated identities,” records all 46 final -values by mechanism. The compatibility commit body records every full old-to- -new digest mapping, including the four lock artifact hashes. - -The requested short release-range note is in `_LANE-NOTES.md` under -“compatibility note for 1.764.6 through 1.819.0.” It flags receipt/take-up and -SNAP changes, OBBBA follow-through, major cash/health/housing and tax formula -changes, school-meal child-support treatment, and newly added state programs. -It is intentionally an owner-facing plausibility scan, not an exhaustive -policy audit. +The producer/consumer defect is fixed. The production stacked route now +produces the same authenticated congressional-district contract that release +preflight requires: the canonical crosswalk SHA, target vintage +`119th_congress`, and positive household +`congressional_district_geoid` support. Geography remains outside source +assembly, and the release guard still fails closed on every mismatch +(`packages/microcosm-build/src/microcosm/build/us_runtime/operator_boundary.py:346-406`; +`tools/build_us_fiscal_refresh_release.py:2565-2613,8608-8612`). + +## Five-part repair + +1. The stacked CLI requires path/SHA pairs for the national PUMA ladder and + canonical 117th-to-119th CD crosswalk. Both declared hashes must equal the + repository pins before the source bytes are authenticated + (`tools/build_us_multispine_pool.py:563-594,897-1004`). The real geography + operator runs immediately after operator-free source assembly and before + gap fill or cloning + (`packages/microcosm-build/src/microcosm/build/us_runtime/operator_boundary.py:346-406`; + `tools/build_us_multispine_pool.py:5290-5334`). + +2. The configured namespace, checkpoint identity, persisted stage receipts, + and terminal manifest bind both byte authorities, source and target + vintages, assignment declaration, overlap algorithm, operator order, seed + site/stream/value, and ordered native-household output receipt + (`tools/build_us_multispine_pool.py:814-928,1300-1469,1711-1930,3530-3635,3989-4027,4133-4215`). + +3. Nullable H5 publication accepts validated root attributes, writes them in + the same temporary HDFStore as the fixed entity tables, verifies their + exact round trip, and atomically replaces the destination only after all + checks pass. The stacked publisher supplies the crosswalk-SHA and target- + vintage attrs + (`packages/microcosm-build/src/microcosm/build/us_runtime/h5_io.py:1463-1699`; + `tools/build_us_multispine_pool.py:4423-4519`). Schema-9 loading also binds + those physical attrs and the live household geography/clone lineage back to + the manifest + (`packages/microcosm-build/src/microcosm/build/us_runtime/h5_io.py:671-923,1032-1193,1321-1364`). + +4. Release preflight now reads root attrs and the household frame from one + HDFStore snapshot through the shared fixed/table-aware reader. Its existing + equality checks for crosswalk SHA, current target vintage, and positive + household district support are unchanged + (`tools/build_us_fiscal_refresh_release.py:2565-2677`). + +5. A tiny fixture invokes the real stacked entry point, post-assembly + geography operator, fixed-H5 publisher, and real release assertion. It + proves both root attrs and positive household district support; consumer + tests reject changed attrs, changed native geography, missing lineage, and + divergent clone geography + (`packages/microcosm-build/tests/test_us_multispine_pool_tool.py:2279-2335`; + `packages/microcosm-build/tests/test_us_multispine_pool_h5_io.py:1357-1440`; + `packages/microcosm-build/tests/test_us_fiscal_refresh_builder.py:1061-1184`). + +## Assignment design and anti-rot + +A one-value deterministic PUMA-to-CD lookup is not defensible because 2020 +PUMAs and congressional districts do not nest. The existing national ladder +preserves observed ACS PUMA, assigns missing ASEC PUMA proportional to 2020 +PUMA population, then draws CD and county within PUMA proportional to block- +population overlap. Stable row/state/PUMA order and the ledgered +`legacy_puma_ladder` / `geography_legacy` / seed `0` contract make the draw +reproducible for fixed inputs +(`packages/microcosm-build/src/microcosm/build/us_runtime/puma_ladder.py:1-20,293-383,638-698`; +`packages/microcosm-build/src/microcosm/build/spec_engine/seeds.py:870-887`; +`packages/microcosm-build/src/microcosm/build/us/spec/spine.yaml:421-432`). + +The spec and schemas now declare the crosswalk source/vintage authority and +geography assignment, while field-usage and inventory owners cover the new +surface exactly +(`packages/microcosm-build/src/microcosm/build/us/spec/geography.yaml:3-37`; +`packages/microcosm-build/src/microcosm/build/us/spec/sources.yaml:69-86`; +`packages/microcosm-build/src/microcosm/build/us/spec/vintages.yaml:45-53`; +`packages/microcosm-build/src/microcosm/build/spec_engine/field_usage.py:387-393,671-692,803-822`; +`packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py:348-380,1649-1712`). + +The generated US spec SHA is +`5378bb9189aec96f50da22aac71e5bd2c3d919e9795f6ef2147e0bc9c739dd8e`. +Coverage is exact at 42,120/42,120 fields, 49 claims, and 41/41 inventory +checks. Principal legitimate movements are pointer inventory +`6d7353c6...` to `bc4a948a...`, full checkpoint `b6a47fac...` to +`a128a85f...`, and the new geography identity `f49425ca...`; executable +versions move checkpoint 11 to 12, terminal manifest 8 to 9, and nullable H5 +materializer 2 to 3 +(`tools/spec_engine_coverage.py:42-45`; +`packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py:348-380`; +`docs/evidence/spec-engine/us-f0-coverage.json:10,775-788,1890,2027-2057,2602`; +`tools/build_us_multispine_pool.py:332-360`; +`packages/microcosm-build/src/microcosm/build/us_runtime/h5_io.py:75-105`). ## Verification -- `microcosm-calibrate`: 203 passed; peak RSS 462,896 KiB. -- `microcosm-data`: 318 passed / 2 skipped; peak RSS 769,408 KiB. -- `microcosm-fit`: 93 passed; peak RSS 872,432 KiB. -- `microcosm-frame`: 295 passed / 36 skipped; peak RSS 6,904,032 KiB. -- `microcosm-build`: exact complete inventory, 6,304 passed / 39 skipped. A - canonical one-process run was green but retained 18,548,960 KiB and was not - accepted under the 15 GiB ceiling. The exact 6,341 collected items were - proven as disjoint 4,161-item and 2,180-item serial fresh-process partitions: - 4,127 passed / 36 skipped at 12,596,384 KiB, then 2,177 passed / 3 skipped at - 13,363,984 KiB. This follows the repository's documented fresh-process shard - rationale (`.github/workflows/test.yml:24-34`) and changes no test assertion - or model behavior. - -Accepted total: **7,213 passed / 77 skipped / 0 failed**, with every accepted -process below 15 GiB RSS. - -Final generated checks are green: - -- release-input manifest: 163 required, 7 reviewed exclusions, 41 reform - probes; -- target parity: 32 compiled, 52 reviewed exclusions; -- `tools/generate_us_bundle_from_constants.py --check`: US spec SHA above; -- `tools/spec_engine_coverage.py --check`: 42,096/42,096 configuration fields - and 40/40 inventory checks. Both check paths reject stale bytes rather than - using a tolerance - (`tools/generate_us_bundle_from_constants.py:355-367,410-437`; +- `microcosm-calibrate`: 203 passed. +- `microcosm-data`: 318 passed / 2 skipped. +- `microcosm-fit`: 93 passed. +- `microcosm-frame`: 295 passed / 36 skipped. +- `microcosm-build` partition A: 4,155 passed / 36 skipped. +- `microcosm-build` partition B: 2,177 passed / 3 skipped. +- Accepted workspace total: **7,241 passed / 77 skipped / 0 failed**. +- The high-risk real integration and H5 consumer cases are included in those + build partitions + (`packages/microcosm-build/tests/test_us_multispine_pool_tool.py:2279-2335`; + `packages/microcosm-build/tests/test_us_multispine_pool_h5_io.py:1357-1440`). +- Bundle freshness reports the spec SHA above; coverage freshness reports + 42,120/42,120 fields and 41/41 inventory checks + (`tools/generate_us_bundle_from_constants.py:355-437`; `tools/spec_engine_coverage.py:378-398`). - -Repository-wide `ruff check .` and `git diff --check` are green. No gate, -threshold, tolerance, or band was tuned. No pool or release build ran, no PR -was opened, and nothing was pushed. - -## Commits and handoff - -- `514964d4` — start the PolicyEngine-US 1.819.0 bump lane and commit the - standing progress journal. -- The compatibility commit containing this report — lock bump, verified - repairs, generated identities, complete digest mapping, and green receipts. - -Next: the owner opens the PR from `bump-policyengine-us`. +- Repository-wide Ruff, smoke-script shell syntax, and `git diff --check` pass. + The source-blind import-graph anti-rot pin now covers the exact 69-module + graph reached by the three new shared validators + (`packages/microcosm-build/tests/test_us_spine_blindness.py:3270-3315`). + +No production pool or release build ran, no push occurred, and +`logbook-pending-chain.txt` was not touched. + +## Candidate handoff + +Stage 1 must add the canonical PUMA-ladder and CD-crosswalk path/SHA pairs +listed exactly at the end of `_LANE-NOTES.md`, run off-chain, and rebuild under +the new checkpoint identity. The old smoke checkpoints and pool publication +are not reusable: configured identity now includes both new input pins and the +geography contract, checkpoint materializer is 12, terminal manifest schema is +9, and nullable H5 materializer is 3 +(`tools/build_us_multispine_pool.py:332-360,1300-1555`; +`packages/microcosm-build/src/microcosm/build/us_runtime/h5_io.py:75-105`). +The six pre-existing immutable source artifacts can be reused; the Stage-1 +checkpoint namespace and pool H5 must be regenerated with the two additional +authenticated authorities (`tools/build_us_multispine_pool.py:931-1004,1292-1319`). diff --git a/PROGRESS.md b/PROGRESS.md index 8f8f577a..3036d509 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -1,56 +1,63 @@ -# Progress: PolicyEngine-US 1.819.0 lock bump +# Progress: stacked-pool to release CD-vintage provenance ## State -Complete and ready for the owner to open the PR. The lock resolves -`policyengine-us==1.819.0` and `policyengine-core==3.31.0`; all required -compatibility repairs, generated contracts, attested identities, package test -shards, generator checks, and Ruff are green. The code-cited mechanism audit, -identity inventory, compatibility note, and validation receipts are in -`_LANE-NOTES.md`. +Implementation and validation are complete. The stacked +producer now authenticates and applies household geography after source +assembly, carries its authority through checkpoint and publication identities, +publishes verified CD-vintage H5 attributes, and reaches the unchanged release +guard through the shared fixed/table-aware reader +(`tools/build_us_multispine_pool.py:814-928,1300-1469,1857-1930,5290-5334`; +`packages/microcosm-build/src/microcosm/build/us_runtime/h5_io.py:1032-1193,1463-1699`; +`tools/build_us_fiscal_refresh_release.py:2565-2677`). -No pool-consumed variable was removed without a verified successor, so no -owner question is pending. No pool/release build, push, gate change, threshold -change, tolerance change, or band change occurred. +No pool/release build, push, guard weakening, operator-boundary weakening, or +`logbook-pending-chain.txt` access has occurred. ## Done -- Started and committed this standing progress log at `514964d4` after - rebasing the lane start onto current `origin/main`. -- Ran the ordered initial sync attempt, documented the sandbox cache/network - limits, and completed the unchanged-lock and upgraded-lock all-package US - syncs from exact official artifacts in a task-local cache. -- Ran `uv lock --upgrade-package policyengine-us`. The complete version - movement is exactly `policyengine-core 3.26.11 -> 3.31.0` and - `policyengine-us 1.764.6 -> 1.819.0`; NumPy 2.4.6 and Torch 2.12.0 did not - move. -- Verified upstream variable reality in the installed PE-US 1.819.0 package, - adapted Microcosm's input ownership and consumer guards, and retained - fail-closed certified-dataset version checks. Each mechanism is cited in - `_LANE-NOTES.md` under “verified upstream compatibility repairs.” -- Regenerated the release-input, parity, source/take-up, engine-ABI, spec, - seed, coverage, and golden identities with repository tools. The final - 46-value inventory is in `_LANE-NOTES.md`; the final commit body carries the - exhaustive old-to-new mapping. -- Added the requested owner-facing PE-US 1.764.6-to-1.819.0 compatibility note - from the installed release changelog, including major SNAP, receipt, - OBBBA-follow-through, cash/health/housing, tax, and new-program changes. -- Passed every package test: 7,213 passed / 77 skipped / 0 failed. Calibrate, - data, fit, and frame ran as individual package shards. The build inventory - was proven as a complete disjoint 4,161-item + 2,180-item partition after - the otherwise-green single process retained more than the binding memory - ceiling; accepted peaks were 12,596,384 and 13,363,984 KiB, both below 15 - GiB. This uses the repository's fresh-process shard rationale - (`.github/workflows/test.yml:24-34`) without changing any assertion or model - behavior. -- Reproduced 163 required release inputs / 7 reviewed exclusions / 41 reform - probes; 32 compiled parity targets / 52 reviewed exclusions; US spec SHA-256 - `3189d90dec95c8ea7090e41b5283fa52b1e6855bed4a776dfa02820f2bd11c62`; - and 42,096/42,096 configuration fields plus 40/40 inventory checks. -- Passed repository-wide Ruff and `git diff --check`; wrote the final handoff - to `FINAL_REPORT.md`. +- Reconciled the worktree with salvage commit `ca26ea21`: its tracked tree was + already exactly present, so the best salvage was retained and audited rather + than reimplemented. +- Completed the required all-package US environment sync from the exact lock + using the writable offline uv cache after the managed sandbox refused the + default cache. +- Added the two required pinned geography authority pairs, the ledgered seeded + PUMA-overlap assignment, schema-12 checkpoint identity, assignment receipts, + schema-9 terminal manifest validation, and schema-3 H5 materializer binding. +- Added atomic nullable-H5 root-attribute write/verification and authenticated + manifest-to-H5 geography/clone-lineage validation. +- Made release preflight read root attributes and fixed household frames in one + `HDFStore` handle while retaining all existing SHA, target-vintage, and + positive-support checks. +- Added a real tiny stacked-pool publication to release-preflight integration + test plus negative attr, digest, missing-lineage, and divergent-clone tests + (`packages/microcosm-build/tests/test_us_multispine_pool_tool.py:2279-2335`; + `packages/microcosm-build/tests/test_us_multispine_pool_h5_io.py:1357-1440`). +- Regenerated and checked the US anti-rot chain: spec SHA + `5378bb9189aec96f50da22aac71e5bd2c3d919e9795f6ef2147e0bc9c739dd8e`, + 42,120/42,120 configuration fields, 49 claims, and 41/41 inventory checks. +- Passed the complete accepted workspace suite: 7,241 passed, 77 skipped, and + 0 failed. The two memory-bounded build partitions were 4,155 passed / 36 + skipped and 2,177 passed / 3 skipped; calibrate, data, fit, and frame account + for the remaining 909 passed / 38 skipped. +- Passed repository-wide Ruff, bundle freshness, coverage freshness, smoke + script syntax, and `git diff --check`. +- Recorded the exact candidate Stage-1 path/SHA additions and the checkpoint + invalidation verdict in `_LANE-NOTES.md` and `FINAL_REPORT.md`. ## Next -The owner opens the PR. No push or additional build is required from this -lane. +1. Candidate Stage 1 adds both authenticated geography authority pairs and + rebuilds under checkpoint materializer 12. +2. Candidate Stage 2 consumes the new schema-9 manifest/materializer-3 H5 and + passes the unchanged release preflight. +3. Do not reuse pre-fix Stage-1 checkpoints or pool publications; immutable + source artifacts remain reusable. + +## Historical prior lane + +The PolicyEngine-US 1.819.0 lock-bump journal previously in this file is +historical: that lane merged into `origin/main` at `7b90bb18` on 2026-08-24. +Its final state remains available at commit `05d254aa` and its detailed +receipts remain in the historical section of `_LANE-NOTES.md`. diff --git a/_LANE-NOTES.md b/_LANE-NOTES.md index fa87b676..b29ab699 100644 --- a/_LANE-NOTES.md +++ b/_LANE-NOTES.md @@ -1,4 +1,44 @@ -# PolicyEngine-US 1.819.0 lock-bump lane notes +# Stacked-pool to release CD-vintage provenance lane notes + +## 2026-08-24 — lane start and environment + +- Branch `pool-cd-vintage-provenance` starts at `origin/main` commit + `7b90bb18`. This lane is limited to the authenticated producer/consumer + contract described in the candidate-chain defect report: post-assembly + household-CD assignment, checkpoint/manifest provenance, atomic nullable-H5 + attributes, format-aware release preflight, and a real tiny integration + test (`tools/build_us_multispine_pool.py:441-577,3477-3786`; + `tools/build_us_fiscal_refresh_release.py:2565-2661`). +- The ordered first `uv sync --all-packages --extra us` could not initialize + the sandbox-inaccessible user cache. A writable-cache retry reached the + locked Jinja2 URL but DNS is disabled. The exact recovery cloned the + completed Python 3.14 environment from the merged PolicyEngine-US 1.819.0 + bump lane copy-on-write, then completed + `UV_CACHE_DIR=/private/tmp/microcosm-scorecard-uv.0rntvY/cache uv sync + --offline --all-packages --extra us`. uv rebuilt all five workspace wheels + and replaced their sibling-worktree editable origins with this worktree. +- The GitNexus debugging skill applies to this cross-surface defect. No + GitNexus query, context, graph, or resource tools are exposed in this + session, so the fallback is direct source tracing, repository generators, + and focused plus suite-wide tests. +- The producer must assign after the source boundary checks: preassembled + `congressional_district_geoid` is an operator output and is rejected + (`packages/microcosm-build/src/microcosm/build/us_runtime/operator_boundary.py:346-353,372-406`; + `tools/build_us_multispine_pool.py:4566-4584`). The release check remains + unconditional and must continue to require matching crosswalk SHA, target + vintage, and positive household support + (`tools/build_us_fiscal_refresh_release.py:2565-2613,8571-8590`). +- No build, release, push, release-guard weakening, operator-boundary + weakening, or `logbook-pending-chain.txt` access occurred at lane start. +- The pre-change suite is green. Fresh-process receipts are: calibrate 203 + passed; data 318 passed / 2 skipped; fit 93 passed; frame 295 passed / 36 + skipped; build partition excluding `test_us_[n-z]*.py` 4,127 passed / 36 + skipped; complementary build partition 2,177 passed / 3 skipped. Combined: + 7,213 passed / 77 skipped / 0 failed. The build split follows the CI + fresh-process rationale (`.github/workflows/test.yml:24-34`) and the same + previously accepted local partition; no assertion or behavior changed. + +# Historical lane notes: PolicyEngine-US 1.819.0 lock bump ## 2026-08-23 — lane start and unchanged-lock environment @@ -857,3 +897,209 @@ decides whether the dense and sparse evidence justifies the flip. - Validation: six new identity/signature/parser tests pass; the standard 10-file affected suite completed to 100% with exit 0; Ruff and `git diff --check` pass. No build ran. + +## 2026-08-24 — stacked-pool CD-vintage provenance contract complete + +### Recovery and environment + +- The live worktree's tracked tree exactly matched salvage commit `ca26ea21`, + so the salvaged implementation was retained and audited rather than + restarted. `PROGRESS.md` and this root journal were already committed at lane + start in `5925f808`. +- The required lock-exact environment command completed offline after the + sandbox rejected the default uv cache: + + ```bash + UV_CACHE_DIR=/private/tmp/microcosm-scorecard-uv.0rntvY/cache \ + uv sync --offline --all-packages --extra us + ``` + + It checked all 100 locked packages. Test execution used + `uv run python -m pytest` so imports resolve to this worktree. + +### Five-part repair + +1. The stacked CLI now requires a national PUMA ladder and the canonical + 117th-to-119th congressional-district crosswalk as explicit path/SHA pairs. + Missing arguments, noncanonical declared pins, or mismatching bytes fail + before any source build + (`tools/build_us_multispine_pool.py:563-594,897-1004`). The real assignment + runs only after operator-free source validation and stack assembly, then + before gap fill and clone operators + (`packages/microcosm-build/src/microcosm/build/us_runtime/operator_boundary.py:346-406`; + `tools/build_us_multispine_pool.py:5290-5334`). + +2. Configured and checkpoint identities bind both byte authorities, the + 117th-to-119th vintage declaration, algorithm + `assign_us_puma_ladder.population_weighted_overlap.v1`, assignment order, + seed site `legacy_puma_ladder`, stream `geography_legacy`, and value `0`. + The ordered native-household assignment receipt persists through every + resumable stage and the schema-9 terminal manifest + (`tools/build_us_multispine_pool.py:814-928,1300-1469,1711-1930,3530-3635,3989-4027,4133-4215`). + +3. Nullable H5 publication now accepts caller-owned root attributes, validates + their names/values, writes them with the fixed entity tables in a temporary + sibling, verifies their exact round trip, and replaces the destination only + after successful verification. The stacked publisher writes the crosswalk + SHA and `119th_congress` + (`packages/microcosm-build/src/microcosm/build/us_runtime/h5_io.py:1463-1699`; + `tools/build_us_multispine_pool.py:4423-4519`). Schema-9 loading binds the + manifest receipt to those physical attrs and recomputes native household ID + and geography digests. It requires integral source/clone roles, exactly one + native per source, positive integral CD values, and coherent PUMA/CD/county + geography across clone roles + (`packages/microcosm-build/src/microcosm/build/us_runtime/h5_io.py:671-923,1032-1193,1321-1364`). + +4. Release preflight reads root attrs and the household frame within one + HDFStore and delegates fixed/table representation handling to the shared + reader. The guard's SHA equality, current-vintage equality, and positive- + household-support checks remain intact + (`tools/build_us_fiscal_refresh_release.py:2565-2677,8608-8612`). + +5. The integration test calls the real stacked entry point with tiny + fixture/stub sources, executes the real post-assembly geography assignment + and publisher, proves fixed household storage and both H5 attrs, then calls + the real release assertion and observes positive district support + (`packages/microcosm-build/tests/test_us_multispine_pool_tool.py:2279-2335`). + H5 consumer tests additionally reject attr drift, native-geography digest + drift, missing clone lineage, and divergent clone geography; writer tests + cover atomic attr round trip and pre-replacement failure + (`packages/microcosm-build/tests/test_us_multispine_pool_h5_io.py:1357-1440`; + `packages/microcosm-build/tests/test_us_acs_multispine_base_builder.py:96-197`; + `packages/microcosm-build/tests/test_us_fiscal_refresh_builder.py:1061-1184`). + +### Assignment decision + +A one-value deterministic PUMA-to-CD allocation is not defensible: 2020 PUMAs +and congressional districts do not nest, while the authority contains the +block-population overlap distribution. Selecting only the largest overlap +would erase supported within-PUMA geography. The existing ladder instead +preserves observed ACS PUMA, assigns missing ASEC PUMA proportional to 2020 +PUMA population, and samples CD/county within PUMA proportional to block +population. Its stable state/PUMA order and single generator make output +reproducible from the bound seed +(`packages/microcosm-build/src/microcosm/build/us_runtime/puma_ladder.py:1-20,293-383,638-698`). +The seed ledger owns `legacy_puma_ladder` on `geography_legacy`, default `0`, +and the generated spine declaration carries the same draw contract +(`packages/microcosm-build/src/microcosm/build/spec_engine/seeds.py:870-887`; +`packages/microcosm-build/src/microcosm/build/us/spec/spine.yaml:421-432`). + +### Anti-rot and identity receipts + +- Canonical US spec SHA: + `3189d90dec95c8ea7090e41b5283fa52b1e6855bed4a776dfa02820f2bd11c62` + to `5378bb9189aec96f50da22aac71e5bd2c3d919e9795f6ef2147e0bc9c739dd8e`. +- Field-pointer inventory SHA: + `6d7353c6c42a6e1dbc6e3a227848e36864526fdc9533d5b284aa469c87dc064f` + to `bc4a948ab632191954600da8474c5b011f977a65e24c399d126f3dc4a79f23e5`. +- Full checkpoint SHA: + `b6a47fac54d7de7aa42ce59dc1950c0765b7a67034f0174ad1531d5bbb06ceef` + to `a128a85f877fb32def9382b841b8b340f974e8a9148ac029c1f04becdc956c18`. +- Pool-code diagnostic SHA: + `044d8a45c4fe42eec9f72f9bedbf403536b9734cf74d8d65b72caf3c6c1d60b7` + to `91c65a9ff36839d575036264c4bf57ffde6457e8fd180a0856f9be712ada371d`. +- New geography-assignment SHA: + `f49425ca8734ac559c73cf44f6458d86d3162a48956b98a27e6e758959361585`. +- Stacked checkpoint materializer moved 11 to 12, terminal manifest schema 8 + to 9, nullable H5 materializer 2 to 3, and geography receipt absent to + schema 1. Pool-stage materializer 7, stacked authority 11, and late-registry + schema 16 did not move; their documentation changes only correct stale prose + (`tools/build_us_multispine_pool.py:332-360`; + `packages/microcosm-build/src/microcosm/build/us_runtime/h5_io.py:75-105`; + `docs/us-multispine-operator-ordering.md:1-120`). + +The generated field ledger moved authored fields 32,331 to 32,351, resolved +bindings 9,765 to 9,769, configuration/consumed fields 42,096 to 42,120, and +claims 47 to 49. Compiler-semantic/front-end-validation/identity-only/legacy +modes moved 27,688/346/128/13,934 to 27,699/348/103/13,970; unused and +multiply-primary-used remain zero. Source records moved 7 to 8, vintage records +15 to 16, resolved references 325 to 334, full/static checkpoint components +12/9 to 13/10, and inventory checks 40 to 41 +(`packages/microcosm-build/src/microcosm/build/spec_engine/field_usage.py:29-31,387-393,671-692,803-822`; +`packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py:348-380,417-453,1649-1712`; +`docs/evidence/spec-engine/us-f0-coverage.json:1-15,775-788,1860-2060,2602-2612`). + +Both generated freshness checks pass byte-for-byte: + +```text +US bundle spec_sha256=5378bb9189aec96f50da22aac71e5bd2c3d919e9795f6ef2147e0bc9c739dd8e +spec-engine coverage: 42120/42120 configuration fields; 41/41 inventory checks +``` + +The new declarations and schema rules are generated in +`us/spec/{geography,sources,vintages,spine}.yaml`; the field-usage ledger and +inventory exact item are their fail-closed consumers +(`packages/microcosm-build/src/microcosm/build/us/spec/geography.yaml:3-37`; +`packages/microcosm-build/src/microcosm/build/us/spec/sources.yaml:69-86`; +`packages/microcosm-build/src/microcosm/build/us/spec/vintages.yaml:45-53`; +`packages/microcosm-build/src/microcosm/build/spec_engine/schema/geography.schema.json:108-150`; +`packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py:1649-1712`). + +### Validation and operational discipline + +- `microcosm-calibrate`: 203 passed. +- `microcosm-data`: 318 passed / 2 skipped. +- `microcosm-fit`: 93 passed. +- `microcosm-frame`: 295 passed / 36 skipped. +- `microcosm-build` partition A: 4,155 passed / 36 skipped. +- `microcosm-build` partition B: 2,177 passed / 3 skipped. +- Accepted total: **7,241 passed / 77 skipped / 0 failed**. +- Repository-wide Ruff, both generator freshness checks, smoke-script shell + syntax, and `git diff --check` pass. The source-blind import-graph pin covers + the exact 69 runtime modules now reached + (`packages/microcosm-build/tests/test_us_spine_blindness.py:3270-3315`). + +No production pool or release build was required: integration evidence is +fixture-sized. No push occurred, the release guard/operator boundary were not +weakened, and `logbook-pending-chain.txt` was not touched. + +## Final candidate Stage-1 flags + +Keep the current six authenticated source pairs, `--sample-fraction 0.25`, +`--sample-seed 578`, `--clone-attachment-fraction 1.0`, +`--clone-attachment-seed 578`, `--checkpoint-root`, and `--out`. Add these +exact runbook variables: + +```bash +PUMA_LADDER="/Users/maxghenis/PolicyEngine/_worktrees/populace-acs-clone/build/us/us_puma_ladder_2020.npz" +PUMA_LADDER_SHA="39a2ab2abeab07a88362af7ab2940e0e1d50a297c919e4bbc6fb65bab51147d8" +CD_CROSSWALK="$WT/packages/microcosm-build/src/microcosm/build/us_runtime/data/congressional_district_vintage_crosswalk.csv" +CD_CROSSWALK_SHA="c7cb040b1f57ca2ea2adcbfe60cc2b250ca23acbc4b640cd421e766fa54c1aec" +``` + +Add these exact `check_pool_inputs` calls: + +```bash +check_sha256 puma-ladder "$PUMA_LADDER" "$PUMA_LADDER_SHA" +check_sha256 cd-crosswalk "$CD_CROSSWALK" "$CD_CROSSWALK_SHA" +``` + +Add these exact `POOL_COMMAND` arguments: + +```bash +--puma-ladder "$PUMA_LADDER" +--puma-ladder-sha256 "$PUMA_LADDER_SHA" +--congressional-district-vintage-crosswalk "$CD_CROSSWALK" +--congressional-district-vintage-crosswalk-sha256 "$CD_CROSSWALK_SHA" +``` + +The parser and canonical-pin validator fail closed on any absent or different +value (`tools/build_us_multispine_pool.py:563-594,897-928`). Preserve the +runbook's wait loop until no other `build_us_*` process is running, invoke the +stage through `env -u POPULACE_LOGBOOK_PREV_ROW_DIGEST`, pass no +`--logbook-prev-row-digest`, and never touch `logbook-pending-chain.txt`. + +## Final checkpoint-reuse verdict + +**No: the existing smoke pool checkpoints and pool H5 are not reusable.** The +configured namespace now binds both new SHA pins, the base identity binds the +complete geography contract, and the checkpoint materializer moved from 11 to +12; old namespaces are not selected, and a relocated stale manifest fails the +exact identity comparison +(`tools/build_us_multispine_pool.py:332-360,1300-1555`). The old schema-8 / +materializer-2 pool H5 also lacks the required physical attrs and household +support binding and is rejected by schema-9 loading +(`packages/microcosm-build/src/microcosm/build/us_runtime/h5_io.py:75-105,671-923,1032-1193`). +The six immutable raw source artifacts remain reusable, but Stage-1 stage +checkpoints, downstream banks, the terminal manifest, and the pool H5 must be +regenerated under the new identity. diff --git a/docs/evidence/spec-engine/us-f0-coverage.json b/docs/evidence/spec-engine/us-f0-coverage.json index 8b2fae90..68f70290 100644 --- a/docs/evidence/spec-engine/us-f0-coverage.json +++ b/docs/evidence/spec-engine/us-f0-coverage.json @@ -6,8 +6,8 @@ "country": "us", "documentation_sha256": "4b39450dbdb8dafb83c3b627123b8026c6f82c660b66fe76f341a67c4f37c77b", "field_usage": { - "authored_normative_field_count": 32331, - "claim_count": 47, + "authored_normative_field_count": 32351, + "claim_count": 49, "claims": [ { "consumer": "loader.typed_resource_manifest", @@ -144,19 +144,35 @@ "verifier": "catalog_columns" }, { - "consumer": "spec_binding.geography_contract", + "consumer": "spec_binding.geography_phase", "generation0_effect": "no_generation0_effect", - "id": "geography", + "id": "geography_phase", "legacy_sinks": [], "mode": "identity_only", "pointer_class": "all", - "pointer_count": 26, - "pointer_sha256": "d07260bec0a904155b3786234b878f4010dc9a6cf05882eef8c6417dfe283156", - "rationale": "F0 normalizes the phase=legacy geography contract but the constants-era executor retains today's PUMA-ladder authority until F1", + "pointer_count": 1, + "pointer_sha256": "e6f1851db8754554dcbab4b09ca911a63ddba82fdd9743a7af542f216037bf9d", + "rationale": "the migration phase labels the mirrored contract but does not change the generation-0 stacked geography executor", "relative_sink_prefix": null, - "source_prefix": "/authored/spec~1geography.yaml", + "source_prefix": "/authored/spec~1geography.yaml/phase", "verifier": "identity" }, + { + "consumer": "legacy_adapter.stacked_checkpoint_static_components.geography_assignment", + "generation0_effect": "legacy_behavior", + "id": "geography_assignment", + "legacy_sinks": [ + "/stacked_checkpoint_static_components/geography_assignment" + ], + "mode": "legacy_behavior", + "pointer_class": "all", + "pointer_count": 28, + "pointer_sha256": "1eb3eaedce22de09799d9a23cc9034f0c048d7b5cc59d76b96b83c5bb49e1508", + "rationale": null, + "relative_sink_prefix": null, + "source_prefix": "/authored/spec~1geography.yaml/assignment", + "verifier": "legacy" + }, { "consumer": "legacy_adapter.imputation", "generation0_effect": "legacy_behavior", @@ -426,15 +442,31 @@ "source_prefix": "/authored/spec~1selection.yaml", "verifier": "identity" }, + { + "consumer": "legacy_adapter.stacked_checkpoint_static_components.geography_assignment", + "generation0_effect": "legacy_behavior", + "id": "source_geography_identity", + "legacy_sinks": [ + "/stacked_checkpoint_static_components/geography_assignment" + ], + "mode": "legacy_behavior", + "pointer_class": "stacked_geography_source_identity", + "pointer_count": 8, + "pointer_sha256": "364a30f79a717a3627ae5e972c753d98a7727ec30d41682ac5b3f5d8027b6274", + "rationale": null, + "relative_sink_prefix": null, + "source_prefix": "/authored/spec~1sources.yaml/sources", + "verifier": "legacy" + }, { "consumer": "loader.sources_schema_and_resolver.source_registry", "generation0_effect": "no_generation0_effect", "id": "source_pins", "legacy_sinks": [], "mode": "front_end_validation", - "pointer_class": "all", - "pointer_count": 72, - "pointer_sha256": "6daa8c007103e65f4485b5ab22c5579876e09da9ba9d182fbf1bbd854f29240e", + "pointer_class": "source_validation", + "pointer_count": 74, + "pointer_sha256": "a6100cd55d2e4f450208b6b0d3513f3d56c5068f74a54dfe62705d9a5f33ef07", "rationale": null, "relative_sink_prefix": null, "source_prefix": "/authored/spec~1sources.yaml/sources", @@ -499,7 +531,7 @@ "mode": "legacy_behavior", "pointer_class": "all", "pointer_count": 88, - "pointer_sha256": "c6e63034c73b2b3ad06df26d03fbf7d63d52aec3a7c3a176b228bacf08d18975", + "pointer_sha256": "ae011455154bae0df3913ca9a056058a909d913439b292cb46c11f63c7d0d9a3", "rationale": null, "relative_sink_prefix": null, "source_prefix": "/authored/spec~1spine.yaml/pipeline_contract", @@ -673,8 +705,8 @@ "legacy_sinks": [], "mode": "compiler_semantic", "pointer_class": "all", - "pointer_count": 105, - "pointer_sha256": "6c3b9c137149c764eae5ca925e3b789ba3f6f589cd25fbc1da58d21a081c7b5e", + "pointer_count": 112, + "pointer_sha256": "c69a923e893fde8a929ca087d29ded607fa9c04fd14a39d0a401ad1db845e6de", "rationale": null, "relative_sink_prefix": null, "source_prefix": "/authored/spec~1vintages.yaml", @@ -732,29 +764,29 @@ "legacy_sinks": [], "mode": "compiler_semantic", "pointer_class": "all", - "pointer_count": 59, - "pointer_sha256": "f9cde9945a03fa6b98685cafbda71efd947dc42f706e047f1d693e0750b85a7a", + "pointer_count": 63, + "pointer_sha256": "5f2edfe9826e1200e2b8706780c9f53b7c61f01aad8cab5907f08c37a0b1ead2", "rationale": null, "relative_sink_prefix": null, "source_prefix": "/resolved/vintage_authorities", "verifier": "vintages" } ], - "configuration_field_count": 42096, - "consumed_field_count": 42096, + "configuration_field_count": 42120, + "consumed_field_count": 42120, "generation0_effect_counts": { - "legacy_behavior": 38407, - "no_generation0_effect": 3689 + "legacy_behavior": 38443, + "no_generation0_effect": 3677 }, "mode_counts": { - "compiler_semantic": 27688, - "front_end_validation": 346, - "identity_only": 128, - "legacy_behavior": 13934 + "compiler_semantic": 27699, + "front_end_validation": 348, + "identity_only": 103, + "legacy_behavior": 13970 }, "multiple_primary_use_field_count": 0, - "pointer_inventory_sha256": "6d7353c6c42a6e1dbc6e3a227848e36864526fdc9533d5b284aa469c87dc064f", - "resolved_binding_field_count": 9765, + "pointer_inventory_sha256": "bc4a948ab632191954600da8474c5b011f977a65e24c399d126f3dc4a79f23e5", + "resolved_binding_field_count": 9769, "unused_field_count": 0 }, "inventory_coverage": { @@ -782,16 +814,16 @@ "producer_nodes": 38, "producer_virtual_resources": 75, "release_rungs": 5, - "resolved_references": 325, + "resolved_references": 334, "seed_owner_bindings": 112, "seed_owner_rows": 54, "seed_sites": 53, "seed_streams": 14, "source_operators": 16, "source_stages": 37, - "stacked_checkpoint_full_components": 12, + "stacked_checkpoint_full_components": 13, "stacked_checkpoint_pool_code_components": 19, - "stacked_checkpoint_static_components": 9, + "stacked_checkpoint_static_components": 10, "tail_control_fields": 934, "take_up_pipeline_steps": 28, "take_up_programs": 17, @@ -800,7 +832,7 @@ "typed_entities": 8, "typed_scopes": 7 }, - "covered_item_count": 40, + "covered_item_count": 41, "items": { "acs_group_predictors_exact": { "bundle_home_match_counts": { @@ -1838,6 +1870,7 @@ "field_names": [ "artifact_kind", "clone_attachment", + "geography_assignment", "inputs", "materializer_version", "model_seed", @@ -1854,13 +1887,14 @@ "alpha", "zeta" ], - "sha256": "b6a47fac54d7de7aa42ce59dc1950c0765b7a67034f0174ad1531d5bbb06ceef" + "sha256": "a128a85f877fb32def9382b841b8b340f974e8a9148ac029c1f04becdc956c18" }, "failures": [], "observed": { "field_names": [ "artifact_kind", "clone_attachment", + "geography_assignment", "inputs", "materializer_version", "model_seed", @@ -1877,7 +1911,7 @@ "alpha", "zeta" ], - "sha256": "b6a47fac54d7de7aa42ce59dc1950c0765b7a67034f0174ad1531d5bbb06ceef" + "sha256": "a128a85f877fb32def9382b841b8b340f974e8a9148ac029c1f04becdc956c18" }, "status": "covered" }, @@ -1941,7 +1975,7 @@ "take_up_contract", "us_qbi_reconciliation_contract" ], - "sha256": "044d8a45c4fe42eec9f72f9bedbf403536b9734cf74d8d65b72caf3c6c1d60b7" + "sha256": "91c65a9ff36839d575036264c4bf57ffde6457e8fd180a0856f9be712ada371d" }, "status": "covered" }, @@ -1962,6 +1996,7 @@ "expected": { "field_names": [ "artifact_kind", + "geography_assignment", "materializer_version", "model_seed", "period", @@ -1976,6 +2011,7 @@ "observed": { "field_names": [ "artifact_kind", + "geography_assignment", "materializer_version", "model_seed", "period", @@ -1988,6 +2024,41 @@ }, "status": "covered" }, + "stacked_geography_assignment_exact": { + "bundle_home_match_counts": { + "/geography/assignment": 1, + "/sources/sources": 1, + "/spine/seed_site_bindings": 1, + "/vintages/records": 1 + }, + "bundle_homes": [ + "/geography/assignment", + "/sources/sources", + "/vintages/records", + "/spine/seed_site_bindings" + ], + "compiler_consumers": [ + "legacy_adapter.stacked_checkpoint_static_components.geography_assignment" + ], + "expected": { + "authority_roles": [ + "congressional_district_vintage_crosswalk", + "puma_ladder" + ], + "sha256": "f49425ca8734ac559c73cf44f6458d86d3162a48956b98a27e6e758959361585", + "target_vintage": "119th_congress" + }, + "failures": [], + "observed": { + "authority_roles": [ + "congressional_district_vintage_crosswalk", + "puma_ladder" + ], + "sha256": "f49425ca8734ac559c73cf44f6458d86d3162a48956b98a27e6e758959361585", + "target_vintage": "119th_congress" + }, + "status": "covered" + }, "take_up_identity_exact": { "bundle_home_match_counts": { "/take_up": 1 @@ -2521,14 +2592,14 @@ "missing_item_count": 0, "missing_items": [], "report_schema_version": 1, - "required_item_count": 40, + "required_item_count": 41, "spec_binding": { "attestation": "mirror-attested", "canonicalizer_version": 1, "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "3189d90dec95c8ea7090e41b5283fa52b1e6855bed4a776dfa02820f2bd11c62" + "spec_sha256": "5378bb9189aec96f50da22aac71e5bd2c3d919e9795f6ef2147e0bc9c739dd8e" } }, "report_schema_version": 3, @@ -2538,7 +2609,7 @@ "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "3189d90dec95c8ea7090e41b5283fa52b1e6855bed4a776dfa02820f2bd11c62" + "spec_sha256": "5378bb9189aec96f50da22aac71e5bd2c3d919e9795f6ef2147e0bc9c739dd8e" }, "status": "pass" } diff --git a/docs/us-multispine-operator-ordering.md b/docs/us-multispine-operator-ordering.md index 67b5b93e..9a0054b8 100644 --- a/docs/us-multispine-operator-ordering.md +++ b/docs/us-multispine-operator-ordering.md @@ -80,9 +80,13 @@ declared SHA-256 values: - the ACS household and person PUMS archives, whose caller-supplied hashes must also match the checked-in ACS source manifest; - the canonical ACS 2022 rent donor used by the post-assembly housing - operator; and + operator; - the processed PUF H5 and source-year PUF CSV used by the existing donor - loader. + loader; +- the canonical 2020-PUMA population-overlap ladder used by the post-assembly + household-geography operator; and +- the packaged 117th-to-119th-Congress crosswalk that authenticates the + ladder's current congressional-district target universe. The raw artifact is a second producer output, not a relabeling of `pre_clone_enrichment`. It contains pooled ASEC unit structure and measured raw @@ -107,11 +111,12 @@ are allowed only when named by the ACS native-input receipt. ### Default sequence The exact outer `US_STACKED_POOL_OPERATOR_ORDER` is byte-stable and contains -these ten entries. The numbered paragraphs below explain those phases; nested +these eleven entries. The numbered paragraphs below explain those phases; nested callbacks are not additional outer entries. ```text assemble_stacked_spine +assign_us_puma_ladder prepare_multispine_source_inputs_for_clone gap_fill_stacked_spine run_stacked_late_producer_dag @@ -126,13 +131,24 @@ by_origin_battery 1. `assemble_stacked_spine(...)` selects whole households independently from both survey arms with the single `sample_fraction` and `sample_seed`, restores each sample to its full-source design-weight mass, and assembles - one origin-labeled frame. Standard rungs are `f001`, `f010`, and `f100`; + one origin-labeled frame. Standard rungs are `f001`, `f004`, `f010`, `f025`, + and `f100`; the manifest binds the fraction, seed, exact realized ASEC/ACS counts, selected-lineage digests, the complete ordered native ACS household and person support/raw/household-parent/classification mappings, and the sampled native ACS TYPEHUGQ 2/3 household and person lineage digests. The full PUF remains a donor and is never sampled. -2. The spine-blind source-preparation chain derives the native predictors and +2. `assign_us_puma_ladder(...)` runs on that first shared frame, after both + source-boundary checks and before source preparation. It authenticates the + pinned 2020-PUMA population-overlap ladder and the packaged + 117th-to-119th-Congress crosswalk, preserves observed ACS PUMAs, and uses the + ledgered `geography_legacy` seed stream to assign missing PUMAs plus household + congressional districts and counties. Its ordered household/geography + digest, positive-support counts, target universe, authority hashes, + algorithm, and seed are bound into every stacked checkpoint and the terminal + manifest; publication writes the crosswalk hash and `119th_congress` target + vintage into the nullable H5 root attributes. +3. The spine-blind source-preparation chain derives the native predictors and pre-clone operator outputs needed by the early declared cross-origin fills. Historical kernels run on the raw-`PERIDNUM` CPS/ASEC availability projection and merge only their declared outputs back into the stack. In @@ -142,7 +158,7 @@ by_origin_battery Targets produced only by the later PUF pass or late source producers are excluded from this early authority surface. No population operator selects behavior from the source-channel labels. -3. `gap_fill_stacked_spine(...)` runs the two immutable directions over the +4. `gap_fill_stacked_spine(...)` runs the two immutable directions over the same frame: ASEC survey fields fill ACS nulls, then ASEC-produced housing rent fills ACS housing-unit nulls in a separately banked direction. Activation authority is source-and-role exact, observed zero is not @@ -159,7 +175,7 @@ by_origin_battery finish with zero `unmodeled_rows` and zero residual nulls: transfer accounting alone is not terminal absence authority. The #608 per-target banks sit beneath the stack-bound checkpoint identity. -4. The derived second node of `run_stacked_late_producer_dag(...)` invokes +5. The derived second node of `run_stacked_late_producer_dag(...)` invokes `run_stacked_puf_pass(...)`; it is not a second outer operator-order entry. The callback attaches the separately controlled PUF clone arm (`clone_attachment_fraction`, default `1.0`) and runs one primary QRF pass @@ -229,14 +245,14 @@ by_origin_battery The authority versions distinguish the two contracts. The primary-QRF root and target checkpoint schema remains version 6. The capital-gains tail manifest uses schema version 2 and binds its support contract and receipt. - The canonical stacked authority remains version 9, the outer stacked - checkpoint materializer uses version 10, and the stacked pool stage - checkpoint materializer uses version 5. + The canonical stacked authority is version 11, the outer stacked checkpoint + materializer uses version 12, and the stacked pool stage checkpoint + materializer uses version 7. The outer base identity binds primary-QRF version 6, the ACS universe and QBI reconciliation contracts, the tail schema and support contract, and - late-producer registry schema version 14, including the signed static and + late-producer registry schema version 16, including the signed static and derivation-mode semantics of every virtual DAG resource. The companion pool - manifest uses schema version 7. + manifest uses schema version 9. Older outer authority or materializer payloads are stale; primary-QRF version 6 remains current. @@ -252,7 +268,7 @@ by_origin_battery transfer. The declared batch-5-to-adult-care edge below therefore derives the repair from data dependency rather than installing another manual ordering exception. -5. One declared late-producer DAG schedules the ACS earnings-universe +6. One declared late-producer DAG schedules the ACS earnings-universe materializer, primary PUF/tail pass, all 16 post-clone source operators, their once-only finalizer, and all 19 bounded transfer groups. Each node declares every effective input and output. A callback cannot run until each @@ -278,7 +294,7 @@ by_origin_battery filled from QRF predictions. No blanket null-to-zero synthesis occurs, every producer cell stays byte-identical, and zero residual nulls are required. -6. The transferred checkpoint records the early gap-fill banks, 19 distinct +7. The transferred checkpoint records the early gap-fill banks, 19 distinct late-transfer banks, the primary-QRF bank, the complete 38-node DAG receipt, tail manifest and its per-status support receipt, weights audit, stack-manifest digest, fraction/seed, clone controls, and the channel-aware @@ -302,7 +318,7 @@ by_origin_battery bytes, and checkpoint discovery positively accepts the current identity but rejects a stale source asset/config even when every engine and sampling pin is otherwise identical. -7. Schedule-D preparation, deterministic derivation, seeded inputs, and +8. Schedule-D preparation, deterministic derivation, seeded inputs, and batched simulation run on the transferred stack. QBI reconciliation uses the same source declaration: it fails on any in-universe self-employment null and preserves the receipted ACS under-15 base self-employment zero in @@ -337,7 +353,7 @@ by_origin_battery authority. The tool retains the existing `assembled`, `transferred`, and `simulated` #599 boundaries. -8. A fresh `us_stacked_completeness` gate proves every declared input is +9. A fresh `us_stacked_completeness` gate proves every declared input is observed or has exact source-by-role absence authority. The terminal `us_by_origin_battery` then evaluates all 134 declared targets (114 person, 12 tax-unit, 8 SPM-unit), plus joint immigration structure, using an @@ -354,7 +370,7 @@ by_origin_battery digest fails closed. At small rungs, comparisons outside the validity domain receipt `insufficient_support`; tolerances do not widen. -9. Only after both gates run does publication write the nullable H5, +10. Only after both gates run does publication write the nullable H5, diagnostics, and readiness manifest. Success, failed gate, and exception paths each append a durable Logbook spool row beside the output, with the fraction token, seed, code/input/identity pins, phases, gate-receipt @@ -954,9 +970,10 @@ cannot bypass late-DAG validation: those post-assembly results. Existing measured/native cells remain byte-for-byte unchanged, and transfer receipts record fitted and imputed rows. The three SCF/SIPP financial-asset leaves are excluded only from this - pool-local transfer plan because none of the six pinned pool inputs is their - donor. They remain hard release requirements and legacy ACS-transfer - targets. The pool persists each as a typed all-null column with an explicit + pool-local transfer plan because none of the six pinned model-data inputs is + their donor; the two additional pinned inputs are geography authorities, not + financial-asset donors. They remain hard release requirements and legacy + ACS-transfer targets. The pool persists each as a typed all-null column with an explicit `with_us_scf_wealth_inputs` deferral receipt; the disposable SSI agreement view separately receipts the engine-default fills it needs for evaluation. 6. Schedule-D and QBI deterministic reconciliation run over that same pool. diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/field_usage.py b/packages/microcosm-build/src/microcosm/build/spec_engine/field_usage.py index 606e3852..a5dd282a 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/field_usage.py +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/field_usage.py @@ -26,9 +26,9 @@ ) from .schemas import load_schema_registry -EXPECTED_AUTHORED_FIELD_COUNT = 32_331 -EXPECTED_RESOLVED_BINDING_FIELD_COUNT = 9_765 -EXPECTED_CONFIGURATION_FIELD_COUNT = 42_096 +EXPECTED_AUTHORED_FIELD_COUNT = 32_351 +EXPECTED_RESOLVED_BINDING_FIELD_COUNT = 9_769 +EXPECTED_CONFIGURATION_FIELD_COUNT = 42_120 class FieldUsageError(AssertionError): @@ -204,6 +204,58 @@ def is_concept_validation(pointer: str) -> bool: return [row for row in rows if not is_concept_validation(row[0])] if claim.pointer_class == "family_concept_validation": return [row for row in rows if is_concept_validation(row[0])] + + def is_stacked_geography_source_identity(pointer: str) -> bool: + if not isinstance(subtree, (list, tuple)): + return False + relative = pointer.removeprefix(claim.source_prefix).removeprefix("/") + tokens = relative.split("/") + if len(tokens) < 2: + return False + try: + source_index = int(tokens[0]) + except ValueError: + return False + if source_index < 0 or source_index >= len(subtree): + return False + source = subtree[source_index] + if not isinstance(source, Mapping): + return False + source_id = source.get("id") + vintage_id = { + "us_puma_ladder_2020": "cd_119", + "us_congressional_district_vintage_crosswalk_117_to_119": "cd_117", + }.get(source_id) + if vintage_id is None: + return False + if len(tokens) == 2: + return tokens[1] in {"id", "sha256"} + if len(tokens) != 4 or tokens[1] != "vintage_authorities": + return False + try: + vintage_index = int(tokens[2]) + except ValueError: + return False + authorities = source.get("vintage_authorities") + if not isinstance(authorities, (list, tuple)) or not ( + 0 <= vintage_index < len(authorities) + ): + return False + authority = authorities[vintage_index] + return ( + isinstance(authority, Mapping) + and authority.get("id") == vintage_id + and tokens[3] in {"id", "value"} + ) + + if claim.pointer_class == "stacked_geography_source_identity": + return [ + row for row in rows if is_stacked_geography_source_identity(row[0]) + ] + if claim.pointer_class == "source_validation": + return [ + row for row in rows if not is_stacked_geography_source_identity(row[0]) + ] raise FieldUsageError(f"{claim.id}: unknown pointer class {claim.pointer_class!r}") @@ -332,9 +384,13 @@ def _path_inventory(rows: Sequence[tuple[str, object]]) -> tuple[int, str]: 8_606, "3f20975597d93f7313583a944eeb9d6437651c4ff20e67628bf6bf4c5aa9f004", ), - "geography": ( - 26, - "d07260bec0a904155b3786234b878f4010dc9a6cf05882eef8c6417dfe283156", + "geography_assignment": ( + 28, + "1eb3eaedce22de09799d9a23cc9034f0c048d7b5cc59d76b96b83c5bb49e1508", + ), + "geography_phase": ( + 1, + "e6f1851db8754554dcbab4b09ca911a63ddba82fdd9743a7af542f216037bf9d", ), "imputation_chaining": ( 20, @@ -409,16 +465,20 @@ def _path_inventory(rows: Sequence[tuple[str, object]]) -> tuple[int, str]: "368fc19a8f07c4abf38ebcf2fbc4414d33c403c0f1f9881c2d2e3f5d6160feb6", ), "resolved_vintage_authorities": ( - 59, - "f9cde9945a03fa6b98685cafbda71efd947dc42f706e047f1d693e0750b85a7a", + 63, + "5f2edfe9826e1200e2b8706780c9f53b7c61f01aad8cab5907f08c37a0b1ead2", ), "selection": ( 87, "a1e8d271197566eb3cf23b309156c2efb563ae10fe21968a3cfe9b9d827da2db", ), + "source_geography_identity": ( + 8, + "364a30f79a717a3627ae5e972c753d98a7727ec30d41682ac5b3f5d8027b6274", + ), "source_pins": ( - 72, - "6daa8c007103e65f4485b5ab22c5579876e09da9ba9d182fbf1bbd854f29240e", + 74, + "a6100cd55d2e4f450208b6b0d3513f3d56c5068f74a54dfe62705d9a5f33ef07", ), "source_stage_asset": ( 2, @@ -450,7 +510,7 @@ def _path_inventory(rows: Sequence[tuple[str, object]]) -> tuple[int, str]: ), "spine_pipeline_contract": ( 88, - "c6e63034c73b2b3ad06df26d03fbf7d63d52aec3a7c3a176b228bacf08d18975", + "ae011455154bae0df3913ca9a056058a909d913439b292cb46c11f63c7d0d9a3", ), "spine_sampling": ( 17, @@ -477,8 +537,8 @@ def _path_inventory(rows: Sequence[tuple[str, object]]) -> tuple[int, str]: "d53096196db4c34da260cce2f35af8e7ba67f978448656c602ee5e17529dc4e0", ), "vintage_records": ( - 105, - "6c3b9c137149c764eae5ca925e3b789ba3f6f589cd25fbc1da58d21a081c7b5e", + 112, + "c69a923e893fde8a929ca087d29ded607fa9c04fd14a39d0a401ad1db845e6de", ), } @@ -608,15 +668,26 @@ def default_usage_claims() -> tuple[UsageClaim, ...]: "catalog_columns", ), _claim( - "geography", - f"{_A}/spec~1geography.yaml", + "geography_phase", + f"{_A}/spec~1geography.yaml/phase", UsageMode.IDENTITY_ONLY, _NO_EFFECT, - "spec_binding.geography_contract", + "spec_binding.geography_phase", "identity", rationale=( - "F0 normalizes the phase=legacy geography contract but the " - "constants-era executor retains today's PUMA-ladder authority until F1" + "the migration phase labels the mirrored contract but does not " + "change the generation-0 stacked geography executor" + ), + ), + _claim( + "geography_assignment", + f"{_A}/spec~1geography.yaml/assignment", + UsageMode.LEGACY_BEHAVIOR, + _LEGACY, + "legacy_adapter.stacked_checkpoint_static_components.geography_assignment", + "legacy", + legacy_sinks=( + "/stacked_checkpoint_static_components/geography_assignment", ), ), ] @@ -728,6 +799,18 @@ def default_usage_claims() -> tuple[UsageClaim, ...]: "not change generation-0 selection behavior in F0" ), ), + _claim( + "source_geography_identity", + f"{_A}/spec~1sources.yaml/sources", + UsageMode.LEGACY_BEHAVIOR, + _LEGACY, + "legacy_adapter.stacked_checkpoint_static_components.geography_assignment", + "legacy", + legacy_sinks=( + "/stacked_checkpoint_static_components/geography_assignment", + ), + pointer_class="stacked_geography_source_identity", + ), _claim( "source_pins", f"{_A}/spec~1sources.yaml/sources", @@ -735,6 +818,7 @@ def default_usage_claims() -> tuple[UsageClaim, ...]: _NO_EFFECT, "loader.sources_schema_and_resolver.source_registry", "source_pins", + pointer_class="source_validation", ), _claim( "source_stage_asset", diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py b/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py index 56f3c6ec..b48afbf5 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py @@ -59,6 +59,7 @@ "pipeline", "policyengine_us_version", "pool_code", + "geography_assignment", "schema_version", "stacked_authority", } @@ -349,9 +350,10 @@ "acs_person_predictors": "878c788a6f037d7aca12b3586ea034eff04f3034ffa11935a736493042551f25", "authority": "3a980927227704d0589f246eef9cd825c2ae84f3a4134ac835e0e5ed39a563ac", "early_families": "4aa9f736fd76e83955477ad1667e58f48f264783f05bdc7f0102cd32d61323bd", - "full_checkpoint": "b6a47fac54d7de7aa42ce59dc1950c0765b7a67034f0174ad1531d5bbb06ceef", + "full_checkpoint": "a128a85f877fb32def9382b841b8b340f974e8a9148ac029c1f04becdc956c18", "gap_fill_schedule": "1c31f9868f7884347cc19cf1ff65da43f950b9114941a715bab168246db414a7", "graph_nodes": "7125ad28ae2c69f22094a574bbf6ed2ddf1682a2c2c3b416f8f49304b7016ce7", + "geography_assignment": "f49425ca8734ac559c73cf44f6458d86d3162a48956b98a27e6e758959361585", "late_families": "d91f9ff0eb52f43e7b6eed3d5c58c37abe1620c3a11021da15dae9c10e16d382", "late_resource_semantics": "3850554eb804fde5e4f86a34ac1bb8a7a07aafff7e8b48396a3d5fca844798e8", "late_schedule": "dcf3c6d2eade3449836c49a1dc4d3b8cd395aab9142db700c3c60598fa9c1c79", @@ -375,6 +377,7 @@ "early_gap_fill_plan_exact", "early_transfer_surface_exact", "gap_fill_schedule_receipt_exact", + "stacked_geography_assignment_exact", "itemization_declared_splits_exact", "late_schedule_receipt_exact", "late_split_ledger_exact", @@ -430,16 +433,16 @@ "producer_nodes": 38, "producer_virtual_resources": 75, "release_rungs": 5, - "resolved_references": 325, + "resolved_references": 334, "seed_owner_bindings": 112, "seed_owner_rows": 54, "seed_sites": 53, "seed_streams": 14, "source_operators": 16, "source_stages": 37, - "stacked_checkpoint_full_components": 12, + "stacked_checkpoint_full_components": 13, "stacked_checkpoint_pool_code_components": 19, - "stacked_checkpoint_static_components": 9, + "stacked_checkpoint_static_components": 10, "tail_control_fields": 934, "take_up_pipeline_steps": 28, "take_up_programs": 17, @@ -620,6 +623,7 @@ def build_inventory_coverage( } imputation = _mapping(domains["imputation"], "imputation") sources = _mapping(domains["sources"], "sources") + geography = _mapping(domains["geography"], "geography") spine = _mapping(domains["spine"], "spine") take_up = _mapping(domains["take_up"], "take_up") calibration = _mapping(domains["calibration"], "calibration") @@ -647,6 +651,14 @@ def build_inventory_coverage( "expected checkpoint static", ) pool_code = _mapping(static.get("pool_code"), "checkpoint pool_code") + geography_assignment = _mapping( + static.get("geography_assignment"), + "checkpoint geography assignment", + ) + expected_geography_assignment = _mapping( + expected_static.get("geography_assignment"), + "expected checkpoint geography assignment", + ) family_rows = [ _mapping(value, "imputation family") @@ -1634,6 +1646,70 @@ def add( observed={"field_names": sorted(static)}, expected={"field_names": sorted(EXPECTED_CHECKPOINT_TOP_LEVEL)}, ) + geography_authorities = _mapping( + geography_assignment.get("authorities"), + "checkpoint geography authorities", + ) + geography_seed = _mapping( + geography_assignment.get("seed"), + "checkpoint geography seed", + ) + add( + "stacked_geography_assignment_exact", + clauses={ + "checkpoint geography assignment differs": _json_equal( + geography_assignment, + expected_geography_assignment, + ), + "geography declaration is not embedded exactly": _json_equal( + geography_assignment.get("declaration"), + geography.get("assignment"), + ), + "geography assignment authorities differ": set(geography_authorities) + == { + "puma_ladder", + "congressional_district_vintage_crosswalk", + }, + "geography assignment seed differs": geography_seed + == { + "site": "legacy_puma_ladder", + "stream": "geography_legacy", + "value_source": "run_request.build_model_seed", + "value": 0, + }, + "geography assignment digest differs": _operational_free_sha256( + geography_assignment + ) + == EXPECTED_HASHES["geography_assignment"], + }, + homes=( + "/geography/assignment", + "/sources/sources", + "/vintages/records", + "/spine/seed_site_bindings", + ), + consumers=( + "legacy_adapter.stacked_checkpoint_static_components.geography_assignment", + ), + observed={ + "sha256": _operational_free_sha256(geography_assignment), + "authority_roles": sorted(geography_authorities), + "target_vintage": _mapping( + geography_authorities.get( + "congressional_district_vintage_crosswalk" + ), + "checkpoint geography crosswalk authority", + ).get("target_vintage"), + }, + expected={ + "sha256": EXPECTED_HASHES["geography_assignment"], + "authority_roles": [ + "congressional_district_vintage_crosswalk", + "puma_ladder", + ], + "target_vintage": "119th_congress", + }, + ) add( "stacked_checkpoint_pool_code_exact", clauses={ diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/resolver.py b/packages/microcosm-build/src/microcosm/build/spec_engine/resolver.py index 1eba714b..e5316f61 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/resolver.py +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/resolver.py @@ -110,6 +110,7 @@ def contract_only_ids(self) -> frozenset[str]: "joint_count_calibration", "load_acs_2022_rent_donor", "load_asec_raw_stage_checkpoint", + "load_congressional_district_vintage_crosswalk", "load_puf_tax_unit_donor", "load_us_puma_ladder", "marketplace_assignment", diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/schema/geography.schema.json b/packages/microcosm-build/src/microcosm/build/spec_engine/schema/geography.schema.json index 05337adb..1806de21 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/schema/geography.schema.json +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/schema/geography.schema.json @@ -105,6 +105,26 @@ "identified_county_source": { "$ref": "defs.schema.json#/$defs/source_ref" }, + "congressional_district_vintage_crosswalk": { + "type": "object", + "additionalProperties": false, + "required": [ + "source_ref", + "source_vintage", + "target_vintage" + ], + "properties": { + "source_ref": { + "$ref": "defs.schema.json#/$defs/source_ref" + }, + "source_vintage": { + "$ref": "defs.schema.json#/$defs/vintage_ref" + }, + "target_vintage": { + "$ref": "defs.schema.json#/$defs/vintage_ref" + } + } + }, "derive": { "type": "array", "minItems": 1, diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/schema/spine.schema.json b/packages/microcosm-build/src/microcosm/build/spec_engine/schema/spine.schema.json index f84467d8..dc4c7955 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/schema/spine.schema.json +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/schema/spine.schema.json @@ -288,7 +288,7 @@ "const": "populace_us_stacked_pool_checkpoint_identity" }, "schema_version": {"const": 1}, - "materializer_version": {"const": 11}, + "materializer_version": {"const": 12}, "pipeline": {"const": "us-stacked-pool"} } }, diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/stacked_authority_semantics.py b/packages/microcosm-build/src/microcosm/build/spec_engine/stacked_authority_semantics.py index 2c26a666..f3e2e7f6 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/stacked_authority_semantics.py +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/stacked_authority_semantics.py @@ -107,6 +107,7 @@ def _us_domains(spec: ResolvedSpec) -> dict[str, dict[str, object]]: for kind in ( "battery", "bundle", + "geography", "imputation", "publication", "sources", @@ -116,6 +117,167 @@ def _us_domains(spec: ResolvedSpec) -> dict[str, dict[str, object]]: } +def _source_record( + sources: Mapping[str, object], + source_ref: str, +) -> Mapping[str, object]: + source_id = source_ref.removeprefix("source:") + matches = [ + row + for index, value in enumerate( + _array(sources.get("sources"), location="sources/sources") + ) + for row in [_mapping(value, location=f"sources/sources/{index}")] + if row.get("id") == source_id + ] + if len(matches) != 1: + raise SpecValidationError( + f"sources/sources: source ref {source_ref!r} must resolve exactly once" + ) + return matches[0] + + +def _source_vintage_value( + source: Mapping[str, object], + vintage_ref: str, +) -> str: + vintage_id = vintage_ref.removeprefix("vintage:") + matches = [ + authority + for index, value in enumerate( + _array( + source.get("vintage_authorities"), + location="sources/source/vintage_authorities", + ) + ) + for authority in [ + _mapping( + value, + location=f"sources/source/vintage_authorities/{index}", + ) + ] + if authority.get("id") == vintage_id + ] + if len(matches) != 1: + raise SpecValidationError( + f"source vintage authority {vintage_ref!r} must resolve exactly once" + ) + return _nonempty_string( + matches[0].get("value"), + location=f"sources/source/vintage_authorities/{vintage_id}/value", + ) + + +def _stacked_geography_assignment_contract( + spec: ResolvedSpec, + domains: Mapping[str, Mapping[str, object]], +) -> dict[str, object]: + assignment = _mapping( + domains["geography"].get("assignment"), + location="geography/assignment", + ) + kernels = _mapping( + assignment.get("kernels"), + location="geography/assignment/kernels", + ) + assign_kernel_ref = _nonempty_string( + kernels.get("assign"), + location="geography/assignment/kernels/assign", + ) + if assign_kernel_ref != "kernel:assign_us_puma_ladder": + raise SpecValidationError( + "geography/assignment/kernels/assign: stacked runtime requires " + "kernel:assign_us_puma_ladder" + ) + order = _nonempty_string( + assignment.get("order"), location="geography/assignment/order" + ) + if order != "before_gap_fill": + raise SpecValidationError( + "geography/assignment/order: stacked runtime requires before_gap_fill" + ) + ladder_ref = _nonempty_string( + assignment.get("ladder_source"), + location="geography/assignment/ladder_source", + ) + crosswalk = _mapping( + assignment.get("congressional_district_vintage_crosswalk"), + location=( + "geography/assignment/congressional_district_vintage_crosswalk" + ), + ) + crosswalk_ref = _nonempty_string( + crosswalk.get("source_ref"), + location=( + "geography/assignment/congressional_district_vintage_crosswalk/" + "source_ref" + ), + ) + source_vintage_ref = _nonempty_string( + crosswalk.get("source_vintage"), + location=( + "geography/assignment/congressional_district_vintage_crosswalk/" + "source_vintage" + ), + ) + target_vintage_ref = _nonempty_string( + crosswalk.get("target_vintage"), + location=( + "geography/assignment/congressional_district_vintage_crosswalk/" + "target_vintage" + ), + ) + ladder_source = _source_record(domains["sources"], ladder_ref) + crosswalk_source = _source_record(domains["sources"], crosswalk_ref) + seed_site = spec.seed_protocol.site("legacy_puma_ladder") + if seed_site.default is None: + raise SpecValidationError( + "seed_protocol/legacy_puma_ladder: default seed required" + ) + return { + "declaration": deepcopy(dict(assignment)), + "algorithm": { + "id": "assign_us_puma_ladder.population_weighted_overlap.v1", + "kernel": assign_kernel_ref.removeprefix("kernel:"), + "operator": "assign_us_puma_ladder", + "order": order, + "assign_tract": assignment.get("assign_tract"), + }, + "authorities": { + "puma_ladder": { + "input_role": "puma_ladder", + "source_ref": ladder_ref, + "sha256": _nonempty_string( + ladder_source.get("sha256"), + location="sources/us_puma_ladder_2020/sha256", + ), + }, + "congressional_district_vintage_crosswalk": { + "input_role": "congressional_district_vintage_crosswalk", + "source_ref": crosswalk_ref, + "sha256": _nonempty_string( + crosswalk_source.get("sha256"), + location="sources/CD vintage crosswalk/sha256", + ), + "source_vintage_ref": source_vintage_ref, + "source_vintage": _source_vintage_value( + crosswalk_source, source_vintage_ref + ), + "target_vintage_ref": target_vintage_ref, + "target_vintage": _source_vintage_value( + ladder_source, target_vintage_ref + ), + }, + }, + "seed": { + "site": seed_site.id, + "stream": seed_site.stream, + "value_source": seed_site.value_source, + "value": seed_site.default, + }, + } + + def _generated_engine_lock(spec: ResolvedSpec) -> dict[str, object]: generated = thaw_json(spec.generated_authorities) if not isinstance(generated, dict): # pragma: no cover - model invariant @@ -1038,6 +1200,9 @@ def project_stacked_checkpoint_base_identity( "fraction": attachment_fraction, "seed": attachment_seed_value, }, + "geography_assignment": _stacked_geography_assignment_contract( + spec, domains + ), "stacked_authority": project_stacked_authority_receipt(spec), "pool_code": { "operator_order": deepcopy( @@ -1214,6 +1379,7 @@ def project_stacked_checkpoint_static_components( "period": full["period"], "model_seed": full["model_seed"], "policyengine_us_version": full["policyengine_us_version"], + "geography_assignment": deepcopy(full["geography_assignment"]), "stacked_authority": deepcopy(full["stacked_authority"]), "pool_code": deepcopy(full["pool_code"]), } diff --git a/packages/microcosm-build/src/microcosm/build/us/spec/geography.yaml b/packages/microcosm-build/src/microcosm/build/us/spec/geography.yaml index 80c1b1c9..219bee5e 100644 --- a/packages/microcosm-build/src/microcosm/build/us/spec/geography.yaml +++ b/packages/microcosm-build/src/microcosm/build/us/spec/geography.yaml @@ -2,7 +2,7 @@ phase: legacy assignment: anchor: puma - order: legacy_post_transfer + order: before_gap_fill kernels: assign: kernel:assign_us_puma_ladder validate: kernel:us_puma_ladder_gate @@ -24,6 +24,10 @@ assignment: - observed_acs_puma_preserved - geography_state_prefix_consistent ladder_source: source:us_puma_ladder_2020 + congressional_district_vintage_crosswalk: + source_ref: source:us_congressional_district_vintage_crosswalk_117_to_119 + source_vintage: vintage:cd_117 + target_vintage: vintage:cd_119 seed: stream:geography_legacy default_seed: 0 assign_tract: false diff --git a/packages/microcosm-build/src/microcosm/build/us/spec/sources.yaml b/packages/microcosm-build/src/microcosm/build/us/spec/sources.yaml index da7159fb..a160533f 100644 --- a/packages/microcosm-build/src/microcosm/build/us/spec/sources.yaml +++ b/packages/microcosm-build/src/microcosm/build/us/spec/sources.yaml @@ -84,6 +84,18 @@ sources: - id: puma_2020 kind: geography_vintage value: 2020_puma +- id: us_congressional_district_vintage_crosswalk_117_to_119 + role: congressional_district_vintage_crosswalk + sha256: c7cb040b1f57ca2ea2adcbfe60cc2b250ca23acbc4b640cd421e766fa54c1aec + byte_size: 77935 + loader: kernel:load_congressional_district_vintage_crosswalk + vintages: + - vintage:cd_117 + - vintage:cd_119 + vintage_authorities: + - id: cd_117 + kind: geography_vintage + value: 117th_congress stage_asset: id: source_stages path: microcosm.build.us/source_stages.json diff --git a/packages/microcosm-build/src/microcosm/build/us/spec/spine.yaml b/packages/microcosm-build/src/microcosm/build/us/spec/spine.yaml index 851394da..e932429a 100644 --- a/packages/microcosm-build/src/microcosm/build/us/spec/spine.yaml +++ b/packages/microcosm-build/src/microcosm/build/us/spec/spine.yaml @@ -3,10 +3,11 @@ pipeline_contract: artifact_protocol: artifact_kind: populace_us_stacked_pool_checkpoint_identity schema_version: 1 - materializer_version: 11 + materializer_version: 12 pipeline: us-stacked-pool stacked_operator_order: - assemble_stacked_spine + - assign_us_puma_ladder - prepare_multispine_source_inputs_for_clone - gap_fill_stacked_spine - run_stacked_late_producer_dag @@ -44,7 +45,6 @@ pipeline_contract: - _complete_schedule_d_input - with_us_qbi_input_reconciliation auxiliary_operations: - - assign_us_puma_ladder - calibrate - select_exact_k qbi_reconciliation: diff --git a/packages/microcosm-build/src/microcosm/build/us/spec/vintages.yaml b/packages/microcosm-build/src/microcosm/build/us/spec/vintages.yaml index a70f9c5b..a5b70b90 100644 --- a/packages/microcosm-build/src/microcosm/build/us/spec/vintages.yaml +++ b/packages/microcosm-build/src/microcosm/build/us/spec/vintages.yaml @@ -42,6 +42,14 @@ records: compatible_with: - vintage:asec_2023 - vintage:target_2024 +- id: cd_117 + kind: geography_vintage_ref + authority_ref: + kind: source_record + source: source:us_congressional_district_vintage_crosswalk_117_to_119 + authority: cd_117 + compatible_with: + - vintage:cd_119 - id: cd_119 kind: geography_vintage_ref authority_ref: @@ -49,6 +57,7 @@ records: source: source:us_puma_ladder_2020 authority: cd_119 compatible_with: + - vintage:cd_117 - vintage:census_2020 - vintage:puma_2020 - id: census_2020 diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/h5_io.py b/packages/microcosm-build/src/microcosm/build/us_runtime/h5_io.py index 706ca354..c63fac0b 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/h5_io.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/h5_io.py @@ -26,6 +26,18 @@ canonicalize_frame_string_dtypes, canonicalize_table_string_dtypes, ) +from microcosm.build.us_runtime.congressional_district_geography import ( + CONGRESSIONAL_DISTRICT_GEOID_COLUMN, +) +from microcosm.build.us_runtime.congressional_district_vintage import ( + CONGRESSIONAL_DISTRICT_VINTAGE_CROSSWALK_SHA256_ATTR, + CONGRESSIONAL_DISTRICT_VINTAGE_TARGET_ATTR, + CURRENT_CONGRESSIONAL_DISTRICT_VINTAGE, +) +from microcosm.build.us_runtime.support_provenance import ( + support_clone_index_column, + support_source_id_column, +) from microcosm.frame import ( Frame, WeightKind, @@ -60,6 +72,8 @@ US_MULTISPINE_AGREEMENT_DIAGNOSTICS_ARTIFACT_KIND = ( "populace_us_multispine_agreement_diagnostics" ) +# 9 binds the post-assembly household-geography assignment receipt and its +# authenticated release-vintage authorities. # 8 binds the nullable-boolean-capable physical H5 materializer in both the # stacked manifest receipt and the H5's frozen metadata key. # 7 binds the complete late-producer resource semantics and removes the PUF @@ -68,9 +82,9 @@ # authority and restores its immutable Frame-metadata anchor on H5 load. # Schema 5 can authenticate the DAG receipt's structure, but cannot prove that # the published receipt is the one authorized by the generating transition. -US_MULTISPINE_POOL_MANIFEST_SCHEMA_VERSION = 8 -US_MULTISPINE_POOL_H5_MATERIALIZER_VERSION = 2 -"""Stacked terminal H5 materializer; version 2 handles pandas BooleanDtype.""" +US_MULTISPINE_POOL_MANIFEST_SCHEMA_VERSION = 9 +US_MULTISPINE_POOL_H5_MATERIALIZER_VERSION = 3 +"""Version 3 atomically binds release CD provenance attrs; v2 added BooleanDtype.""" _LEGACY_MULTISPINE_POOL_MANIFEST_SCHEMA_VERSION = 4 _METADATA_KEY = "_populace_staging_metadata" _TIME_PERIOD_KEY = "_time_period" @@ -78,6 +92,7 @@ _STACKED_PIPELINE = "us-stacked-pool" US_STACKED_POOL_OPERATOR_ORDER = ( "assemble_stacked_spine", + "assign_us_puma_ladder", "prepare_multispine_source_inputs_for_clone", "gap_fill_stacked_spine", "run_stacked_late_producer_dag", @@ -109,6 +124,7 @@ "release_id", "sampling", "clone_attachment", + "geography_assignment", "input_pins_digest", "late_producer_transition_authority_sha256", "stack_manifest", @@ -119,6 +135,7 @@ { "pipeline", "operator_order", + "geography_assignment", "stage_receipts", } ) @@ -595,7 +612,7 @@ def _validate_stacked_late_dag_manifest_binding( *, manifest_path: Path, ) -> None: - """Make schema-8 stacked consumers authenticate the published DAG proof.""" + """Make schema-9 consumers authenticate geography and late-DAG proofs.""" if manifest.get("pipeline") != "us-stacked-pool": return @@ -604,6 +621,10 @@ def _validate_stacked_late_dag_manifest_binding( f"US stacked pool manifest {manifest_path} does not bind the " "canonical late-DAG operator order." ) + _validate_stacked_geography_assignment_manifest_binding( + manifest, + manifest_path=manifest_path, + ) stage_receipts = manifest.get("stage_receipts") impute = ( stage_receipts.get("impute") if isinstance(stage_receipts, Mapping) else None @@ -647,6 +668,531 @@ def _validate_stacked_late_dag_manifest_binding( ) +def _validate_stacked_geography_assignment_manifest_binding( + manifest: Mapping[str, object], + *, + manifest_path: Path, +) -> None: + """Authenticate the post-assembly household-CD authority and receipt.""" + + assignment = manifest.get("geography_assignment") + if not isinstance(assignment, Mapping): + raise ValueError( + f"US stacked pool manifest {manifest_path} has no geography " + "assignment receipt." + ) + stage_receipts = manifest.get("stage_receipts") + stage_assignment = ( + stage_receipts.get("geography_assignment") + if isinstance(stage_receipts, Mapping) + else None + ) + if stage_assignment != assignment: + raise ValueError( + f"US stacked pool manifest {manifest_path} geography assignment " + "differs from its assembled-stage receipt." + ) + if ( + assignment.get("artifact_kind") + != ("populace_us_stacked_household_geography_assignment") + or assignment.get("schema_version") != 1 + ): + raise ValueError( + f"US stacked pool manifest {manifest_path} has an unsupported " + "geography assignment receipt." + ) + contract = assignment.get("contract") + if not isinstance(contract, Mapping): + raise ValueError( + f"US stacked pool manifest {manifest_path} geography assignment " + "has no contract." + ) + expected_declaration = { + "anchor": "puma", + "order": "before_gap_fill", + "kernels": { + "assign": "kernel:assign_us_puma_ladder", + "validate": "kernel:us_puma_ladder_gate", + }, + "draw": { + "asec": { + "universe": "puma_within_state", + "weight": "puma_population_2020", + }, + "congressional_district": { + "universe": "congressional_district_within_puma", + "weight": "block_population_overlap", + }, + "county": { + "universe": "county_within_puma", + "weight": "block_population_overlap", + }, + }, + "derive": ["puma", "congressional_district_geoid", "county_fips"], + "assertions": [ + "observed_acs_puma_preserved", + "geography_state_prefix_consistent", + ], + "ladder_source": "source:us_puma_ladder_2020", + "congressional_district_vintage_crosswalk": { + "source_ref": ( + "source:us_congressional_district_vintage_crosswalk_117_to_119" + ), + "source_vintage": "vintage:cd_117", + "target_vintage": "vintage:cd_119", + }, + "seed": "stream:geography_legacy", + "default_seed": 0, + "assign_tract": False, + "layer_vintages": { + "congressional_district": "vintage:cd_119", + "county": "vintage:census_2020", + "puma": "vintage:puma_2020", + "tract": "vintage:census_2020", + }, + "validation": ["puma_ladder_gate", "vintage_refusal"], + } + if contract.get("declaration") != expected_declaration: + raise ValueError( + f"US stacked pool manifest {manifest_path} geography declaration changed." + ) + algorithm = contract.get("algorithm") + expected_algorithm = { + "id": "assign_us_puma_ladder.population_weighted_overlap.v1", + "kernel": "assign_us_puma_ladder", + "operator": "assign_us_puma_ladder", + "order": "before_gap_fill", + "assign_tract": False, + } + if algorithm != expected_algorithm: + raise ValueError( + f"US stacked pool manifest {manifest_path} geography assignment " + "algorithm changed." + ) + seed = contract.get("seed") + expected_seed = { + "site": "legacy_puma_ladder", + "stream": "geography_legacy", + "value_source": "run_request.build_model_seed", + "value": manifest.get("random_seed"), + } + if seed != expected_seed or manifest.get("random_seed") != 0: + raise ValueError( + f"US stacked pool manifest {manifest_path} geography assignment " + "seed changed." + ) + authorities = contract.get("authorities") + expected_roles = { + "puma_ladder", + "congressional_district_vintage_crosswalk", + } + if not isinstance(authorities, Mapping) or set(authorities) != expected_roles: + raise ValueError( + f"US stacked pool manifest {manifest_path} geography authorities changed." + ) + provenance_pins = manifest.get("provenance_pins") + if not isinstance(provenance_pins, Mapping): + raise ValueError( + f"US stacked pool manifest {manifest_path} has no provenance pins." + ) + for role in sorted(expected_roles): + authority = authorities.get(role) + pin = provenance_pins.get(role) + if ( + not isinstance(authority, Mapping) + or authority.get("input_role") != role + or not isinstance(pin, Mapping) + or authority.get("sha256") != pin.get("actual_sha256") + or pin.get("expected_sha256") != pin.get("actual_sha256") + ): + raise ValueError( + f"US stacked pool manifest {manifest_path} geography authority " + f"{role!r} differs from its authenticated input pin." + ) + puma = authorities["puma_ladder"] + if not isinstance(puma, Mapping) or puma.get("source_ref") != ( + "source:us_puma_ladder_2020" + ): + raise ValueError( + f"US stacked pool manifest {manifest_path} PUMA-ladder source " + "authority changed." + ) + crosswalk = authorities["congressional_district_vintage_crosswalk"] + if ( + not isinstance(crosswalk, Mapping) + or crosswalk.get("source_ref") + != "source:us_congressional_district_vintage_crosswalk_117_to_119" + or crosswalk.get("source_vintage_ref") != "vintage:cd_117" + or crosswalk.get("source_vintage") != "117th_congress" + or crosswalk.get("target_vintage_ref") != "vintage:cd_119" + or crosswalk.get("target_vintage") != "119th_congress" + ): + raise ValueError( + f"US stacked pool manifest {manifest_path} congressional-district " + "vintage authority changed." + ) + output = assignment.get("output") + if not isinstance(output, Mapping): + raise ValueError( + f"US stacked pool manifest {manifest_path} geography output is missing." + ) + rows = output.get("household_rows") + positive_rows = output.get("positive_congressional_district_rows") + unique_values = output.get("unique_congressional_district_values") + if ( + isinstance(rows, bool) + or not isinstance(rows, int) + or rows < 1 + or positive_rows != rows + or isinstance(unique_values, bool) + or not isinstance(unique_values, int) + or unique_values < 1 + ): + raise ValueError( + f"US stacked pool manifest {manifest_path} does not prove positive " + "household congressional-district support." + ) + order = assignment.get("pre_assignment_household_order") + if ( + not isinstance(order, Mapping) + or order.get("column") != "household_id" + or order.get("codec") != "int64_little_endian.v1" + or order.get("row_count") != rows + or not isinstance(order.get("sha256"), str) + or _LOWERCASE_SHA256.fullmatch(str(order["sha256"])) is None + ): + raise ValueError( + f"US stacked pool manifest {manifest_path} has an invalid seeded " + "household-order receipt." + ) + assigned_geography = assignment.get("assigned_household_geography") + if ( + not isinstance(assigned_geography, Mapping) + or assigned_geography.get("columns") + != [ + "household_id", + "puma", + "congressional_district_geoid", + "county_fips", + ] + or assigned_geography.get("codec") != "column_major_int64_little_endian.v1" + or assigned_geography.get("row_count") != rows + or not isinstance(assigned_geography.get("sha256"), str) + or _LOWERCASE_SHA256.fullmatch(str(assigned_geography["sha256"])) is None + ): + raise ValueError( + f"US stacked pool manifest {manifest_path} has an invalid ordered " + "household-geography output receipt." + ) + summary = assignment.get("summary") + if ( + not isinstance(summary, Mapping) + or summary.get("applied") is not True + or summary.get("household_rows") != rows + ): + raise ValueError( + f"US stacked pool manifest {manifest_path} has an invalid geography " + "assignment summary." + ) + gate = assignment.get("gate") + gates = gate.get("gates") if isinstance(gate, Mapping) else None + puma_gate = gates.get("us_puma_ladder") if isinstance(gates, Mapping) else None + if ( + not isinstance(gate, Mapping) + or gate.get("passed") is not True + or not isinstance(puma_gate, Mapping) + or puma_gate.get("passed") is not True + ): + raise ValueError( + f"US stacked pool manifest {manifest_path} does not carry a passed " + "PUMA-ladder gate." + ) + universe = assignment.get("target_universe") + if ( + not isinstance(universe, Mapping) + or isinstance(universe.get("district_count"), bool) + or not isinstance(universe.get("district_count"), int) + or universe["district_count"] < 1 + or not isinstance(universe.get("geoids_sha256"), str) + or _LOWERCASE_SHA256.fullmatch(str(universe["geoids_sha256"])) is None + ): + raise ValueError( + f"US stacked pool manifest {manifest_path} has an invalid " + "congressional-district target-universe receipt." + ) + + +def _ordered_household_id_receipt( + household: pd.DataFrame, + *, + boundary: str, +) -> dict[str, object]: + """Recompute the assignment receipt's ordered native-household identity.""" + + column = "household_id" + if column not in household: + raise ValueError(f"{boundary} has no {column!r} column.") + numeric = pd.to_numeric(household[column], errors="coerce").to_numpy( + dtype=np.float64, + na_value=np.nan, + ) + valid = np.isfinite(numeric) & (numeric == np.floor(numeric)) + if not valid.all(): + raise ValueError(f"{boundary} {column} values must be integral.") + ordered = numeric.astype(" np.ndarray: + if CONGRESSIONAL_DISTRICT_GEOID_COLUMN not in household: + raise ValueError( + f"{boundary} has no {CONGRESSIONAL_DISTRICT_GEOID_COLUMN!r} column." + ) + numeric = pd.to_numeric( + household[CONGRESSIONAL_DISTRICT_GEOID_COLUMN], + errors="coerce", + ).to_numpy(dtype=np.float64, na_value=np.nan) + valid = np.isfinite(numeric) & (numeric > 0) & (numeric == np.floor(numeric)) + if not valid.all(): + raise ValueError( + f"{boundary} requires a positive integral " + f"{CONGRESSIONAL_DISTRICT_GEOID_COLUMN} on every household." + ) + return numeric.astype(np.int64) + + +def _ordered_household_geography_receipt( + household: pd.DataFrame, + *, + boundary: str, +) -> dict[str, object]: + """Recompute the producer's ordered native-household geography digest.""" + + columns = ( + "household_id", + "puma", + CONGRESSIONAL_DISTRICT_GEOID_COLUMN, + "county_fips", + ) + missing = [column for column in columns if column not in household] + if missing: + raise ValueError(f"{boundary} is missing geography column(s): {missing}.") + + household_ids = pd.to_numeric( + household["household_id"], + errors="coerce", + ).to_numpy(dtype=np.float64, na_value=np.nan) + valid_ids = np.isfinite(household_ids) & (household_ids == np.floor(household_ids)) + if not valid_ids.all(): + raise ValueError(f"{boundary} household_id values must be integral.") + + def fixed_width_values(column: str, width: int) -> np.ndarray: + text = household[column].astype(str) + if not text.str.fullmatch(rf"[0-9]{{{width}}}").all(): + raise ValueError( + f"{boundary} {column!r} must contain exactly {width}-digit codes." + ) + return text.astype(np.int64).to_numpy(dtype=" None: + """Bind schema-9 manifest geography claims to the authenticated H5.""" + + if manifest.get("pipeline") != _STACKED_PIPELINE: + return + assignment = _mapping( + manifest.get("geography_assignment"), + label=f"US stacked pool manifest {manifest_path}.geography_assignment", + ) + contract = _mapping( + assignment.get("contract"), + label=( + f"US stacked pool manifest {manifest_path}.geography_assignment.contract" + ), + ) + authorities = _mapping( + contract.get("authorities"), + label=( + "US stacked pool manifest " + f"{manifest_path}.geography_assignment.contract.authorities" + ), + ) + crosswalk = _mapping( + authorities.get("congressional_district_vintage_crosswalk"), + label=( + "US stacked pool manifest " + f"{manifest_path}.geography_assignment crosswalk authority" + ), + ) + expected_attributes = { + CONGRESSIONAL_DISTRICT_VINTAGE_CROSSWALK_SHA256_ATTR: crosswalk.get("sha256"), + CONGRESSIONAL_DISTRICT_VINTAGE_TARGET_ATTR: crosswalk.get("target_vintage"), + } + if ( + expected_attributes[CONGRESSIONAL_DISTRICT_VINTAGE_TARGET_ATTR] + != CURRENT_CONGRESSIONAL_DISTRICT_VINTAGE + ): + raise ValueError( + f"US stacked pool manifest {manifest_path} does not name the current " + "congressional-district target vintage." + ) + mismatched_attributes = { + key: {"expected": expected, "actual": root_attributes.get(key)} + for key, expected in expected_attributes.items() + if root_attributes.get(key) != expected + } + if mismatched_attributes: + raise ValueError( + f"US stacked pool H5 {pool_path} congressional-district root " + f"attributes do not match its manifest: {mismatched_attributes}." + ) + + boundary = f"US stacked pool H5 {pool_path}" + clone_column = support_clone_index_column("household") + source_column = support_source_id_column("household") + missing_lineage = [ + column for column in (source_column, clone_column) if column not in household + ] + if missing_lineage: + raise ValueError( + f"{boundary} is missing household clone-lineage column(s): " + f"{missing_lineage}." + ) + clone_index = pd.to_numeric(household[clone_column], errors="coerce") + clone_numeric = clone_index.to_numpy(dtype=np.float64, na_value=np.nan) + valid_clone_index = ( + np.isfinite(clone_numeric) + & (clone_numeric >= 0) + & (clone_numeric == np.floor(clone_numeric)) + ) + if not valid_clone_index.all(): + raise ValueError(f"{boundary} has invalid {clone_column!r} values.") + source_ids = pd.to_numeric(household[source_column], errors="coerce") + source_numeric = source_ids.to_numpy(dtype=np.float64, na_value=np.nan) + valid_source_ids = np.isfinite(source_numeric) & ( + source_numeric == np.floor(source_numeric) + ) + if not valid_source_ids.all(): + raise ValueError(f"{boundary} has invalid {source_column!r} values.") + lineage = pd.DataFrame( + { + "source_id": source_numeric.astype(np.int64), + "clone_index": clone_numeric.astype(np.int64), + }, + index=household.index, + ) + if lineage.duplicated(["source_id", "clone_index"]).any(): + raise ValueError(f"{boundary} has duplicate household clone-lineage roles.") + native_counts = lineage["clone_index"].eq(0).groupby(lineage["source_id"]).sum() + if not native_counts.eq(1).all(): + raise ValueError( + f"{boundary} requires exactly one native household for every clone lineage." + ) + native_mask = lineage["clone_index"].eq(0) + native_household = household.loc[native_mask] + native_ids = pd.to_numeric( + native_household["household_id"], errors="coerce" + ).to_numpy(dtype=np.float64, na_value=np.nan) + native_source_ids = lineage.loc[native_mask, "source_id"].to_numpy(dtype=np.float64) + if not np.array_equal(native_ids, native_source_ids): + raise ValueError( + f"{boundary} native household IDs differ from their clone-lineage " + "source IDs." + ) + + expected_order = assignment.get("pre_assignment_household_order") + actual_order = _ordered_household_id_receipt( + native_household, + boundary=boundary, + ) + if actual_order != expected_order: + raise ValueError( + f"{boundary} native household order differs from its manifest " + "geography assignment receipt." + ) + expected_geography = assignment.get("assigned_household_geography") + actual_geography = _ordered_household_geography_receipt( + native_household, + boundary=boundary, + ) + if actual_geography != expected_geography: + raise ValueError( + f"{boundary} native household geography differs from its manifest " + "assignment output receipt." + ) + + native_districts = _positive_integral_district_values( + native_household, + boundary=boundary, + ) + _positive_integral_district_values(household, boundary=boundary) + actual_output = { + "household_rows": len(native_household), + "positive_congressional_district_rows": len(native_districts), + "unique_congressional_district_values": int(len(np.unique(native_districts))), + } + if assignment.get("output") != actual_output: + raise ValueError( + f"{boundary} household geography counts differ from its manifest " + "assignment output receipt." + ) + + geography_columns = ( + "puma", + CONGRESSIONAL_DISTRICT_GEOID_COLUMN, + "county_fips", + ) + grouped = household.groupby(source_column, sort=False, dropna=False) + for column in geography_columns: + incoherent = grouped[column].nunique(dropna=False) > 1 + if bool(incoherent.any()): + raise ValueError( + f"{boundary} cloned household rows disagree on assigned " + f"geography column {column!r}." + ) + + def _stacked_late_transition_binding( manifest: Mapping[str, object], *, @@ -789,6 +1335,14 @@ def _load_us_multispine_pool( for entity in US_SCHEMA.entities } period = store[_TIME_PERIOD_KEY] + root_attribute_set = store.get_node("/")._v_attrs + geography_root_attributes = { + key: _hdf_root_attribute_text(root_attribute_set, key) + for key in ( + CONGRESSIONAL_DISTRICT_VINTAGE_CROSSWALK_SHA256_ATTR, + CONGRESSIONAL_DISTRICT_VINTAGE_TARGET_ATTR, + ) + } if len(period) != 1 or period.tolist() != [manifest.get("period")]: raise ValueError( @@ -801,6 +1355,13 @@ def _load_us_multispine_pool( f"US multispine pool H5 {pool_path} household table has no " "household_weight column." ) + _validate_stacked_geography_h5_binding( + manifest, + household, + geography_root_attributes, + manifest_path=manifest_path, + pool_path=pool_path, + ) household_weights = household.pop("household_weight").to_numpy(dtype=np.float64) tables["household"] = household late_transition = _stacked_late_transition_binding( @@ -907,13 +1468,15 @@ def write_nullable_us_h5( artifact_kind: str, publication_run_id: str | None = None, materializer_version: int | None = None, + root_attributes: Mapping[str, str] | None = None, ) -> None: """Atomically write and verify a nullable US single-year H5. The destination is replaced only after a temporary sibling has round-trip verified every nonempty entity table, household weights, period metadata, - fixed-format storage, and the caller-declared ``artifact_kind``. A failed - write or verification leaves any existing destination bytes untouched. + fixed-format storage, caller-declared root attributes, and the + ``artifact_kind``. A failed write or verification leaves any existing + destination bytes untouched. """ if not isinstance(frame, Frame): @@ -928,6 +1491,7 @@ def write_nullable_us_h5( type(materializer_version) is not int or materializer_version <= 0 ): raise ValueError("materializer_version must be a positive integer when set.") + normalized_root_attributes = _validated_root_attributes(root_attributes) for entity in US_SCHEMA.entities: canonicalize_table_string_dtypes( @@ -948,6 +1512,7 @@ def write_nullable_us_h5( artifact_kind=artifact_kind, publication_run_id=publication_run_id, materializer_version=materializer_version, + root_attributes=normalized_root_attributes, ) _verify_nullable_us_h5( frame, @@ -956,6 +1521,7 @@ def write_nullable_us_h5( artifact_kind=artifact_kind, publication_run_id=publication_run_id, materializer_version=materializer_version, + root_attributes=normalized_root_attributes, ) os.replace(temporary, output) except BaseException: @@ -971,6 +1537,7 @@ def _write_nullable_us_h5_file( artifact_kind: str, publication_run_id: str | None, materializer_version: int | None, + root_attributes: tuple[tuple[str, str], ...], ) -> None: with pd.HDFStore(path, mode="w") as store: for entity in frame.entities: @@ -1005,6 +1572,9 @@ def _write_nullable_us_h5_file( ), format="table", ) + root_node_attributes = store.get_node("/")._v_attrs + for key, value in root_attributes: + root_node_attributes[key] = value def _verify_nullable_us_h5( @@ -1015,6 +1585,7 @@ def _verify_nullable_us_h5( artifact_kind: str, publication_run_id: str | None, materializer_version: int | None, + root_attributes: tuple[tuple[str, str], ...], ) -> None: with pd.HDFStore(path, mode="r") as store: for entity in frame.entities: @@ -1075,6 +1646,65 @@ def _verify_nullable_us_h5( "Nullable US H5 round trip changed artifact metadata: " f"expected {expected_metadata}, got {stored_metadata}." ) + stored_root_attributes = store.get_node("/")._v_attrs + for key, expected_value in root_attributes: + try: + stored_value = stored_root_attributes[key] + except KeyError as exc: + raise RuntimeError( + f"Nullable US H5 round trip omitted root attribute {key!r}." + ) from exc + actual_value = _h5_root_attribute_text(stored_value) + if actual_value != expected_value: + raise RuntimeError( + "Nullable US H5 round trip changed root attribute " + f"{key!r}: expected {expected_value!r}, got {actual_value!r}." + ) + + +def _validated_root_attributes( + root_attributes: Mapping[str, str] | None, +) -> tuple[tuple[str, str], ...]: + if root_attributes is None: + return () + if not isinstance(root_attributes, Mapping): + raise TypeError("root_attributes must be a mapping of string names to strings.") + + pytables_owned_names = frozenset( + {"CLASS", "PYTABLES_FORMAT_VERSION", "TITLE", "VERSION"} + ) + normalized: list[tuple[str, str]] = [] + for key, value in root_attributes.items(): + if not isinstance(key, str): + raise TypeError("root attribute names must be strings.") + if not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", key): + raise ValueError( + f"root attribute names must be non-empty HDF identifiers; got {key!r}." + ) + if key in pytables_owned_names: + raise ValueError( + f"root attribute {key!r} is owned by the HDF materializer." + ) + if not isinstance(value, str): + raise TypeError(f"root attribute {key!r} must have a string value.") + if not value: + raise ValueError(f"root attribute {key!r} must not be empty.") + normalized.append((key, value)) + return tuple(sorted(normalized)) + + +def _h5_root_attribute_text(value: object) -> str: + if isinstance(value, bytes): + return value.decode() + return str(value) + + +def _hdf_root_attribute_text(attributes: object, key: str) -> str | None: + try: + value = attributes[key] # type: ignore[index] + except KeyError: + return None + return _h5_root_attribute_text(value) def _export_table(frame: Frame, entity: str) -> pd.DataFrame: diff --git a/packages/microcosm-build/tests/test_inspect_us_fiscal_refresh_checkpoints.py b/packages/microcosm-build/tests/test_inspect_us_fiscal_refresh_checkpoints.py index c2f6c7c9..90dcae54 100644 --- a/packages/microcosm-build/tests/test_inspect_us_fiscal_refresh_checkpoints.py +++ b/packages/microcosm-build/tests/test_inspect_us_fiscal_refresh_checkpoints.py @@ -1,6 +1,9 @@ import importlib.util from pathlib import Path +import pandas as pd +import pytest + def _load_inspector_module(): root = Path(__file__).resolve().parents[3] @@ -181,6 +184,52 @@ def test__given_default_reader_gets_invalid_h5__then_payload_reports_read_error( assert "Could not read H5 file" in result["error"] +def test__given_fixed_household_frame__then_default_reader_finds_cd_support( + tmp_path, +) -> None: + pytest.importorskip("tables", reason="pandas fixed-format fixture needs PyTables (absent in the base wheel gate; the US-extra test jobs run this fully)") + inspector = _load_inspector_module() + support_h5 = tmp_path / "support.h5" + + with pd.HDFStore(support_h5, mode="w") as store: + store.put( + "household", + pd.DataFrame({"congressional_district_geoid": [0, 101, 101, 202]}), + format="fixed", + ) + assert store.get_storer("household").is_table is False + attrs = store.get_node("/")._v_attrs + attrs[inspector.CONGRESSIONAL_DISTRICT_VINTAGE_CROSSWALK_SHA256_ATTR] = ( + "crosswalk-sha" + ) + attrs[inspector.CONGRESSIONAL_DISTRICT_VINTAGE_TARGET_ATTR] = ( + inspector.CURRENT_CONGRESSIONAL_DISTRICT_VINTAGE + ) + + result = inspector.read_h5_provenance(support_h5) + + assert result == { + "readable": True, + "attrs": { + inspector.CONGRESSIONAL_DISTRICT_VINTAGE_CROSSWALK_SHA256_ATTR: ( + "crosswalk-sha" + ), + inspector.CONGRESSIONAL_DISTRICT_VINTAGE_TARGET_ATTR: ( + inspector.CURRENT_CONGRESSIONAL_DISTRICT_VINTAGE + ), + }, + "household_congressional_district_geoid": { + "exists": True, + "table": "household", + "column": "congressional_district_geoid", + "rows": 4, + "positive_unique_count": 2, + }, + "read_error_kind": None, + "error": None, + } + + def test__given_missing_crosswalk__then_support_status_says_requested_missing( tmp_path, ) -> None: diff --git a/packages/microcosm-build/tests/test_spec_engine_compiler_ir.py b/packages/microcosm-build/tests/test_spec_engine_compiler_ir.py index 3f83fd0e..393d98dd 100644 --- a/packages/microcosm-build/tests/test_spec_engine_compiler_ir.py +++ b/packages/microcosm-build/tests/test_spec_engine_compiler_ir.py @@ -145,7 +145,7 @@ def test_ir_retains_all_typed_resolution_products(compiled_us: CompiledSpecIR) - assert len(inventory["columns"]) == 176 assert len(inventory["artifacts"]) == 84 assert len(inventory["scopes"]) == 7 - assert len(inventory["references"]) == 325 + assert len(inventory["references"]) == 334 assert "engine_abi_lock" in compiled_us.generated_authorities assert "records" in compiled_us.vintage_authorities assert set(compiled_us.surfaces) == { diff --git a/packages/microcosm-build/tests/test_spec_engine_coverage_tool.py b/packages/microcosm-build/tests/test_spec_engine_coverage_tool.py index af65676c..7a4e443b 100644 --- a/packages/microcosm-build/tests/test_spec_engine_coverage_tool.py +++ b/packages/microcosm-build/tests/test_spec_engine_coverage_tool.py @@ -52,27 +52,27 @@ def test_us_coverage_is_exact_complete_and_honest( assert_coverage_complete(coverage_report) assert coverage_report["status"] == "pass" fields = coverage_report["field_usage"] - assert fields["configuration_field_count"] == 42_096 - assert fields["authored_normative_field_count"] == 32_331 - assert fields["resolved_binding_field_count"] == 9_765 - assert fields["consumed_field_count"] == 42_096 + assert fields["configuration_field_count"] == 42_120 + assert fields["authored_normative_field_count"] == 32_351 + assert fields["resolved_binding_field_count"] == 9_769 + assert fields["consumed_field_count"] == 42_120 assert fields["unused_field_count"] == 0 assert fields["multiple_primary_use_field_count"] == 0 - assert fields["claim_count"] == 47 + assert fields["claim_count"] == 49 assert fields["mode_counts"] == { - "legacy_behavior": 13_934, - "compiler_semantic": 27_688, - "front_end_validation": 346, - "identity_only": 128, + "legacy_behavior": 13_970, + "compiler_semantic": 27_699, + "front_end_validation": 348, + "identity_only": 103, } assert fields["generation0_effect_counts"] == { - "legacy_behavior": 38_407, - "no_generation0_effect": 3_689, + "legacy_behavior": 38_443, + "no_generation0_effect": 3_677, } inventory = coverage_report["inventory_coverage"] - assert inventory["required_item_count"] == 40 - assert inventory["covered_item_count"] == 40 + assert inventory["required_item_count"] == 41 + assert inventory["covered_item_count"] == 41 assert inventory["missing_item_count"] == 0 assert inventory["missing_items"] == [] assert inventory["counts"]["producer_inputs"] == 2_742 diff --git a/packages/microcosm-build/tests/test_spec_engine_field_usage.py b/packages/microcosm-build/tests/test_spec_engine_field_usage.py index 039db46c..77fad494 100644 --- a/packages/microcosm-build/tests/test_spec_engine_field_usage.py +++ b/packages/microcosm-build/tests/test_spec_engine_field_usage.py @@ -92,22 +92,22 @@ def _mutated_bundle( def test_exact_complete_ledger_has_one_primary_mode_per_pointer(field_ledger) -> None: - assert len(field_ledger.fields) == EXPECTED_CONFIGURATION_FIELD_COUNT == 42_096 + assert len(field_ledger.fields) == EXPECTED_CONFIGURATION_FIELD_COUNT == 42_120 assert field_ledger.source_counts == { - "authored": 32_331, - "resolved_bindings": 9_765, + "authored": 32_351, + "resolved_bindings": 9_769, } assert field_ledger.mode_counts == { - "legacy_behavior": 13_934, - "compiler_semantic": 27_688, - "front_end_validation": 346, - "identity_only": 128, + "legacy_behavior": 13_970, + "compiler_semantic": 27_699, + "front_end_validation": 348, + "identity_only": 103, } assert field_ledger.generation0_effect_counts == { - "legacy_behavior": 38_407, - "no_generation0_effect": 3_689, + "legacy_behavior": 38_443, + "no_generation0_effect": 3_677, } - assert len({field.pointer for field in field_ledger.fields}) == 42_096 + assert len({field.pointer for field in field_ledger.fields}) == 42_120 def test_eligibility_concepts_are_validation_not_generation0_behavior( @@ -139,6 +139,24 @@ def test_source_pins_channels_and_dtype_policy_are_honest_validation_routes( assert field.generation0_effect is Generation0Effect.NO_GENERATION0_EFFECT +def test_stacked_geography_authority_fields_name_the_checkpoint_sink( + field_ledger, +) -> None: + pointers = ( + "/authored/spec~1geography.yaml/assignment/draw/asec/weight", + "/authored/spec~1sources.yaml/sources/6/sha256", + "/authored/spec~1sources.yaml/sources/7/vintage_authorities/0/value", + ) + for pointer in pointers: + field = field_ledger.field(pointer) + assert field.mode is UsageMode.LEGACY_BEHAVIOR + assert field.generation0_effect is Generation0Effect.LEGACY_BEHAVIOR + assert ( + "/stacked_checkpoint_static_components/geography_assignment" + in field.sink_pointers + ) + + def test_spine_assembly_mass_share_fields_name_exact_adapter_sinks( field_ledger, ) -> None: @@ -348,3 +366,78 @@ def mutate(document: dict[str, object]) -> None: f"/spine_assembly/household_mass_shares/{channel}" in field.sink_pointers ) + + +def test_geography_declaration_mutation_changes_checkpoint_identity( + tmp_path: Path, + resolved_us: ResolvedSpec, + legacy_us: dict[str, object], +) -> None: + def mutate(document: dict[str, object]) -> None: + assignment = _mapping(document["assignment"]) + draw = _mapping(assignment["draw"]) + asec = _mapping(draw["asec"]) + asec["weight"] = "mutated_population_weight" + + mutated = _mutated_bundle( + resolved_us, + tmp_path, + ResourceKind.GEOGRAPHY, + mutate, + ) + mutated_legacy = compile_to_legacy_payload(mutated) + diff_paths = { + difference.path + for difference in diff_legacy_payloads(legacy_us, mutated_legacy) + } + pointer = "/authored/spec~1geography.yaml/assignment/draw/asec/weight" + assert ( + "/stacked_checkpoint_static_components/geography_assignment/" + "declaration/draw/asec/weight" + ) in diff_paths + field = build_field_usage_ledger( + mutated, + legacy_payload=mutated_legacy, + ).field(pointer) + assert field.claim_id == "geography_assignment" + assert field.mode is UsageMode.LEGACY_BEHAVIOR + + +def test_geography_source_pin_mutation_changes_checkpoint_identity( + tmp_path: Path, + resolved_us: ResolvedSpec, + legacy_us: dict[str, object], +) -> None: + def mutate(document: dict[str, object]) -> None: + sources = document["sources"] + assert isinstance(sources, list) + crosswalk = next( + source + for source in sources + if isinstance(source, dict) + and source.get("id") + == "us_congressional_district_vintage_crosswalk_117_to_119" + ) + crosswalk["sha256"] = "b" * 64 + + mutated = _mutated_bundle( + resolved_us, + tmp_path, + ResourceKind.SOURCES, + mutate, + ) + mutated_legacy = compile_to_legacy_payload(mutated) + diff_paths = { + difference.path + for difference in diff_legacy_payloads(legacy_us, mutated_legacy) + } + assert ( + "/stacked_checkpoint_static_components/geography_assignment/authorities/" + "congressional_district_vintage_crosswalk/sha256" + ) in diff_paths + field = build_field_usage_ledger( + mutated, + legacy_payload=mutated_legacy, + ).field("/authored/spec~1sources.yaml/sources/7/sha256") + assert field.claim_id == "source_geography_identity" + assert field.mode is UsageMode.LEGACY_BEHAVIOR diff --git a/packages/microcosm-build/tests/test_spec_engine_identity_contracts.py b/packages/microcosm-build/tests/test_spec_engine_identity_contracts.py index b4693a0e..a2a682f6 100644 --- a/packages/microcosm-build/tests/test_spec_engine_identity_contracts.py +++ b/packages/microcosm-build/tests/test_spec_engine_identity_contracts.py @@ -63,7 +63,7 @@ def test_pipeline_contract_is_an_exact_generation_zero_projection() -> None: == { "artifact_kind": "populace_us_stacked_pool_checkpoint_identity", "schema_version": 1, - "materializer_version": 11, + "materializer_version": 12, "pipeline": "us-stacked-pool", } ) diff --git a/packages/microcosm-build/tests/test_spec_engine_inventory_coverage.py b/packages/microcosm-build/tests/test_spec_engine_inventory_coverage.py index 0732becc..8f4e2673 100644 --- a/packages/microcosm-build/tests/test_spec_engine_inventory_coverage.py +++ b/packages/microcosm-build/tests/test_spec_engine_inventory_coverage.py @@ -56,6 +56,7 @@ "stacked_checkpoint_base_identity_exact", "stacked_checkpoint_pool_code_exact", "stacked_checkpoint_top_level_exact", + "stacked_geography_assignment_exact", "take_up_identity_exact", "take_up_legacy_contract_exact", "take_up_pipeline_steps_exact", @@ -82,16 +83,16 @@ "producer_nodes": 38, "producer_virtual_resources": 75, "release_rungs": 5, - "resolved_references": 325, + "resolved_references": 334, "seed_owner_bindings": 112, "seed_owner_rows": 54, "seed_sites": 53, "seed_streams": 14, "source_operators": 16, "source_stages": 37, - "stacked_checkpoint_full_components": 12, + "stacked_checkpoint_full_components": 13, "stacked_checkpoint_pool_code_components": 19, - "stacked_checkpoint_static_components": 9, + "stacked_checkpoint_static_components": 10, "tail_control_fields": 934, "take_up_pipeline_steps": 28, "take_up_programs": 17, @@ -160,8 +161,8 @@ def test_us_inventory_is_structure_exact_and_complete( ) assert_inventory_coverage_complete(report) - assert report["required_item_count"] == 40 - assert report["covered_item_count"] == 40 + assert report["required_item_count"] == 41 + assert report["covered_item_count"] == 41 assert report["missing_item_count"] == 0 assert report["missing_items"] == [] assert set(report["items"]) == EXPECTED_CHECKS diff --git a/packages/microcosm-build/tests/test_spec_engine_stacked_authority_semantics.py b/packages/microcosm-build/tests/test_spec_engine_stacked_authority_semantics.py index 1c517b8d..c378f3db 100644 --- a/packages/microcosm-build/tests/test_spec_engine_stacked_authority_semantics.py +++ b/packages/microcosm-build/tests/test_spec_engine_stacked_authority_semantics.py @@ -127,6 +127,20 @@ def test_checkpoint_projection_is_field_and_byte_identical_to_live_oracle( actual_sha256="a" * 64, size_bytes=17, ), + "puma_ladder": pool_tool._VerifiedInput( + role="puma_ladder", + path=Path("unused-puma-ladder"), + expected_sha256=pool_tool._STACKED_PUMA_LADDER_SHA256, + actual_sha256=pool_tool._STACKED_PUMA_LADDER_SHA256, + size_bytes=29, + ), + "congressional_district_vintage_crosswalk": pool_tool._VerifiedInput( + role="congressional_district_vintage_crosswalk", + path=Path("unused-cd-crosswalk"), + expected_sha256=pool_tool._STACKED_CD_CROSSWALK_SHA256, + actual_sha256=pool_tool._STACKED_CD_CROSSWALK_SHA256, + size_bytes=77_935, + ), } pins = { role: {"sha256": pin.actual_sha256, "size_bytes": pin.size_bytes} @@ -169,7 +183,12 @@ def test_checkpoint_projection_is_field_and_byte_identical_to_live_oracle( assert projected == live assert stacked_identity_bytes(projected) == _canonical_bytes(live) - assert list(projected["inputs"]) == ["alpha", "zeta"] + assert list(projected["inputs"]) == [ + "alpha", + "congressional_district_vintage_crosswalk", + "puma_ladder", + "zeta", + ] assert ( projected["pool_code"]["late_producer_schedule"]["schedule_sha256"] == "dcf3c6d2eade3449836c49a1dc4d3b8cd395aab9142db700c3c60598fa9c1c79" @@ -203,6 +222,7 @@ def test_static_projection_selects_exact_defaulted_live_identity_components( "period", "model_seed", "policyengine_us_version", + "geography_assignment", "stacked_authority", "pool_code", ) diff --git a/packages/microcosm-build/tests/test_us_acs_multispine_base_builder.py b/packages/microcosm-build/tests/test_us_acs_multispine_base_builder.py index 08fd3aec..58647311 100644 --- a/packages/microcosm-build/tests/test_us_acs_multispine_base_builder.py +++ b/packages/microcosm-build/tests/test_us_acs_multispine_base_builder.py @@ -97,14 +97,20 @@ def test_nullable_writer_round_trips_fixed_tables_and_caller_artifact_kind( tmp_path: Path, ) -> None: pytest.importorskip("tables") + h5py = pytest.importorskip("h5py") frame = _frame() output = tmp_path / "pool.h5" + root_attributes = { + "populace_congressional_district_vintage_crosswalk_sha256": "a" * 64, + "populace_congressional_district_vintage_target": "119th_congress", + } write_nullable_us_h5( frame, output, period=2024, artifact_kind="nullable_multispine_pool_h5", + root_attributes=root_attributes, ) with pd.HDFStore(output, mode="r") as store: @@ -113,6 +119,14 @@ def test_nullable_writer_round_trips_fixed_tables_and_caller_artifact_kind( assert pd.isna(store["person"]["is_snap_abawd_discretionary_exempt"].iloc[1]) assert store["household"]["household_weight"].tolist() == [40.0, 60.0] assert store["_time_period"].tolist() == [2024] + stored_root_attributes = store.get_node("/")._v_attrs + assert { + key: str(stored_root_attributes[key]) for key in root_attributes + } == root_attributes + with h5py.File(output, mode="r") as h5: + assert { + key: h5.attrs[key].decode() for key in root_attributes + } == root_attributes assert _stored_metadata(output) == { "artifact_kind": "nullable_multispine_pool_h5", "entity_hdf_format": "fixed_nullable", @@ -127,24 +141,62 @@ def test_nullable_writer_keeps_existing_bytes_when_verification_fails( pytest.importorskip("tables") output = tmp_path / "existing.h5" output.write_bytes(b"existing-artifact") + original_write = h5_io._write_nullable_us_h5_file - def fail_verification(*_args, **_kwargs): - raise RuntimeError("injected verification failure") + def omit_root_attributes(*args, **kwargs): + kwargs["root_attributes"] = () + original_write(*args, **kwargs) - monkeypatch.setattr(h5_io, "_verify_nullable_us_h5", fail_verification) + monkeypatch.setattr( + h5_io, + "_write_nullable_us_h5_file", + omit_root_attributes, + ) - with pytest.raises(RuntimeError, match="injected verification failure"): + with pytest.raises(RuntimeError, match="omitted root attribute"): write_nullable_us_h5( _frame(), output, period=2024, artifact_kind="fixture", + root_attributes={"fixture_provenance": "verified-before-replace"}, ) assert output.read_bytes() == b"existing-artifact" assert list(tmp_path.glob(f".{output.name}.*.tmp")) == [] +@pytest.mark.parametrize( + ("root_attributes", "expected_error"), + [ + ([("fixture", "value")], TypeError), + ({1: "value"}, TypeError), + ({"bad-name": "value"}, ValueError), + ({"CLASS": "value"}, ValueError), + ({"fixture": 1}, TypeError), + ({"fixture": ""}, ValueError), + ], +) +def test_nullable_writer_rejects_invalid_root_attributes_before_writing( + root_attributes, + expected_error, + tmp_path: Path, +) -> None: + pytest.importorskip("tables") + output = tmp_path / "invalid-root-attrs.h5" + + with pytest.raises(expected_error): + write_nullable_us_h5( + _frame(), + output, + period=2024, + artifact_kind="fixture", + root_attributes=root_attributes, + ) + + assert not output.exists() + + def test_shim_preserves_legacy_write_signature_and_default( tmp_path: Path, ) -> None: diff --git a/packages/microcosm-build/tests/test_us_bundle_core_contracts.py b/packages/microcosm-build/tests/test_us_bundle_core_contracts.py index 28ef469e..1ab09588 100644 --- a/packages/microcosm-build/tests/test_us_bundle_core_contracts.py +++ b/packages/microcosm-build/tests/test_us_bundle_core_contracts.py @@ -36,6 +36,7 @@ CORE_SPEC.loader.exec_module(CORE_MODULE) build_catalogs = CORE_MODULE.build_catalogs build_bundle = CORE_MODULE.build_bundle +build_geography = CORE_MODULE.build_geography build_publication = CORE_MODULE.build_publication build_sources = CORE_MODULE.build_sources build_vintages = CORE_MODULE.build_vintages @@ -175,6 +176,54 @@ def test_source_schema_rejects_non_hex_content_digests() -> None: load_schema_registry().validate(sources, "sources.schema.json") +def test_cd_vintage_crosswalk_source_and_geography_authority_are_pinned() -> None: + crosswalk_source = next( + row + for row in build_sources()["sources"] + if row["id"] + == "us_congressional_district_vintage_crosswalk_117_to_119" + ) + assert crosswalk_source == { + "id": "us_congressional_district_vintage_crosswalk_117_to_119", + "role": "congressional_district_vintage_crosswalk", + "sha256": ( + "c7cb040b1f57ca2ea2adcbfe60cc2b250ca23acbc4b640cd421e766fa54c1aec" + ), + "byte_size": 77_935, + "loader": "kernel:load_congressional_district_vintage_crosswalk", + "vintages": ["vintage:cd_117", "vintage:cd_119"], + "vintage_authorities": [ + { + "id": "cd_117", + "kind": "geography_vintage", + "value": "117th_congress", + } + ], + } + + assignment = build_geography()["assignment"] + assert assignment["order"] == "before_gap_fill" + assert assignment["congressional_district_vintage_crosswalk"] == { + "source_ref": ( + "source:us_congressional_district_vintage_crosswalk_117_to_119" + ), + "source_vintage": "vintage:cd_117", + "target_vintage": "vintage:cd_119", + } + load_schema_registry().validate(build_geography(), "geography.schema.json") + + +def test_cd_vintage_crosswalk_source_reference_is_resolved() -> None: + resources = {**_vintage_resources(), "geography": build_geography()} + _resolve_vintages(resources) + + resources["geography"]["assignment"][ + "congressional_district_vintage_crosswalk" + ]["source_ref"] = "source:missing_crosswalk" + with pytest.raises(SpecResolutionError, match="dangling source reference"): + _resolve_vintages(resources) + + def test_source_operational_locator_is_outside_spec_hash() -> None: sources = build_sources() original_hash, _, original_surfaces = _source_surface_hashes(sources) @@ -228,7 +277,7 @@ def test_all_us_vintages_have_reciprocal_reviewed_compatibility() -> None: sources = build_sources() vintages = build_vintages() records = vintages["records"] - assert len(records) == 15 + assert len(records) == 16 by_id = {record["id"]: record for record in records} assert set(by_id) == { "acs_2022", @@ -236,6 +285,7 @@ def test_all_us_vintages_have_reciprocal_reviewed_compatibility() -> None: "asec_2022", "asec_2023", "asec_2024", + "cd_117", "cd_119", "census_2020", "org_2024", @@ -263,7 +313,7 @@ def test_all_us_vintages_have_reciprocal_reviewed_compatibility() -> None: } ) resolved_wire = thaw_json(resolved.vintage_authorities) - assert len(resolved_wire["records"]) == 15 + assert len(resolved_wire["records"]) == 16 assert ( resolved_wire["records"]["policyengine_us_surface"]["value"] == (ENGINE_LOCK["engine"]["version"]) diff --git a/packages/microcosm-build/tests/test_us_fiscal_refresh_builder.py b/packages/microcosm-build/tests/test_us_fiscal_refresh_builder.py index 6295b949..48486968 100644 --- a/packages/microcosm-build/tests/test_us_fiscal_refresh_builder.py +++ b/packages/microcosm-build/tests/test_us_fiscal_refresh_builder.py @@ -1,4 +1,3 @@ -import builtins import hashlib import importlib.util import inspect @@ -1076,24 +1075,57 @@ def test_cd_vintage_support_provenance_requires_matching_h5_attrs( with pytest.raises(ValueError, match="crosswalk provenance mismatch"): builder._assert_cd_vintage_support_matches(h5_path, metadata) - monkeypatch.setattr( - builder, - "_read_cd_vintage_support_provenance", - lambda path: { - builder.CONGRESSIONAL_DISTRICT_VINTAGE_CROSSWALK_SHA256_ATTR: ( - "crosswalk-sha" - ), - builder.CONGRESSIONAL_DISTRICT_VINTAGE_TARGET_ATTR: ( - builder.CURRENT_CONGRESSIONAL_DISTRICT_VINTAGE - ), - "household_congressional_district_geoid": { - "exists": True, - "positive_unique_count": 436, - }, - }, - ) +def test_cd_vintage_support_provenance_reads_fixed_household_frame( + monkeypatch, + tmp_path, +) -> None: + pytest.importorskip("tables") + builder = _load_builder_module() + h5_path = tmp_path / "support.h5" + metadata = {"sha256": "crosswalk-sha"} + + with pd.HDFStore(h5_path, mode="w") as store: + store.put( + "household", + pd.DataFrame({"congressional_district_geoid": [0, 101, 101, 202]}), + format="fixed", + ) + assert store.get_storer("household").is_table is False + attrs = store.get_node("/")._v_attrs + attrs[builder.CONGRESSIONAL_DISTRICT_VINTAGE_CROSSWALK_SHA256_ATTR] = ( + "crosswalk-sha" + ) + attrs[builder.CONGRESSIONAL_DISTRICT_VINTAGE_TARGET_ATTR] = ( + builder.CURRENT_CONGRESSIONAL_DISTRICT_VINTAGE + ) + + original_hdf_store = builder.pd.HDFStore + opened_paths: list[Path] = [] + + def tracked_hdf_store(path, *args, **kwargs): + opened_paths.append(Path(path)) + return original_hdf_store(path, *args, **kwargs) + + monkeypatch.setattr(builder.pd, "HDFStore", tracked_hdf_store) + provenance = builder._read_cd_vintage_support_provenance(h5_path) + + assert opened_paths == [h5_path] + assert provenance == { + builder.CONGRESSIONAL_DISTRICT_VINTAGE_CROSSWALK_SHA256_ATTR: ("crosswalk-sha"), + builder.CONGRESSIONAL_DISTRICT_VINTAGE_TARGET_ATTR: ( + builder.CURRENT_CONGRESSIONAL_DISTRICT_VINTAGE + ), + "household_congressional_district_geoid": { + "exists": True, + "table": "household", + "column": "congressional_district_geoid", + "rows": 4, + "positive_unique_count": 2, + }, + } builder._assert_cd_vintage_support_matches(h5_path, metadata) + assert opened_paths == [h5_path, h5_path] def test_cd_vintage_support_provenance_rejects_missing_cd_lookup( @@ -1136,20 +1168,17 @@ def test_cd_vintage_support_provenance_counts_only_positive_numeric_lookup() -> ) -def test_cd_vintage_support_provenance_names_us_extra_when_h5py_missing( +def test_cd_vintage_support_provenance_names_us_extra_when_pytables_missing( monkeypatch, tmp_path ) -> None: builder = _load_builder_module() h5_path = tmp_path / "support.h5" h5_path.write_text("") - original_import = builtins.__import__ - def fake_import(name, *args, **kwargs): - if name == "h5py": - raise ModuleNotFoundError("No module named 'h5py'") - return original_import(name, *args, **kwargs) + def missing_hdf_store(*args, **kwargs): + raise ImportError("Missing optional dependency 'tables'.") - monkeypatch.setattr(builtins, "__import__", fake_import) + monkeypatch.setattr(builder.pd, "HDFStore", missing_hdf_store) with pytest.raises(RuntimeError) as excinfo: builder._read_cd_vintage_support_provenance(h5_path) diff --git a/packages/microcosm-build/tests/test_us_multispine_pool_h5_io.py b/packages/microcosm-build/tests/test_us_multispine_pool_h5_io.py index 0190ff69..54a7fe18 100644 --- a/packages/microcosm-build/tests/test_us_multispine_pool_h5_io.py +++ b/packages/microcosm-build/tests/test_us_multispine_pool_h5_io.py @@ -20,6 +20,11 @@ canonicalize_frame_string_dtypes, canonicalize_table_string_dtypes, ) +from microcosm.build.us_runtime.congressional_district_vintage import ( + CONGRESSIONAL_DISTRICT_VINTAGE_CROSSWALK_SHA256_ATTR, + CONGRESSIONAL_DISTRICT_VINTAGE_TARGET_ATTR, + CURRENT_CONGRESSIONAL_DISTRICT_VINTAGE, +) from microcosm.build.us_runtime.h5_io import ( US_MULTISPINE_AGREEMENT_DIAGNOSTICS_ARTIFACT_KIND, US_MULTISPINE_POOL_H5_ARTIFACT_KIND, @@ -32,6 +37,10 @@ load_simulation_ready_us_multispine_pool, write_nullable_us_h5, ) +from microcosm.build.us_runtime.support_provenance import ( + support_clone_index_column, + support_source_id_column, +) from microcosm.frame import US_SCHEMA, Frame, WeightKind, Weights @@ -70,7 +79,10 @@ def _pool_frame() -> Frame: ) -def _pool_frame_with_object_strings_on_every_entity() -> Frame: +def _pool_frame_with_object_strings_on_every_entity( + *, + stacked: bool = False, +) -> Frame: """Match the assembled pool's object-backed source-string shape.""" frame = _pool_frame() @@ -88,6 +100,21 @@ def _pool_frame_with_object_strings_on_every_entity() -> Frame: ), ) tables[entity] = table + if stacked: + household = tables["household"].copy() + household["puma"] = ["0600101", "0600102", "0600102"] + household["congressional_district_geoid"] = np.asarray( + [601, 601, 601], + dtype=np.int64, + ) + household["county_fips"] = ["06001", "06001", "06001"] + household[support_source_id_column("household")] = np.asarray( + [10, 20, 20], dtype=np.int64 + ) + household[support_clone_index_column("household")] = np.asarray( + [0, 0, 1], dtype=np.int64 + ) + tables["household"] = household return Frame( tables, frame.schema, @@ -387,6 +414,166 @@ def test_pool_h5_load_boundary_canonicalizes_under_pyarrow_default( assert pd.isna(canonical["PERIDNUM"].iloc[1]) +def _fixture_geography_declaration() -> dict[str, object]: + return { + "anchor": "puma", + "order": "before_gap_fill", + "kernels": { + "assign": "kernel:assign_us_puma_ladder", + "validate": "kernel:us_puma_ladder_gate", + }, + "draw": { + "asec": { + "universe": "puma_within_state", + "weight": "puma_population_2020", + }, + "congressional_district": { + "universe": "congressional_district_within_puma", + "weight": "block_population_overlap", + }, + "county": { + "universe": "county_within_puma", + "weight": "block_population_overlap", + }, + }, + "derive": ["puma", "congressional_district_geoid", "county_fips"], + "assertions": [ + "observed_acs_puma_preserved", + "geography_state_prefix_consistent", + ], + "ladder_source": "source:us_puma_ladder_2020", + "congressional_district_vintage_crosswalk": { + "source_ref": ( + "source:us_congressional_district_vintage_crosswalk_117_to_119" + ), + "source_vintage": "vintage:cd_117", + "target_vintage": "vintage:cd_119", + }, + "seed": "stream:geography_legacy", + "default_seed": 0, + "assign_tract": False, + "layer_vintages": { + "congressional_district": "vintage:cd_119", + "county": "vintage:census_2020", + "puma": "vintage:puma_2020", + "tract": "vintage:census_2020", + }, + "validation": ["puma_ladder_gate", "vintage_refusal"], + } + + +def _fixture_geography_assignment( + household: pd.DataFrame, +) -> dict[str, object]: + clone_column = support_clone_index_column("household") + native = household.loc[household[clone_column].eq(0)] + household_ids = native["household_id"].to_numpy(dtype=" Path: run_id = "fixture-publication" pool_path = tmp_path / "pool.h5" @@ -403,8 +590,14 @@ def _write_ready_pool(tmp_path: Path, *, stacked: bool = False) -> Path: }, } schema_version = US_MULTISPINE_POOL_MANIFEST_SCHEMA_VERSION if stacked else 4 + pool_frame = _pool_frame_with_object_strings_on_every_entity(stacked=stacked) + geography_assignment = ( + _fixture_geography_assignment(pool_frame.table("household")) + if stacked + else None + ) write_nullable_us_h5( - _pool_frame_with_object_strings_on_every_entity(), + pool_frame, pool_path, period=2024, artifact_kind=US_MULTISPINE_POOL_H5_ARTIFACT_KIND, @@ -412,6 +605,20 @@ def _write_ready_pool(tmp_path: Path, *, stacked: bool = False) -> Path: materializer_version=( US_MULTISPINE_POOL_H5_MATERIALIZER_VERSION if stacked else None ), + root_attributes=( + { + CONGRESSIONAL_DISTRICT_VINTAGE_CROSSWALK_SHA256_ATTR: ( + geography_assignment["contract"]["authorities"][ + "congressional_district_vintage_crosswalk" + ]["sha256"] + ), + CONGRESSIONAL_DISTRICT_VINTAGE_TARGET_ATTR: ( + CURRENT_CONGRESSIONAL_DISTRICT_VINTAGE + ), + } + if geography_assignment is not None + else None + ), ) diagnostics = { "artifact_kind": (US_MULTISPINE_AGREEMENT_DIAGNOSTICS_ARTIFACT_KIND), @@ -477,25 +684,40 @@ def _write_ready_pool(tmp_path: Path, *, stacked: bool = False) -> Path: } if stacked: dag = _canonical_stacked_late_dag_receipt() + assert geography_assignment is not None transition_authority = ( stacked_spine_module._late_producer_transition_authority_receipt(dag) ) manifest.update( { "pipeline": "us-stacked-pool", + "random_seed": 0, + "geography_assignment": geography_assignment, + "provenance_pins": { + role: { + "path": f"/fixture/{role}", + "expected_sha256": authority["sha256"], + "actual_sha256": authority["sha256"], + "size_bytes": 1, + } + for role, authority in geography_assignment["contract"][ + "authorities" + ].items() + }, "late_producer_transition_authority_sha256": ( transition_authority["sha256"] ), "terminal_gates": agreement_gate, "operator_order": list(US_STACKED_POOL_OPERATOR_ORDER), "stage_receipts": { + "geography_assignment": geography_assignment, "impute": { "source_operator_chain": { "late_dag_completion": dag["source_completion"], }, "stacked_late_producer_dag": dag, "stacked_post_puf_transfer": dag["post_puf_transfer"], - } + }, }, } ) @@ -1112,7 +1334,7 @@ def test_ready_stacked_pool_loader_binds_terminal_gate_aliases( frame, manifest, _ = load_simulation_ready_us_multispine_pool(manifest_path) assert manifest["terminal_gates"] == manifest["agreement_gate"] - assert manifest["schema_version"] == 8 + assert manifest["schema_version"] == 9 assert ( manifest["pool_h5"]["materializer_version"] == US_MULTISPINE_POOL_H5_MATERIALIZER_VERSION @@ -1132,6 +1354,92 @@ def test_ready_stacked_pool_loader_binds_terminal_gate_aliases( ) +def test_ready_stacked_pool_loader_binds_h5_cd_vintage_attrs( + tmp_path: Path, +) -> None: + pytest.importorskip("tables") + manifest_path = _write_ready_pool(tmp_path, stacked=True) + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + pool_path = Path(manifest["pool_h5"]["path"]) + with pd.HDFStore(pool_path, mode="a") as store: + store.get_node("/")._v_attrs[CONGRESSIONAL_DISTRICT_VINTAGE_TARGET_ATTR] = ( + "117th_congress" + ) + manifest["pool_h5"]["sha256"] = _sha256(pool_path) + manifest["pool_h5"]["size_bytes"] = pool_path.stat().st_size + manifest_path.write_text(json.dumps(manifest), encoding="utf-8") + + with pytest.raises(ValueError, match="root attributes do not match"): + load_simulation_ready_us_multispine_pool(manifest_path) + + +def test_ready_stacked_pool_loader_binds_h5_household_geography_digest( + tmp_path: Path, +) -> None: + pytest.importorskip("tables") + manifest_path = _write_ready_pool(tmp_path, stacked=True) + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + pool_path = Path(manifest["pool_h5"]["path"]) + with pd.HDFStore(pool_path, mode="a") as store: + household = h5_io.read_frame_table(store, "household") + household.loc[household.index[0], "congressional_district_geoid"] = 602 + store.put("household", household, format="fixed") + manifest["pool_h5"]["sha256"] = _sha256(pool_path) + manifest["pool_h5"]["size_bytes"] = pool_path.stat().st_size + manifest_path.write_text(json.dumps(manifest), encoding="utf-8") + + with pytest.raises(ValueError, match="household geography differs"): + load_simulation_ready_us_multispine_pool(manifest_path) + + +@pytest.mark.parametrize( + "column", + ( + support_source_id_column("household"), + support_clone_index_column("household"), + ), +) +def test_ready_stacked_pool_loader_requires_h5_clone_lineage( + tmp_path: Path, + column: str, +) -> None: + pytest.importorskip("tables") + manifest_path = _write_ready_pool(tmp_path, stacked=True) + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + pool_path = Path(manifest["pool_h5"]["path"]) + with pd.HDFStore(pool_path, mode="a") as store: + household = h5_io.read_frame_table(store, "household").drop(columns=column) + store.put("household", household, format="fixed") + manifest["pool_h5"]["sha256"] = _sha256(pool_path) + manifest["pool_h5"]["size_bytes"] = pool_path.stat().st_size + manifest_path.write_text(json.dumps(manifest), encoding="utf-8") + + with pytest.raises(ValueError, match="missing household clone-lineage"): + load_simulation_ready_us_multispine_pool(manifest_path) + + +def test_ready_stacked_pool_loader_rejects_divergent_clone_geography( + tmp_path: Path, +) -> None: + pytest.importorskip("tables") + manifest_path = _write_ready_pool(tmp_path, stacked=True) + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + pool_path = Path(manifest["pool_h5"]["path"]) + clone_column = support_clone_index_column("household") + with pd.HDFStore(pool_path, mode="a") as store: + household = h5_io.read_frame_table(store, "household") + clone_row = household[clone_column].eq(1) + assert int(clone_row.sum()) == 1 + household.loc[clone_row, "puma"] = "0600103" + store.put("household", household, format="fixed") + manifest["pool_h5"]["sha256"] = _sha256(pool_path) + manifest["pool_h5"]["size_bytes"] = pool_path.stat().st_size + manifest_path.write_text(json.dumps(manifest), encoding="utf-8") + + with pytest.raises(ValueError, match="cloned household rows disagree.*puma"): + load_simulation_ready_us_multispine_pool(manifest_path) + + def test_scoring_pool_loader_authenticates_failed_stacked_terminal_receipt( tmp_path: Path, ) -> None: @@ -1307,6 +1615,7 @@ def test_ready_stacked_pool_cannot_be_stripped_into_legacy_shape( "release_id", "sampling", "clone_attachment", + "geography_assignment", "input_pins_digest", "late_producer_transition_authority_sha256", "stack_manifest", diff --git a/packages/microcosm-build/tests/test_us_multispine_pool_tool.py b/packages/microcosm-build/tests/test_us_multispine_pool_tool.py index 7f67da31..d17303b3 100644 --- a/packages/microcosm-build/tests/test_us_multispine_pool_tool.py +++ b/packages/microcosm-build/tests/test_us_multispine_pool_tool.py @@ -58,7 +58,7 @@ load_take_up_contract, take_up_contract_identity, ) -from microcosm.frame import US_SCHEMA, Frame, WeightKind, Weights +from microcosm.frame import US_SCHEMA, Frame, WeightKind, Weights, read_frame_table _FIXTURE_SEED_PERSON_COLUMN = "takes_up_medicaid_if_eligible" @@ -78,6 +78,21 @@ def pool_tool() -> ModuleType: return module +@pytest.fixture(scope="module") +def release_tool() -> ModuleType: + root = Path(__file__).resolve().parents[3] + path = root / "tools" / "build_us_fiscal_refresh_release.py" + spec = importlib.util.spec_from_file_location( + "build_us_fiscal_refresh_release_pool_fixture", + path, + ) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + def _source_frame( *, measured_offset: float = 0.0, @@ -123,6 +138,8 @@ def _many_household_source_frame( *, count: int = 100, measured_offset: float = 0.0, + state_fips: str | None = None, + puma: str | None = None, ) -> Frame: ids = np.arange(1, count + 1, dtype=np.int64) person = pd.DataFrame( @@ -148,6 +165,18 @@ def _many_household_source_frame( } if measured_offset: tables["household"]["TYPEHUGQ"] = 1 + if state_fips is not None: + tables["household"]["state_fips"] = pd.Series( + state_fips, + index=tables["household"].index, + dtype="string", + ) + if puma is not None: + tables["household"]["puma"] = pd.Series( + puma, + index=tables["household"].index, + dtype="string", + ) return Frame( tables, US_SCHEMA, @@ -217,6 +246,89 @@ def _with_fixture_pre_clone_strike_benefits(frame: Frame) -> Frame: return _replace_person(frame, person) +def _with_fixture_household_geography(frame: Frame) -> Frame: + tables = {entity: frame.table(entity).copy() for entity in frame.entities} + household = tables["household"] + household["puma"] = pd.Series("0600100", index=household.index, dtype="string") + household["congressional_district_geoid"] = np.full( + len(household), 601, dtype=np.int64 + ) + household["county_fips"] = pd.Series("06001", index=household.index, dtype="string") + tables.update({link: frame.link(link) for link in frame.links}) + return Frame( + tables, + frame.schema, + {entity: frame.weights_for(entity) for entity in frame.weighted_entities}, + frame.strata, + mass_log=frame.mass_log, + metadata=frame.metadata, + ) + + +def _fixture_geography_assignment_receipt( + pool_tool: ModuleType, + frame: Frame, + *, + gate: GateResult | None = None, +) -> dict[str, object]: + household = frame.table("household") + fixture_gate = gate or GateResult( + name="us_puma_ladder", + passed=True, + details={"fixture": True}, + ) + return { + "artifact_kind": "populace_us_stacked_household_geography_assignment", + "schema_version": 1, + "contract": pool_tool._stacked_geography_assignment_contract(), + "pre_assignment_household_order": pool_tool._ordered_household_id_receipt( + household + ), + "assigned_household_geography": ( + pool_tool._ordered_household_geography_receipt(household) + ), + "target_universe": ( + pool_tool._target_congressional_district_universe_receipt((601,)) + ), + "output": { + "household_rows": len(household), + "positive_congressional_district_rows": len(household), + "unique_congressional_district_values": 1, + }, + "summary": {"applied": True, "household_rows": len(household)}, + "gate": GateReport((fixture_gate,)).to_manifest(), + } + + +def _fixture_puma_ladder(pool_tool: ModuleType): + puma = np.asarray([600_100], dtype=np.int64) + population = np.asarray([100.0]) + return pool_tool.UsPumaLadder( + puma=puma, + puma_population=population, + cd_overlap_puma=puma.copy(), + cd_overlap_cd=np.asarray([601], dtype=np.int64), + cd_overlap_population=population.copy(), + county_overlap_puma=puma.copy(), + county_overlap_county=np.asarray([6_001], dtype=np.int32), + county_overlap_population=population.copy(), + tract_overlap_puma=puma.copy(), + tract_overlap_tract=np.asarray([6_001_000_100], dtype=np.int64), + tract_overlap_population=population.copy(), + metadata={ + "schema_version": 1, + "kind": "us_puma_ladder", + "puma_vintage": "2020_puma", + "sampling_basis": "population", + "layers": { + "congressional_district": {"vintage": "119th_congress"}, + "county": {"vintage": "2020_census"}, + "tract": {"vintage": "2020_census"}, + }, + }, + ) + + def _semantic_string_columns(table: pd.DataFrame) -> tuple[str, ...]: return tuple( column @@ -488,13 +600,20 @@ def _verified_inputs_fixture(pool_tool: ModuleType, root: Path): "acs_rent_donor", "processed_puf", "puf_source_year", + "puma_ladder", + "congressional_district_vintage_crosswalk", ), start=1, ): path = root / f"{role}.stacked-fixture" path.parent.mkdir(parents=True, exist_ok=True) path.write_bytes(f"stacked-input-{index}".encode()) - digest = hashlib.sha256(path.read_bytes()).hexdigest() + digest = { + "puma_ladder": pool_tool._STACKED_PUMA_LADDER_SHA256, + "congressional_district_vintage_crosswalk": ( + pool_tool._STACKED_CD_CROSSWALK_SHA256 + ), + }.get(role, hashlib.sha256(path.read_bytes()).hexdigest()) verified[role] = pool_tool._VerifiedInput( role=role, path=path, @@ -886,6 +1005,18 @@ def _stacked_main_argv( ): arguments.extend([f"--{option}", str(tmp_path / option)]) arguments.extend([f"--{option}-sha256", "1" * 64]) + arguments.extend( + [ + "--puma-ladder", + str(tmp_path / "puma-ladder"), + "--puma-ladder-sha256", + "39a2ab2abeab07a88362af7ab2940e0e1d50a297c919e4bbc6fb65bab51147d8", + "--congressional-district-vintage-crosswalk", + str(tmp_path / "congressional-district-vintage-crosswalk"), + "--congressional-district-vintage-crosswalk-sha256", + "c7cb040b1f57ca2ea2adcbfe60cc2b250ca23acbc4b640cd421e766fa54c1aec", + ] + ) arguments.extend( [ "--sample-fraction", @@ -1515,14 +1646,21 @@ def _install_stacked_entrypoint_stubs( *, terminal: str, post_puf_authority: Mapping[str, object] | None = None, + real_geography_assignment: bool = False, ) -> tuple[list[str], int]: order: list[str] = [] verified = _verified_inputs_fixture(pool_tool, tmp_path / "pins") source_manifest = pool_tool.load_acs_source_manifest() puf_donor = pd.DataFrame({"fixture": np.arange(7)}) loaded = pool_tool._LoadedInputs( - asec=_many_household_source_frame(), - acs=_many_household_source_frame(measured_offset=1_000.0), + asec=_many_household_source_frame( + state_fips="06" if real_geography_assignment else None, + ), + acs=_many_household_source_frame( + measured_offset=1_000.0, + state_fips="06" if real_geography_assignment else None, + puma="0600100" if real_geography_assignment else None, + ), acs_rent_donor=pd.DataFrame({"fixture": [1.0]}), puf_donor=puf_donor, asec_raw_stage_checkpoint={"artifact": "fixture-raw-stage"}, @@ -1551,6 +1689,36 @@ def _install_stacked_entrypoint_stubs( lambda _args: (loaded.puf_donor, loaded.puf_donor_build), ) monkeypatch.setattr(pool_tool, "_git_code_pin", lambda: "a" * 40) + crosswalk = pd.DataFrame( + { + "source_geography_id": ["5001700US0601"], + "target_geography_id": ["5001900US0601"], + "weight": [1.0], + } + ) + monkeypatch.setattr( + pool_tool, + "load_congressional_district_vintage_crosswalk", + lambda _path: crosswalk, + ) + fixture_ladder = ( + _fixture_puma_ladder(pool_tool) if real_geography_assignment else object() + ) + monkeypatch.setattr( + pool_tool, + "load_us_puma_ladder", + lambda _path: fixture_ladder, + ) + fixture_puma_gate = GateResult( + name="us_puma_ladder", + passed=True, + details={"fixture": True}, + ) + monkeypatch.setattr( + pool_tool, + "us_puma_ladder_gate", + lambda *_args, **_kwargs: fixture_puma_gate, + ) real_stack = pool_tool.assemble_stacked_spine @@ -1560,6 +1728,26 @@ def stack(*args, **kwargs): monkeypatch.setattr(pool_tool, "assemble_stacked_spine", stack) + real_assign_geography = pool_tool._assign_stacked_household_geography + + def assign_geography(frame: Frame, **kwargs: object): + order.append("geography") + if real_geography_assignment: + return real_assign_geography(frame, **kwargs) + assigned = _with_fixture_household_geography(frame) + receipt = _fixture_geography_assignment_receipt( + pool_tool, + assigned, + gate=fixture_puma_gate, + ) + return assigned, receipt, (601,) + + monkeypatch.setattr( + pool_tool, + "_assign_stacked_household_geography", + assign_geography, + ) + real_build_stacked_pool = pool_tool.build_stacked_pool def build_stacked_pool(*args, **kwargs): @@ -1967,12 +2155,20 @@ def test_stacked_tool_entrypoint_fixture_e2e_emits_one_logbook_row_at_every_term assert "f001-s578-asec1-acs1" in row.build_id assert full_puf_rows == 7 if terminal == "error": - assert order == ["stack", "build_stacked_pool", "prepare", "gap", "puf"] + assert order == [ + "stack", + "geography", + "build_stacked_pool", + "prepare", + "gap", + "puf", + ] assert row.gate_verdicts["pipeline_error"]["verdict"] == "error" assert row.artifact_location is None else: assert order == [ "stack", + "geography", "build_stacked_pool", "prepare", "gap", @@ -2060,6 +2256,7 @@ def test_stacked_tool_entrypoint_fixture_e2e_emits_one_logbook_row_at_every_term ) assert manifest["operator_order"] == [ "assemble_stacked_spine", + "assign_us_puma_ladder", "prepare_multispine_source_inputs_for_clone", "gap_fill_stacked_spine", "run_stacked_late_producer_dag", @@ -2079,6 +2276,65 @@ def test_stacked_tool_entrypoint_fixture_e2e_emits_one_logbook_row_at_every_term } +def test_stacked_pool_fixed_h5_reaches_release_cd_vintage_preflight( + pool_tool: ModuleType, + release_tool: ModuleType, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + h5py = pytest.importorskip("h5py", exc_type=ModuleNotFoundError) + pytest.importorskip("tables", exc_type=ModuleNotFoundError) + order, _full_puf_rows = _install_stacked_entrypoint_stubs( + pool_tool, + monkeypatch, + tmp_path, + terminal="success", + real_geography_assignment=True, + ) + + assert pool_tool.main(_stacked_main_argv(tmp_path)) == 0 + assert order.index("stack") < order.index("geography") < order.index("prepare") + + pool_h5 = tmp_path / "stacked-pool.h5" + with h5py.File(pool_h5, mode="r") as h5: + assert ( + release_tool._h5_attr_text( + h5.attrs, + release_tool.CONGRESSIONAL_DISTRICT_VINTAGE_CROSSWALK_SHA256_ATTR, + ) + == pool_tool._STACKED_CD_CROSSWALK_SHA256 + ) + assert ( + release_tool._h5_attr_text( + h5.attrs, + release_tool.CONGRESSIONAL_DISTRICT_VINTAGE_TARGET_ATTR, + ) + == pool_tool.CURRENT_CONGRESSIONAL_DISTRICT_VINTAGE + ) + with pd.HDFStore(pool_h5, mode="r") as store: + assert store.get_storer("household").format_type == "fixed" + household = read_frame_table(store, "household") + districts = pd.to_numeric( + household["congressional_district_geoid"], + errors="raise", + ) + assert districts.gt(0).all() + assert set(districts.astype(int)) == {601} + + release_tool._assert_cd_vintage_support_matches( + pool_h5, + {"sha256": pool_tool._STACKED_CD_CROSSWALK_SHA256}, + ) + preflight = release_tool._read_cd_vintage_support_provenance(pool_h5) + assert preflight["household_congressional_district_geoid"] == { + "exists": True, + "table": "household", + "column": "congressional_district_geoid", + "rows": len(household), + "positive_unique_count": 1, + } + + def test_stacked_config_authority_defaults_to_constants_without_loading_bundle( pool_tool: ModuleType, monkeypatch: pytest.MonkeyPatch, @@ -2209,7 +2465,7 @@ def capture_equality(expected: object, actual: object) -> None: "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "3189d90dec95c8ea7090e41b5283fa52b1e6855bed4a776dfa02820f2bd11c62", + "spec_sha256": "5378bb9189aec96f50da22aac71e5bd2c3d919e9795f6ef2147e0bc9c739dd8e", }, } @@ -2412,7 +2668,7 @@ def test_constants_adapter_fixture_checkpoints_are_byte_identical_and_only_recei "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "3189d90dec95c8ea7090e41b5283fa52b1e6855bed4a776dfa02820f2bd11c62", + "spec_sha256": "5378bb9189aec96f50da22aac71e5bd2c3d919e9795f6ef2147e0bc9c739dd8e", } def run_fixture(root: Path, *, config_authority: str) -> dict[str, object]: @@ -2524,6 +2780,7 @@ def test_stacked_entrypoint_rejects_noncanonical_post_puf_transfer_receipt( assert order == [ "stack", + "geography", "build_stacked_pool", "prepare", "gap", @@ -2547,7 +2804,10 @@ def test_stacked_checkpoint_emission_propagates_and_authenticates_late_authority stage="transferred", frame=authorized, assembly_receipt={}, - stage_receipts={"impute": impute}, + stage_receipts={ + "geography_assignment": {"fixture": True}, + "impute": impute, + }, late_producer_transition_authority_sha256=(transition_authority_sha256), ) @@ -2564,7 +2824,10 @@ def test_stacked_checkpoint_emission_propagates_and_authenticates_late_authority stage="transferred", frame=authorized, assembly_receipt={}, - stage_receipts={"impute": impute}, + stage_receipts={ + "geography_assignment": {"fixture": True}, + "impute": impute, + }, late_producer_transition_authority_sha256="0" * 64, ) assert len(captured) == 1 @@ -3197,7 +3460,7 @@ def test_legacy_checkpoint_identity_excludes_stacked_late_producer_schedule( assert changed == current -def test_stacked_checkpoint_identity_binds_v11_semantic_contracts( +def test_stacked_checkpoint_identity_binds_v12_semantic_contracts( pool_tool: ModuleType, monkeypatch: pytest.MonkeyPatch, tmp_path: Path, @@ -3230,10 +3493,20 @@ def identity() -> dict[str, object]: current = identity() pool_code = current["pool_code"] - assert current["materializer_version"] == 11 + assert current["materializer_version"] == 12 assert current["stacked_authority"]["version"] == 11 + assert current["geography_assignment"] == ( + pool_tool._stacked_geography_assignment_contract() + ) + assert current["geography_assignment"]["seed"] == { + "site": "legacy_puma_ladder", + "stream": "geography_legacy", + "value_source": "run_request.build_model_seed", + "value": 0, + } assert pool_code["operator_order"] == [ "assemble_stacked_spine", + "assign_us_puma_ladder", "prepare_multispine_source_inputs_for_clone", "gap_fill_stacked_spine", "run_stacked_late_producer_dag", @@ -3448,7 +3721,7 @@ def changed_source_stage_binding( ) ) - assert current["materializer_version"] == stale_qrf["materializer_version"] == 11 + assert current["materializer_version"] == stale_qrf["materializer_version"] == 12 assert stale_qrf["pool_code"]["primary_qrf_checkpoint_schema_version"] == 5 assert ( pool_tool._discover_stacked_checkpoint_identity( @@ -3561,6 +3834,70 @@ def identity() -> dict[str, object]: assert "unsupported binding" in capsys.readouterr().out +def test_geography_assignment_receipt_rejects_divergent_clone_geography( + pool_tool: ModuleType, +) -> None: + stack = pool_tool.assemble_stacked_spine( + _many_household_source_frame(count=2), + _many_household_source_frame(count=2, measured_offset=1_000.0), + sample_fraction=1.0, + sample_seed=578, + ) + assigned = _with_fixture_household_geography(stack.frame) + household = assigned.table("household") + source_column = pool_tool.support_source_id_column("household") + household.loc[household.index[1], source_column] = household.loc[ + household.index[0], source_column + ] + household.loc[household.index[1], "congressional_district_geoid"] = 602 + receipt = _fixture_geography_assignment_receipt(pool_tool, assigned) + receipt["target_universe"] = ( + pool_tool._target_congressional_district_universe_receipt((601, 602)) + ) + receipt["output"]["unique_congressional_district_values"] = 2 + + with pytest.raises( + ValueError, + match="cloned household support rows disagree.*congressional_district_geoid", + ): + pool_tool._validate_stacked_geography_assignment_receipt( + assigned, + receipt, + target_districts=(601, 602), + boundary="fixture clone divergence", + require_exact_assembled_rows=False, + ) + + +def test_geography_assignment_receipt_rejects_coherent_valid_target_rewrite( + pool_tool: ModuleType, +) -> None: + stack = pool_tool.assemble_stacked_spine( + _many_household_source_frame(count=2), + _many_household_source_frame(count=2, measured_offset=1_000.0), + sample_fraction=1.0, + sample_seed=578, + ) + assigned = _with_fixture_household_geography(stack.frame) + receipt = _fixture_geography_assignment_receipt(pool_tool, assigned) + receipt["target_universe"] = ( + pool_tool._target_congressional_district_universe_receipt((601, 602)) + ) + assigned.table("household")["congressional_district_geoid"] = 602 + + with pytest.raises( + ValueError, + match="ordered native household geography differs", + ): + pool_tool._validate_stacked_geography_assignment_receipt( + assigned, + receipt, + target_districts=(601, 602), + boundary="fixture coherent geography rewrite", + require_exact_assembled_rows=False, + ) + + @pytest.mark.parametrize( ("route", "stage_receipts"), ( @@ -3639,7 +3976,7 @@ def test_qbi_receipt_route_resolution_rejects_wrong_or_ambiguous_paths( ) -@pytest.mark.parametrize("legacy_version", (1, 2, 3, 4, 5, 6, 7, 8, 9, 10)) +@pytest.mark.parametrize("legacy_version", (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11)) def test_legacy_stacked_materializer_checkpoint_is_not_discovered( pool_tool: ModuleType, monkeypatch: pytest.MonkeyPatch, @@ -3693,7 +4030,7 @@ def test_legacy_stacked_materializer_checkpoint_is_not_discovered( ) ) - assert pool_tool._STACKED_CHECKPOINT_MATERIALIZER_VERSION == 11 + assert pool_tool._STACKED_CHECKPOINT_MATERIALIZER_VERSION == 12 assert ( pool_tool._discover_stacked_checkpoint_identity( checkpoint_root, @@ -3718,17 +4055,22 @@ def test_stacked_resume_rejects_noncanonical_post_puf_transfer_receipt( sample_fraction=0.10, sample_seed=578, ) + assigned = _with_fixture_household_geography(stack.frame) + geography_receipt = _fixture_geography_assignment_receipt(pool_tool, assigned) noncanonical = _noncanonical_post_puf_authority_receipt() authorized, impute, transition_authority_sha256 = _authorized_late_impute_fixture( pool_tool, - stack.frame, + assigned, authority=noncanonical, ) resume = pool_tool.MultispinePoolCheckpoint( stage="transferred", frame=authorized, assembly_receipt=stack.frame.metadata[pool_tool.SPINE_ASSEMBLY_MANIFEST_KEY], - stage_receipts={"impute": impute}, + stage_receipts={ + "geography_assignment": geography_receipt, + "impute": impute, + }, late_producer_transition_authority_sha256=transition_authority_sha256, ) @@ -3740,8 +4082,10 @@ def test_stacked_resume_rejects_noncanonical_post_puf_transfer_receipt( ), ): pool_tool.build_stacked_pool( - stack.frame, + assigned, expected_stack_receipt=stack.receipt, + geography_assignment_receipt=None, + geography_target_districts=(601,), release_id=( "populace-us-2024-stacked-f010-s578-asec4-acs1-" "20260807T000000Z-deadbeef" @@ -3816,6 +4160,7 @@ def test_stacked_entrypoint_resumes_each_checkpoint_boundary( assert cold_order == [ "stack", + "geography", "build_stacked_pool", "prepare", "gap", @@ -4089,7 +4434,7 @@ def deterministic_fixture_h5( outputs = pool_tool._output_paths(output, checkpoint_root=checkpoint_root) manifest = pool_tool._read_json_object(outputs.manifest) diagnostics = pool_tool._read_json_object(outputs.agreement_diagnostics) - assert pool_tool.POOL_MANIFEST_SCHEMA_VERSION == 8 + assert pool_tool.POOL_MANIFEST_SCHEMA_VERSION == 9 assert pool_tool.POOL_STAGE_CHECKPOINT_MATERIALIZER_VERSION == 7 assert manifest["schema_version"] == 4 assert diagnostics["schema_version"] == 4 @@ -4182,7 +4527,7 @@ def test_main_refuses_constants_adapter_with_legacy_two_spine( assert calls == [] -def test_parser_exposes_six_pinned_inputs_out_and_checkpoint_root( +def test_parser_exposes_eight_pinned_inputs_out_and_checkpoint_root( pool_tool: ModuleType, ) -> None: parser = pool_tool._parser() @@ -4196,6 +4541,11 @@ def test_parser_exposes_six_pinned_inputs_out_and_checkpoint_root( ("acs_rent_h5", "acs_rent_h5_sha256"), ("puf_h5", "puf_h5_sha256"), ("puf_source_year_csv", "puf_source_year_csv_sha256"), + ("puma_ladder", "puma_ladder_sha256"), + ( + "congressional_district_vintage_crosswalk", + "congressional_district_vintage_crosswalk_sha256", + ), ) expected_destinations = {destination for pair in pairs for destination in pair} | { "config_authority", @@ -4219,6 +4569,10 @@ def test_parser_exposes_six_pinned_inputs_out_and_checkpoint_root( "logbook_prev_row_digest", "clone_attachment_fraction", "clone_attachment_seed", + "puma_ladder", + "puma_ladder_sha256", + "congressional_district_vintage_crosswalk", + "congressional_district_vintage_crosswalk_sha256", "legacy_two_spine", "sample_fraction", "sample_seed", @@ -4255,6 +4609,71 @@ def test_parser_exposes_six_pinned_inputs_out_and_checkpoint_root( ) +@pytest.mark.parametrize( + ("destination", "option"), + [ + ("puma_ladder", "--puma-ladder"), + ("puma_ladder_sha256", "--puma-ladder-sha256"), + ( + "congressional_district_vintage_crosswalk", + "--congressional-district-vintage-crosswalk", + ), + ( + "congressional_district_vintage_crosswalk_sha256", + "--congressional-district-vintage-crosswalk-sha256", + ), + ], +) +def test_stacked_route_requires_each_pinned_geography_authority( + pool_tool: ModuleType, + tmp_path: Path, + destination: str, + option: str, +) -> None: + args = SimpleNamespace( + puma_ladder=tmp_path / "puma-ladder.npz", + puma_ladder_sha256=pool_tool._STACKED_PUMA_LADDER_SHA256, + congressional_district_vintage_crosswalk=tmp_path / "crosswalk.csv", + congressional_district_vintage_crosswalk_sha256=( + pool_tool._STACKED_CD_CROSSWALK_SHA256 + ), + ) + setattr(args, destination, None) + + with pytest.raises(ValueError, match=option): + pool_tool._require_stacked_geography_arguments(args) + + +@pytest.mark.parametrize( + ("destination", "option"), + [ + ("puma_ladder_sha256", "--puma-ladder-sha256"), + ( + "congressional_district_vintage_crosswalk_sha256", + "--congressional-district-vintage-crosswalk-sha256", + ), + ], +) +def test_stacked_route_rejects_noncanonical_geography_authority_pins( + pool_tool: ModuleType, + tmp_path: Path, + destination: str, + option: str, +) -> None: + args = SimpleNamespace( + puma_ladder=tmp_path / "puma-ladder.npz", + puma_ladder_sha256=pool_tool._STACKED_PUMA_LADDER_SHA256, + congressional_district_vintage_crosswalk=tmp_path / "crosswalk.csv", + congressional_district_vintage_crosswalk_sha256=( + pool_tool._STACKED_CD_CROSSWALK_SHA256 + ), + ) + setattr(args, destination, "f" * 64) + + with pytest.raises(ValueError, match=option): + pool_tool._require_stacked_geography_arguments(args) + + def test_pool_tool_structurally_accepts_only_the_raw_stage_loader( pool_tool: ModuleType, ) -> None: @@ -4767,6 +5186,18 @@ def unexpected_write(*_args, **_kwargs): else "0" * 64 ) argv.extend([f"--{option}-sha256", digest]) + argv.extend( + [ + "--puma-ladder", + str(tmp_path / "puma-ladder.npz"), + "--puma-ladder-sha256", + pool_tool._STACKED_PUMA_LADDER_SHA256, + "--congressional-district-vintage-crosswalk", + str(tmp_path / "congressional-district-vintage-crosswalk.csv"), + "--congressional-district-vintage-crosswalk-sha256", + pool_tool._STACKED_CD_CROSSWALK_SHA256, + ] + ) argv.extend(["--out", str(output)]) with pytest.raises(ValueError, match="ASEC raw-stage.*SHA-256 mismatch"): diff --git a/packages/microcosm-build/tests/test_us_spec_bundle.py b/packages/microcosm-build/tests/test_us_spec_bundle.py index 03ac7d37..58690780 100644 --- a/packages/microcosm-build/tests/test_us_spec_bundle.py +++ b/packages/microcosm-build/tests/test_us_spec_bundle.py @@ -296,7 +296,7 @@ def test_constant_derived_domain_counts_are_complete( selection = _domain(resolved_us_spec, ResourceKind.SELECTION) catalogs = _domain(resolved_us_spec, ResourceKind.CATALOGS) - assert len(sources["sources"]) == 7 + assert len(sources["sources"]) == 8 assert len(sources["stages"]) == 37 families = imputation["families"] @@ -934,8 +934,17 @@ def test_legacy_seed_vintage_and_publication_grammars_are_pinned( ) assert geography["phase"] == "legacy" assert geography["assignment"]["anchor"] == "puma" - assert geography["assignment"]["order"] == "legacy_post_transfer" + assert geography["assignment"]["order"] == "before_gap_fill" assert geography["assignment"]["assign_tract"] is False + assert geography["assignment"][ + "congressional_district_vintage_crosswalk" + ] == { + "source_ref": ( + "source:us_congressional_district_vintage_crosswalk_117_to_119" + ), + "source_vintage": "vintage:cd_117", + "target_vintage": "vintage:cd_119", + } engine_pins = [ record diff --git a/packages/microcosm-build/tests/test_us_spine_blindness.py b/packages/microcosm-build/tests/test_us_spine_blindness.py index 692b9913..aa6b8016 100644 --- a/packages/microcosm-build/tests/test_us_spine_blindness.py +++ b/packages/microcosm-build/tests/test_us_spine_blindness.py @@ -3279,8 +3279,8 @@ def test_pool_build_tool_import_graph_is_source_spine_blind() -> None: for tool in _SPINE_BLIND_BUILD_TOOLS: runtime_graph, missing_modules = _us_runtime_import_graph(tool) - assert len(runtime_graph) == 66, ( - f"{tool.name} must reach the pinned 66-module runtime graph; " + assert len(runtime_graph) == 69, ( + f"{tool.name} must reach the pinned 69-module runtime graph; " f"reached {len(runtime_graph)}" ) assert not missing_modules, ( diff --git a/tools/build_us_fiscal_refresh_release.py b/tools/build_us_fiscal_refresh_release.py index e5eb90ab..b00c5189 100644 --- a/tools/build_us_fiscal_refresh_release.py +++ b/tools/build_us_fiscal_refresh_release.py @@ -288,7 +288,7 @@ from microcosm.data.us_critical_targets import ( US_SOI_TABLE_1_4_NATIONAL_DOLLAR_FIT_REQUIREMENT as SHARED_US_SOI_TABLE_1_4_NATIONAL_DOLLAR_FIT_REQUIREMENT, ) -from microcosm.frame import Frame, MassChange, WeightKind, Weights +from microcosm.frame import Frame, MassChange, WeightKind, Weights, read_frame_table from microcosm.frame.adapters.policyengine_us import ( PolicyEngineUSEngine, PolicyEngineUSVariableMetadataIndex, @@ -2615,43 +2615,59 @@ def _assert_cd_vintage_support_matches( def _read_cd_vintage_support_provenance(h5_path: Path) -> dict[str, object]: try: - import h5py - except ModuleNotFoundError as exc: + with pd.HDFStore(h5_path, mode="r") as store: + root_attributes = store.get_node("/")._v_attrs + crosswalk_sha256 = _h5_attr_text( + root_attributes, + CONGRESSIONAL_DISTRICT_VINTAGE_CROSSWALK_SHA256_ATTR, + ) + target_vintage = _h5_attr_text( + root_attributes, + CONGRESSIONAL_DISTRICT_VINTAGE_TARGET_ATTR, + ) + cd_lookup = _hdf_store_frame_column_status( + store, + table="household", + column="congressional_district_geoid", + ) + except ImportError as exc: raise RuntimeError( - "Reading congressional-district support provenance requires h5py. " + "Reading congressional-district support provenance requires PyTables. " "Run the fiscal refresh builder with the US extra, for example " "`uv run --python 3.13 --package microcosm-build --extra us --group " "dev python tools/build_us_fiscal_refresh_release.py ...`. This " "preflight is intentionally before calibration or donor imputation." ) from exc + return { + CONGRESSIONAL_DISTRICT_VINTAGE_CROSSWALK_SHA256_ATTR: crosswalk_sha256, + CONGRESSIONAL_DISTRICT_VINTAGE_TARGET_ATTR: target_vintage, + "household_congressional_district_geoid": cd_lookup, + } - with h5py.File(h5_path, "r") as h5: - return { - CONGRESSIONAL_DISTRICT_VINTAGE_CROSSWALK_SHA256_ATTR: _h5_attr_text( - h5.attrs, CONGRESSIONAL_DISTRICT_VINTAGE_CROSSWALK_SHA256_ATTR - ), - CONGRESSIONAL_DISTRICT_VINTAGE_TARGET_ATTR: _h5_attr_text( - h5.attrs, CONGRESSIONAL_DISTRICT_VINTAGE_TARGET_ATTR - ), - "household_congressional_district_geoid": ( - _h5_table_column_status( - h5, - table="household", - column="congressional_district_geoid", - ) - ), - } +def _hdf_frame_column_status( + h5_path: Path, + *, + table: str, + column: str, +) -> dict[str, object]: + with pd.HDFStore(h5_path, mode="r") as store: + return _hdf_store_frame_column_status(store, table=table, column=column) -def _h5_table_column_status(h5, *, table: str, column: str) -> dict[str, object]: - table_path = f"{table}/table" - if table_path not in h5: + +def _hdf_store_frame_column_status( + store: pd.HDFStore, + *, + table: str, + column: str, +) -> dict[str, object]: + try: + frame_table = read_frame_table(store, table) + except KeyError: return {"exists": False, "table": table, "column": column} - dataset = h5[table_path] - names = getattr(dataset.dtype, "names", None) or () - if column not in names: + if column not in frame_table.columns: return {"exists": False, "table": table, "column": column} - values = np.asarray(dataset[column]) + values = frame_table[column].to_numpy(copy=False) return { "exists": True, "table": table, @@ -2673,8 +2689,14 @@ def _positive_numeric_unique_count(values: np.ndarray) -> int: return len(positive) -def _h5_attr_text(attrs: Mapping[str, object], key: str) -> str | None: - value = attrs.get(key) +def _h5_attr_text(attrs: object, key: str) -> str | None: + try: + if isinstance(attrs, Mapping): + value = attrs.get(key) + else: + value = attrs[key] # type: ignore[index] + except KeyError: + return None if value is None: return None if isinstance(value, bytes): diff --git a/tools/build_us_multispine_pool.py b/tools/build_us_multispine_pool.py index 4cd14c64..23e0de84 100644 --- a/tools/build_us_multispine_pool.py +++ b/tools/build_us_multispine_pool.py @@ -3,7 +3,7 @@ The default production path is the stacked pipeline: -``stack -> gap-fill -> PUF pass + tail -> late DAG -> derive -> seed -> simulate -> gates``. +``stack -> geography -> gap-fill -> PUF pass + tail -> late DAG -> derive -> seed -> simulate -> gates``. Both survey arms use one composition-preserving ``--sample-fraction``; PUF donors always remain full. The terminal completeness gate plus by-origin @@ -33,7 +33,11 @@ --acs-rent-h5 "$RENT_H5" --acs-rent-h5-sha256 "$RENT_SHA" \\ --puf-h5 "$PUF_H5" --puf-h5-sha256 "$PUF_SHA" \\ --puf-source-year-csv "$PUF_CSV" \\ - --puf-source-year-csv-sha256 "$PUF_CSV_SHA" --out "$OUT" + --puf-source-year-csv-sha256 "$PUF_CSV_SHA" \\ + --puma-ladder "$PUMA_LADDER" --puma-ladder-sha256 "$PUMA_LADDER_SHA" \\ + --congressional-district-vintage-crosswalk "$CD_CROSSWALK" \\ + --congressional-district-vintage-crosswalk-sha256 "$CD_CROSSWALK_SHA" \\ + --out "$OUT" """ from __future__ import annotations @@ -102,6 +106,16 @@ from microcosm.build.us_runtime.asec_checkpoint import ( load_asec_raw_stage_checkpoint, ) +from microcosm.build.us_runtime.congressional_district_geography import ( + CONGRESSIONAL_DISTRICT_GEOID_COLUMN, +) +from microcosm.build.us_runtime.congressional_district_vintage import ( + CONGRESSIONAL_DISTRICT_VINTAGE_CROSSWALK_SHA256_ATTR, + CONGRESSIONAL_DISTRICT_VINTAGE_TARGET_ATTR, + CURRENT_CONGRESSIONAL_DISTRICT_PREFIX, + CURRENT_CONGRESSIONAL_DISTRICT_VINTAGE, + load_congressional_district_vintage_crosswalk, +) from microcosm.build.us_runtime.h5_io import ( US_MULTISPINE_AGREEMENT_DIAGNOSTICS_ARTIFACT_KIND, US_MULTISPINE_POOL_H5_ARTIFACT_KIND, @@ -160,6 +174,13 @@ PUF_SUPPORT_MAX_CLONE_SAFE_SOURCE_ID, US_PUF_SUPPORT_FIT_NAME, ) +from microcosm.build.us_runtime.puma_ladder import ( + UsPumaLadder, + load_us_puma_ladder, + us_puma_ladder_assignment_summary, + us_puma_ladder_gate, + with_household_us_puma_ladder, +) from microcosm.build.us_runtime.qbi_inputs import ( us_qbi_post_reconciliation_person_columns, us_qbi_reconciliation_contract_identity, @@ -195,6 +216,8 @@ BASE_ASEC_SUPPORT_CHANNEL, SPINE_ASSEMBLY_MANIFEST_KEY, spine_provenance_counts, + support_clone_index_column, + support_source_id_column, validate_assembly_provenance, ) from microcosm.build.us_runtime.take_up_contract import take_up_contract_identity @@ -306,16 +329,40 @@ _STACKED_CHECKPOINT_IDENTITY_ARTIFACT_KIND = ( "populace_us_stacked_pool_checkpoint_identity" ) -# Version 11 additionally binds the primary-PUF whole-pool universe semantics. +# Version 12 binds the post-assembly household geography assignment authority, +# target vintage, algorithm, operator order, and seed. Earlier checkpoints +# predate the congressional-district support required by release preflight. +# Version 11 additionally bound the primary-PUF whole-pool universe semantics. # Earlier checkpoints must rebuild rather than resume with a nullable # s_corp_income leaf. Version 10 bound the complete late-resource semantics and # corrected outer order (the primary PUF callback is nested inside the DAG). -_STACKED_CHECKPOINT_MATERIALIZER_VERSION = 11 +_STACKED_CHECKPOINT_MATERIALIZER_VERSION = 12 _STACKED_RELEASE_ID_PATTERN = re.compile( r"^populace-us-2024-stacked-f(?:001|004|010|025|100)-s[0-9]+-" r"asec[0-9]+-acs[0-9]+-[0-9]{8}T[0-9]{6}Z-[0-9a-f]{8}$" ) +_STACKED_GEOGRAPHY_ASSIGNMENT_ALGORITHM = ( + "assign_us_puma_ladder.population_weighted_overlap.v1" +) +_STACKED_GEOGRAPHY_ASSIGNMENT_ORDER = "before_gap_fill" +_STACKED_GEOGRAPHY_ASSIGNMENT_SEED_SITE = "legacy_puma_ladder" +_STACKED_PUMA_LADDER_INPUT_ROLE = "puma_ladder" +_STACKED_PUMA_LADDER_SOURCE_REF = "source:us_puma_ladder_2020" +_STACKED_PUMA_LADDER_SHA256 = ( + "39a2ab2abeab07a88362af7ab2940e0e1d50a297c919e4bbc6fb65bab51147d8" +) +_STACKED_CD_CROSSWALK_INPUT_ROLE = "congressional_district_vintage_crosswalk" +_STACKED_CD_CROSSWALK_SOURCE_REF = ( + "source:us_congressional_district_vintage_crosswalk_117_to_119" +) +_STACKED_CD_CROSSWALK_SHA256 = ( + "c7cb040b1f57ca2ea2adcbfe60cc2b250ca23acbc4b640cd421e766fa54c1aec" +) +_STACKED_CD_CROSSWALK_SOURCE_VINTAGE_REF = "vintage:cd_117" +_STACKED_CD_CROSSWALK_SOURCE_VINTAGE = "117th_congress" +_STACKED_CD_CROSSWALK_TARGET_VINTAGE_REF = "vintage:cd_119" + type PoolOperator = Callable[[Frame], PoolStageOutput] @@ -512,6 +559,39 @@ def _parser() -> argparse.ArgumentParser: type=_sha256_argument, help="Expected SHA-256 of --puf-source-year-csv.", ) + parser.add_argument( + "--puma-ladder", + type=Path, + help=( + "Pinned national PUMA-overlap ladder. Required by the default " + "stacked pipeline; unused by --legacy-two-spine." + ), + ) + parser.add_argument( + "--puma-ladder-sha256", + type=_sha256_argument, + help=( + "Canonical SHA-256 of --puma-ladder. Required by the default " + "stacked pipeline; unused by --legacy-two-spine." + ), + ) + parser.add_argument( + "--congressional-district-vintage-crosswalk", + type=Path, + help=( + "Pinned 117th-to-119th congressional-district crosswalk. Required " + "by the default stacked pipeline; unused by --legacy-two-spine." + ), + ) + parser.add_argument( + "--congressional-district-vintage-crosswalk-sha256", + type=_sha256_argument, + help=( + "Canonical SHA-256 of --congressional-district-vintage-crosswalk. " + "Required by the default stacked pipeline; unused by " + "--legacy-two-spine." + ), + ) parser.add_argument( "--out", required=True, @@ -731,10 +811,129 @@ def _verify_acs_file( return verified +def _stacked_geography_assignment_contract() -> dict[str, object]: + """Return the closed household-geography authority bound by every cache.""" + + return { + "declaration": { + "anchor": "puma", + "order": _STACKED_GEOGRAPHY_ASSIGNMENT_ORDER, + "kernels": { + "assign": "kernel:assign_us_puma_ladder", + "validate": "kernel:us_puma_ladder_gate", + }, + "draw": { + "asec": { + "universe": "puma_within_state", + "weight": "puma_population_2020", + }, + "congressional_district": { + "universe": "congressional_district_within_puma", + "weight": "block_population_overlap", + }, + "county": { + "universe": "county_within_puma", + "weight": "block_population_overlap", + }, + }, + "derive": [ + "puma", + CONGRESSIONAL_DISTRICT_GEOID_COLUMN, + "county_fips", + ], + "assertions": [ + "observed_acs_puma_preserved", + "geography_state_prefix_consistent", + ], + "ladder_source": _STACKED_PUMA_LADDER_SOURCE_REF, + "congressional_district_vintage_crosswalk": { + "source_ref": _STACKED_CD_CROSSWALK_SOURCE_REF, + "source_vintage": _STACKED_CD_CROSSWALK_SOURCE_VINTAGE_REF, + "target_vintage": _STACKED_CD_CROSSWALK_TARGET_VINTAGE_REF, + }, + "seed": "stream:geography_legacy", + "default_seed": POOL_RANDOM_SEED, + "assign_tract": False, + "layer_vintages": { + "congressional_district": _STACKED_CD_CROSSWALK_TARGET_VINTAGE_REF, + "county": "vintage:census_2020", + "puma": "vintage:puma_2020", + "tract": "vintage:census_2020", + }, + "validation": ["puma_ladder_gate", "vintage_refusal"], + }, + "algorithm": { + "id": _STACKED_GEOGRAPHY_ASSIGNMENT_ALGORITHM, + "kernel": "assign_us_puma_ladder", + "operator": "assign_us_puma_ladder", + "order": _STACKED_GEOGRAPHY_ASSIGNMENT_ORDER, + "assign_tract": False, + }, + "authorities": { + _STACKED_PUMA_LADDER_INPUT_ROLE: { + "input_role": _STACKED_PUMA_LADDER_INPUT_ROLE, + "source_ref": _STACKED_PUMA_LADDER_SOURCE_REF, + "sha256": _STACKED_PUMA_LADDER_SHA256, + }, + _STACKED_CD_CROSSWALK_INPUT_ROLE: { + "input_role": _STACKED_CD_CROSSWALK_INPUT_ROLE, + "source_ref": _STACKED_CD_CROSSWALK_SOURCE_REF, + "sha256": _STACKED_CD_CROSSWALK_SHA256, + "source_vintage_ref": _STACKED_CD_CROSSWALK_SOURCE_VINTAGE_REF, + "source_vintage": _STACKED_CD_CROSSWALK_SOURCE_VINTAGE, + "target_vintage_ref": _STACKED_CD_CROSSWALK_TARGET_VINTAGE_REF, + "target_vintage": CURRENT_CONGRESSIONAL_DISTRICT_VINTAGE, + }, + }, + "seed": { + "site": _STACKED_GEOGRAPHY_ASSIGNMENT_SEED_SITE, + "stream": "geography_legacy", + "value_source": "run_request.build_model_seed", + "value": POOL_RANDOM_SEED, + }, + } + + +def _require_stacked_geography_arguments(args: argparse.Namespace) -> None: + """Require both authenticated geography authorities on the stacked route.""" + + required = { + "--puma-ladder": getattr(args, "puma_ladder", None), + "--puma-ladder-sha256": getattr(args, "puma_ladder_sha256", None), + "--congressional-district-vintage-crosswalk": getattr( + args, "congressional_district_vintage_crosswalk", None + ), + "--congressional-district-vintage-crosswalk-sha256": getattr( + args, "congressional_district_vintage_crosswalk_sha256", None + ), + } + missing = [option for option, value in required.items() if value is None] + if missing: + raise ValueError( + "The stacked pipeline requires pinned household-geography " + f"authorities; missing {', '.join(missing)}." + ) + canonical_pins = { + "--puma-ladder-sha256": _STACKED_PUMA_LADDER_SHA256, + "--congressional-district-vintage-crosswalk-sha256": ( + _STACKED_CD_CROSSWALK_SHA256 + ), + } + for option, expected in canonical_pins.items(): + value = required[option] + if value != expected: + raise ValueError( + f"{option} differs from the canonical US authority pin: " + f"got {value}, expected {expected}." + ) + + def _verify_inputs( args: argparse.Namespace, outputs: PoolBuildOutputs, ) -> tuple[dict[str, _VerifiedInput], AcsSourceManifest]: + if not args.legacy_two_spine: + _require_stacked_geography_arguments(args) source_paths = _configured_source_paths(args) _validate_checkpoint_path_layout(outputs, source_paths=source_paths) @@ -788,13 +987,28 @@ def _verify_inputs( args.puf_source_year_csv_sha256, ), } + if not args.legacy_two_spine: + verified.update( + { + _STACKED_PUMA_LADDER_INPUT_ROLE: _verify_file( + "US PUMA ladder", + args.puma_ladder, + args.puma_ladder_sha256, + ), + _STACKED_CD_CROSSWALK_INPUT_ROLE: _verify_file( + "congressional-district vintage crosswalk", + args.congressional_district_vintage_crosswalk, + args.congressional_district_vintage_crosswalk_sha256, + ), + } + ) return verified, acs_source_manifest def _configured_source_paths(args: argparse.Namespace) -> set[Path]: - """Resolve the six immutable input locations without opening them.""" + """Resolve immutable input locations without opening them.""" - return { + paths = { Path(args.asec_raw_stage_h5).resolve(), Path(args.acs_household_zip).resolve(), Path(args.acs_person_zip).resolve(), @@ -802,6 +1016,15 @@ def _configured_source_paths(args: argparse.Namespace) -> set[Path]: Path(args.puf_h5).resolve(), Path(args.puf_source_year_csv).resolve(), } + if not args.legacy_two_spine: + _require_stacked_geography_arguments(args) + paths.update( + { + Path(args.puma_ladder).resolve(), + Path(args.congressional_district_vintage_crosswalk).resolve(), + } + ) + return paths def _validate_checkpoint_path_layout( @@ -1049,7 +1272,8 @@ def _stacked_rung(sample_fraction: float) -> str: return _STACKED_SAMPLE_RUNG_TOKENS[float(sample_fraction)] except KeyError as exc: raise ValueError( - "Stacked sample_fraction must be one standard rung: 0.01, 0.04, 0.10, or 1.0." + "Stacked sample_fraction must be one standard rung: " + "0.01, 0.04, 0.10, 0.25, or 1.0." ) from exc @@ -1082,9 +1306,42 @@ def _configured_input_pins_digest(args: argparse.Namespace) -> str: "processed_puf": args.puf_h5_sha256, "puf_source_year": args.puf_source_year_csv_sha256, } + if not args.legacy_two_spine: + _require_stacked_geography_arguments(args) + payload.update( + { + _STACKED_PUMA_LADDER_INPUT_ROLE: args.puma_ladder_sha256, + _STACKED_CD_CROSSWALK_INPUT_ROLE: ( + args.congressional_district_vintage_crosswalk_sha256 + ), + } + ) return hashlib.sha256(_canonical_json_bytes(payload)).hexdigest() +def _assert_stacked_geography_verified_inputs( + verified_inputs: Mapping[str, _VerifiedInput], +) -> None: + expected = { + _STACKED_PUMA_LADDER_INPUT_ROLE: _STACKED_PUMA_LADDER_SHA256, + _STACKED_CD_CROSSWALK_INPUT_ROLE: _STACKED_CD_CROSSWALK_SHA256, + } + for role, canonical_sha256 in expected.items(): + pin = verified_inputs.get(role) + if pin is None: + raise ValueError( + f"Stacked checkpoint identity requires verified input role {role!r}." + ) + if ( + pin.expected_sha256 != canonical_sha256 + or pin.actual_sha256 != canonical_sha256 + ): + raise ValueError( + f"Stacked verified input {role!r} differs from its canonical " + f"SHA-256 {canonical_sha256}." + ) + + def _stacked_checkpoint_base_identity( verified_inputs: Mapping[str, _VerifiedInput], *, @@ -1097,6 +1354,10 @@ def _stacked_checkpoint_base_identity( ) -> dict[str, object]: """Bind #599/#608 caches to the live stack and both scale controls.""" + # The empty mapping is reserved for the pure static-component oracle. + # Every configured/runtime identity carries verified inputs. + if verified_inputs: + _assert_stacked_geography_verified_inputs(verified_inputs) fraction_token = _stacked_rung(sample_fraction) if isinstance(sample_seed, bool) or sample_seed < 0: raise ValueError("sample_seed must be a non-negative integer.") @@ -1132,6 +1393,7 @@ def _stacked_checkpoint_base_identity( "fraction": float(clone_attachment_fraction), "seed": clone_attachment_seed, }, + "geography_assignment": _stacked_geography_assignment_contract(), "stacked_authority": stacked_spine_authority_receipt(), "pool_code": { "operator_order": list(US_STACKED_POOL_OPERATOR_ORDER), @@ -1204,6 +1466,7 @@ def _configured_stacked_identity(args: argparse.Namespace) -> dict[str, object]: "sample_seed": args.sample_seed, "clone_attachment_fraction": float(args.clone_attachment_fraction), "clone_attachment_seed": args.clone_attachment_seed, + "geography_assignment": _stacked_geography_assignment_contract(), "stacked_authority": stacked_spine_authority_receipt(), } @@ -1292,6 +1555,381 @@ def _discover_stacked_checkpoint_identity( return None +def _crosswalk_target_congressional_districts( + crosswalk: pd.DataFrame, +) -> tuple[int, ...]: + """Return the authenticated crosswalk's sorted 119th-Congress universe.""" + + column = "target_geography_id" + if column not in crosswalk: + raise ValueError( + "Congressional-district vintage crosswalk has no target_geography_id." + ) + target = crosswalk[column].astype(str) + prefix = CURRENT_CONGRESSIONAL_DISTRICT_PREFIX + if not target.str.startswith(prefix, na=False).all(): + raise ValueError( + "Congressional-district vintage crosswalk contains a non-current " + "target geography prefix." + ) + suffix = target.str.removeprefix(prefix) + if not suffix.str.fullmatch(r"[0-9]{4}").all(): + raise ValueError( + "Congressional-district vintage crosswalk target geoids must have " + "four-digit state-plus-district suffixes." + ) + districts = tuple(sorted(int(value) for value in suffix.unique())) + if not districts or districts[0] <= 0: + raise ValueError( + "Congressional-district vintage crosswalk has no positive targets." + ) + return districts + + +def _target_congressional_district_universe_receipt( + districts: tuple[int, ...], +) -> dict[str, object]: + return { + "district_count": len(districts), + "geoids_sha256": hashlib.sha256( + _canonical_json_bytes(list(districts)) + ).hexdigest(), + } + + +def _ordered_household_id_receipt(household: pd.DataFrame) -> dict[str, object]: + column = "household_id" + if column not in household: + raise ValueError( + "Stacked geography assignment requires ordered household_id values." + ) + numeric = pd.to_numeric(household[column], errors="coerce").to_numpy( + dtype=np.float64, + na_value=np.nan, + ) + valid = np.isfinite(numeric) & (numeric == np.floor(numeric)) + if not valid.all(): + raise ValueError( + "Stacked geography assignment household_id values must be integral." + ) + ordered = numeric.astype(" dict[str, object]: + """Digest ordered native IDs and every assigned launch-geography value.""" + + id_column = "household_id" + columns = ( + id_column, + "puma", + CONGRESSIONAL_DISTRICT_GEOID_COLUMN, + "county_fips", + ) + missing = [column for column in columns if column not in household] + if missing: + raise ValueError( + "Stacked geography output digest is missing household column(s): " + f"{missing}." + ) + + household_ids = pd.to_numeric(household[id_column], errors="coerce").to_numpy( + dtype=np.float64, + na_value=np.nan, + ) + valid_ids = np.isfinite(household_ids) & ( + household_ids == np.floor(household_ids) + ) + if not valid_ids.all(): + raise ValueError("Stacked geography output household IDs must be integral.") + + def fixed_width_values(column: str, width: int) -> np.ndarray: + text = household[column].astype(str) + if not text.str.fullmatch(rf"[0-9]{{{width}}}").all(): + raise ValueError( + f"Stacked geography output {column!r} must contain exactly " + f"{width}-digit codes." + ) + return text.astype(np.int64).to_numpy(dtype=" np.ndarray: + if CONGRESSIONAL_DISTRICT_GEOID_COLUMN not in household: + raise ValueError( + f"{boundary}: household table has no " + f"{CONGRESSIONAL_DISTRICT_GEOID_COLUMN!r}." + ) + numeric = pd.to_numeric( + household[CONGRESSIONAL_DISTRICT_GEOID_COLUMN], + errors="coerce", + ).to_numpy(dtype=np.float64, na_value=np.nan) + valid = np.isfinite(numeric) & (numeric > 0) & (numeric == np.floor(numeric)) + if not valid.all(): + raise ValueError( + f"{boundary}: every household must carry a positive integral " + f"{CONGRESSIONAL_DISTRICT_GEOID_COLUMN}." + ) + return numeric.astype(np.int64) + + +def _validate_stacked_geography_assignment_receipt( + frame: Frame, + receipt: Mapping[str, object], + *, + target_districts: tuple[int, ...], + boundary: str, + require_exact_assembled_rows: bool, +) -> None: + """Bind a live assembled-or-later Frame to its assignment receipt.""" + + if receipt.get("artifact_kind") != ( + "populace_us_stacked_household_geography_assignment" + ) or receipt.get("schema_version") != 1: + raise ValueError(f"{boundary}: geography assignment receipt is unsupported.") + expected_contract = _stacked_geography_assignment_contract() + if _json_ready(receipt.get("contract")) != _json_ready(expected_contract): + raise ValueError(f"{boundary}: geography assignment contract changed.") + expected_universe = _target_congressional_district_universe_receipt( + target_districts + ) + if receipt.get("target_universe") != expected_universe: + raise ValueError( + f"{boundary}: congressional-district target universe changed." + ) + output = receipt.get("output") + if not isinstance(output, Mapping): + raise ValueError(f"{boundary}: geography assignment output is missing.") + assigned_rows = output.get("household_rows") + if ( + isinstance(assigned_rows, bool) + or not isinstance(assigned_rows, int) + or assigned_rows < 1 + ): + raise ValueError( + f"{boundary}: geography assignment household row count is invalid." + ) + + household = frame.table("household") + if len(household) < assigned_rows or ( + require_exact_assembled_rows and len(household) != assigned_rows + ): + raise ValueError( + f"{boundary}: live household rows do not match the assembled " + "geography assignment." + ) + native_household = household + clone_column = support_clone_index_column("household") + if clone_column in household: + clone_index = pd.to_numeric(household[clone_column], errors="coerce") + native_household = household.loc[clone_index.eq(0)] + expected_order = receipt.get("pre_assignment_household_order") + if ( + len(native_household) != assigned_rows + or expected_order != _ordered_household_id_receipt(native_household) + ): + raise ValueError( + f"{boundary}: ordered native household IDs differ from the seeded " + "geography assignment receipt." + ) + if receipt.get("assigned_household_geography") != ( + _ordered_household_geography_receipt(native_household) + ): + raise ValueError( + f"{boundary}: ordered native household geography differs from the " + "assignment output receipt." + ) + values = _assigned_congressional_district_values( + household, + boundary=boundary, + ) + target = np.asarray(target_districts, dtype=np.int64) + outside = ~np.isin(values, target) + if outside.any(): + raise ValueError( + f"{boundary}: household congressional-district values fall outside " + f"the authenticated 119th-Congress crosswalk target universe: " + f"{sorted(set(values[outside].tolist()))[:5]}." + ) + if require_exact_assembled_rows: + expected_output = { + "household_rows": len(household), + "positive_congressional_district_rows": len(values), + "unique_congressional_district_values": int(len(np.unique(values))), + } + if output != expected_output: + raise ValueError( + f"{boundary}: assembled geography assignment counts changed." + ) + + summary = receipt.get("summary") + if ( + not isinstance(summary, Mapping) + or summary.get("applied") is not True + or summary.get("household_rows") != assigned_rows + ): + raise ValueError(f"{boundary}: geography assignment summary is invalid.") + gate_receipt = receipt.get("gate") + gates = gate_receipt.get("gates") if isinstance(gate_receipt, Mapping) else None + puma_gate = gates.get("us_puma_ladder") if isinstance(gates, Mapping) else None + if ( + not isinstance(gate_receipt, Mapping) + or gate_receipt.get("passed") is not True + or not isinstance(puma_gate, Mapping) + or puma_gate.get("passed") is not True + ): + raise ValueError(f"{boundary}: PUMA-ladder gate receipt did not pass.") + if require_exact_assembled_rows: + live_gate = GateReport( + ( + us_puma_ladder_gate( + household, + frame.weights_for("household").values, + assign_tract=False, + ), + ) + ).to_manifest() + if gate_receipt != live_gate: + raise ValueError( + f"{boundary}: PUMA-ladder gate receipt changed from live output." + ) + + provenance_columns = ( + support_source_id_column("household"), + support_clone_index_column("household"), + ) + geography_columns = ( + "puma", + CONGRESSIONAL_DISTRICT_GEOID_COLUMN, + "county_fips", + ) + if all(column in household for column in (*provenance_columns, *geography_columns)): + source_column, _clone_column = provenance_columns + grouped = household.groupby( + source_column, + sort=False, + dropna=False, + ) + for column in geography_columns: + incoherent = grouped[column].nunique(dropna=False) > 1 + if bool(incoherent.any()): + raise ValueError( + f"{boundary}: cloned household support rows disagree on " + f"assigned geography column {column!r}." + ) + + +def _assign_stacked_household_geography( + frame: Frame, + *, + ladder: UsPumaLadder, + crosswalk: pd.DataFrame, +) -> tuple[Frame, dict[str, object], tuple[int, ...]]: + """Apply the pinned geography operator after legal source assembly.""" + + target_districts = _crosswalk_target_congressional_districts(crosswalk) + ladder_districts = tuple( + sorted(int(value) for value in np.unique(ladder.cd_overlap_cd).tolist()) + ) + if ladder_districts != target_districts: + raise ValueError( + "US PUMA ladder congressional-district universe differs from the " + "authenticated vintage-crosswalk target universe." + ) + pre_assignment_order = _ordered_household_id_receipt(frame.table("household")) + assigned = with_household_us_puma_ladder( + frame, + ladder, + seed=POOL_RANDOM_SEED, + assign_tract=False, + expected_congressional_district_vintage=( + CURRENT_CONGRESSIONAL_DISTRICT_VINTAGE + ), + ) + household = assigned.table("household") + values = _assigned_congressional_district_values( + household, + boundary="stacked post-assembly geography assignment", + ) + gate = us_puma_ladder_gate( + household, + assigned.weights_for("household").values, + assign_tract=False, + ) + if not gate.passed: + raise ValueError( + "Stacked post-assembly PUMA-ladder gate failed:\n " + + "\n ".join(gate.failures) + ) + receipt = { + "artifact_kind": "populace_us_stacked_household_geography_assignment", + "schema_version": 1, + "contract": _stacked_geography_assignment_contract(), + "pre_assignment_household_order": pre_assignment_order, + "assigned_household_geography": _ordered_household_geography_receipt( + household + ), + "target_universe": _target_congressional_district_universe_receipt( + target_districts + ), + "output": { + "household_rows": len(household), + "positive_congressional_district_rows": len(values), + "unique_congressional_district_values": int(len(np.unique(values))), + }, + "summary": us_puma_ladder_assignment_summary( + household, + ladder, + weight_values=assigned.weights_for("household").values, + assign_tract=False, + ), + "gate": GateReport((gate,)).to_manifest(), + } + _validate_stacked_geography_assignment_receipt( + assigned, + receipt, + target_districts=target_districts, + boundary="stacked post-assembly geography assignment", + require_exact_assembled_rows=True, + ) + return assigned, receipt, target_districts + + def _stacked_realized_counts( stack_receipt: Mapping[str, object], ) -> tuple[int, int]: @@ -1907,7 +2545,11 @@ def _load(self, stage: str) -> MultispinePoolCheckpoint | None: raise ValueError(f"{stage} checkpoint stage_receipts must be an object") if not isinstance(input_receipts, Mapping): raise ValueError(f"{stage} checkpoint input_receipts must be an object") - _validate_checkpoint_receipt_prefix(stage, stage_receipts) + _validate_checkpoint_receipt_prefix( + stage, + stage_receipts, + stacked=_checkpoint_qbi_route(self._base_identity) == "stacked", + ) restored_stage_receipts, receipts_record = ( self._load_operational_stage_receipts( stage, @@ -2127,11 +2769,14 @@ def _validate_checkpoint_metadata( def _validate_checkpoint_receipt_prefix( stage: str, stage_receipts: Mapping[str, object], + *, + stacked: bool, ) -> None: + geography = frozenset({"geography_assignment"}) if stacked else frozenset() expected = { - "assembled": frozenset(), - "transferred": frozenset({"impute"}), - "simulated": frozenset({"impute", "derive", "seed", "simulate"}), + "assembled": geography, + "transferred": geography | {"impute"}, + "simulated": geography | {"impute", "derive", "seed", "simulate"}, }[stage] observed = frozenset(stage_receipts) allowed = expected | ({"clone"} if stage != "assembled" else set()) @@ -2842,6 +3487,11 @@ def _emit_stacked_checkpoint( qbi_transition_authority_sha256: str | None = None, late_producer_transition_authority_sha256: str | None = None, ) -> None: + _validate_checkpoint_receipt_prefix( + stage, + stage_receipts, + stacked=True, + ) if stage in {"transferred", "simulated"}: _validate_stacked_post_puf_stage_receipt( frame, @@ -2881,6 +3531,8 @@ def build_stacked_pool( assembled: Frame, *, expected_stack_receipt: Mapping[str, object], + geography_assignment_receipt: Mapping[str, object] | None, + geography_target_districts: tuple[int, ...], release_id: str, puf_donor: pd.DataFrame | None, acs_rent_donor: pd.DataFrame | None, @@ -2910,13 +3562,26 @@ def mark_phase(name: str) -> None: raise ValueError("Fresh stacked assembly receipt changed before execution.") if resume is None: + if not isinstance(geography_assignment_receipt, Mapping): + raise ValueError( + "A cold stacked build requires its live geography assignment receipt." + ) + _validate_stacked_geography_assignment_receipt( + assembled, + geography_assignment_receipt, + target_districts=geography_target_districts, + boundary="stacked pool fresh assembly", + require_exact_assembled_rows=True, + ) current = canonicalize_frame_string_dtypes( assembled, boundary="stacked pool assembled checkpoint", in_place=True, ) assembly_receipt = current.metadata[SPINE_ASSEMBLY_MANIFEST_KEY] - receipts: dict[str, Mapping[str, object]] = {} + receipts: dict[str, Mapping[str, object]] = { + "geography_assignment": dict(geography_assignment_receipt) + } qbi_transition_authority_sha256: str | None = None late_producer_transition_authority_sha256: str | None = None resume_stage: str | None = None @@ -2929,6 +3594,11 @@ def mark_phase(name: str) -> None: ) mark_phase("assembled") else: + if geography_assignment_receipt is not None: + raise ValueError( + "A resumed stacked build must use its checkpointed geography " + "assignment receipt rather than redraw geography." + ) current = canonicalize_frame_string_dtypes( resume.frame, boundary=f"stacked pool {resume.stage} resume", @@ -2950,6 +3620,19 @@ def mark_phase(name: str) -> None: receipts = { name: dict(receipt) for name, receipt in resume.stage_receipts.items() } + resumed_geography_receipt = receipts.get("geography_assignment") + if not isinstance(resumed_geography_receipt, Mapping): + raise ValueError( + f"Stacked {resume.stage!r} checkpoint has no geography " + "assignment receipt." + ) + _validate_stacked_geography_assignment_receipt( + current, + resumed_geography_receipt, + target_districts=geography_target_districts, + boundary=f"stacked pool {resume.stage} resume", + require_exact_assembled_rows=resume.stage == "assembled", + ) qbi_transition_authority_sha256 = resume.qbi_transition_authority_sha256 late_producer_transition_authority_sha256 = ( resume.late_producer_transition_authority_sha256 @@ -3319,6 +4002,16 @@ def primary_puf_producer(primary_input: Frame): current, boundary="stacked pool terminal input-only output", ) + terminal_geography_receipt = receipts.get("geography_assignment") + if not isinstance(terminal_geography_receipt, Mapping): # pragma: no cover + raise AssertionError("Stacked terminal output lost geography provenance.") + _validate_stacked_geography_assignment_receipt( + current, + terminal_geography_receipt, + target_districts=geography_target_districts, + boundary="stacked pool terminal input-only output", + require_exact_assembled_rows=False, + ) return StackedPoolBuildResult( frame=current, stack_receipt=dict(expected_stack_receipt), @@ -3472,6 +4165,16 @@ def _stacked_manifest_payload( puf_donor_receipt = input_receipts.get("puf_donor") if not isinstance(puf_donor_receipt, Mapping): raise ValueError("Stacked pool input receipts have no PUF donor object.") + geography_assignment = result.stage_receipts.get("geography_assignment") + if not isinstance(geography_assignment, Mapping): + raise ValueError( + "Stacked pool stage receipts have no geography assignment object." + ) + if _json_ready(geography_assignment.get("contract")) != _json_ready( + _stacked_geography_assignment_contract() + ): + raise ValueError("Stacked pool geography assignment contract changed.") + _assert_stacked_geography_verified_inputs(verified_inputs) gates = _stacked_gate_payload(result) stack_manifest = _json_ready(result.stack_receipt) return { @@ -3505,6 +4208,7 @@ def _stacked_manifest_payload( "fraction": float(clone_attachment_fraction), "seed": clone_attachment_seed, }, + "geography_assignment": _json_ready(geography_assignment), "provenance_pins": { role: pin.to_manifest() for role, pin in verified_inputs.items() }, @@ -3783,6 +4487,16 @@ def _write_stacked_outputs( artifact_kind=POOL_H5_ARTIFACT_KIND, publication_run_id=publication_run_id, materializer_version=US_MULTISPINE_POOL_H5_MATERIALIZER_VERSION, + root_attributes={ + CONGRESSIONAL_DISTRICT_VINTAGE_CROSSWALK_SHA256_ATTR: ( + verified_inputs[ + _STACKED_CD_CROSSWALK_INPUT_ROLE + ].actual_sha256 + ), + CONGRESSIONAL_DISTRICT_VINTAGE_TARGET_ATTR: ( + CURRENT_CONGRESSIONAL_DISTRICT_VINTAGE + ), + }, ) _atomic_write_json(temporary_diagnostics, diagnostics) os.replace(temporary_h5, outputs.pool_h5) @@ -3917,6 +4631,7 @@ def _live_constants_adapter_gate() -> dict[str, object]: "period", "model_seed", "policyengine_us_version", + "geography_assignment", "stacked_authority", "pool_code", ) @@ -4497,6 +5212,14 @@ def _main_stacked(args: argparse.Namespace) -> int: verified_inputs, acs_source_manifest = _verify_inputs(args, outputs) state.input_pins_digest = _input_pins_digest(verified_inputs) _append_phase(state, "inputs_verified") + congressional_district_crosswalk = ( + load_congressional_district_vintage_crosswalk( + args.congressional_district_vintage_crosswalk + ) + ) + geography_target_districts = _crosswalk_target_congressional_districts( + congressional_district_crosswalk + ) stacked_checkpoint_root = _stacked_checkpoint_root( outputs, configured_identity, @@ -4525,6 +5248,7 @@ def _main_stacked(args: argparse.Namespace) -> int: stack_receipt: Mapping[str, object] | None = None puf_donor: pd.DataFrame | None = None acs_rent_donor: pd.DataFrame | None = None + geography_assignment_receipt: Mapping[str, object] | None = None if checkpoint_identity is not None: checkpoint_store = _PoolStageCheckpointStore( outputs.checkpoint_root, @@ -4581,7 +5305,22 @@ def _main_stacked(args: argparse.Namespace) -> int: sample_fraction=args.sample_fraction, sample_seed=args.sample_seed, ) - stack_frame = stack.frame + puma_ladder = load_us_puma_ladder(args.puma_ladder) + ( + stack_frame, + geography_assignment_receipt, + assigned_target_districts, + ) = _assign_stacked_household_geography( + stack.frame, + ladder=puma_ladder, + crosswalk=congressional_district_crosswalk, + ) + if assigned_target_districts != geography_target_districts: + raise ValueError( + "Post-assembly geography assignment changed the authenticated " + "crosswalk target universe." + ) + _append_phase(state, "geography_assigned") stack_receipt = stack.receipt puf_donor = loaded.puf_donor acs_rent_donor = loaded.acs_rent_donor @@ -4623,6 +5362,8 @@ def _main_stacked(args: argparse.Namespace) -> int: result = build_stacked_pool( stack_frame, expected_stack_receipt=stack_receipt, + geography_assignment_receipt=geography_assignment_receipt, + geography_target_districts=geography_target_districts, release_id=state.build_id, puf_donor=puf_donor, acs_rent_donor=acs_rent_donor, diff --git a/tools/inspect_us_fiscal_refresh_checkpoints.py b/tools/inspect_us_fiscal_refresh_checkpoints.py index 0b1ec534..57071ca3 100644 --- a/tools/inspect_us_fiscal_refresh_checkpoints.py +++ b/tools/inspect_us_fiscal_refresh_checkpoints.py @@ -15,11 +15,14 @@ from collections.abc import Callable, Mapping from pathlib import Path +import pandas as pd + from microcosm.build.us_runtime import ( CONGRESSIONAL_DISTRICT_VINTAGE_CROSSWALK_SHA256_ATTR, CONGRESSIONAL_DISTRICT_VINTAGE_TARGET_ATTR, CURRENT_CONGRESSIONAL_DISTRICT_VINTAGE, ) +from microcosm.frame import read_frame_table CALIBRATION_FILENAME = "populace_us_2024_calibration.npz" TARGET_MATERIALIZATION_CACHE_DIRNAME = "target_materialization_cache" @@ -156,11 +159,11 @@ def read_h5_provenance(path: Path) -> Mapping[str, object]: CONGRESSIONAL_DISTRICT_VINTAGE_TARGET_ATTR, ), } - cd_lookup = _h5_table_column_status( - h5, - table="household", - column="congressional_district_geoid", - ) + cd_lookup = _hdf_frame_column_status( + path, + table="household", + column="congressional_district_geoid", + ) except OSError as exc: return { "readable": False, @@ -438,19 +441,23 @@ def _h5_attr_text(attrs: Mapping[str, object], key: str) -> str | None: return str(value) -def _h5_table_column_status(h5, *, table: str, column: str) -> dict[str, object]: - table_path = f"{table}/table" - if table_path not in h5: +def _hdf_frame_column_status( + h5_path: Path, + *, + table: str, + column: str, +) -> dict[str, object]: + try: + with pd.HDFStore(h5_path, mode="r") as store: + frame_table = read_frame_table(store, table) + except KeyError: return {"exists": False, "table": table, "column": column} - dataset = h5[table_path] - names = getattr(dataset.dtype, "names", None) or () - if column not in names: + if column not in frame_table.columns: return {"exists": False, "table": table, "column": column} - values = dataset[column][:] - positive_unique_count = 0 - if values.size: - positive = [value for value in values if _is_positive_number(value)] - positive_unique_count = len(set(positive)) + values = frame_table[column].to_numpy(copy=False) + positive_unique_count = len( + {float(value) for value in values.ravel() if _is_positive_number(value)} + ) return { "exists": True, "table": table, diff --git a/tools/spec_engine_coverage.py b/tools/spec_engine_coverage.py index ad89873c..7a7a1786 100644 --- a/tools/spec_engine_coverage.py +++ b/tools/spec_engine_coverage.py @@ -41,7 +41,7 @@ REPORT_SCHEMA_VERSION = 3 EXPECTED_POINTER_INVENTORY_SHA256 = ( - "6d7353c6c42a6e1dbc6e3a227848e36864526fdc9533d5b284aa469c87dc064f" + "bc4a948ab632191954600da8474c5b011f977a65e24c399d126f3dc4a79f23e5" ) DEFAULT_REPORT_PATH = ( Path(__file__).resolve().parents[1] diff --git a/tools/us_bundle_generation/core.py b/tools/us_bundle_generation/core.py index 35bb6d25..aa6448f1 100644 --- a/tools/us_bundle_generation/core.py +++ b/tools/us_bundle_generation/core.py @@ -54,6 +54,12 @@ _REPOSITORY_ROOT = Path(__file__).resolve().parents[2] _CANONICAL_INPUT_PIN_BUILD_ID = "populace-us-2024-pool-inc2-run7" +_CD_CROSSWALK_PACKAGE = "microcosm.build.us_runtime.data" +_CD_CROSSWALK_RESOURCE = "congressional_district_vintage_crosswalk.csv" +_CD_CROSSWALK_PROVENANCE_RESOURCE = f"{_CD_CROSSWALK_RESOURCE}.provenance.json" +_CD_CROSSWALK_SOURCE_ID = ( + "us_congressional_district_vintage_crosswalk_117_to_119" +) _SHA256 = re.compile(r"^[0-9a-f]{64}$") _EXPECTED_INPUT_ROLES = frozenset( { @@ -235,6 +241,50 @@ def _puma_provenance() -> dict[str, Any]: return payload +def _congressional_district_vintage_crosswalk_provenance() -> dict[str, Any]: + """Verify and return the packaged 117th-to-119th CD authority receipt.""" + + from microcosm.build.us_runtime.congressional_district_vintage import ( + CURRENT_CONGRESSIONAL_DISTRICT_VINTAGE, + load_default_congressional_district_vintage_crosswalk, + ) + + payload = _json_package_resource( + _CD_CROSSWALK_PACKAGE, + _CD_CROSSWALK_PROVENANCE_RESOURCE, + ) + expected_digest = payload.get("crosswalk_sha256") + if not isinstance(expected_digest, str) or not _SHA256.fullmatch( + expected_digest + ): + raise ValueError("US CD-vintage crosswalk has no valid crosswalk_sha256.") + raw = ( + importlib_resources.files(_CD_CROSSWALK_PACKAGE) + .joinpath(_CD_CROSSWALK_RESOURCE) + .read_bytes() + ) + actual_digest = hashlib.sha256(raw).hexdigest() + if actual_digest != expected_digest: + raise ValueError( + "US CD-vintage crosswalk differs from its packaged provenance: " + f"expected={expected_digest}, actual={actual_digest}." + ) + if payload.get("source_geography_vintage") != "117th_congress": + raise ValueError("US CD-vintage crosswalk source must be 117th_congress.") + if payload.get("target_geography_vintage") != ( + CURRENT_CONGRESSIONAL_DISTRICT_VINTAGE + ): + raise ValueError( + "US CD-vintage crosswalk target differs from the runtime current " + "congressional-district vintage." + ) + + # This validates the supported assignment input's columns, district rosters, + # weights, and population-conservation contract in addition to its byte pin. + load_default_congressional_district_vintage_crosswalk() + return {**payload, "byte_size": len(raw)} + + def canonical_input_pins() -> dict[str, dict[str, int | str]]: """Return the six immutable file identities used by the canonical stack. @@ -414,6 +464,24 @@ def build_sources() -> dict[str, Any]: ], } ) + crosswalk = _congressional_district_vintage_crosswalk_provenance() + rows.append( + { + "id": _CD_CROSSWALK_SOURCE_ID, + "role": "congressional_district_vintage_crosswalk", + "sha256": crosswalk["crosswalk_sha256"], + "byte_size": crosswalk["byte_size"], + "loader": "kernel:load_congressional_district_vintage_crosswalk", + "vintages": ["vintage:cd_117", "vintage:cd_119"], + "vintage_authorities": [ + { + "id": "cd_117", + "kind": "geography_vintage", + "value": crosswalk["source_geography_vintage"], + } + ], + } + ) stage_manifest = _source_stage_compatibility() stages = stage_manifest.get("stages") if not isinstance(stages, list): @@ -568,6 +636,7 @@ def build_geography() -> dict[str, Any]: from microcosm.build.us_runtime.puma_ladder import assign_us_puma_ladder puma = _puma_provenance() + crosswalk = _congressional_district_vintage_crosswalk_provenance() layer_vintages = puma.get("layer_vintages") if not isinstance(layer_vintages, Mapping): raise ValueError("US PUMA provenance has no layer_vintages object.") @@ -576,6 +645,7 @@ def build_geography() -> dict[str, Any]: ): raise ValueError("US PUMA and runtime congressional-district vintages differ.") vintage_ref_by_value = { + "117th_congress": "vintage:cd_117", "119th_congress": "vintage:cd_119", "2020_census": "vintage:census_2020", "2020_puma": "vintage:puma_2020", @@ -596,7 +666,7 @@ def build_geography() -> dict[str, Any]: "phase": "legacy", "assignment": { "anchor": "puma", - "order": "legacy_post_transfer", + "order": "before_gap_fill", "kernels": { "assign": "kernel:assign_us_puma_ladder", "validate": "kernel:us_puma_ladder_gate", @@ -621,6 +691,15 @@ def build_geography() -> dict[str, Any]: "geography_state_prefix_consistent", ], "ladder_source": "source:us_puma_ladder_2020", + "congressional_district_vintage_crosswalk": { + "source_ref": f"source:{_CD_CROSSWALK_SOURCE_ID}", + "source_vintage": vintage_ref_by_value[ + str(crosswalk["source_geography_vintage"]) + ], + "target_vintage": vintage_ref_by_value[ + str(crosswalk["target_geography_vintage"]) + ], + }, "seed": "stream:geography_legacy", "default_seed": default_seed, "assign_tract": False, @@ -758,6 +837,17 @@ def build_vintages() -> dict[str, Any]: "authority": "asec_2024", }, }, + { + "id": "cd_117", + "kind": "geography_vintage_ref", + "authority_ref": { + "kind": "source_record", + "source": ( + "source:us_congressional_district_vintage_crosswalk_117_to_119" + ), + "authority": "cd_117", + }, + }, { "id": "cd_119", "kind": "geography_vintage_ref", @@ -860,7 +950,8 @@ def build_vintages() -> dict[str, Any]: "asec_2022": {"target_2024"}, "asec_2023": {"asec_2024", "target_2024"}, "asec_2024": {"asec_2023", "target_2024"}, - "cd_119": {"census_2020", "puma_2020"}, + "cd_117": {"cd_119"}, + "cd_119": {"cd_117", "census_2020", "puma_2020"}, "census_2020": {"cd_119", "puma_2020"}, "org_2024": {"target_2024"}, "policyengine_us_surface": {"target_2024"}, diff --git a/tools/us_bundle_generation/identity_contracts.py b/tools/us_bundle_generation/identity_contracts.py index a384613e..06b2bf04 100644 --- a/tools/us_bundle_generation/identity_contracts.py +++ b/tools/us_bundle_generation/identity_contracts.py @@ -32,7 +32,6 @@ _AUXILIARY_OPERATIONS = ( - "assign_us_puma_ladder", "calibrate", "select_exact_k", ) diff --git a/tools/us_stacked_pool_smoke.sh.example b/tools/us_stacked_pool_smoke.sh.example index 8c667301..121bd4e5 100644 --- a/tools/us_stacked_pool_smoke.sh.example +++ b/tools/us_stacked_pool_smoke.sh.example @@ -16,6 +16,10 @@ set -euo pipefail : "${PUF_SHA:?}" : "${PUF_CSV:?}" : "${PUF_CSV_SHA:?}" +: "${PUMA_LADDER:?}" +: "${PUMA_LADDER_SHA:?}" +: "${CD_CROSSWALK:?}" +: "${CD_CROSSWALK_SHA:?}" : "${OUT:?}" env \ @@ -37,4 +41,8 @@ env \ --puf-h5-sha256 "${PUF_SHA}" \ --puf-source-year-csv "${PUF_CSV}" \ --puf-source-year-csv-sha256 "${PUF_CSV_SHA}" \ + --puma-ladder "${PUMA_LADDER}" \ + --puma-ladder-sha256 "${PUMA_LADDER_SHA}" \ + --congressional-district-vintage-crosswalk "${CD_CROSSWALK}" \ + --congressional-district-vintage-crosswalk-sha256 "${CD_CROSSWALK_SHA}" \ --out "${OUT}"